import { useCallback, useEffect, useRef, useState } from "react"; import { useServerFn } from "@tanstack/react-start"; import { askEcho, saveEchoCall } from "@/lib/gemini.functions"; import type { LiveEvent } from "./use-live-voice"; type Recognition = { lang: string; continuous: boolean; interimResults: boolean; start(): void; abort(): void; onresult: ((event: { results: ArrayLike> }) => void) | null; onerror: ((event: { error: string }) => void) | null; onend: (() => void) | null; }; type SpeechWindow = Window & { SpeechRecognition?: new () => Recognition; webkitSpeechRecognition?: new () => Recognition }; type Message = { role: "user" | "assistant"; text: string }; export function useBrowserVoice({ onEvent }: { onEvent: (event: LiveEvent) => void }) { const ask = useServerFn(askEcho); const save = useServerFn(saveEchoCall); const latest = useRef({ ask, save, onEvent }); latest.current = { ask, save, onEvent }; const [status, setStatus] = useState<"idle" | "connecting" | "connected" | "stopping" | "closed">("idle"); const [phase, setPhase] = useState<"listening" | "thinking" | "speaking">("listening"); const [error, setError] = useState(null); const [muted, setMute] = useState(false); const state = useRef({ active: false, muted: false, busy: false, generation: 0, messages: [] as Message[], startedAt: "", recognition: null as Recognition | null, timer: undefined as ReturnType | undefined }); const mounted = useRef(true); const stop = useCallback(() => { const s = state.current; if (!s.active) return; s.active = false; s.generation++; clearTimeout(s.timer); s.recognition?.abort(); s.recognition = null; window.speechSynthesis?.cancel(); if (mounted.current) setStatus("stopping"); const messages = [...s.messages]; const finish = async () => { try { if (messages.some(m => m.role === "user")) await latest.current.save({ data: { messages, startedAt: s.startedAt } }); } catch { if (mounted.current) setError("The conversation could not be saved."); } finally { if (mounted.current && !state.current.active) { setStatus("closed"); setMute(false); latest.current.onEvent({ type: "app.closed" }); } } }; void finish(); }, []); const listen = useRef<() => void>(() => {}); listen.current = () => { const s = state.current; if (!s.active || s.muted || s.busy) return; try { s.recognition?.start(); setPhase("listening"); } catch { /* A recognition already starting must not create a duplicate listener. */ } }; const start = useCallback((greeting?: unknown) => { const s = state.current; if (s.active) return; const browser = window as SpeechWindow; const Constructor = browser.SpeechRecognition ?? browser.webkitSpeechRecognition; if (!Constructor || !window.speechSynthesis) { setError("Voice is not supported in this browser. Try Chrome or Edge."); return; } setError(null); setStatus("connecting"); setMute(false); s.active = true; s.muted = false; s.busy = false; s.messages = []; s.startedAt = new Date().toISOString(); s.generation++; latest.current.onEvent({ type: "app.started" }); const recognition = new Constructor(); s.recognition = recognition; recognition.lang = navigator.language || "en-US"; recognition.continuous = false; recognition.interimResults = false; recognition.onresult = event => { if (!s.active || s.muted || s.busy) return; const text = Array.from(event.results).map(r => r[0]?.transcript ?? "").join(" ").trim(); if (!text) return; if (s.messages.length >= 48) { setError("This call is full. Start a new call to continue."); stop(); return; } s.busy = true; recognition.abort(); setPhase("thinking"); const generation = s.generation; s.messages.push({ role: "user", text }); latest.current.onEvent({ type: "session.input_transcript.delta", delta: text }); void latest.current.ask({ data: { messages: [...s.messages] } }).then(result => { if (!s.active || generation !== s.generation) return; if (result.error || !result.answer) { setError(result.error ?? "Echo returned no answer."); stop(); return; } s.messages.push({ role: "assistant", text: result.answer }); latest.current.onEvent({ type: "session.output_transcript.delta", delta: result.answer }); const spoken = result.answer.replace(/```[\s\S]*?```/g, "The code is shown on screen.").replace(/[*#`]/g, ""); const utterance = new SpeechSynthesisUtterance(spoken); utterance.lang = recognition.lang; const resume = () => { if (!s.active || generation !== s.generation) return; s.busy = false; s.timer = setTimeout(() => listen.current(), 250); }; utterance.onend = resume; utterance.onerror = () => { if (s.active && generation === s.generation) { setError("Voice playback failed."); resume(); } }; setPhase("speaking"); window.speechSynthesis.speak(utterance); }).catch(() => { if (s.active && generation === s.generation) { setError("Echo could not answer. Start a new call later."); stop(); } }); }; recognition.onerror = event => { if (!s.active || event.error === "aborted" || event.error === "no-speech") return; setError(event.error === "not-allowed" ? "Microphone access was denied. Allow microphone access and try again." : "Speech recognition failed. Check your connection and try again."); stop(); }; recognition.onend = () => { if (s.active && !s.muted && !s.busy) s.timer = setTimeout(() => listen.current(), 250); }; const begin = () => { try { recognition.start(); setStatus("connected"); setPhase("listening"); } catch { setError("Microphone startup failed. Try a new call."); stop(); } }; if (typeof greeting === "string" && greeting) { const generation = s.generation; s.busy = true; setStatus("connected"); setPhase("speaking"); s.messages.push({ role: "assistant", text: greeting }); latest.current.onEvent({ type: "session.output_transcript.delta", delta: greeting }); const utterance = new SpeechSynthesisUtterance(greeting); utterance.lang = recognition.lang; const after = () => { if (!s.active || generation !== s.generation) return; s.busy = false; begin(); }; utterance.onend = after; utterance.onerror = after; window.speechSynthesis.cancel(); window.speechSynthesis.speak(utterance); } else begin(); }, [stop]); const setMuted = useCallback((value: boolean) => { state.current.muted = value; setMute(value); if (value) { clearTimeout(state.current.timer); state.current.recognition?.abort(); } else listen.current(); }, []); useEffect(() => { mounted.current = true; window.addEventListener("pagehide", stop); return () => { mounted.current = false; window.removeEventListener("pagehide", stop); stop(); }; }, [stop]); return { status, phase, error, muted, start, stop, setMuted }; }