import { useCallback, useEffect, useRef, useState } from "react"; import { useServerFn } from "@tanstack/react-start"; import { buildEchoSite, deployEchoBuild, listEchoProjects, lookupEchoHistory, openEchoProject, saveEchoCall, startEchoLive } from "@/lib/gemini.functions"; import { connectCustomDomain } from "@/lib/codehub.functions"; import { ECHO_LIVE_MODEL } from "@/lib/gemini-policy"; import type { LiveEvent } from "./use-live-voice"; type Message = { role: "user" | "assistant"; text: string }; type Status = "idle" | "connecting" | "connected" | "stopping" | "closed"; export type GeminiToolCall = { name: string; id: string; args?: Record }; type VoiceOptions = { onEvent: (event: LiveEvent) => void; startSession?: (name: string) => Promise<{ token: string } | { error: string }>; onToolCall?: (call: GeminiToolCall) => Promise; saveCall?: boolean; assistantName?: string; }; const WS_BASE = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained"; // Captures mic audio and posts Float32 frames to the main thread. const WORKLET = `class P extends AudioWorkletProcessor{process(i){const c=i[0]&&i[0][0];if(c)this.port.postMessage(c.slice(0));return true}}registerProcessor("echo-mic",P)`; function toPcm16Base64(input: Float32Array, fromRate: number) { const ratio = fromRate / 16000; const length = Math.floor(input.length / ratio); const out = new Int16Array(length); for (let i = 0; i < length; i++) { const s = Math.max(-1, Math.min(1, input[Math.floor(i * ratio)] ?? 0)); out[i] = s < 0 ? s * 0x8000 : s * 0x7fff; } const bytes = new Uint8Array(out.buffer); let bin = ""; for (let i = 0; i < bytes.length; i += 0x8000) bin += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); return btoa(bin); } function fromPcm16Base64(data: string) { const bin = atob(data); const view = new DataView(new ArrayBuffer(bin.length)); for (let i = 0; i < bin.length; i++) view.setUint8(i, bin.charCodeAt(i)); const out = new Float32Array(bin.length / 2); for (let i = 0; i < out.length; i++) out[i] = view.getInt16(i * 2, true) / 0x8000; return out; } export function useGeminiLive(options: VoiceOptions) { const startFn = useServerFn(startEchoLive); const save = useServerFn(saveEchoCall); const lookupHistory = useServerFn(lookupEchoHistory); const build = useServerFn(buildEchoSite); const deployFn = useServerFn(deployEchoBuild); const listProjects = useServerFn(listEchoProjects); const openProject = useServerFn(openEchoProject); const connectDomain = useServerFn(connectCustomDomain); const latest = useRef({ startFn, save, lookupHistory, build, deployFn, listProjects, openProject, connectDomain, ...options }); latest.current = { startFn, save, lookupHistory, build, deployFn, listProjects, openProject, connectDomain, ...options }; const [sessionId, setSessionId] = useState(null); const [status, setStatus] = useState("idle"); const [phase, setPhase] = useState<"listening" | "speaking">("listening"); const [error, setError] = useState(null); const [muted, setMute] = useState(false); const s = useRef({ active: false, generation: 0, deadline: undefined as ReturnType | undefined, muted: false, ws: null as WebSocket | null, stream: null as MediaStream | null, micCtx: null as AudioContext | null, outCtx: null as AudioContext | null, sources: new Set(), playAt: 0, messages: [] as Message[], startedAt: "", sessionId: "", lastHtml: "", lastBuildId: "", }); // Builds (or fixes) the page and keeps it as the base for the next voice change. const runBuild = async (request: string, fix = false) => { const c = s.current; if (!c.sessionId) return { error: "Start a call first." } as const; latest.current.onEvent({ type: "app.build.start", request, fix }); try { const r = await latest.current.build({ data: { sessionId: c.sessionId, request, previous: c.lastHtml || undefined } }); if ("error" in r) { latest.current.onEvent({ type: "app.build.error", message: r.error }); return { error: r.error } as const; } c.lastHtml = r.html; c.lastBuildId = r.id; latest.current.onEvent({ type: "app.build.done", html: r.html, id: r.id, fix }); return { ok: true } as const; } catch { latest.current.onEvent({ type: "app.build.error", message: "Build failed." }); return { error: "Build failed." } as const; } }; const runDeploy = async () => { const c = s.current; if (!c.lastBuildId) return { error: "Nothing has been built yet in this call." } as const; latest.current.onEvent({ type: "app.deploy.start", id: c.lastBuildId }); try { const r = await latest.current.deployFn({ data: { buildId: c.lastBuildId, apiBase: window.location.origin } }); if ("error" in r) { latest.current.onEvent({ type: "app.deploy.error", message: r.error }); return { error: r.error } as const; } latest.current.onEvent({ type: "app.deploy.done", url: r.url, id: c.lastBuildId }); return { url: r.url } as const; } catch { latest.current.onEvent({ type: "app.deploy.error", message: "Publishing failed." }); return { error: "Publishing failed." } as const; } }; const flushAudio = () => { for (const src of s.current.sources) { try { src.stop(); } catch { /* already stopped */ } } s.current.sources.clear(); s.current.playAt = 0; }; const addText = (role: Message["role"], text: string) => { if (!text) return; const msgs = s.current.messages; const last = msgs[msgs.length - 1]; if (last && last.role === role) last.text += text; else msgs.push({ role, text }); latest.current.onEvent({ type: role === "user" ? "session.input_transcript.delta" : "session.output_transcript.delta", delta: text, }); }; const teardown = () => { const c = s.current; c.active = false; c.generation += 1; clearTimeout(c.deadline); flushAudio(); c.ws?.close(); c.ws = null; c.stream?.getTracks().forEach((t) => t.stop()); c.stream = null; void c.micCtx?.close().catch(() => {}); void c.outCtx?.close().catch(() => {}); c.micCtx = null; c.outCtx = null; }; const stop = useCallback(() => { const c = s.current; if (!c.active) return; teardown(); setMute(false); c.muted = false; setStatus("stopping"); const messages = c.messages.map((m) => ({ ...m, text: m.text.trim() })).filter((m) => m.text).slice(-50); void (async () => { try { if (messages.length && latest.current.saveCall !== false) await latest.current.save({ data: { messages, startedAt: c.startedAt, sessionId: c.sessionId } }); } catch { setError("The call ended, but it could not be saved."); } setStatus("closed"); latest.current.onEvent({ type: "app.closed" }); })(); }, []); const start = useCallback(async (name: string) => { const c = s.current; if (c.active) return; c.active = true; const generation = ++c.generation; const current = () => c.active && c.generation === generation; let setupReceived = false; const cancelledTools = new Set(); c.messages = []; c.startedAt = new Date().toISOString(); c.sessionId = crypto.randomUUID(); c.lastHtml = ""; c.lastBuildId = ""; setSessionId(c.sessionId); setError(null); setStatus("connecting"); setPhase("listening"); latest.current.onEvent({ type: "app.started" }); const fail = (message: string) => { if (!current()) return; teardown(); c.muted = false; setMute(false); setError(message); setStatus("closed"); }; try { // Created inside the tap so browsers allow mic + sound. const outCtx = new AudioContext({ sampleRate: 24000 }); const micCtx = new AudioContext(); c.outCtx = outCtx; c.micCtx = micCtx; void outCtx.resume(); void micCtx.resume(); if (!navigator.mediaDevices?.getUserMedia) return fail("This browser can't use the microphone. Open the site in Chrome or Safari."); const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } }); if (!current()) { stream.getTracks().forEach(t => t.stop()); return; } c.stream = stream; const result = await (latest.current.startSession ? latest.current.startSession(name) : latest.current.startFn({ data: { name } })); if (!current()) return; if ("error" in result) return fail(result.error ?? "Live voice could not start."); const ws = new WebSocket(`${WS_BASE}?access_token=${encodeURIComponent(result.token)}`); c.ws = ws; c.deadline = setTimeout(() => fail("Gemini voice did not connect. Start a new call to retry."), 20000); ws.onopen = () => { if (current()) ws.send(JSON.stringify({ setup: {} })); }; ws.onerror = () => fail("Lost connection to Gemini live voice."); ws.onclose = (e) => { if (!current() || c.ws !== ws) return; if (e.code === 1000) stop(); else fail(e.reason?.includes("quota") ? "Today's free Gemini voice limit is reached." : "Live voice disconnected. Tap Reconnect."); }; ws.onmessage = async (event) => { try { const raw = typeof event.data === "string" ? event.data : await (event.data as Blob).text(); if (!current() || c.ws !== ws) return; const msg = JSON.parse(raw) as { setupComplete?: unknown; toolCall?: { functionCalls?: GeminiToolCall[] }; toolCallCancellation?: { ids?: string[] }; serverContent?: { modelTurn?: { parts?: { inlineData?: { data?: string }; functionCall?: { name?: string; id?: string; args?: { request?: string } } }[] }; inputTranscription?: { text?: string }; outputTranscription?: { text?: string }; interrupted?: boolean; turnComplete?: boolean; }; }; for (const id of msg.toolCallCancellation?.ids ?? []) cancelledTools.add(id); if (msg.setupComplete) { if (setupReceived) return; setupReceived = true; const workletUrl = URL.createObjectURL(new Blob([WORKLET], { type: "text/javascript" })); try { await micCtx.audioWorklet.addModule(workletUrl); } finally { URL.revokeObjectURL(workletUrl); } if (!current()) return; const node = new AudioWorkletNode(micCtx, "echo-mic"); micCtx.createMediaStreamSource(stream).connect(node); let buffer: number[] = []; node.port.onmessage = (e: MessageEvent) => { if (!current() || c.muted || ws.readyState !== WebSocket.OPEN) return; for (const v of e.data) buffer.push(v); if (buffer.length < micCtx.sampleRate / 10) return; // ~100ms chunks const data = toPcm16Base64(Float32Array.from(buffer), micCtx.sampleRate); buffer = []; ws.send(JSON.stringify({ realtimeInput: { audio: { data, mimeType: "audio/pcm;rate=16000" } } })); }; setStatus("connected"); clearTimeout(c.deadline); // Trigger one opening after media capture is ready. ws.send(JSON.stringify({ realtimeInput: { text: `(${name} just connected. Greet them now.)` } })); return; } if (msg.toolCall?.functionCalls && latest.current.onToolCall) { const responses = []; for (const call of msg.toolCall.functionCalls) { if (!current()) return; if (typeof call.id !== "string" || typeof call.name !== "string") throw new Error("Invalid voice tool request."); if (cancelledTools.has(call.id)) continue; let response: unknown; try { response = await latest.current.onToolCall(call); } catch { response = { error: "That security request could not complete. Please try from the workspace." }; } if (!cancelledTools.has(call.id)) responses.push({ id: call.id, name: call.name, response }); } if (responses.length && current() && c.ws === ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ toolResponse: { functionResponses: responses } })); } const sc = msg.serverContent; if (!sc && !msg.toolCall) return; if (sc?.interrupted) { flushAudio(); setPhase("listening"); } addText("user", sc?.inputTranscription?.text ?? ""); addText("assistant", sc?.outputTranscription?.text ?? ""); const calls = latest.current.onToolCall ? [] : msg.toolCall?.functionCalls?.map(functionCall => ({ functionCall })) ?? (sc?.modelTurn?.parts ?? []).filter((p) => p.functionCall?.name); if (calls.length && ws.readyState === WebSocket.OPEN) { void (async () => { const functionResponses = await Promise.all( calls.map(async (p) => { const name = p.functionCall?.name ?? ""; const id = p.functionCall?.id; if (name === "build_website") { const request = String(p.functionCall?.args?.request ?? "").slice(0, 4000); if (!request) return { id, name, response: { error: "No description given." } }; const r = await runBuild(request); return { id, name, response: "error" in r ? { error: r.error } : { result: "The live preview is now on the user's screen. A live test result will follow shortly as a note." } }; } if (name === "deploy_website") { const r = await runDeploy(); return { id, name, response: "error" in r ? { error: r.error } : { result: `Published. Live address: ${r.url} (it can take up to a minute to appear).` } }; } if (name === "connect_domain") { const domain = String((p.functionCall?.args as Record | undefined)?.["domain"] ?? "").slice(0, 253); if (!c.sessionId) return { id, name, response: { error: "Open or build and publish a project first." } }; if (!domain) return { id, name, response: { error: "Say which domain to connect." } }; try { const r = await latest.current.connectDomain({ data: { kind: "echo", id: c.sessionId, domain } }); if ("error" in r) return { id, name, response: { error: r.error } }; const dns = r.dns === "manual" ? `The caller must add a CNAME record for ${r.domain} pointing to ${r.target} at their domain provider.` : "DNS was set up automatically in Cloudflare."; return { id, name, response: { result: `Connected ${r.domain} (status ${r.status}). ${dns}` } }; } catch { return { id, name, response: { error: "The domain could not be connected. Check the spelling, e.g. mywebsite.com." } }; } } if (name === "list_projects") { try { const r = await latest.current.listProjects(); if ("error" in r) return { id, name, response: { error: r.error } }; return { id, name, response: { projects: r.projects.slice(0, 15).map((p, i) => ({ number: i + 1, title: p.title, versions: p.versions, updated: p.updated })) } }; } catch { return { id, name, response: { error: "Projects are unavailable right now." } }; } } if (name === "open_project") { const query = String((p.functionCall?.args as Record | undefined)?.["query"] ?? "").slice(0, 200); if (!query) return { id, name, response: { error: "Say which project to open." } }; try { const r = await latest.current.openProject({ data: { query } }); if ("error" in r) return { id, name, response: { error: r.error } }; c.sessionId = r.project.project_id; c.lastHtml = r.build.html; c.lastBuildId = r.build.id; setSessionId(c.sessionId); latest.current.onEvent({ type: "app.project.opened", sessionId: c.sessionId, title: r.project.title }); latest.current.onEvent({ type: "app.build.done", html: r.build.html, id: r.build.id, fix: true }); return { id, name, response: { result: `Opened "${r.project.title}" (${r.project.versions} versions). It is on screen and new changes continue it.` } }; } catch { return { id, name, response: { error: "That project could not be opened." } }; } } if (name !== "get_conversation_history") return { id, name, response: { error: "Unsupported action." } }; try { return { id, name, response: await latest.current.lookupHistory({}) }; } catch { return { id, name, response: { error: "History is unavailable right now." } }; } }), ); if (current() && c.ws === ws && ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ toolResponse: { functionResponses } })); })(); } for (const part of sc?.modelTurn?.parts ?? []) { const data = part.inlineData?.data; if (!data) continue; const samples = fromPcm16Base64(data); const buf = outCtx.createBuffer(1, samples.length, 24000); buf.copyToChannel(samples, 0); const src = outCtx.createBufferSource(); src.buffer = buf; src.connect(outCtx.destination); const at = Math.max(outCtx.currentTime, c.playAt); src.start(at); c.playAt = at + buf.duration; c.sources.add(src); setPhase("speaking"); src.onended = () => { c.sources.delete(src); if (current() && c.sources.size === 0) setPhase("listening"); }; } } catch { fail("The voice connection received an invalid response. Start a new call."); } }; } catch (err) { const denied = err instanceof DOMException && err.name === "NotAllowedError"; fail(denied ? "Microphone access was blocked. Allow the mic and tap Reconnect." : "Live voice could not start. Try again."); } }, [stop]); const setMuted = useCallback((value: boolean) => { s.current.muted = value; setMute(value); }, []); useEffect(() => { const hide = () => stop(); window.addEventListener("pagehide", hide); return () => { window.removeEventListener("pagehide", hide); teardown(); }; }, [stop]); const fixBuild = useCallback((errors: string[]) => runBuild(`Live testing found these errors in the current page. Fix them and keep everything else the same:\n${errors.slice(0, 10).join("\n")}`, true), []); const deploy = useCallback(() => runDeploy(), []); // Tells Echo the live test outcome so he can say it out loud. const reportTest = useCallback((summary: string) => { const ws = s.current.ws; if (s.current.active && ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ realtimeInput: { text: `(Live test note: ${summary.slice(0, 600)})` } })); }, []); return { status, phase, error, muted, start, stop, setMuted, sessionId, fixBuild, deploy, reportTest, model: ECHO_LIVE_MODEL }; }