import { createOpenAI } from "@ai-sdk/openai"; import { stepCountIs, streamText, tool, type ModelMessage } from "ai"; import process from "node:process"; import { z } from "zod"; export type LiveConfig = { baseURL: string; key: string; liveModel: string; backendModel: string; openingInstructions?: string; }; // Fill from co-loaded knowledge: the gateway URL, the Live model and the resolved chat model. const liveSettings = { baseURL: "https://ai.gateway.lovable.dev/v1", liveModel: "openai/gpt-live-1", backendModel: "openai/gpt-6-astra", }; export type LiveSocket = { readonly readyState: number; send(data: string): void; close(code?: number, reason?: string): void; onMessage(handler: (data: unknown) => void): void; onClose(handler: () => void): void; onError(handler: () => void): void; }; export type LiveExecutionContext = { waitUntil(task: Promise): void }; export type LiveConnector = (url: string, headers: Record, signal: AbortSignal) => Promise; type WorkerSocket = WebSocket & { accept(): void }; declare const WebSocketPair: { new (): { 0: WorkerSocket; 1: WorkerSocket } }; type Transcript = { role: "user" | "assistant"; text: string; start_ms: number; end_ms: number; listeningSound: boolean; }; type ProviderEvent = { type: string; client_event_id?: string; error?: { client_event_id?: string; message?: string }; session?: { id: string }; delta?: string; start_ms?: number; end_ms?: number; offset_ms?: number; delegation?: { id: string; target: string }; }; export function getLiveConfig(): LiveConfig { const config: LiveConfig = { ...liveSettings, key: process.env["LOVABLE_API_KEY"] ?? "", }; if ([config.baseURL, config.key, config.liveModel, config.backendModel].some((value) => !value)) { throw new Error("Missing Live relay configuration"); } return config; } function gatewayAPIBase(baseURL: string) { return `${baseURL.replace(/\/+$/, "").replace(/\/v1$/, "")}/v1`; } async function gatewayRejection(response: Response) { const body = (await response.json().catch(() => null)) as { message?: unknown } | null; const message = typeof body?.message === "string" ? body.message.slice(0, 300) : ""; return new Error(message || `Voice gateway rejected the connection (${response.status})`); } export function validateLiveUpgrade(request: Request, options: { allowMissingOrigin?: boolean } = {}): Response | null { const origin = request.headers.get("origin"); if (origin === null ? !options.allowMissingOrigin : origin !== new URL(request.url).origin) { return new Response("Voice connection origin rejected", { status: 403 }); } if (request.method !== "GET" || request.headers.get("upgrade")?.toLowerCase() !== "websocket") { return new Response("WebSocket required", { status: 426 }); } return null; } function workerSocket(socket: WorkerSocket): LiveSocket { return { get readyState() { return socket.readyState; }, send: (data) => socket.send(data), close: (code, reason) => socket.close(code, reason), onMessage: (handler) => socket.addEventListener("message", (event) => handler(event.data)), onClose: (handler) => socket.addEventListener("close", handler), onError: (handler) => socket.addEventListener("error", handler), }; } export function handleLiveRequest(request: Request): Response { const waitUntil = (request as Request & Partial).waitUntil; if (!waitUntil) return new Response("Live runtime unavailable", { status: 503 }); const config = getLiveConfig(); const rejected = validateLiveUpgrade(request); if (rejected) return rejected; const pair = new WebSocketPair(); pair[1].accept(); bindLiveConnection(workerSocket(pair[1]), config, { waitUntil }, async (url, headers, signal) => { const response = await fetch(url, { headers: { ...headers, Upgrade: "websocket" }, signal }); const socket = (response as Response & { webSocket?: WorkerSocket | null }).webSocket; if (!socket) throw await gatewayRejection(response); socket.accept(); return workerSocket(socket); }); const response: ResponseInit & { webSocket: WebSocket } = { status: 101, webSocket: pair[0] }; return new Response(null, response); } const conversationInstructions = `You are Echo, the hands-free voice assistant of this website. Speak naturally in short, friendly replies. Reply in the language the visitor speaks. Backchannel policy: Use light listening sounds without taking over. Interruption policy: Stop your answer and listen when the user interrupts. Delegation policy: Backend tools: Look up the saved history of past voice conversations shown on the dashboard. Delegate to the backend when: The user asks about the dashboard, past conversations, history, statistics, what someone asked before, or anything needing careful reasoning. Do not delegate to the backend when: Greeting, small talk, or clarifying a question. Wait for the backend result before presenting its answer. Never invent history.`; const historyInput = z .object({ limit: z.number().int().min(1).max(50), search: z.string().nullable(), }) .strict(); export async function lookupHistory(args: z.infer) { const { supabaseAdmin } = await import("@/integrations/supabase/client.server"); const { data: conversations, error } = await supabaseAdmin .from("voice_conversations") .select("id, started_at, ended_at, turn_count, voice_messages(position, role, content)") .order("started_at", { ascending: false }) .limit(args.limit); if (error) throw new Error("History lookup failed"); const { count } = await supabaseAdmin.from("voice_conversations").select("id", { count: "exact", head: true }); const term = args.search?.trim().toLowerCase(); const rows = (conversations ?? []) .map((c) => ({ started_at: c.started_at, ended_at: c.ended_at, turns: c.turn_count, messages: [...(c.voice_messages ?? [])] .sort((a, b) => a.position - b.position) .map((m) => `${m.role}: ${m.content.slice(0, 300)}`), })) .filter((c) => !term || c.messages.some((m) => m.toLowerCase().includes(term))); return { total_conversations: count ?? rows.length, conversations: rows }; } export async function saveConversation(transcripts: Transcript[], startedAt: Date, sessionId?: string, userId?: string) { const turns: Array<{ role: string; content: string }> = []; for (const t of [...transcripts].sort((a, b) => a.start_ms - b.start_ms)) { const last = turns[turns.length - 1]; if (last && last.role === t.role) last.content += t.text; else turns.push({ role: t.role, content: t.text }); } const cleaned = turns.map((t) => ({ ...t, content: t.content.trim() })).filter((t) => t.content); if (!cleaned.some((t) => t.role === "user")) return; const { supabaseAdmin } = await import("@/integrations/supabase/client.server"); const { data, error } = await supabaseAdmin .from("voice_conversations") .insert({ started_at: startedAt.toISOString(), ended_at: new Date().toISOString(), turn_count: cleaned.length, session_id: sessionId ?? null, user_id: userId ?? null }) .select("id") .single(); if (error || !data) throw new Error("Saving conversation failed"); const { error: msgError } = await supabaseAdmin .from("voice_messages") .insert(cleaned.map((t, i) => ({ conversation_id: data.id, position: i, role: t.role, content: t.content }))); if (msgError) throw new Error("Saving messages failed"); } function isListeningSound(text: string) { const normalized = text.toLowerCase().replace(/[\s\p{Pd}]/gu, ""); return /^(?:m+hm+|uhhuh)[.,!]*$/.test(normalized); } async function answerQuestion( messages: ModelMessage[], config: LiveConfig, correlation: { runID: string; sessionID: string | undefined; delegationID: string }, signal: AbortSignal, consumeInput: () => void, ) { signal.throwIfAborted(); const provider = createOpenAI({ baseURL: gatewayAPIBase(config.baseURL), apiKey: config.key, headers: { "Lovable-API-Key": config.key, "X-Lovable-AIG-SDK": "vercel-ai-sdk", }, }); let responseCursor = 0; consumeInput(); const result = streamText({ model: provider.responses(config.backendModel), abortSignal: signal, maxRetries: 0, stopWhen: stepCountIs(50), includeRawChunks: true, prepareStep() { consumeInput(); return { messages: [...messages] }; }, onStepFinish(step) { messages.push(...step.response.messages.slice(responseCursor)); responseCursor = step.response.messages.length; }, headers: { "X-Lovable-AIG-Run-ID": correlation.runID, "X-Lovable-AIG-Metadata": JSON.stringify({ live_session_id: correlation.sessionID, delegation_id: correlation.delegationID, }), }, providerOptions: { openai: { store: false, ...(config.backendModel !== "openai/chat-latest" ? { forceReasoning: true, reasoningEffort: "medium", reasoningSummary: "auto", include: ["reasoning.encrypted_content"], } : {}), }, }, system: "Help a spoken website voice assistant answer the latest user question. " + "Transcripts may be incomplete or corrected. Use the latest correction. " + "Continue from completed tool results; do not repeat completed actions. " + "Return verified facts and useful next steps in at most 150 words. " + "For questions about the dashboard, history, or past conversations, call get_conversation_history. " + "Use only data the tool returns; never invent history. Answer in plain spoken sentences without markdown. " + "Reply in the user's language. You have no other tools.", messages, tools: { get_conversation_history: tool({ description: "Read saved voice conversations from the dashboard, newest first, with total count. Optionally filter by a search word.", inputSchema: historyInput, execute: async (args) => { signal.throwIfAborted(); return lookupHistory(args); }, }), }, }); let completed = false; let stepCompleted = false; let failed = false; // Drain through HTTP EOF so successful work is not recorded as cancelled by the Gateway. for await (const part of result.fullStream) { if (part.type === "start-step") stepCompleted = false; if (part.type === "raw" && part.rawValue && typeof part.rawValue === "object" && "type" in part.rawValue) { if (part.rawValue.type === "response.completed") stepCompleted = true; if (part.rawValue.type === "response.failed" || part.rawValue.type === "response.incomplete") failed = true; } if (part.type === "finish-step" && !stepCompleted) failed = true; if (part.type === "error" || part.type === "abort") failed = true; if (part.type === "finish") completed = part.finishReason === "stop"; } signal.throwIfAborted(); if (failed || !completed) throw new Error("The backend response did not complete"); const answer = await result.text; if (!answer.trim()) throw new Error("The backend response had no answer"); return answer; } function* commentaryChunks(content: string) { const encoder = new TextEncoder(); let chunk = ""; let bytes = 0; for (const [word] of content.matchAll(/\S+\s*|\s+/gu)) { // Chunks stay well inside the provider's per-append size limit. if (chunk && bytes + encoder.encode(word).length > 480) { yield chunk; chunk = ""; bytes = 0; } for (const character of word) { const size = encoder.encode(character).length; if (bytes + size > 480) { yield chunk; chunk = ""; bytes = 0; } chunk += character; bytes += size; } } if (chunk) yield chunk; } export function bindLiveConnection( browser: LiveSocket, configuration: LiveConfig, execution: LiveExecutionContext, connect: LiveConnector, ): void { const config = { ...configuration }; const runID = crypto.randomUUID(); const setupAbort = new AbortController(); let gateway: LiveSocket | undefined; let sessionID: string | undefined; let starting = false; let closing = false; let finished = false; let revision = 0; let task: AbortController | undefined; const pendingDelegations: Array<{ event: ProviderEvent; }> = []; let completedDelegation: (typeof pendingDelegations)[number] | undefined; const backendMessages: ModelMessage[] = []; let transcriptCursor = 0; let restartTimer: ReturnType | undefined; let startupTimer: ReturnType | undefined; let closeTimer: ReturnType | undefined; let finishDrain: (() => void) | undefined; let browserReady = false; let greetingRequested = false; let greetingCommand: { id: string; type: "instructions" | "commentary" } | undefined; let greetingTimer: ReturnType | undefined; const startTimer = setTimeout(() => { emit({ type: "app.error", error: { message: "Voice startup message timed out" } }); stop(); }, 5000); const transcripts: Transcript[] = []; const startedAt = new Date(); let saved = false; const delegations = new Set(); function emit(event: object) { if (browser.readyState !== 1) return; try { browser.send(JSON.stringify(event)); } catch { stop(); } } function close(socket?: LiveSocket) { if (!socket || socket.readyState === 3) return; try { socket.close(1000, "Call ended"); } catch { return; } } function clearGreeting() { clearTimeout(greetingTimer); greetingCommand = undefined; } function requestGreeting() { const content = ( config.openingInstructions ?? "Greet the visitor in one short sentence as Echo, say they can ask about the dashboard history or anything else, then listen." ).trim(); if (!content || !browserReady || !sessionID || greetingRequested || closing) return; greetingRequested = true; if (new TextEncoder().encode(content).length > 480) { emit({ type: "app.greeting.error", error: { message: "The opening instructions are too long" } }); return; } greetingCommand = { id: crypto.randomUUID(), type: "instructions" }; greetingTimer = setTimeout(() => { clearGreeting(); emit({ type: "app.greeting.error", error: { message: "The opening could not be confirmed" } }); }, 10_000); gateway?.send( JSON.stringify({ type: "session.instructions.append", event_id: greetingCommand.id, delegation_id: null, content, }), ); } function discardPendingWork() { pendingDelegations.length = 0; completedDelegation = undefined; clearTimeout(restartTimer); task?.abort(); } function finish() { if (finished) return; finished = closing = true; if (!saved) { saved = true; execution.waitUntil(saveConversation(transcripts, startedAt).catch((e) => console.error(e))); } clearTimeout(startTimer); clearTimeout(startupTimer); clearTimeout(closeTimer); clearTimeout(restartTimer); clearGreeting(); discardPendingWork(); setupAbort.abort(); close(gateway); close(browser); finishDrain?.(); } function stop() { if (closing) return; closing = true; discardPendingWork(); clearTimeout(startupTimer); clearTimeout(startTimer); clearGreeting(); if (gateway?.readyState === 1) { execution.waitUntil( new Promise((resolve) => { finishDrain = resolve; }), ); closeTimer = setTimeout(finish, 15_000); try { gateway.send(JSON.stringify({ type: "session.close" })); } catch { finish(); } } else { finish(); } } function scheduleDelegation() { clearTimeout(restartTimer); if (closing || task || pendingDelegations.length === 0) return; restartTimer = setTimeout(() => void runDelegation(), 300); } function deliverResult(delegationID: string, answer: string) { if (closing) return; if (gateway?.readyState !== 1) return stop(); try { for (const content of commentaryChunks(answer)) { gateway.send( JSON.stringify({ type: "session.commentary.append", event_id: crypto.randomUUID(), delegation_id: delegationID, content, }), ); } completedDelegation = pendingDelegations.shift(); } catch { stop(); } } async function runDelegation() { const pending = pendingDelegations[0]; const event = pending?.event; const id = event?.delegation?.id; if (!pending || !event || !id || closing || task) return; if (!transcripts.some(({ role, text }) => role === "user" && text.trim())) return; const controller = new AbortController(); task = controller; let taskRevision = revision; try { const answer = await answerQuestion( backendMessages, config, { runID, sessionID, delegationID: id }, controller.signal, () => { const updates = transcripts.slice(transcriptCursor); backendMessages.push(...updates.map(({ role, text }) => ({ role, content: text }))); transcriptCursor = transcripts.length; taskRevision = revision; }, ); if (closing || controller.signal.aborted) return; if (taskRevision !== revision) return; deliverResult(id, answer.trim()); } catch { if (closing || controller.signal.aborted) return; backendMessages.push({ role: "assistant", content: "The backend attempt failed. Completed tool results remain valid; verify uncertain external actions before any retry.", }); if (taskRevision !== revision) return; deliverResult(id, "The backend could not finish the answer. Ask whether the user wants to try again."); } finally { if (task === controller) task = undefined; scheduleDelegation(); } } function queueDelegation(event: ProviderEvent) { pendingDelegations.push({ event }); emit({ type: "app.delegation.pending", delegation_id: event.delegation?.id }); scheduleDelegation(); } function receiveGateway(data: unknown) { try { if (typeof data !== "string" || data.length > 1024 * 1024) throw new Error("Invalid voice event"); const event: ProviderEvent = JSON.parse(data); if (browser.readyState === 1) browser.send(data); if (event.type === "session.closed") return finish(); if (event.type === "gateway.session.closing" || event.type === "gateway.error" || event.type === "app.error") { return stop(); } if (closing) return; if (event.type === "gateway.session.created") { sessionID = event.session?.id; clearTimeout(startupTimer); requestGreeting(); } if ( greetingCommand && event.type === `session.${greetingCommand.type}.appended` && event.client_event_id === greetingCommand.id ) { if (greetingCommand.type === "instructions") { greetingCommand = { id: crypto.randomUUID(), type: "commentary" }; gateway?.send( JSON.stringify({ type: "session.commentary.append", event_id: greetingCommand.id, delegation_id: null, content: "Begin the conversation now, following the instructions provided.", }), ); } else { clearGreeting(); emit({ type: "app.greeting.accepted" }); } } else if (greetingCommand && event.type === "error" && event.error?.client_event_id === greetingCommand.id) { clearGreeting(); emit({ type: "app.greeting.error", error: { message: event.error.message ?? "The opening was rejected" } }); } if (event.type === "session.input_transcript.delta" || event.type === "session.output_transcript.delta") { const role = event.type === "session.input_transcript.delta" ? "user" : "assistant"; if (!event.delta?.trim()) return; const listeningSound = role === "user" && Boolean(task) && isListeningSound(event.delta); transcripts.push({ role, text: event.delta, start_ms: event.start_ms ?? 0, end_ms: event.end_ms ?? 0, listeningSound, }); if (listeningSound) return; if (role === "user") { revision++; const offset = completedDelegation?.event.offset_ms; // Live has no transcript watermark; completed handoffs can receive late context. if ( pendingDelegations.length === 0 && completedDelegation && typeof offset === "number" && Number.isFinite(offset) && offset >= 0 && typeof event.start_ms === "number" && Number.isFinite(event.start_ms) && event.start_ms >= 0 && event.start_ms <= offset ) { pendingDelegations.push(completedDelegation); completedDelegation = undefined; } if (pendingDelegations.length) { emit({ type: "app.delegation.pending", delegation_id: pendingDelegations[0]?.event.delegation?.id }); } } scheduleDelegation(); } else if (event.type === "session.delegation.created") { const id = event.delegation?.id; if (id && event.delegation?.target === "client" && !delegations.has(id)) { delegations.add(id); queueDelegation(event); } } } catch { emit({ type: "app.error", error: { message: "Invalid voice event or lost connection" } }); stop(); } } async function startSession(sdp: string) { if (closing || browser.readyState !== 1) return; clearTimeout(startTimer); startupTimer = setTimeout(() => { emit({ type: "app.error", error: { message: "Voice startup timed out" } }); stop(); }, 40_000); const accepted = await connect( new URL(`${gatewayAPIBase(config.baseURL)}/live/sessions`).href, { "Lovable-API-Key": config.key, "X-Lovable-AIG-SDK": "fetch", "X-Lovable-AIG-Run-ID": runID, }, setupAbort.signal, ); if (closing || browser.readyState !== 1) { close(accepted); return; } gateway = accepted; gateway.onMessage(receiveGateway); gateway.onClose(finish); gateway.onError(() => { emit({ type: "app.error", error: { message: "Voice gateway connection failed" } }); stop(); }); gateway.send( JSON.stringify({ type: "session.start", session: { model: config.liveModel, instructions: conversationInstructions, audio: { output: { voice: "marin" } }, delegation: { type: "client" }, }, transport: { type: "webrtc", sdp }, }), ); } browser.onMessage((data) => { try { if (typeof data !== "string" || data.length > 64 * 1024) throw new Error("Invalid client message"); const event = JSON.parse(data); if (event.type === "session.close") return stop(); if (closing) return; if (!gateway) { if (starting || event.type !== "app.start" || typeof event.sdp !== "string" || !event.sdp.trim()) { throw new Error("Voice startup message required"); } starting = true; execution.waitUntil( startSession(event.sdp).catch((error) => { if (!closing) emit({ type: "app.error", error: { message: error instanceof Error ? error.message : "Voice startup failed" }, }); stop(); }), ); return; } if (event.type === "app.ready") { browserReady = true; requestGreeting(); return; } if (event.type !== "gateway.heartbeat") { throw new Error("Unsupported client event"); } if (gateway.readyState !== 1) return stop(); gateway.send(data); } catch { emit({ type: "app.error", error: { message: "Voice connection could not be started or continued" } }); stop(); } }); browser.onClose(stop); browser.onError(stop); }