diff --git a/.changeset/fresh-live-voice.md b/.changeset/fresh-live-voice.md new file mode 100644 index 0000000..d7115ba --- /dev/null +++ b/.changeset/fresh-live-voice.md @@ -0,0 +1,11 @@ +--- +'orb-ui': minor +--- + +Add `createOpenAILiveAdapter` for GPT-Live WebRTC sessions, full-duplex input/output metering, +application event forwarding, and graceful session finalization. Make GPT-Live the primary OpenAI +documentation and homepage example while preserving the existing Realtime adapter. + +Live uses a server-side `/v1/live/sessions` SDP exchange through `createSession`; it is not a model +substitution for Realtime's `getClientSecret` flow. See the new Live guide for provider migration, +delegation, playback, and final usage handling. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1951dbf..cc25004 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,6 +83,13 @@ held in page memory only and exchanged for provider session credentials; they ar local storage or accepted through `VITE_*` variables. Self-hosted Pipecat SmallWebRTC only needs a public `/api/offer` URL. +GPT-Live is also available in the playground. `pnpm dev:demo` serves its session endpoint locally. +Provide `OPENAI_API_KEY` to the dev server process to use a saved server-side credential, or paste +a test key in the GPT-Live panel for the current page session. Never use a `VITE_*` variable for +the key. Server credentials are accepted only on same-origin loopback requests. The deployed +endpoint requires each caller's own key. Select **OpenAI GPT-Live**, then click the orb to start +and click it again to end the billed session. Model settings persist; pasted Live keys do not. + Non-secret playground values are saved in browser local storage for that origin, and the Clear button removes the selected provider's values. To prefill the fields during local development, copy `demo/.env.example` to `demo/.env.local`, fill in non-secret `VITE_*` defaults, and restart the diff --git a/README.md b/README.md index b21b77e..5e0aacf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Voice agent UI that feels alive.** -Expressive, accessible React components for realtime voice agents. Connect Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live, or your own voice stack through one consistent UI layer. +Expressive, accessible React components for realtime voice agents. Connect Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, or your own voice stack through one consistent UI layer.

@@ -55,7 +55,7 @@ npm install orb-ui livekit-client # Pipecat (choose the transport used by your agent) npm install orb-ui @pipecat-ai/client-js @pipecat-ai/small-webrtc-transport -# OpenAI Realtime uses browser WebRTC and has no additional client SDK +# OpenAI Live and Realtime use browser WebRTC and need no additional client SDK npm install orb-ui # Gemini Live @@ -80,6 +80,7 @@ The only difference is how the adapter obtains a provider session: | [ElevenLabs guide](https://orb-ui.com/docs/adapters/elevenlabs) | Pass `Conversation` plus an `agentId`, signed URL, or conversation token | | [LiveKit guide](https://orb-ui.com/docs/adapters/livekit) | Provide a token endpoint and optional agent name | | [Pipecat guide](https://orb-ui.com/docs/adapters/pipecat) | Pass a configured `PipecatClient` plus its connect callback | +| [OpenAI Live guide](https://orb-ui.com/docs/adapters/openai-live) | Exchange an SDP offer through your server with `createSession` | | [OpenAI Realtime guide](https://orb-ui.com/docs/adapters/openai-realtime) | Return a fresh short-lived client secret from `getClientSecret` | | [Gemini Live guide](https://orb-ui.com/docs/adapters/gemini-live) | Open the official Google Live session in `connect` | @@ -166,6 +167,29 @@ The Pipecat adapter consumes the standard RTVI event surface and meters the clie a browser fallback, so it works with Pipecat Cloud, Daily, SmallWebRTC, and transports that emit sparse audio-level events. See the [Pipecat guide](https://orb-ui.com/docs/adapters/pipecat). +### With OpenAI GPT-Live + +```tsx +import { createOpenAILiveAdapter } from 'orb-ui/adapters' + +const adapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() + }, +}) +``` + +Your server creates a `gpt-live-1` session through `/v1/live/sessions` and returns the session ID +and SDP answer. Live handles listening and speaking concurrently; configure backend delegation +on your server. See the [OpenAI Live guide](https://orb-ui.com/docs/adapters/openai-live). + ### With OpenAI Realtime ```jsx @@ -348,6 +372,7 @@ the orb with the typed style variable: | [ElevenLabs](https://elevenlabs.io/conversational-ai) | `createElevenLabsAdapter` from `orb-ui/adapters` | | [LiveKit](https://livekit.io) | `createLiveKitAdapter` from `orb-ui/adapters` | | [Pipecat](https://pipecat.ai) | `createPipecatAdapter` from `orb-ui/adapters` | +| [OpenAI GPT-Live](https://developers.openai.com/api/docs/guides/live) | `createOpenAILiveAdapter` from `orb-ui/adapters` | | [OpenAI Realtime](https://developers.openai.com/api/docs/guides/realtime) | `createOpenAIRealtimeAdapter` from `orb-ui/adapters` | | [Gemini Live](https://ai.google.dev/gemini-api/docs/live-api) | `createGeminiLiveAdapter` from `orb-ui/adapters` | | Custom | Use controlled mode with a directional `signal` or build an adapter | diff --git a/ROADMAP.md b/ROADMAP.md index f77c600..e073f86 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -67,6 +67,19 @@ Cloud/Daily, self-hosted SmallWebRTC, and custom client transports. It normalize audio levels while leaving agent deployment and connection credentials in the application. Direct browser-track metering fills gaps when a transport does not emit frequent RTVI audio-level events. +### OpenAI GPT-Live adapter — implemented + +A separate Live adapter owns browser WebRTC, concurrent input/output metering, event forwarding, +and graceful session finalization. GPT-Live is the primary OpenAI documentation path. Session +creation and delegation configuration stay on the application's server; Realtime remains supported. +Protocol and lifecycle tests cover startup, playback, cancellation, failure, and shutdown. Real +GPT-Live WebRTC sessions validate audio, interruption, delegated responses, same-adapter restart, +and final usage on graceful shutdown using synthetic spoken input. A Chrome playground session +also verifies physical microphone input and assistant playback; independent Live voice calibration +remains outstanding. +The provider playground includes GPT-Live with model settings, directional calibration, and a +local session endpoint that can use a server-side test credential. + ### OpenAI Realtime adapter — complete The OpenAI Realtime adapter owns browser WebRTC, audio playback, input/output metering, and current diff --git a/demo/api/openai-live-session.ts b/demo/api/openai-live-session.ts new file mode 100644 index 0000000..de6476e --- /dev/null +++ b/demo/api/openai-live-session.ts @@ -0,0 +1,64 @@ +function json(data: unknown, status = 200) { + return Response.json(data, { status, headers: { 'Cache-Control': 'no-store' } }) +} + +/** A local dev server may supply its key; deployed playgrounds require the caller's own key. */ +export async function createLiveSession(request: Request, localApiKey?: string) { + if (request.method !== 'POST') return json({ error: 'Method not allowed.' }, 405) + let body: Record + try { + body = await request.json() + if (!body || typeof body !== 'object' || Array.isArray(body)) throw new Error('Invalid body') + } catch { + return json({ error: 'Expected a JSON object.' }, 400) + } + const value = (key: string) => (typeof body[key] === 'string' ? body[key].trim() : '') + const apiKey = value('apiKey') || localApiKey + if (!apiKey) return json({ error: 'An OpenAI API key is required.' }, 400) + // SDP is line-oriented; preserve the browser's trailing CRLF. + const sdp = typeof body.sdp === 'string' ? body.sdp : '' + if (!sdp.trim() || sdp.length > 64000) + return json({ error: 'A valid SDP offer is required.' }, 400) + + try { + const response = await fetch('https://api.openai.com/v1/live/sessions', { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + signal: AbortSignal.any([request.signal, AbortSignal.timeout(30000)]), + body: JSON.stringify({ + session: { + model: value('model') || 'gpt-live-1', + store: false, + instructions: + value('instructions') || + 'Be a friendly voice assistant. Honor interruptions and delegate factual questions to the backend.', + delegation: { + type: 'responses', + responses: { + model: value('backendModel') || 'gpt-5.6-terra', + instructions: + 'Answer accurately and concisely for a spoken conversation. Follow the latest caller request.', + }, + }, + }, + transport: { type: 'webrtc', sdp }, + }), + }) + const payload = await response.json() + if (!response.ok) + return json( + { + error: `OpenAI Live session creation failed (${response.status}). Check your key and model access.`, + }, + response.status, + ) + if (!payload.session?.id || payload.transport?.type !== 'webrtc' || !payload.transport.sdp) { + return json({ error: 'OpenAI returned an incomplete Live session.' }, 502) + } + return json({ session: { id: payload.session.id }, transport: payload.transport }) + } catch { + return json({ error: 'OpenAI Live session request failed or timed out.' }, 502) + } +} + +export default { fetch: (request: Request) => createLiveSession(request) } diff --git a/demo/openai-live-dev.ts b/demo/openai-live-dev.ts new file mode 100644 index 0000000..27d3304 --- /dev/null +++ b/demo/openai-live-dev.ts @@ -0,0 +1,58 @@ +import type { Plugin } from 'vite' +import { env } from 'node:process' +import { createLiveSession } from './api/openai-live-session' + +/** Keep the optional server credential confined to same-origin loopback requests. */ +export function openAILiveDevPlugin(): Plugin { + return { + name: 'orb-ui-openai-live-local', + configureServer(server) { + server.middlewares.use(async (req, res, next) => { + const path = req.url?.split('?')[0] + if (path !== '/api/openai-live-session' && path !== '/api/openai-live-status') return next() + const reply = (status: number, data: unknown) => { + res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }) + res.end(JSON.stringify(data)) + } + const origin = `http://${req.headers.host}` + let hostname: string + try { + hostname = new URL(origin).hostname + } catch { + return reply(403, { error: 'Local access only.' }) + } + if (!['localhost', '127.0.0.1', '[::1]'].includes(hostname)) + return reply(403, { error: 'Local access only.' }) + if (path === '/api/openai-live-status' && req.method === 'GET') { + return reply(200, { configured: Boolean(env.OPENAI_API_KEY) }) + } + if (req.method !== 'POST' || req.headers.origin !== origin) + return reply(403, { error: 'Same-origin POST required.' }) + const controller = new AbortController() + res.on('close', () => { + if (!res.writableEnded) controller.abort() + }) + try { + let body = '' + for await (const chunk of req) { + body += chunk + if (body.length > 100000) return reply(413, { error: 'Request too large.' }) + } + const result = await createLiveSession( + new Request(`${origin}${path}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body, + signal: controller.signal, + }), + env.OPENAI_API_KEY, + ) + res.writeHead(result.status, Object.fromEntries(result.headers)) + res.end(await result.text()) + } catch { + reply(502, { error: 'Local Live session request failed.' }) + } + }) + }, + } +} diff --git a/demo/src/App.tsx b/demo/src/App.tsx index 025270e..8f19e19 100644 --- a/demo/src/App.tsx +++ b/demo/src/App.tsx @@ -108,12 +108,18 @@ export function VoiceOrb() { }` const OPENAI_CODE = `import { Orb } from "orb-ui" -import { createOpenAIRealtimeAdapter } from "orb-ui/adapters" - -const adapter = createOpenAIRealtimeAdapter({ - getClientSecret: async () => { - const response = await fetch("/api/openai-realtime-token", { method: "POST" }) - return (await response.json()).value +import { createOpenAILiveAdapter } from "orb-ui/adapters" + +const adapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch("/api/openai-live-session", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sdp }), + signal + }) + if (!response.ok) throw new Error("Could not create a Live session") + return response.json() } }) @@ -226,7 +232,7 @@ const NAV_LINKS = [ ] as const const PROOF_POINTS = [ - { value: '6+', label: 'Provider paths', detail: 'Plus controlled mode' }, + { value: '7', label: 'Provider paths', detail: 'Plus controlled mode' }, { value: '2-way', label: 'Audio response', detail: 'Input and output levels' }, { value: 'A11y', label: 'Accessible controls', detail: 'Keyboard-ready semantics' }, { value: 'MIT', label: 'Open source', detail: 'Use it anywhere' }, @@ -265,8 +271,8 @@ const CODE_OPTIONS: ReadonlyArray<{ { id: 'openai', label: 'OpenAI', - detail: 'Realtime', - description: 'Use native browser WebRTC with short-lived client secrets.', + detail: 'GPT-Live', + description: 'Connect GPT-Live through a server-created WebRTC session.', }, { id: 'gemini', @@ -293,7 +299,7 @@ const SEO_SECTIONS = [ { id: 'adapters', title: 'Provider adapters', - copy: 'Use adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, and Gemini Live.', + copy: 'Use adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, and Gemini Live.', link: '/docs/adapters/overview', linkLabel: 'Explore adapters', }, @@ -315,7 +321,7 @@ const SEO_SECTIONS = [ id: 'roadmap', title: 'Native realtime voice adapters', copy: 'Drive the UI from managed browser audio, provider state, and separate input/output levels.', - link: '/docs/adapters/openai-realtime', + link: '/docs/adapters/openai-live', linkLabel: 'OpenAI setup', }, ] as const @@ -325,6 +331,7 @@ const PROVIDER_GUIDES = [ { href: '/docs/adapters/elevenlabs', label: 'ElevenLabs', detail: 'Conversational AI' }, { href: '/docs/adapters/livekit', label: 'LiveKit', detail: 'Agents' }, { href: '/docs/adapters/pipecat', label: 'Pipecat', detail: 'RTVI' }, + { href: '/docs/adapters/openai-live', label: 'OpenAI GPT-Live', detail: 'Full duplex' }, { href: '/docs/adapters/openai-realtime', label: 'OpenAI Realtime', detail: 'WebRTC' }, { href: '/docs/adapters/gemini-live', label: 'Gemini Live', detail: 'Live API' }, { href: '/docs/adapters/custom', label: 'Custom voice stack', detail: 'Controlled mode' }, @@ -1784,11 +1791,13 @@ export default function App() {

Native paths - {PROVIDER_GUIDES.slice(0, 6).map((provider) => ( - - {provider.label} - - ))} + {PROVIDER_GUIDES.filter((provider) => provider.href !== '/docs/adapters/custom').map( + (provider) => ( + + {provider.label} + + ), + )}
diff --git a/demo/src/provider-playground.tsx b/demo/src/provider-playground.tsx index 8338efb..78c8576 100644 --- a/demo/src/provider-playground.tsx +++ b/demo/src/provider-playground.tsx @@ -15,6 +15,7 @@ import { createElevenLabsAdapter, createGeminiLiveAdapter, createLiveKitAdapter, + createOpenAILiveAdapter, createOpenAIRealtimeAdapter, createPipecatAdapter, createVapiAdapter, @@ -31,7 +32,15 @@ import type { } from 'orb-ui/adapters' import './provider-playground.css' -type ProviderId = 'manual' | 'vapi' | 'elevenlabs' | 'livekit' | 'pipecat' | 'openai' | 'gemini' +type ProviderId = + | 'manual' + | 'vapi' + | 'elevenlabs' + | 'livekit' + | 'pipecat' + | 'openai-live' + | 'openai' + | 'gemini' type LiveKitConnectionMode = 'sandbox' | 'endpoint' | 'raw' type PipecatConnectionMode = 'cloud' | 'small-webrtc' type CalibratableProviderId = Exclude @@ -56,6 +65,10 @@ interface ProviderConfig { pipecatAgentName: string pipecatWebrtcUrl: string openAIApiKey: string + openAILiveApiKey: string + openAILiveModel: string + openAILiveBackendModel: string + openAILiveInstructions: string openAIModel: string openAIVoice: string openAIInstructions: string @@ -81,6 +94,7 @@ const PROVIDERS: Array<{ id: ProviderId; label: string }> = [ { id: 'elevenlabs', label: 'ElevenLabs' }, { id: 'livekit', label: 'LiveKit' }, { id: 'pipecat', label: 'Pipecat' }, + { id: 'openai-live', label: 'OpenAI GPT-Live' }, { id: 'openai', label: 'OpenAI Realtime' }, { id: 'gemini', label: 'Gemini Live' }, ] @@ -106,6 +120,10 @@ const THEME_MODES: Array<{ id: ThemeMode; label: string }> = [ const STATES: OrbState[] = ['idle', 'connecting', 'listening', 'thinking', 'speaking', 'error'] const DEFAULT_LIVEKIT_ROOM_PREFIX = 'orb-ui-playground' const DEFAULT_OPENAI_MODEL = 'gpt-realtime-2.1' +const DEFAULT_OPENAI_LIVE_MODEL = 'gpt-live-1' +const DEFAULT_OPENAI_LIVE_BACKEND = 'gpt-5.6-terra' +const DEFAULT_OPENAI_LIVE_INSTRUCTIONS = + 'You are a concise, friendly voice assistant. Respond naturally, honor interruptions, and delegate factual questions to the backend.' const DEFAULT_OPENAI_VOICE = 'marin' const DEFAULT_GEMINI_MODEL = 'gemini-3.1-flash-live-preview' const DEFAULT_GEMINI_VOICE = 'Kore' @@ -120,6 +138,7 @@ const CALIBRATABLE_PROVIDERS: CalibratableProviderId[] = [ 'elevenlabs', 'livekit', 'pipecat', + 'openai-live', 'openai', 'gemini', ] @@ -259,7 +278,9 @@ function isCalibratableProvider(provider: ProviderId): provider is CalibratableP } function supportsDirection(provider: CalibratableProviderId, direction: VolumeDirection) { - return Boolean(PROVIDER_VOLUME_CALIBRATIONS[provider][direction]) + return Boolean( + PROVIDER_VOLUME_CALIBRATIONS[provider === 'openai-live' ? 'openai' : provider][direction], + ) } function copyCalibration(calibration: VolumeCalibration): VolumeCalibration { @@ -284,6 +305,7 @@ function copyProviderCalibrations(): CalibrationByProvider { elevenlabs: copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.elevenlabs), livekit: copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.livekit), pipecat: copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.pipecat), + 'openai-live': copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.openai), openai: copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.openai), gemini: copyDirectionalCalibration(PROVIDER_VOLUME_CALIBRATIONS.gemini), } @@ -436,6 +458,12 @@ function readStoredConfig(): Partial { if (!isRecord(parsed)) return {} const storedConfig: Partial = {} + if (typeof parsed.openAILiveModel === 'string') + storedConfig.openAILiveModel = parsed.openAILiveModel + if (typeof parsed.openAILiveBackendModel === 'string') + storedConfig.openAILiveBackendModel = parsed.openAILiveBackendModel + if (typeof parsed.openAILiveInstructions === 'string') + storedConfig.openAILiveInstructions = parsed.openAILiveInstructions if (typeof parsed.vapiPublicKey === 'string') storedConfig.vapiPublicKey = parsed.vapiPublicKey if (typeof parsed.vapiAssistantId === 'string') { storedConfig.vapiAssistantId = parsed.vapiAssistantId @@ -542,7 +570,14 @@ function writeStoredConfig( try { storage.setItem( CONFIG_STORAGE_KEY, - JSON.stringify({ ...config, provider, theme, themePreset, themeMode }), + JSON.stringify({ + ...config, + openAILiveApiKey: undefined, + provider, + theme, + themePreset, + themeMode, + }), ) } catch { // Storage can be disabled or full in some browser modes. @@ -567,6 +602,10 @@ function readEnvConfig(): ProviderConfig { pipecatAgentName: import.meta.env.VITE_PIPECAT_AGENT_NAME ?? '', pipecatWebrtcUrl: import.meta.env.VITE_PIPECAT_WEBRTC_URL ?? '', openAIApiKey: '', + openAILiveApiKey: '', + openAILiveModel: DEFAULT_OPENAI_LIVE_MODEL, + openAILiveBackendModel: DEFAULT_OPENAI_LIVE_BACKEND, + openAILiveInstructions: DEFAULT_OPENAI_LIVE_INSTRUCTIONS, openAIModel: import.meta.env.VITE_OPENAI_REALTIME_MODEL ?? DEFAULT_OPENAI_MODEL, openAIVoice: import.meta.env.VITE_OPENAI_REALTIME_VOICE ?? DEFAULT_OPENAI_VOICE, openAIInstructions: import.meta.env.VITE_OPENAI_REALTIME_INSTRUCTIONS ?? DEFAULT_INSTRUCTIONS, @@ -601,6 +640,11 @@ function normalizeConfig(config: ProviderConfig): ProviderConfig { pipecatAgentName: (config.pipecatAgentName ?? '').trim(), pipecatWebrtcUrl: (config.pipecatWebrtcUrl ?? '').trim(), openAIApiKey: (config.openAIApiKey ?? '').trim(), + openAILiveApiKey: (config.openAILiveApiKey ?? '').trim(), + openAILiveModel: config.openAILiveModel.trim() || DEFAULT_OPENAI_LIVE_MODEL, + openAILiveBackendModel: config.openAILiveBackendModel.trim() || DEFAULT_OPENAI_LIVE_BACKEND, + openAILiveInstructions: + config.openAILiveInstructions.trim() || DEFAULT_OPENAI_LIVE_INSTRUCTIONS, openAIModel: (config.openAIModel ?? '').trim() || DEFAULT_OPENAI_MODEL, openAIVoice: (config.openAIVoice ?? '').trim() || DEFAULT_OPENAI_VOICE, openAIInstructions: (config.openAIInstructions ?? '').trim() || DEFAULT_INSTRUCTIONS, @@ -621,7 +665,7 @@ function createManualSignal(state: OrbState, inputVolume: number, outputVolume: return { state, inputVolume: 0, outputVolume: 0 } } -function getProviderReady(provider: ProviderId, config: ProviderConfig) { +function getProviderReady(provider: ProviderId, config: ProviderConfig, localOpenAIKey = false) { if (provider === 'manual') return true if (provider === 'vapi') return Boolean(config.vapiPublicKey && config.vapiAssistantId) if (provider === 'elevenlabs') return Boolean(config.elevenLabsAgentId) @@ -631,6 +675,7 @@ function getProviderReady(provider: ProviderId, config: ProviderConfig) { : Boolean(config.pipecatWebrtcUrl) } if (provider === 'openai') return Boolean(config.openAIApiKey) + if (provider === 'openai-live') return localOpenAIKey || Boolean(config.openAILiveApiKey) if (provider === 'gemini') return Boolean(config.geminiApiKey) if (config.liveKitConnectionMode === 'sandbox') { return Boolean(config.liveKitSandboxId && config.liveKitAgentName) @@ -644,11 +689,13 @@ function getProviderReady(provider: ProviderId, config: ProviderConfig) { async function postProviderJson( endpoint: string, body: Record, + signal?: AbortSignal, ): Promise { const response = await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body), + signal, }) const payload = (await response.json()) as TResponse & { error?: string } if (!response.ok) { @@ -732,7 +779,30 @@ function createProviderAdapter( onInputSample: (sample: VolumeSample) => void onOutputSample: (sample: VolumeSample) => void }, + localOpenAIKey = false, ): OrbAdapter | undefined { + if (provider === 'openai-live' && getProviderReady(provider, config, localOpenAIKey)) { + return createLazyAdapter(() => + createOpenAILiveAdapter({ + inputVolumeCalibration: volumeCalibration?.getInput, + outputVolumeCalibration: volumeCalibration?.getOutput, + onInputVolumeSample: volumeCalibration?.onInputSample, + onOutputVolumeSample: volumeCalibration?.onOutputSample, + createSession: (sdp, signal) => + postProviderJson( + '/api/openai-live-session', + { + sdp, + apiKey: config.openAILiveApiKey || undefined, + model: config.openAILiveModel, + backendModel: config.openAILiveBackendModel, + instructions: config.openAILiveInstructions, + }, + signal, + ), + }), + ) + } if (provider === 'vapi' && getProviderReady(provider, config)) { return createLazyAdapter(async () => { const vapiModule = await import('@vapi-ai/web') @@ -1050,11 +1120,51 @@ function ProviderConfigFields({ config, provider, updateConfig, + localOpenAIKey, }: { config: ProviderConfig provider: Exclude updateConfig: UpdateProviderConfig + localOpenAIKey: boolean }) { + if (provider === 'openai-live') { + return ( + <> + {localOpenAIKey ? ( +

Saved local OpenAI key is ready. Start a session to talk.

+ ) : ( + updateConfig('openAILiveApiKey', value)} + /> + )} + updateConfig('openAILiveModel', value)} + /> + updateConfig('openAILiveBackendModel', value)} + /> + updateConfig('openAILiveInstructions', value)} + /> +

+ Speak naturally and interrupt at any time. End the session when finished. +

+ + ) + } if (provider === 'vapi') { return ( <> @@ -1289,10 +1399,24 @@ function ProviderConfigFields({ function ProviderReadinessRows({ config, provider, + localOpenAIKey, }: { config: ProviderConfig provider: Exclude + localOpenAIKey: boolean }) { + if (provider === 'openai-live') { + return ( + <> + + + + + ) + } if (provider === 'vapi') { return ( <> @@ -1358,6 +1482,16 @@ function ProviderReadinessRows({ function ProviderPlayground() { const [config, setConfig] = useState(() => readConfig()) + const [localOpenAIKey, setLocalOpenAIKey] = useState(false) + useEffect(() => { + if (!import.meta.env.DEV) return + const controller = new AbortController() + void fetch('/api/openai-live-status', { signal: controller.signal }) + .then((response) => (response.ok ? response.json() : undefined)) + .then((result) => setLocalOpenAIKey(result?.configured === true)) + .catch(() => {}) + return () => controller.abort() + }, []) const [storedSelection] = useState(() => readStoredSelection()) const [provider, setProvider] = useState(storedSelection.provider) const [theme, setTheme] = useState(storedSelection.theme) @@ -1471,7 +1605,7 @@ function ProviderPlayground() { : undefined, [themeMode], ) - const providerReady = getProviderReady(provider, activeConfig) + const providerReady = getProviderReady(provider, activeConfig, localOpenAIKey) const providerAdapter = useMemo( () => createProviderAdapter( @@ -1485,8 +1619,16 @@ function ProviderPlayground() { onOutputSample: recordOutputSample, } : undefined, + localOpenAIKey, ), - [activeConfig, getActiveCalibration, provider, recordInputSample, recordOutputSample], + [ + activeConfig, + getActiveCalibration, + provider, + recordInputSample, + recordOutputSample, + localOpenAIKey, + ], ) const updateConfig = useCallback(function updateConfig( @@ -1500,6 +1642,14 @@ function ProviderPlayground() { const defaultConfig = readEnvConfig() setConfig((current) => { + if (provider === 'openai-live') + return { + ...current, + openAILiveApiKey: '', + openAILiveModel: DEFAULT_OPENAI_LIVE_MODEL, + openAILiveBackendModel: DEFAULT_OPENAI_LIVE_BACKEND, + openAILiveInstructions: DEFAULT_OPENAI_LIVE_INSTRUCTIONS, + } if (provider === 'vapi') { return { ...current, @@ -1561,6 +1711,14 @@ function ProviderPlayground() { const clearProviderConfig = useCallback(() => { setConfig((current) => { + if (provider === 'openai-live') + return { + ...current, + openAILiveApiKey: '', + openAILiveModel: DEFAULT_OPENAI_LIVE_MODEL, + openAILiveBackendModel: DEFAULT_OPENAI_LIVE_BACKEND, + openAILiveInstructions: DEFAULT_OPENAI_LIVE_INSTRUCTIONS, + } if (provider === 'vapi') { return { ...current, vapiPublicKey: '', vapiAssistantId: '' } } @@ -2005,11 +2163,13 @@ function ProviderPlayground() { config={config} provider={provider} updateConfig={updateConfig} + localOpenAIKey={localOpenAIKey} />

- Configuration, including credentials, is saved in this browser for this exact - playground URL. Use Clear to remove the current provider's saved values. + {provider === 'openai-live' + ? 'Model settings are saved in this browser. Pasted Live keys stay in page memory; the saved local key stays on the server.' + : 'Configuration, including credentials, is saved in this browser for this exact playground URL. Use Clear to remove the current provider’s saved values.'}

- +
) : null} diff --git a/demo/vite.config.ts b/demo/vite.config.ts index b7d068a..d10febe 100644 --- a/demo/vite.config.ts +++ b/demo/vite.config.ts @@ -1,6 +1,7 @@ import { defineConfig, type Plugin } from 'vite' import react from '@vitejs/plugin-react' import { fileURLToPath, URL } from 'node:url' +import { openAILiveDevPlugin } from './openai-live-dev' function resolveInput(path: string) { return fileURLToPath(new URL(path, import.meta.url)) @@ -34,7 +35,7 @@ function playgroundRoutePlugin(): Plugin { } export default defineConfig({ - plugins: [react(), playgroundRoutePlugin()], + plugins: [react(), playgroundRoutePlugin(), openAILiveDevPlugin()], build: { rollupOptions: { input: { diff --git a/docs/adapters/custom.mdx b/docs/adapters/custom.mdx index 85e315b..4eb8771 100644 --- a/docs/adapters/custom.mdx +++ b/docs/adapters/custom.mdx @@ -40,7 +40,7 @@ Controlled mode works well with: - telephony systems - internal speech pipelines - provider SDK wrappers -- experimental OpenAI Realtime or Gemini Live API prototypes +- experimental OpenAI Live, OpenAI Realtime, or Gemini Live API prototypes ## Normalize state once diff --git a/docs/adapters/openai-live.mdx b/docs/adapters/openai-live.mdx new file mode 100644 index 0000000..a6375bd --- /dev/null +++ b/docs/adapters/openai-live.mdx @@ -0,0 +1,160 @@ +--- +title: OpenAI GPT-Live Voice UI for React +description: Connect GPT-Live to an audio-reactive React voice UI with managed WebRTC, full-duplex audio metering, and orb-ui. +--- + +Start here for new OpenAI GPT-Live integrations. `createOpenAILiveAdapter` owns microphone capture, +WebRTC negotiation, audio playback, input/output metering, and session cleanup. Your server owns +the OpenAI API key, voice model, instructions, and delegation configuration. + +GPT-Live uses a different API from Realtime. Use the [Realtime adapter](/adapters/openai-realtime) +for existing Realtime sessions; changing its model or endpoint does not make it compatible with Live. + +## Create a session on your server + +The browser sends an SDP offer to your authenticated application endpoint. The server exchanges +it with `POST /v1/live/sessions` and returns the session ID and SDP answer. This example uses +`gpt-live-1` for speech and a Responses backend for delegated work. + +```ts +export async function POST(request: Request) { + // Authenticate and authorize the caller using your application's session here. + const { sdp } = await request.json() + if (typeof sdp !== 'string' || !sdp.trim()) { + return Response.json({ error: 'An SDP offer is required' }, { status: 400 }) + } + const response = await fetch('https://api.openai.com/v1/live/sessions', { + method: 'POST', + headers: { + Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + session: { + model: 'gpt-live-1', + instructions: 'Be concise and friendly. Delegate factual questions to the backend.', + delegation: { + type: 'responses', + responses: { + model: 'gpt-5.6-terra', + instructions: 'Return concise, accurate answers for a spoken conversation.', + }, + }, + }, + transport: { type: 'webrtc', sdp }, + }), + }) + if (!response.ok) { + return Response.json({ error: 'Could not create a Live session' }, { status: response.status }) + } + return Response.json(await response.json(), { headers: { 'Cache-Control': 'no-store' } }) +} +``` + +Keep `OPENAI_API_KEY` server-only and protect session creation with your application's +authentication and request limits. WebRTC negotiates the audio format; omit `audio.format`. +Live sessions are billed by duration, including an initialization minimum; see the +[official WebRTC guide](https://developers.openai.com/api/docs/guides/voice-webrtc?api=live). + +## Create the adapter + +```tsx +import { Orb } from 'orb-ui' +import { createOpenAILiveAdapter } from 'orb-ui/adapters' + +const adapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() + }, +}) + +export function OpenAILiveVoiceUI() { + return +} +``` + +`createSession` runs for each start and returns +`{ session: { id }, transport: { type: 'webrtc', sdp } }`. Forward the provided `AbortSignal` to +your fetch. No client SDK or Realtime client secret is required. `start()` resolves after both +SDP negotiation and `session.started`; opening the data channel alone does not mean Live is ready. + +## Full-duplex state and volume + +- Connection setup → `connecting` +- `session.started` and quiet assistant output → `listening` +- Audible remote-track activity → `speaking` +- Provider, playback, or connection failure → `error` +- Confirmed `session.closed` or cancelled startup → `idle` + +Live can listen and speak at the same time. Both volume fields remain active, and microphone +activity does not suppress the speaking indicator. The single orb state shows assistant playback +when both sides are talking. + +Live has no per-utterance audio completion event. The adapter estimates speech activity from the +remote audio track, with a short silence hold; captions and delegated `response.event` completion +do not start or stop speech. It does not infer `thinking` from backend events, since backend work +and speech run independently. Use application state alongside the orb for task progress. + +The adapter starts with the shared OpenAI WebRTC RMS calibration, not a separately measured Live +voice profile. `inputVolumeCalibration`, `outputVolumeCalibration`, `onInputVolumeSample`, and +`onOutputVolumeSample` support [directional tuning](/guides/volume-calibration). If Web Audio +metering is unavailable or disabled, volume and inferred speaking state are unavailable; session +events still work. + +## Captions and delegation + +Pass `onEvent(event)` to receive all JSON server events unchanged, including transcript timing, +delegation IDs, nested Responses events, errors, usage updates, and `session.closed`. +After startup, use `adapter.send({ type: ..., ... })` for application commands allowed by your +server's frontend permissions. Handle custom tools on your trusted backend or sideband connection. +The adapter does not execute tools or manage backend task state. + +For client delegation, supply conversation context to your own agent and return results with +`session.commentary.append`. For Responses delegation, read inner `response.output_item.done` +events, return function outputs with `response.item.create`, and continue with `response.create`. +See the [delegation guide](https://developers.openai.com/api/docs/guides/live-delegation) for full +event schemas, frontend permissions, and backend responsibilities. + +Do not send `session.start` on WebRTC: the HTTP request already starts the session. +Use `adapter.stop()` to send `session.close`. It waits for `session.closed` before releasing media, +so `onEvent` can collect final usage. Finish required delegated work before calling stop. +On timeout or connection loss it releases resources and rejects with an error indicating that +final usage is unconfirmed. Muting the microphone does not end a billed Live session. + +## Migrating from Realtime + +1. Replace `createOpenAIRealtimeAdapter({ getClientSecret })` with + `createOpenAILiveAdapter({ createSession })`. +2. Replace your client-secret endpoint with the server-side JSON session exchange above. +3. Split voice instructions from backend instructions and choose Responses or client delegation. +4. Replace Realtime captions and tool handling with Live events. Remove manual audio commits and + voice-response triggers. Keep existing Realtime users on the unchanged Realtime adapter. + +This is an additive orb-ui API; no existing imports need to change. See OpenAI's +[migration guide](https://developers.openai.com/api/docs/guides/live-migration) for the provider +behavior changes. + +## Runtime options and troubleshooting + +`mediaStreamConstraints`, `getUserMedia`, `createPeerConnection`, `createAudioElement`, and +`createAudioContext` allow custom browser runtimes and tests. `startTimeoutMs` defaults to 30 seconds; +`closeTimeoutMs` defaults to 15 seconds and may need increasing for stored sessions. + +**No audio:** Start from a user action on HTTPS or localhost. Check microphone permission and +autoplay; playback failures surface as an error. If you provide an audio element, keep it unmuted. + +**Startup fails:** Verify project access to `gpt-live-1`, backend model access, your authenticated +session endpoint, and its JSON response. The adapter waits for ICE gathering and `session.started`. + +**Speaking never ends:** Do not drive playback from transcript deltas or Responses completion. +Inspect remote audio activity and calibration; Live has no Realtime-style output buffer end event. + +**Stop fails:** The connection ended or timed out before `session.closed`. Resources are released, +but final usage is unconfirmed. Start a fresh session to reconnect. diff --git a/docs/adapters/openai-realtime.mdx b/docs/adapters/openai-realtime.mdx index 058f208..64742e2 100644 --- a/docs/adapters/openai-realtime.mdx +++ b/docs/adapters/openai-realtime.mdx @@ -7,6 +7,9 @@ description: Connect OpenAI's Realtime API to an audio-reactive React voice UI w remote audio playback, input/output metering, interruption-aware state, and cleanup. Your server still owns the standard OpenAI API key and creates a short-lived Realtime client secret. +For GPT-Live, start with the [OpenAI Live adapter](/adapters/openai-live). Live has its own session +endpoint and event lifecycle; this Realtime adapter cannot connect to it by changing the model. + The adapter targets the GA Realtime API. New browser integrations should use `/v1/realtime/client_secrets` and `/v1/realtime/calls`, not the older beta session flow. @@ -116,11 +119,6 @@ guided profile generator. `createAudioElement`, and `createAudioContext` are available for custom browser wrappers and tests. Most applications only need `getClientSecret`. -## ChatGPT Live - -This adapter targets the public OpenAI Realtime API. It does not wrap consumer ChatGPT voice -features that do not expose a corresponding developer API. - ## Troubleshooting **The browser receives `401` or cannot connect.** Verify that the server endpoint creates a fresh diff --git a/docs/adapters/overview.mdx b/docs/adapters/overview.mdx index ccbdda5..3d18511 100644 --- a/docs/adapters/overview.mdx +++ b/docs/adapters/overview.mdx @@ -1,6 +1,6 @@ --- title: React voice agent adapters -description: Compare orb-ui adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live, and custom React voice agent UIs. +description: Compare orb-ui adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, and custom React voice agent UIs. --- Choose an orb-ui adapter when a provider SDK owns your voice session but your React UI needs one @@ -29,13 +29,14 @@ including normalized input and output volume when the provider exposes audio. | ElevenLabs | The `Conversation` class and agent/session credential | Session lifecycle, state mapping, playback, and both volume levels | | LiveKit | A token endpoint and optional agent name | SDK setup, room lifecycle, agent discovery, playback, and both volume levels | | Pipecat | A configured `PipecatClient` and its connect callback | RTVI state mapping, remote playback, and both volume levels | +| OpenAI GPT-Live | A callback that exchanges SDP through your server | Browser WebRTC, full-duplex metering, playback, and session finalization | | OpenAI Realtime | A callback that returns a fresh client secret | Browser WebRTC, microphone, playback, state, and both volume levels | | Gemini Live | A callback that opens the official Google Live session | Microphone PCM streaming, turn detection, playback, state, and both volume levels | Choose based on who should own the session: - If your app already uses Vapi or Pipecat's browser client, wrap that existing client. -- If you want orb-ui to own browser media and connection lifecycle, use OpenAI Realtime or a +- If you want orb-ui to own browser media and connection lifecycle, use OpenAI Live, OpenAI Realtime, or a managed ElevenLabs or LiveKit setup. - If the provider requires an app-owned SDK connection, use Gemini Live's `connect` callback. - If your application already normalizes session state, use [controlled mode](/adapters/custom). @@ -43,7 +44,7 @@ Choose based on who should own the session: [signal architecture guide](/guides/signal-based-voice-agent-ui). Provider authentication stays explicit. Standard OpenAI and Gemini API keys belong on your server; -the browser should receive only short-lived provider credentials. LiveKit participant tokens should +the browser receives the Live SDP answer or short-lived Realtime/Gemini credentials. LiveKit participant tokens should also be minted on a server. Pipecat Cloud can use its public agent-start key in the browser while private deployment credentials remain server-side. @@ -79,18 +80,27 @@ const pipecatAdapter = createPipecatAdapter(pipecatClient, { ### Let orb-ui own the browser session -OpenAI Realtime uses native browser WebRTC, so the only required integration seam is a fresh client -secret: +OpenAI GPT-Live uses native browser WebRTC. Exchange the SDP offer through your server: ```tsx -const adapter = createOpenAIRealtimeAdapter({ - getClientSecret: () => - fetch('/api/openai-realtime-token', { method: 'POST' }) - .then((response) => response.json()) - .then((data) => data.value), +const adapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() + }, }) ``` +Start with the [Live guide](/adapters/openai-live) for server configuration. Existing Realtime +sessions continue to use `createOpenAIRealtimeAdapter({ getClientSecret })`; the two APIs have +different endpoints and event lifecycles. + ### Let the official SDK open the session Gemini Live session creation stays in the application because `@google/genai` is an app-owned, @@ -163,6 +173,7 @@ integration. - [ElevenLabs](/adapters/elevenlabs) - [LiveKit](/adapters/livekit) - [Pipecat](/adapters/pipecat) +- [OpenAI GPT-Live](/adapters/openai-live) - [OpenAI Realtime](/adapters/openai-realtime) - [Gemini Live](/adapters/gemini-live) - [Custom integrations and controlled mode](/adapters/custom) diff --git a/docs/docs.json b/docs/docs.json index 1ca6157..7fa964e 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -2,7 +2,7 @@ "$schema": "https://mintlify.com/docs.json", "theme": "maple", "name": "orb-ui", - "description": "React voice agent UI components for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live, and custom voice AI apps.", + "description": "React voice agent UI components for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, and custom voice AI apps.", "colors": { "primary": "#4da3ff", "light": "#9fd2ff", @@ -82,6 +82,7 @@ "adapters/elevenlabs", "adapters/livekit", "adapters/pipecat", + "adapters/openai-live", "adapters/openai-realtime", "adapters/gemini-live", "adapters/custom" diff --git a/docs/guides/voice-agent-platforms.mdx b/docs/guides/voice-agent-platforms.mdx index f54953c..d03776d 100644 --- a/docs/guides/voice-agent-platforms.mdx +++ b/docs/guides/voice-agent-platforms.mdx @@ -1,6 +1,6 @@ --- title: Voice Agent Platforms for React Developers -description: Compare where the UI layer fits across voice agent platforms like Vapi, ElevenLabs, LiveKit, OpenAI Realtime, Gemini Live API, Retell AI, Bland AI, and Synthflow. +description: Compare where the UI layer fits across voice agent platforms like Vapi, ElevenLabs, LiveKit, OpenAI Live, OpenAI Realtime, Gemini Live API, Retell AI, Bland AI, and Synthflow. --- Voice agent platforms can provide speech, realtime audio, call orchestration, tool calls, and deployment. React teams still need a clear frontend layer for state, activity, and trust. @@ -12,7 +12,7 @@ orb-ui is intentionally narrow. It is the React UI layer that can sit on top of | Layer | Examples | orb-ui role | | -------------------- | --------------------------------------------- | ---------------------------------------------- | | Voice agent platform | Vapi, LiveKit, Retell AI, Bland AI, Synthflow | Render the visible state layer | -| Realtime model API | OpenAI Realtime, Gemini Live API | Use the dedicated provider adapter | +| Realtime model API | OpenAI Live, OpenAI Realtime, Gemini Live API | Use the dedicated provider adapter | | Voice generation | ElevenLabs | Render conversational state and audio activity | | Custom backend | WebRTC, WebSocket, telephony | Use controlled mode or write an adapter | @@ -23,7 +23,7 @@ When comparing tools, keep the distinction clear: - Vapi is a voice agent platform. - LiveKit Agents can provide realtime media infrastructure for voice agents. - ElevenLabs can provide voice and conversational agent capabilities. -- OpenAI Realtime and Gemini Live API are lower-level realtime model APIs. +- OpenAI Live, OpenAI Realtime, and Gemini Live API are lower-level realtime model APIs. - orb-ui is the frontend component layer. That distinction is better for trust and more useful for developers than pretending everything is interchangeable. @@ -33,5 +33,6 @@ That distinction is better for trust and more useful for developers than pretend - [Vapi adapter](/adapters/vapi) - [ElevenLabs adapter](/adapters/elevenlabs) - [LiveKit adapter](/adapters/livekit) +- [OpenAI GPT-Live adapter](/adapters/openai-live) - [OpenAI Realtime adapter](/adapters/openai-realtime) - [Gemini Live adapter](/adapters/gemini-live) diff --git a/docs/guides/voice-agent-ui.mdx b/docs/guides/voice-agent-ui.mdx index edf18c1..d80921a 100644 --- a/docs/guides/voice-agent-ui.mdx +++ b/docs/guides/voice-agent-ui.mdx @@ -8,7 +8,7 @@ assistant is doing. It should make connecting, listening, thinking, speaking, in failure understandable without forcing users to interpret an animation on its own. orb-ui gives React teams that layer through animated voice orbs, audio-reactive themes, provider -adapters, and controlled state. Use it with Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, +adapters, and controlled state. Use it with Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, or a custom browser voice stack. ## Build a React voice agent UI @@ -57,7 +57,7 @@ orb-ui models the core lifecycle as: - `speaking` - `error` -Those states map cleanly to Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live, and custom WebRTC or WebSocket voice pipelines. +Those states map cleanly to Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, and custom WebRTC or WebSocket voice pipelines. | State | Question the UI should answer | Useful visual treatment | | ------------ | --------------------------------------- | -------------------------------------------------------- | @@ -180,6 +180,7 @@ evaluating a polished theme. | ElevenLabs | [ElevenLabs Voice Orb UI for React](/adapters/elevenlabs) | Let the adapter manage the conversational session and both audio levels | | LiveKit Agents | [LiveKit Voice UI for React](/adapters/livekit) | Connect through a token endpoint and map agent participant state | | Pipecat | [Pipecat Voice UI for React](/adapters/pipecat) | Reuse a configured Pipecat client across supported transports | +| OpenAI GPT-Live | [OpenAI GPT-Live Voice UI for React](/adapters/openai-live) | Let orb-ui own full-duplex WebRTC and audio playback | | OpenAI Realtime | [OpenAI Realtime Voice UI for React](/adapters/openai-realtime) | Let orb-ui own browser WebRTC and audio playback | | Gemini Live | [Gemini Live Voice UI for React](/adapters/gemini-live) | Open the official Live session with a short-lived token | | Custom stack | [Custom Voice AI UI Integrations](/adapters/custom) | Supply normalized state and volume or build an adapter | diff --git a/docs/index.mdx b/docs/index.mdx index ae97f6f..9928311 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -10,7 +10,7 @@ Use orb-ui when you want users to understand whether a voice agent is idle, conn ## Start here 1. Install `orb-ui`. -2. Choose an adapter for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, or Gemini Live, or use controlled mode. +2. Choose an adapter for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, or Gemini Live, or use controlled mode. 3. Pick a visual theme. 4. Map your voice agent state into the Orb component. @@ -26,13 +26,13 @@ export function VoiceAgentStatus({ state, inputVolume, outputVolume }) { - A small React component for visible voice agent state. - Audio-reactive orb and bars themes. -- Provider adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, and Gemini Live. +- Provider adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, and Gemini Live. - Controlled mode for telephony, custom WebRTC/WebSocket transports, and other speech pipelines. - A stable state model for `idle`, `connecting`, `listening`, `thinking`, `speaking`, and `error`. ## What orb-ui does not handle -orb-ui is not a voice agent platform. It does not host calls, manage prompts, run speech recognition, or replace Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live API, or your own backend. It is the frontend UI layer that makes those systems feel understandable inside a product. +orb-ui is not a voice agent platform. It does not host calls, manage prompts, run speech recognition, or replace Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live API, or your own backend. It is the frontend UI layer that makes those systems feel understandable inside a product. ## Next steps @@ -46,6 +46,7 @@ orb-ui is not a voice agent platform. It does not host calls, manage prompts, ru - [ElevenLabs adapter](/adapters/elevenlabs) - [LiveKit adapter](/adapters/livekit) - [Pipecat adapter](/adapters/pipecat) +- [OpenAI GPT-Live adapter](/adapters/openai-live) - [OpenAI Realtime adapter](/adapters/openai-realtime) - [Gemini Live adapter](/adapters/gemini-live) - [Custom integrations](/adapters/custom) diff --git a/docs/installation.mdx b/docs/installation.mdx index 4bb27ff..6e28ec3 100644 --- a/docs/installation.mdx +++ b/docs/installation.mdx @@ -36,7 +36,7 @@ npm install orb-ui livekit-client # Pipecat (add the transport used by your agent) npm install orb-ui @pipecat-ai/client-js @pipecat-ai/small-webrtc-transport -# OpenAI Realtime uses browser WebRTC and needs no additional client SDK +# OpenAI Live and Realtime use browser WebRTC and need no additional client SDK npm install orb-ui # Gemini Live @@ -47,6 +47,7 @@ npm install orb-ui @google/genai import { createElevenLabsAdapter, createGeminiLiveAdapter, + createOpenAILiveAdapter, createOpenAIRealtimeAdapter, createPipecatAdapter, createVapiAdapter, @@ -65,7 +66,8 @@ Use an adapter when a supported provider owns the voice session. Use controlled - Use the [ElevenLabs adapter](/adapters/elevenlabs) for ElevenLabs conversational agents. - Use the [LiveKit adapter](/adapters/livekit) for LiveKit Agents. - Use the [Pipecat adapter](/adapters/pipecat) with Pipecat Cloud, Daily, or self-hosted SmallWebRTC agents. -- Use the [OpenAI Realtime adapter](/adapters/openai-realtime) for native browser WebRTC voice sessions. +- Start with the [OpenAI GPT-Live adapter](/adapters/openai-live) for full-duplex voice sessions. +- Use the [OpenAI Realtime adapter](/adapters/openai-realtime) for existing Realtime integrations. - Use the [Gemini Live adapter](/adapters/gemini-live) for native-audio Live API sessions. ## Next steps diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 633cf4e..6a610db 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -19,7 +19,7 @@ npm install orb-ui @pipecat-ai/client-js @pipecat-ai/small-webrtc-transport npm install orb-ui @google/genai ``` -OpenAI Realtime uses browser WebRTC and does not require an additional client SDK. Standard OpenAI +OpenAI Live and Realtime use browser WebRTC and do not require an additional client SDK. Standard OpenAI and Gemini API keys stay on your server. See the [adapter overview](/adapters/overview) for a concise comparison of what each provider setup requires. @@ -130,27 +130,34 @@ export function PipecatVoiceUI() { See the [Pipecat adapter guide](/adapters/pipecat) for Pipecat Cloud and Daily. -## OpenAI Realtime +## OpenAI GPT-Live ```tsx import { Orb } from 'orb-ui' -import { createOpenAIRealtimeAdapter } from 'orb-ui/adapters' - -const adapter = createOpenAIRealtimeAdapter({ - getClientSecret: async () => { - const response = await fetch('/api/openai-realtime-token', { method: 'POST' }) - const data = await response.json() - return data.value +import { createOpenAILiveAdapter } from 'orb-ui/adapters' + +const adapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() }, }) -export function OpenAIRealtimeVoiceUI() { - return +export function OpenAILiveVoiceUI() { + return } ``` -The endpoint must mint short-lived credentials with your server-only API key. See the -[OpenAI Realtime adapter guide](/adapters/openai-realtime). +Your server creates a `gpt-live-1` session and returns its session ID and SDP answer. Follow the +[OpenAI Live guide](/adapters/openai-live) for the server endpoint and delegation setup. +For existing Realtime sessions, keep using +[`createOpenAIRealtimeAdapter({ getClientSecret })`](/adapters/openai-realtime). ## Gemini Live diff --git a/package.json b/package.json index 5ebd5c0..41f2e6d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "orb-ui", "version": "0.8.1", - "description": "React voice agent UI components with audio-reactive adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Realtime, Gemini Live, and custom voice AI apps.", + "description": "React voice agent UI components with audio-reactive adapters for Vapi, ElevenLabs, LiveKit, Pipecat, OpenAI Live, OpenAI Realtime, Gemini Live, and custom voice AI apps.", "type": "module", "main": "./dist/orb-ui.cjs", "module": "./dist/orb-ui.js", @@ -101,6 +101,7 @@ "elevenlabs", "livekit", "pipecat", + "openai-live", "openai-realtime", "gemini-live" ], diff --git a/src/adapters/index.ts b/src/adapters/index.ts index 8678e7c..4fae9be 100644 --- a/src/adapters/index.ts +++ b/src/adapters/index.ts @@ -28,6 +28,13 @@ export { type PipecatParticipantLike, type PipecatTracksLike, } from './pipecat' +export { + createOpenAILiveAdapter, + type OpenAILiveAdapterConfig, + type OpenAILiveEvent, + type OpenAILiveOrbAdapter, + type OpenAILiveSessionResponse, +} from './openai-live' export { createOpenAIRealtimeAdapter, type OpenAIRealtimeAdapterConfig, diff --git a/src/adapters/openai-live/index.test.ts b/src/adapters/openai-live/index.test.ts new file mode 100644 index 0000000..deefe6e --- /dev/null +++ b/src/adapters/openai-live/index.test.ts @@ -0,0 +1,276 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { createOpenAILiveAdapter, type OpenAILiveAdapterConfig } from './index' +import type { OrbSignal } from '../types' + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() +}) + +function setup(overrides: Partial = {}) { + const microphone = { kind: 'audio', stop: vi.fn() } + const speaker = { kind: 'audio', stop: vi.fn() } + const levels = new Map([ + [microphone, 0.1], + [speaker, 0], + ]) + class Stream { + constructor(public tracks: unknown[]) {} + getTracks() { + return this.tracks + } + getAudioTracks() { + return this.tracks + } + } + vi.stubGlobal('MediaStream', Stream) + const stream = new Stream([microphone]) as unknown as MediaStream + const channel = { + readyState: 'open', + send: vi.fn(), + close: vi.fn(), + onmessage: null as ((event: MessageEvent) => void) | null, + onclose: null as (() => void) | null, + } + const peer = Object.assign(new EventTarget(), { + iceGatheringState: 'complete', + connectionState: 'connected', + localDescription: { sdp: 'gathered-offer' }, + ontrack: null as ((event: RTCTrackEvent) => void) | null, + onconnectionstatechange: null as (() => void) | null, + addTrack: vi.fn(), + createDataChannel: vi.fn(() => channel), + createOffer: vi.fn(async () => ({ type: 'offer', sdp: 'initial-offer' })), + setLocalDescription: vi.fn(async () => undefined), + setRemoteDescription: vi.fn(async () => undefined), + close: vi.fn(), + }) + const audio = { + autoplay: false, + paused: false, + muted: false, + volume: 1, + play: vi.fn(async () => undefined), + pause: vi.fn(), + remove: vi.fn(), + srcObject: null, + } + const contexts: Array<{ close: ReturnType }> = [] + const createSession = vi.fn(async () => ({ + session: { id: 'live_test' }, + transport: { type: 'webrtc' as const, sdp: 'answer' }, + })) + const onEvent = vi.fn() + const adapter = createOpenAILiveAdapter({ + createSession, + onEvent, + getUserMedia: async () => stream, + createPeerConnection: () => peer as unknown as RTCPeerConnection, + createAudioElement: () => audio as unknown as HTMLAudioElement, + createAudioContext: () => { + let track = microphone + const context = { + state: 'running', + close: vi.fn(async () => undefined), + createMediaStreamSource: (source: Stream) => { + track = source.tracks[0] as typeof microphone + return { connect: vi.fn(), disconnect: vi.fn() } + }, + createAnalyser: () => ({ + fftSize: 512, + smoothingTimeConstant: 0, + getFloatTimeDomainData: (samples: Float32Array) => samples.fill(levels.get(track) ?? 0), + disconnect: vi.fn(), + }), + } + contexts.push(context) + return context as unknown as AudioContext + }, + ...overrides, + }) + const signals: OrbSignal[] = [] + adapter.subscribe((signal) => signals.push(signal)) + const event = (type: string, extra = {}) => + channel.onmessage?.({ data: JSON.stringify({ type, ...extra }) } as MessageEvent) + const start = async () => { + const pending = adapter.start() + await vi.waitFor(() => expect(peer.setRemoteDescription).toHaveBeenCalled()) + event('session.started', { session: { id: 'live_test' } }) + await pending + peer.ontrack?.({ streams: [new Stream([speaker])], track: speaker } as unknown as RTCTrackEvent) + } + const stop = async () => { + const pending = adapter.stop() + event('session.closed', { usage: { seconds: 12 } }) + await pending + } + return { + adapter, + peer, + channel, + audio, + contexts, + microphone, + speaker, + levels, + createSession, + onEvent, + signals, + event, + start, + stop, + } +} + +describe('GPT-Live WebRTC adapter', () => { + it('waits for gathered SDP and session.started, forwards events and commands, and finalizes before cleanup', async () => { + const s = setup() + s.peer.iceGatheringState = 'gathering' + const first = s.adapter.start() + expect(s.adapter.start()).toBe(first) + await vi.waitFor(() => expect(s.peer.setLocalDescription).toHaveBeenCalled()) + expect(s.createSession).not.toHaveBeenCalled() + s.peer.iceGatheringState = 'complete' + s.peer.dispatchEvent(new Event('icegatheringstatechange')) + await vi.waitFor(() => + expect(s.peer.setRemoteDescription).toHaveBeenCalledWith({ type: 'answer', sdp: 'answer' }), + ) + expect(s.createSession).toHaveBeenCalledWith('gathered-offer', expect.any(AbortSignal)) + expect(s.signals.at(-1)?.state).toBe('connecting') + expect(() => s.adapter.send({ type: 'session.update' })).toThrow('not ready') + s.event('session.started') + await first + expect(s.signals.at(-1)?.state).toBe('listening') + expect(s.channel.send).not.toHaveBeenCalled() + const caption = { + type: 'session.output_transcript.delta', + delta: 'Hello', + start_ms: 10, + end_ms: 30, + } + s.event(caption.type, caption) + expect(s.onEvent).toHaveBeenCalledWith(caption) + const command = { type: 'session.commentary.append', delegation_id: null, content: 'Done' } + s.adapter.send(command) + expect(s.channel.send).toHaveBeenCalledWith(JSON.stringify(command)) + expect(() => s.adapter.send({ type: 'session.start' })).toThrow('Use start()') + const stopping = s.adapter.stop() + expect(s.adapter.stop()).toBe(stopping) + expect(s.channel.send).toHaveBeenLastCalledWith('{"type":"session.close"}') + expect(s.microphone.stop).not.toHaveBeenCalled() + await expect(s.adapter.start()).rejects.toThrow('Wait for stop') + s.event('session.closed', { usage: { seconds: 12 }, reason: 'close_requested' }) + await stopping + expect(s.onEvent).toHaveBeenLastCalledWith(expect.objectContaining({ usage: { seconds: 12 } })) + expect(s.microphone.stop).toHaveBeenCalledOnce() + expect(s.peer.close).toHaveBeenCalledOnce() + expect(s.signals.at(-1)).toEqual({ state: 'idle', inputVolume: 0, outputVolume: 0 }) + }) + + it('meters simultaneous speech; transcripts and backend completion never end playback', async () => { + vi.useFakeTimers() + const s = setup() + await s.start() + s.levels.set(s.speaker, 0.1) + await vi.advanceTimersByTimeAsync(330) + expect(s.signals.at(-1)).toMatchObject({ state: 'speaking' }) + expect(s.signals.at(-1)!.inputVolume).toBeGreaterThan(0) + expect(s.signals.at(-1)!.outputVolume).toBeGreaterThan(0) + s.event('session.input_transcript.delta', { delta: 'interrupt' }) + s.event('response.event', { event: { type: 'response.completed' } }) + s.event('response.done') + expect(s.signals.at(-1)?.state).toBe('speaking') + s.levels.set(s.speaker, 0) + await vi.advanceTimersByTimeAsync(300) + expect(s.signals.at(-1)?.state).toBe('listening') + s.levels.set(s.speaker, 0.1) + s.audio.muted = true + await vi.advanceTimersByTimeAsync(300) + expect(s.signals.at(-1)?.state).toBe('listening') + await s.stop() + expect(s.contexts.every((context) => context.close.mock.calls.length === 1)).toBe(true) + const count = s.signals.length + await vi.advanceTimersByTimeAsync(1000) + expect(s.signals).toHaveLength(count) + await s.start() + expect(s.createSession).toHaveBeenCalledTimes(2) + await s.stop() + }) + + it('reports negotiation failures and releases media', async () => { + const s = setup({ + createSession: async () => { + throw new Error('Access denied') + }, + }) + await expect(s.adapter.start()).rejects.toThrow('Access denied') + expect(s.microphone.stop).toHaveBeenCalledOnce() + expect(s.peer.close).toHaveBeenCalledOnce() + expect(s.signals.at(-1)?.state).toBe('error') + }) + + it('cancels pending microphone access and stops tracks when permission eventually resolves', async () => { + let grant!: (stream: MediaStream) => void + const s = setup({ + getUserMedia: () => + new Promise((resolve) => { + grant = resolve + }), + }) + const pending = s.adapter.start() + const rejected = expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + await s.adapter.stop() + grant({ getTracks: () => [s.microphone] } as unknown as MediaStream) + await rejected + expect(s.microphone.stop).toHaveBeenCalledOnce() + expect(s.createSession).not.toHaveBeenCalled() + expect(s.signals.at(-1)?.state).toBe('idle') + }) + + it('times out startup without session.started and cleans up', async () => { + vi.useFakeTimers() + const s = setup({ startTimeoutMs: 100 }) + const pending = expect(s.adapter.start()).rejects.toThrow( + 'Timed out waiting for session.started', + ) + await vi.advanceTimersByTimeAsync(100) + await pending + expect(s.peer.close).toHaveBeenCalledOnce() + }) + + it('reports unconfirmed finalization on timeout and disconnect', async () => { + vi.useFakeTimers() + const s = setup({ closeTimeoutMs: 100 }) + await s.start() + const pending = expect(s.adapter.stop()).rejects.toThrow('final usage is unconfirmed') + await vi.advanceTimersByTimeAsync(100) + await pending + expect(s.microphone.stop).toHaveBeenCalledOnce() + expect(s.signals.at(-1)?.state).toBe('error') + const other = setup() + await other.start() + const disconnected = expect(other.adapter.stop()).rejects.toThrow('Connection lost') + other.channel.onclose?.() + await disconnected + expect(other.peer.close).toHaveBeenCalledOnce() + }) + + it('surfaces autoplay errors and ignores malformed data', async () => { + const s = setup() + s.audio.play.mockRejectedValueOnce(new Error('Playback blocked')) + await s.start() + await Promise.resolve() + expect(s.signals.at(-1)?.state).toBe('error') + for (const data of ['not json', 'null', '{}']) s.channel.onmessage?.({ data } as MessageEvent) + await s.stop() + }) + + it('does not report a successful stop when the event channel is already closing', async () => { + const s = setup() + await s.start() + s.channel.readyState = 'closing' + await expect(s.adapter.stop()).rejects.toThrow('final usage is unconfirmed') + expect(s.microphone.stop).toHaveBeenCalledOnce() + expect(s.signals.at(-1)?.state).toBe('error') + }) +}) diff --git a/src/adapters/openai-live/index.ts b/src/adapters/openai-live/index.ts new file mode 100644 index 0000000..3d34c97 --- /dev/null +++ b/src/adapters/openai-live/index.ts @@ -0,0 +1,379 @@ +import type { OrbAdapter, OrbSignal, OrbSignalListener, OrbState } from '../types' +import { + createMediaStreamTrackVolumeMeter, + createVolumeNormalizer, + type MediaStreamTrackVolumeMeter, +} from '../audio-level' +import type { OpenAIRealtimeAdapterConfig } from '../openai-realtime' +import { PROVIDER_VOLUME_CALIBRATIONS } from '../volume-presets' + +/** JSON events are deliberately open-ended so apps can handle new Live events. */ +export interface OpenAILiveEvent { + type: string + [key: string]: unknown +} + +export interface OpenAILiveSessionResponse { + session: { id: string } + transport: { type: 'webrtc'; sdp: string } +} + +export interface OpenAILiveAdapterConfig extends Pick< + OpenAIRealtimeAdapterConfig, + | 'mediaStreamConstraints' + | 'getUserMedia' + | 'createPeerConnection' + | 'createAudioElement' + | 'createAudioContext' + | 'inputVolumeCalibration' + | 'outputVolumeCalibration' + | 'onInputVolumeSample' + | 'onOutputVolumeSample' +> { + /** Exchange the SDP offer on your server using POST /v1/live/sessions. */ + createSession(sdp: string, signal: AbortSignal): Promise + /** Receives captions, delegation, usage, errors, and final session events unchanged. */ + onEvent?: (event: OpenAILiveEvent) => void + /** Startup deadline, including microphone permission and session.started. Default: 30 seconds. */ + startTimeoutMs?: number + /** How long stop waits for session.closed. Default: 15 seconds. */ + closeTimeoutMs?: number +} + +export interface OpenAILiveOrbAdapter extends OrbAdapter { + /** Resolves after SDP negotiation and session.started. */ + start(): Promise + /** Finalizes the session; rejects if finalization cannot be confirmed. */ + stop(): Promise + /** Send application commands after startup, subject to server-configured client permissions. */ + send(event: OpenAILiveEvent): void +} + +interface Session { + abort: AbortController + pc?: RTCPeerConnection + channel?: RTCDataChannel + stream?: MediaStream + audio?: HTMLAudioElement + inputMeter?: MediaStreamTrackVolumeMeter + outputMeter?: MediaStreamTrackVolumeMeter + ready: boolean + closing: boolean + disposed: boolean + closed: boolean + silenceTicks: number + started?: () => void + finalized?: (error?: Error) => void +} + +const failure = (message: string) => new Error(`[orb-ui/openai-live] ${message}`) + +/** Managed GPT-Live WebRTC. Project keys and model/delegation configuration stay on your server. */ +export function createOpenAILiveAdapter(config: OpenAILiveAdapterConfig): OpenAILiveOrbAdapter { + const listeners = new Set() + let signal: OrbSignal = { state: 'idle', inputVolume: 0, outputVolume: 0 } + let active: Session | undefined + let starting: Promise | undefined + let stopping: Promise | undefined + // Shared WebRTC RMS baseline; applications can calibrate Live voices independently. + const input = createVolumeNormalizer( + PROVIDER_VOLUME_CALIBRATIONS.openai.input, + config.inputVolumeCalibration, + ) + const output = createVolumeNormalizer( + PROVIDER_VOLUME_CALIBRATIONS.openai.output, + config.outputVolumeCalibration, + ) + + function emit(next: OrbSignal) { + signal = next + listeners.forEach((listener) => listener(next)) + } + + function state(next: OrbState, error?: unknown) { + if (next === signal.state && error === undefined) return + const reset = next === 'idle' || next === 'connecting' || next === 'error' + if (reset) { + input.reset() + output.reset() + } + emit({ + state: next, + inputVolume: reset ? 0 : signal.inputVolume, + outputVolume: reset ? 0 : signal.outputVolume, + ...(error === undefined ? {} : { error }), + }) + } + + async function cleanup(session: Session) { + if (session.disposed) return + session.disposed = true + session.abort.abort() + if (active === session) active = undefined + if (session.channel) { + session.channel.onmessage = null + session.channel.onclose = null + session.channel.onerror = null + session.channel.close() + } + if (session.pc) { + session.pc.ontrack = null + session.pc.onconnectionstatechange = null + session.pc.close() + } + session.stream?.getTracks().forEach((track) => track.stop()) + if (session.audio) { + session.audio.pause() + session.audio.srcObject = null + session.audio.remove() + } + await Promise.allSettled([session.inputMeter?.stop(), session.outputMeter?.stop()]) + } + + function check(session: Session) { + if (session.disposed) throw new DOMException('Session start cancelled.', 'AbortError') + } + + function disconnect(session: Session) { + if (session.disposed || session.closed) return + const error = failure('Connection lost before session.closed; final usage is unconfirmed.') + session.finalized?.(error) + state('error', error) + void cleanup(session) + } + + function handleEvent(session: Session, event: OpenAILiveEvent) { + if (session.disposed) return + if (event.type === 'session.started' && !session.ready) { + session.ready = true + state('listening') + session.started?.() + } else if (event.type === 'session.closed') { + session.closed = true + session.finalized?.() + state('idle') + void cleanup(session) + } else if (event.type === 'error') { + state('error', event.error ?? event) + } + // Captions and delegated Responses events are not voice playback boundaries. + config.onEvent?.(event) + } + + async function connect(session: Session) { + const getUserMedia = + config.getUserMedia ?? ((constraints) => navigator.mediaDevices.getUserMedia(constraints)) + const stream = await getUserMedia(config.mediaStreamConstraints ?? { audio: true }) + if (session.disposed) { + stream.getTracks().forEach((track) => track.stop()) + check(session) + } + session.stream = stream + const pc = (config.createPeerConnection ?? (() => new RTCPeerConnection()))() + session.pc = pc + const audio = (config.createAudioElement ?? (() => new Audio()))() + session.audio = audio + audio.autoplay = true + const createContext = config.createAudioContext ?? (() => new AudioContext()) + const meter = (track: MediaStreamTrack, direction: 'input' | 'output') => + createMediaStreamTrackVolumeMeter(track, createContext, (raw) => { + if (session.disposed || !session.ready || signal.state === 'error') return + const sample = (direction === 'input' ? input : output).sample(raw) + if (direction === 'input') config.onInputVolumeSample?.(sample) + else { + config.onOutputVolumeSample?.(sample) + // Use activity before envelope decay, so a visual tail cannot prolong speech. + // Full duplex: microphone activity never suppresses the assistant's playback. + if (!audio.paused && !audio.muted && audio.volume > 0 && sample.mapped > 0.015) { + session.silenceTicks = 0 + state('speaking') + } else if (++session.silenceTicks >= 8) state('listening') + } + emit({ + ...signal, + [direction === 'input' ? 'inputVolume' : 'outputVolume']: sample.normalized, + }) + }) + const tracks = stream.getAudioTracks() + tracks.forEach((track) => pc.addTrack(track, stream)) + if (tracks[0]) session.inputMeter = meter(tracks[0], 'input') + pc.ontrack = (event) => { + if (session.disposed || event.track.kind !== 'audio') return + audio.srcObject = event.streams[0] ?? new MediaStream([event.track]) + void audio.play().catch((error: unknown) => { + if (!session.disposed) state('error', error) + }) + void session.outputMeter?.stop() + session.outputMeter = meter(event.track, 'output') + } + pc.onconnectionstatechange = () => { + if ( + pc.connectionState === 'failed' || + pc.connectionState === 'closed' || + pc.connectionState === 'disconnected' + ) + disconnect(session) + } + const channel = pc.createDataChannel('oai-events') + session.channel = channel + channel.onmessage = ({ data }) => { + let event: unknown + try { + event = JSON.parse(String(data)) + } catch { + return + } + if ( + !event || + typeof event !== 'object' || + !('type' in event) || + typeof event.type !== 'string' + ) + return + handleEvent(session, event as OpenAILiveEvent) + } + channel.onclose = () => disconnect(session) + channel.onerror = () => disconnect(session) + const offer = await pc.createOffer() + check(session) + await pc.setLocalDescription(offer) + check(session) + if (pc.iceGatheringState !== 'complete') { + await new Promise((resolve, reject) => { + const done = () => { + pc.removeEventListener('icegatheringstatechange', changed) + session.abort.signal.removeEventListener('abort', cancelled) + } + const changed = () => { + if (pc.iceGatheringState === 'complete') { + done() + resolve() + } + } + const cancelled = () => { + done() + reject(new DOMException('Session start cancelled.', 'AbortError')) + } + pc.addEventListener('icegatheringstatechange', changed) + session.abort.signal.addEventListener('abort', cancelled, { once: true }) + changed() + }) + } + check(session) + const sdp = pc.localDescription?.sdp + if (!sdp) throw failure('Missing local SDP offer.') + const result = await config.createSession(sdp, session.abort.signal) + check(session) + if (!result?.session?.id || result.transport?.type !== 'webrtc' || !result.transport.sdp) { + throw failure('createSession must return a session ID and WebRTC SDP answer.') + } + await pc.setRemoteDescription({ type: 'answer', sdp: result.transport.sdp }) + check(session) + } + + return { + subscribe(listener) { + listeners.add(listener) + listener(signal) + return () => listeners.delete(listener) + }, + start() { + if (stopping) return Promise.reject(failure('Wait for stop() before restarting.')) + if (starting) return starting + if (active) return Promise.resolve() + const session: Session = { + abort: new AbortController(), + ready: false, + closing: false, + disposed: false, + closed: false, + silenceTicks: 0, + } + active = session + state('connecting') + starting = (async () => { + let timer: ReturnType | undefined + const ready = new Promise((resolve) => { + session.started = resolve + }) + const cancelled = new Promise((_, reject) => { + session.abort.signal.addEventListener( + 'abort', + () => reject(new DOMException('Session start cancelled.', 'AbortError')), + { once: true }, + ) + timer = setTimeout( + () => reject(failure('Timed out waiting for session.started.')), + config.startTimeoutMs ?? 30_000, + ) + }) + try { + await Promise.race([Promise.all([connect(session), ready]), cancelled]) + } catch (error) { + const wasActive = active === session + await cleanup(session) + if (wasActive) state('error', error) + throw error + } finally { + clearTimeout(timer) + starting = undefined + } + })() + return starting + }, + stop() { + if (stopping) return stopping + const session = active + if (!session) { + state('idle') + return Promise.resolve() + } + session.closing = true + stopping = (async () => { + try { + if (session.ready && session.channel?.readyState !== 'open') { + throw failure('Event channel is unavailable; final usage is unconfirmed.') + } + if (session.ready && session.channel?.readyState === 'open') { + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => + reject( + failure('Timed out waiting for session.closed; final usage is unconfirmed.'), + ), + config.closeTimeoutMs ?? 15_000, + ) + session.finalized = (error) => { + clearTimeout(timer) + if (error) reject(error) + else resolve() + } + try { + session.channel!.send(JSON.stringify({ type: 'session.close' })) + } catch (error) { + clearTimeout(timer) + reject(error) + } + }) + } + await cleanup(session) + state('idle') + } catch (error) { + await cleanup(session) + state('error', error) + throw error + } finally { + stopping = undefined + } + })() + return stopping + }, + send(event) { + if (!active?.ready || active.closing || active.channel?.readyState !== 'open') + throw failure('Session is not ready for commands.') + if (event.type === 'session.start' || event.type === 'session.close') + throw failure('Use start() and stop() to manage the session.') + active.channel.send(JSON.stringify(event)) + }, + } +} diff --git a/tests/demo/openai-live-session.test.ts b/tests/demo/openai-live-session.test.ts new file mode 100644 index 0000000..746aebd --- /dev/null +++ b/tests/demo/openai-live-session.test.ts @@ -0,0 +1,90 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import handler, { createLiveSession } from '../../demo/api/openai-live-session' + +const request = (body: unknown) => + new Request('http://localhost/api/openai-live-session', { + method: 'POST', + body: JSON.stringify(body), + }) + +afterEach(() => vi.unstubAllGlobals()) + +describe('playground GPT-Live session endpoint', () => { + it('keeps the local credential upstream and returns only the session transport', async () => { + const fetch = vi.fn().mockResolvedValue( + Response.json({ + session: { id: 'live-test', private: 'omitted' }, + transport: { type: 'webrtc', sdp: 'answer' }, + extra: 'omitted', + }), + ) + vi.stubGlobal('fetch', fetch) + const response = await createLiveSession( + request({ sdp: 'v=0\r\n', backendModel: 'backend-test', instructions: 'test instructions' }), + 'local-test-key', + ) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + session: { id: 'live-test' }, + transport: { type: 'webrtc', sdp: 'answer' }, + }) + const [url, init] = fetch.mock.calls[0] + expect(url).toBe('https://api.openai.com/v1/live/sessions') + expect(init.headers.Authorization).toBe('Bearer local-test-key') + expect(JSON.parse(init.body)).toMatchObject({ + session: { + model: 'gpt-live-1', + store: false, + instructions: 'test instructions', + delegation: { type: 'responses', responses: { model: 'backend-test' } }, + }, + transport: { type: 'webrtc', sdp: 'v=0\r\n' }, + }) + expect(response.headers.get('Cache-Control')).toBe('no-store') + }) + + it('requires a caller key on the deployed handler and validates SDP before calling OpenAI', async () => { + const fetch = vi.fn() + vi.stubGlobal('fetch', fetch) + expect((await handler.fetch(request({ sdp: 'offer' }))).status).toBe(400) + expect((await handler.fetch(request({ apiKey: 'test', sdp: 42 }))).status).toBe(400) + expect((await handler.fetch(request(null))).status).toBe(400) + expect(fetch).not.toHaveBeenCalled() + }) + + it('redacts upstream errors and rejects incomplete success responses', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce( + Response.json({ error: { message: 'sensitive provider detail' } }, { status: 401 }), + ) + .mockResolvedValueOnce(Response.json({ session: { id: 'incomplete' } })), + ) + const response = await createLiveSession(request({ sdp: 'offer' }), 'test-key') + expect(response.status).toBe(401) + expect(await response.text()).not.toContain('sensitive provider detail') + expect((await createLiveSession(request({ sdp: 'offer' }), 'test-key')).status).toBe(502) + }) + + it('propagates cancellation to the upstream request', async () => { + const controller = new AbortController() + const fetch = vi.fn().mockImplementation((_url, init) => { + controller.abort() + expect(init.signal.aborted).toBe(true) + throw new DOMException('Aborted', 'AbortError') + }) + vi.stubGlobal('fetch', fetch) + const response = await createLiveSession( + new Request('http://localhost/api/openai-live-session', { + method: 'POST', + body: JSON.stringify({ sdp: 'offer' }), + signal: controller.signal, + }), + 'test-key', + ) + expect(response.status).toBe(502) + expect(fetch).toHaveBeenCalledOnce() + }) +}) diff --git a/tests/e2e/fixture/main.tsx b/tests/e2e/fixture/main.tsx index 0e91a79..0cbca30 100644 --- a/tests/e2e/fixture/main.tsx +++ b/tests/e2e/fixture/main.tsx @@ -2,7 +2,12 @@ import { useEffect, useMemo, useState } from 'react' import { createRoot } from 'react-dom/client' import { Orb, OrbThemeProvider } from 'orb-ui' import type { OrbAdapter, OrbSignal, OrbState, OrbThemePreset } from 'orb-ui' -import { createElevenLabsAdapter, createLiveKitAdapter, createVapiAdapter } from 'orb-ui/adapters' +import { + createElevenLabsAdapter, + createLiveKitAdapter, + createOpenAILiveAdapter, + createVapiAdapter, +} from 'orb-ui/adapters' import { createLiveKitAdapter as createManagedLiveKitAdapter } from 'orb-ui/adapters/livekit' const IDLE_SIGNAL: OrbSignal = { @@ -11,6 +16,9 @@ const IDLE_SIGNAL: OrbSignal = { outputVolume: 0, } +// Exercise the published factory against actual browser media and peer connections. +Object.assign(window, { createOpenAILiveAdapter }) + function App() { const [adapterSignal, setAdapterSignal] = useState(IDLE_SIGNAL) const [cloudPreset, setCloudPreset] = useState('balanced') @@ -46,6 +54,7 @@ function App() { typeof createVapiAdapter === 'function' && typeof createElevenLabsAdapter === 'function' && typeof createLiveKitAdapter === 'function' && + typeof createOpenAILiveAdapter === 'function' && typeof createManagedLiveKitAdapter === 'function' return ( diff --git a/tests/e2e/openai-live.spec.ts b/tests/e2e/openai-live.spec.ts new file mode 100644 index 0000000..8d7d038 --- /dev/null +++ b/tests/e2e/openai-live.spec.ts @@ -0,0 +1,103 @@ +import { expect, test } from '@playwright/test' + +test('GPT-Live adapter negotiates browser WebRTC, meters audio, and finalizes', async ({ + page, +}) => { + await page.goto('/') + const result = await page.evaluate(async () => { + const { createOpenAILiveAdapter } = window as unknown as { + createOpenAILiveAdapter: typeof import('orb-ui/adapters').createOpenAILiveAdapter + } + const context = new AudioContext() + await context.resume() + const oscillator = context.createOscillator() + const gain = context.createGain() + gain.gain.value = 0.1 + oscillator.connect(gain) + const media = context.createMediaStreamDestination() + gain.connect(media) + oscillator.start() + const remote = new RTCPeerConnection() + let channel: RTCDataChannel | undefined + const signals: Array<{ state: string; inputVolume?: number; outputVolume?: number }> = [] + const events: string[] = [] + const commands: string[] = [] + const adapter = createOpenAILiveAdapter({ + getUserMedia: async () => media.stream, + createSession: async (sdp) => { + remote.ondatachannel = ({ channel: opened }) => { + channel = opened + opened.onopen = () => + opened.send(JSON.stringify({ type: 'session.started', session: { id: 'live_local' } })) + opened.onmessage = ({ data }) => { + const event = JSON.parse(data) + commands.push(event.type) + if (event.type === 'session.close') { + opened.send(JSON.stringify({ type: 'session.closed', usage: { seconds: 1 } })) + } + } + } + remote.addTrack(media.stream.getAudioTracks()[0], media.stream) + await remote.setRemoteDescription({ type: 'offer', sdp }) + await remote.setLocalDescription(await remote.createAnswer()) + if (remote.iceGatheringState !== 'complete') { + await new Promise((resolve) => { + remote.onicegatheringstatechange = () => { + if (remote.iceGatheringState === 'complete') resolve() + } + }) + } + return { + session: { id: 'live_local' }, + transport: { type: 'webrtc', sdp: remote.localDescription!.sdp }, + } + }, + onEvent: (event) => events.push(event.type), + }) + adapter.subscribe((signal) => signals.push(signal)) + const waitFor = async (predicate: () => boolean) => { + const deadline = performance.now() + 5000 + while (!predicate()) { + if (performance.now() > deadline) + throw new Error('Timed out waiting for browser audio signal') + await new Promise((resolve) => setTimeout(resolve, 33)) + } + } + try { + await adapter.start() + await waitFor(() => + signals.some( + (signal) => + signal.state === 'speaking' && + (signal.inputVolume ?? 0) > 0 && + (signal.outputVolume ?? 0) > 0, + ), + ) + channel!.send( + JSON.stringify({ type: 'response.event', event: { type: 'response.completed' } }), + ) + await waitFor(() => events.includes('response.event')) + const duringBackendCompletion = signals[signals.length - 1].state + gain.gain.value = 0 + await waitFor(() => signals[signals.length - 1].state === 'listening') + await adapter.stop() + return { + duringBackendCompletion, + lastState: signals[signals.length - 1].state, + trackState: media.stream.getAudioTracks()[0].readyState, + events, + commands, + } + } finally { + await adapter.stop().catch(() => undefined) + remote.close() + oscillator.stop() + await context.close() + } + }) + expect(result.duringBackendCompletion).toBe('speaking') + expect(result.lastState).toBe('idle') + expect(result.trackState).toBe('ended') + expect(result.events).toContain('session.closed') + expect(result.commands).toEqual(['session.close']) +}) diff --git a/tests/package-typecheck/package-consumer.tsx b/tests/package-typecheck/package-consumer.tsx index 5a36393..58fe117 100644 --- a/tests/package-typecheck/package-consumer.tsx +++ b/tests/package-typecheck/package-consumer.tsx @@ -4,6 +4,7 @@ import { createElevenLabsAdapter, createGeminiLiveAdapter, createLiveKitAdapter as createAdvancedLiveKitAdapter, + createOpenAILiveAdapter, createOpenAIRealtimeAdapter, createPipecatAdapter, } from 'orb-ui/adapters' @@ -98,6 +99,23 @@ const pipecatAdapter = createPipecatAdapter(pipecatClient, { outputVolumeCalibration: { envelope: { fallTimeMs: 200 } }, }) +const openAILiveAdapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() + }, + onEvent: (event) => { + const type: string = event.type + void type + }, +}) + const openAIRealtimeAdapter = createOpenAIRealtimeAdapter({ getClientSecret: async () => 'short-lived-client-secret', }) @@ -120,6 +138,7 @@ export function PackageConsumerSmoke() { theme="circle" aria-label="Start app-managed LiveKit assistant" /> + diff --git a/tests/typecheck/provider-adapters.tsx b/tests/typecheck/provider-adapters.tsx index 4fc5264..ff7fcfa 100644 --- a/tests/typecheck/provider-adapters.tsx +++ b/tests/typecheck/provider-adapters.tsx @@ -8,6 +8,7 @@ import type { OrbThemeRenderer } from '../../src' import { createElevenLabsAdapter, createGeminiLiveAdapter, + createOpenAILiveAdapter, createOpenAIRealtimeAdapter, createPipecatAdapter, createVapiAdapter, @@ -63,6 +64,23 @@ const pipecatAdapter = createPipecatAdapter(pipecatClient, { onOutputVolumeSample: ({ raw }) => void raw, }) +const openAILiveAdapter = createOpenAILiveAdapter({ + createSession: async (sdp, signal) => { + const response = await fetch('/api/openai-live-session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sdp }), + signal, + }) + if (!response.ok) throw new Error('Could not create a Live session') + return response.json() + }, + onEvent: (event) => { + const type: string = event.type + void type + }, +}) + const openAIRealtimeAdapter = createOpenAIRealtimeAdapter({ getClientSecret: async () => ({ value: 'short-lived-client-secret' }), }) @@ -164,6 +182,7 @@ export function ProviderAdapterSmokeExamples() { aria-label="Start token-based ElevenLabs voice assistant" /> + ) { export default defineConfig({ test: { - include: ['src/**/*.test.{ts,tsx}'], + include: ['src/**/*.test.{ts,tsx}', 'tests/demo/**/*.test.ts'], }, plugins: [ react(),