feat(agents): Add realtime agents support

.changeset/realtime-agents-voice-mode.md

@@ -0,0 +1,9 @@
+---
+'@electric-ax/agents': patch
+'@electric-ax/agents-desktop': patch
+'@electric-ax/agents-runtime': patch
+'@electric-ax/agents-server': patch
+'@electric-ax/agents-server-ui': patch
+---
+
+Add OpenAI realtime voice mode for Electric Agents, backed by durable audio/control streams. Horton can enter realtime mode with normal context and tools, desktop exposes realtime model/voice/reasoning settings, the server/runtime persist session stream refs, transcripts, and audio spans, and the UI adds voice controls, typed-message forwarding, credential gating, input metering, new-session voice startup, and audio capture/playback fixes.

packages/agents-desktop/package.json

@@ -27,6 +27,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@electric-ax/agents-runtime": "workspace:*",
     "@electric-sql/client": "^1.5.20",
     "@mixmark-io/domino": "^2.2.0",
     "better-sqlite3": "^12.9.0",

packages/agents-desktop/src/app/controller.ts

@@ -11,6 +11,7 @@ import * as DesktopIpc from '../ipc/register'
 import { ensureRuntimeEntry as ensureRuntimeEntryInStore } from '../runtime/entries'
 import { createRuntimeController } from '../runtime/controller'
 import * as SettingsBootstrap from '../settings/bootstrap'
+import * as RealtimeSettings from '../settings/realtime'
 import * as ServerSelection from '../settings/selection'
 import { saveDesktopSettings } from '../settings/store'
 import { desktopStateForWindow as desktopStateForWindowImpl } from '../state/desktop-state'
@@ -30,6 +31,7 @@ import type {
   DesktopMenuSection,
   DesktopMenuState,
   DesktopState,
+  RealtimeSettings as RealtimeSettingsConfig,
   RuntimeEntry,
   ServerConfig,
 } from '../shared/types'
@@ -328,6 +330,20 @@ export function createDesktopMainController(ctx: DesktopAppContext) {
     runtime.refreshPowerSaveBlocker()
   }
 
+  const getRealtimeSettingsStatus = async () =>
+    await RealtimeSettings.realtimeSettingsStatus({
+      settings,
+      apiKeys,
+      launchEnv: ctx.envApiKeysSnapshot,
+    })
+
+  const setRealtimeSettings = async (
+    next: RealtimeSettingsConfig
+  ): Promise<void> => {
+    settings.realtime = RealtimeSettings.normalizeRealtimeSettings(next)
+    await saveSettings()
+  }
+
   const syncLaunchAtLoginSetting = async (): Promise<void> => {
     await LoginItems.setLaunchAtLogin(settings.launchAtLogin === true)
   }
@@ -438,6 +454,8 @@ export function createDesktopMainController(ctx: DesktopAppContext) {
     setLaunchAtLogin,
     getPreventAppSuspension,
     setPreventAppSuspension,
+    getRealtimeSettingsStatus,
+    setRealtimeSettings,
   }
 
   const loadSettings = (): Promise<void> =>

packages/agents-desktop/src/ipc/preferences.ts

@@ -2,13 +2,19 @@ import { ipcMain } from 'electron'
 import type {
   LaunchAtLoginStatus,
   PreventAppSuspensionPreference,
+  RealtimeSettings,
+  RealtimeSettingsStatus,
 } from '../shared/types'
 
 export type PreferencesIpcDeps = {
   getLaunchAtLoginStatus: () => Promise<LaunchAtLoginStatus>
   setLaunchAtLogin: (enabled: boolean) => Promise<LaunchAtLoginStatus>
   getPreventAppSuspension: () => PreventAppSuspensionPreference
   setPreventAppSuspension: (enabled: boolean) => Promise<void>
+  getRealtimeSettingsStatus: () =>
+    | RealtimeSettingsStatus
+    | Promise<RealtimeSettingsStatus>
+  setRealtimeSettings: (settings: RealtimeSettings) => Promise<void>
 }
 
 export function registerPreferencesIpcHandlers(deps: PreferencesIpcDeps): void {
@@ -25,4 +31,11 @@ export function registerPreferencesIpcHandlers(deps: PreferencesIpcDeps): void {
     `desktop:set-prevent-app-suspension`,
     (_event, enabled: boolean) => deps.setPreventAppSuspension(Boolean(enabled))
   )
+  ipcMain.handle(`desktop:get-realtime-settings`, () =>
+    deps.getRealtimeSettingsStatus()
+  )
+  ipcMain.handle(
+    `desktop:set-realtime-settings`,
+    (_event, settings: RealtimeSettings) => deps.setRealtimeSettings(settings)
+  )
 }

packages/agents-desktop/src/preload.ts

@@ -21,6 +21,8 @@ import type {
   McpServerConfig,
   OnboardingState,
   PreventAppSuspensionPreference,
+  RealtimeSettings,
+  RealtimeSettingsStatus,
   ServerConfig,
 } from './shared/types'
 import type { CloudAgentServersState } from './cloud/cloud-agent-servers'
@@ -190,6 +192,10 @@ const api = {
     ipcRenderer.invoke(`desktop:get-prevent-app-suspension`),
   setPreventAppSuspension: (enabled: boolean): Promise<void> =>
     ipcRenderer.invoke(`desktop:set-prevent-app-suspension`, enabled),
+  getRealtimeSettings: (): Promise<RealtimeSettingsStatus> =>
+    ipcRenderer.invoke(`desktop:get-realtime-settings`),
+  setRealtimeSettings: (settings: RealtimeSettings): Promise<void> =>
+    ipcRenderer.invoke(`desktop:set-realtime-settings`, settings),
   getWorkingDirectory: (): Promise<string | null> =>
     ipcRenderer.invoke(`desktop:get-working-directory`),
   chooseWorkingDirectory: (): Promise<string | null> =>

packages/agents-desktop/src/settings/realtime.ts

@@ -0,0 +1,145 @@
+import { createHash } from 'node:crypto'
+import type {
+  ApiKeys,
+  DesktopSettings,
+  RealtimeCredentialStatus,
+  RealtimeSettings,
+  RealtimeSettingsStatus,
+} from '../shared/types'
+import {
+  DEFAULT_OPENAI_REALTIME_MODEL,
+  DEFAULT_OPENAI_REALTIME_REASONING_EFFORT,
+  DEFAULT_OPENAI_REALTIME_VOICE,
+  OPENAI_REALTIME_MODELS,
+  OPENAI_REALTIME_REASONING_EFFORTS,
+  OPENAI_REALTIME_VOICES,
+  isOpenAIRealtimeModel,
+  isOpenAIRealtimeReasoningEffort,
+  isOpenAIRealtimeVoice,
+} from '@electric-ax/agents-runtime'
+
+export const DEFAULT_REALTIME_SETTINGS: RealtimeSettings = {
+  provider: `openai`,
+  model: DEFAULT_OPENAI_REALTIME_MODEL,
+  voice: DEFAULT_OPENAI_REALTIME_VOICE,
+  reasoningEffort: DEFAULT_OPENAI_REALTIME_REASONING_EFFORT,
+  interruptResponse: true,
+}
+
+const OPENAI_REALTIME_VALIDATION_TTL_MS = 5 * 60 * 1000
+
+type RealtimeCredentialValidation = {
+  openAIApiKeyStatus: RealtimeCredentialStatus
+  openAIApiKeyError?: string
+}
+
+const validationCache = new Map<
+  string,
+  { expiresAt: number; result: RealtimeCredentialValidation }
+>()
+
+export function normalizeRealtimeSettings(value: unknown): RealtimeSettings {
+  if (!value || typeof value !== `object`) return DEFAULT_REALTIME_SETTINGS
+  const maybe = value as Partial<Record<keyof RealtimeSettings, unknown>>
+  return {
+    provider: `openai`,
+    model: isOpenAIRealtimeModel(maybe.model)
+      ? maybe.model
+      : DEFAULT_REALTIME_SETTINGS.model,
+    voice: isOpenAIRealtimeVoice(maybe.voice)
+      ? maybe.voice
+      : DEFAULT_REALTIME_SETTINGS.voice,
+    reasoningEffort: isOpenAIRealtimeReasoningEffort(maybe.reasoningEffort)
+      ? maybe.reasoningEffort
+      : DEFAULT_REALTIME_SETTINGS.reasoningEffort,
+    interruptResponse:
+      typeof maybe.interruptResponse === `boolean`
+        ? maybe.interruptResponse
+        : DEFAULT_REALTIME_SETTINGS.interruptResponse,
+  }
+}
+
+function validationCacheKey(apiKey: string, model: string): string {
+  const keyHash = createHash(`sha256`).update(apiKey).digest(`hex`)
+  return `${keyHash}:${model}`
+}
+
+async function validateOpenAIRealtimeApiKey(
+  apiKey: string | null | undefined,
+  model: string
+): Promise<RealtimeCredentialValidation> {
+  if (!apiKey) {
+    return { openAIApiKeyStatus: `missing` }
+  }
+
+  const cacheKey = validationCacheKey(apiKey, model)
+  const cached = validationCache.get(cacheKey)
+  if (cached && cached.expiresAt > Date.now()) return cached.result
+
+  let result: RealtimeCredentialValidation
+  try {
+    const response = await fetch(
+      `https://api.openai.com/v1/models/${encodeURIComponent(model)}`,
+      {
+        headers: { Authorization: `Bearer ${apiKey}` },
+      }
+    )
+    if (response.ok) {
+      result = { openAIApiKeyStatus: `valid` }
+    } else if (
+      response.status === 401 ||
+      response.status === 403 ||
+      response.status === 404
+    ) {
+      result = {
+        openAIApiKeyStatus: `invalid`,
+        openAIApiKeyError:
+          response.status === 404
+            ? `OpenAI API key cannot access ${model}.`
+            : `OpenAI API key was rejected (${response.status}).`,
+      }
+    } else {
+      result = {
+        openAIApiKeyStatus: `unknown`,
+        openAIApiKeyError: `OpenAI credential check failed (${response.status}).`,
+      }
+    }
+  } catch (error) {
+    result = {
+      openAIApiKeyStatus: `unknown`,
+      openAIApiKeyError: error instanceof Error ? error.message : String(error),
+    }
+  }
+
+  validationCache.set(cacheKey, {
+    expiresAt: Date.now() + OPENAI_REALTIME_VALIDATION_TTL_MS,
+    result,
+  })
+  return result
+}
+
+export async function realtimeSettingsStatus({
+  settings,
+  apiKeys,
+  launchEnv,
+}: {
+  settings: DesktopSettings
+  apiKeys: ApiKeys
+  launchEnv: ApiKeys
+}): Promise<RealtimeSettingsStatus> {
+  const normalized = normalizeRealtimeSettings(settings.realtime)
+  const apiKey = apiKeys.openai || launchEnv.openai
+  const validation = await validateOpenAIRealtimeApiKey(
+    apiKey,
+    normalized.model
+  )
+  return {
+    settings: normalized,
+    availableModels: [...OPENAI_REALTIME_MODELS],
+    availableVoices: [...OPENAI_REALTIME_VOICES],
+    availableReasoningEfforts: [...OPENAI_REALTIME_REASONING_EFFORTS],
+    hasOpenAIApiKey: Boolean(apiKey),
+    ...validation,
+    codexEnabled: settings.codex?.enabled === true,
+  }
+}

packages/agents-desktop/src/settings/store.ts

@@ -17,11 +17,15 @@ import {
   saveApiKeysToSecret,
 } from '../credentials/api-keys'
 import { normalizeEnabledModelValues } from '../credentials/model-picker'
+import {
+  DEFAULT_REALTIME_SETTINGS,
+  normalizeRealtimeSettings,
+} from './realtime'
 import { normalizeServer, normalizeServers } from './servers'
 
 export { settingsPath } from '../shared/paths'
 
-export const SETTINGS_VERSION = 2
+export const SETTINGS_VERSION = 3
 
 export const DEFAULT_SETTINGS: DesktopSettings = {
   servers: [],
@@ -31,6 +35,7 @@ export const DEFAULT_SETTINGS: DesktopSettings = {
   launchAtLogin: false,
   preventAppSuspension: true,
   codex: { enabled: false, source: null },
+  realtime: DEFAULT_REALTIME_SETTINGS,
 }
 
 export function normalizeCodexSettings(value: unknown): CodexSettings {
@@ -165,6 +170,7 @@ export async function loadDesktopSettings(
       preventAppSuspension: parsed.preventAppSuspension !== false,
       onboardingDismissed: parsed.onboardingDismissed === true,
       codex: normalizeCodexSettings(parsed.codex),
+      realtime: normalizeRealtimeSettings(parsed.realtime),
       enabledModelValues:
         enabledModelValues.length > 0 ? enabledModelValues : undefined,
       mcp: normalizeMcp(parsed.mcp),

packages/agents-desktop/src/shared/types.ts

@@ -4,6 +4,12 @@ import type {
   McpServerConfig,
   RegistrySnapshot,
 } from '@electric-ax/agents'
+import type {
+  OpenAIRealtimeReasoningEffort,
+  RealtimeModelChoice,
+  RealtimeReasoningEffortChoice,
+  RealtimeVoiceChoice,
+} from '@electric-ax/agents-runtime'
 
 export type ServerSource = `manual` | `local-discovery` | `electric-cloud`
 export type ServerDesiredState = `connected` | `disconnected`
@@ -122,6 +128,33 @@ export type CodexSettings = {
   source: CodexAuthSource | null
 }
 
+export type RealtimeProvider = `openai`
+
+export type RealtimeSettings = {
+  provider: RealtimeProvider
+  model: string
+  voice: string
+  reasoningEffort: OpenAIRealtimeReasoningEffort
+  interruptResponse: boolean
+}
+
+export type RealtimeCredentialStatus =
+  | `missing`
+  | `valid`
+  | `invalid`
+  | `unknown`
+
+export type RealtimeSettingsStatus = {
+  settings: RealtimeSettings
+  availableModels: Array<RealtimeModelChoice>
+  availableVoices: Array<RealtimeVoiceChoice>
+  availableReasoningEfforts: Array<RealtimeReasoningEffortChoice>
+  hasOpenAIApiKey: boolean
+  openAIApiKeyStatus: RealtimeCredentialStatus
+  openAIApiKeyError?: string
+  codexEnabled: boolean
+}
+
 export type DesktopSettings = {
   servers: Array<ServerConfig>
   defaultServerId: string | null
@@ -131,6 +164,7 @@ export type DesktopSettings = {
   preventAppSuspension?: boolean
   codex?: CodexSettings
   enabledModelValues?: Array<string>
+  realtime?: RealtimeSettings
   onboardingDismissed?: boolean
   mcp?: { servers: Array<McpServerConfig> }
   seededDefaultMcpServerNames?: Array<string>

packages/agents-runtime/src/agents-client.ts

@@ -4,6 +4,10 @@ import { normalizeObservationSchema } from './observation-schema'
 import { createRuntimeServerClient } from './runtime-server-client'
 import { appendPathToUrl } from './url'
 import type { EntitySignal } from './runtime-server-client'
+import type {
+  RealtimeSessionStartResult,
+  StartRealtimeSessionOptions,
+} from './runtime-server-client'
 import type {
   EntitiesObservationSource,
   EntityObservationSource,
@@ -31,6 +35,9 @@ export interface AgentsClient {
     payload?: unknown
   }) => Promise<{ txid: number }>
   kill: (entityUrl: string, reason?: string) => Promise<{ txid: number }>
+  startRealtimeSession: (
+    options: StartRealtimeSessionOptions
+  ) => Promise<RealtimeSessionStartResult>
 }
 
 export function createAgentsClient(config: AgentsClientConfig): AgentsClient {
@@ -44,6 +51,8 @@ export function createAgentsClient(config: AgentsClientConfig): AgentsClient {
         signal: `SIGKILL`,
         reason,
       }),
+    startRealtimeSession: (options) =>
+      serverClient.startRealtimeSession(options),
     async observe(source) {
       if (source.sourceType === `entity`) {
         const info = await serverClient.getEntity(