From 041cc038ed69dfe10f02d78e19baba66fec43271 Mon Sep 17 00:00:00 2001 From: Wang Date: Mon, 7 Sep 2026 18:45:54 +0800 Subject: [PATCH 01/23] feat(desktop): add a scoped assistant with visible UI control Generated-by: Codex --- apps/desktop/renderer-architecture.json | 1 + .../main/__tests__/desktop-assistant.test.ts | 98 +++++++ .../src/main/desktop-assistant-state.ts | 98 +++++++ apps/desktop/src/main/desktop-assistant-ui.ts | 202 +++++++++++++ apps/desktop/src/main/desktop-assistant.ts | 277 ++++++++++++++++++ apps/desktop/src/main/runtime-host-boot.ts | 26 ++ apps/desktop/src/main/runtime-host-client.ts | 2 +- apps/desktop/src/preload/bridge-contract.d.ts | 1 + apps/desktop/src/preload/desktop-assistant.ts | 38 +++ apps/desktop/src/preload/preload.ts | 2 + apps/desktop/src/renderer/app-shell.tsx | 9 +- .../composition/desktop-feature-services.tsx | 5 +- .../features/desktop-assistant/index.tsx | 140 +++++++++ .../create-desktop-assistant-services.ts | 23 ++ .../settings/appearance-settings-page.tsx | 1 + .../personalization-settings-section.tsx | 9 +- .../settings/settings-expandable-row.tsx | 3 + .../renderer/settings/settings-surface.tsx | 3 +- apps/desktop/src/renderer/styles.css | 2 + .../src/renderer/styles/desktop-assistant.css | 55 ++++ .../desktop/src/shared/desktop-assistant.d.ts | 62 ++++ packages/core/src/session.ts | 6 +- .../hosted-execution-tool-profile.test.ts | 22 ++ .../interactive-run-composer.test.ts | 31 ++ packages/runtime-host/src/protocol/index.ts | 3 +- .../server/hosted-execution-tool-profile.ts | 16 + .../src/server/interactive-run-composer.ts | 6 +- packages/ui/src/session-sidebar-nav.tsx | 1 + 28 files changed, 1126 insertions(+), 16 deletions(-) create mode 100644 apps/desktop/src/main/__tests__/desktop-assistant.test.ts create mode 100644 apps/desktop/src/main/desktop-assistant-state.ts create mode 100644 apps/desktop/src/main/desktop-assistant-ui.ts create mode 100644 apps/desktop/src/main/desktop-assistant.ts create mode 100644 apps/desktop/src/preload/desktop-assistant.ts create mode 100644 apps/desktop/src/renderer/features/desktop-assistant/index.tsx create mode 100644 apps/desktop/src/renderer/platform/desktop/create-desktop-assistant-services.ts create mode 100644 apps/desktop/src/renderer/styles/desktop-assistant.css create mode 100644 apps/desktop/src/shared/desktop-assistant.d.ts diff --git a/apps/desktop/renderer-architecture.json b/apps/desktop/renderer-architecture.json index 4712d5e0c1..e1f88445fe 100644 --- a/apps/desktop/renderer-architecture.json +++ b/apps/desktop/renderer-architecture.json @@ -832,6 +832,7 @@ "./features/app-update/index.js": 1, "./features/conversation": 1, "./features/conversation/index.js": 1, + "./features/desktop-assistant": 1, "./features/goals": 1, "./features/module-hub": 1, "./features/session-collaboration": 1, diff --git a/apps/desktop/src/main/__tests__/desktop-assistant.test.ts b/apps/desktop/src/main/__tests__/desktop-assistant.test.ts new file mode 100644 index 0000000000..5107a0a7b3 --- /dev/null +++ b/apps/desktop/src/main/__tests__/desktop-assistant.test.ts @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { IpcMain, WebContents } from 'electron'; +import { createDefaultSettings } from '@maka/core/settings'; +import type { SessionCatalogProjection } from '@maka/runtime-host/protocol'; +import { createDesktopAssistant } from '../desktop-assistant.js'; +import { ASSISTANT_RETENTION_MS, DesktopAssistantState } from '../desktop-assistant-state.js'; + +test('assistant retention removes only expired owned sessions on the connected Host', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'maka-assistant-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const state = new DesktopAssistantState(join(directory, 'state.json')); + const now = 2 * ASSISTANT_RETENTION_MS; + for (const id of ['expired', 'active', 'recent-host-activity', 'renamed-label', 'protected']) await state.touch('host', id, 1); + await state.touch('host', 'recent', now - 1); + await state.touch('disconnected-host', 'remote', 1); + const queried: string[] = []; + const removed: string[] = []; + const client = { + hostId: 'host', + getSession: async (id: string) => { + queried.push(id); + return { + id, labels: id === 'renamed-label' ? [] : ['mode:desktop_assistant'], + activityAt: id === 'recent-host-activity' ? now : 1, + ...(id === 'active' ? { liveRunState: {} } : {}), + } as unknown as SessionCatalogProjection; + }, + removeSession: async (id: string) => { removed.push(id); return { disposition: 'removed' as const, archivedSubtaskCount: 0 }; }, + }; + assert.deepEqual(await state.cleanup(client, 'protected', now), ['expired']); + assert.deepEqual(removed, ['expired']); + assert.equal(queried.includes('recent'), false); + assert.equal(queried.includes('remote'), false); + assert.equal(queried.includes('protected'), false); + const reopened = new DesktopAssistantState(join(directory, 'state.json')); + assert.deepEqual(await reopened.cleanup(client, 'protected', now), []); +}); + +test('assistant model preferences survive restart and stay scoped to their Host', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'maka-assistant-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'state.json'); + const state = new DesktopAssistantState(path); + const selected = { connectionId: 'connection', connectionSlug: 'provider', model: 'model' }; + await state.selectModel('host', selected); + const reopened = new DesktopAssistantState(path); + assert.deepEqual(await reopened.model('host'), selected); + assert.equal(await reopened.model('another-host'), undefined); +}); + +test('assistant IPC rejects other renderers and its tool rejects unowned Sessions', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'maka-assistant-')); + t.after(() => rm(directory, { recursive: true, force: true })); + let command!: Parameters[1]; + const frame = {}; + const window = { mainFrame: frame } as WebContents; + const assistant = createDesktopAssistant({ + ipcMain: { handle: (_channel: string, handler: typeof command) => { command = handler; } } as IpcMain, + statePath: join(directory, 'state.json'), + window: () => window, + readSettings: async () => createDefaultSettings(), + host: async () => { throw new Error('No Host should be contacted'); }, + clients: () => [], + isCurrent: () => false, + }); + t.after(() => assistant.close()); + await assert.rejects(command({ sender: {}, senderFrame: frame } as Electron.IpcMainInvokeEvent, 'snapshot'), /main window/); + await assert.rejects(command({ sender: window, senderFrame: {} } as Electron.IpcMainInvokeEvent, 'submit', 'change language'), /main window/); + const entry = assistant.group.tools[0]!; + const tool = 'tool' in entry ? entry.tool : entry; + await assert.rejects(async () => tool.impl({ operation: 'act', actions: [{ kind: 'set', target: 'language', value: 'en' }] }, { + sessionId: 'ordinary-session', turnId: 'turn', toolCallId: 'call', cwd: directory, + abortSignal: new AbortController().signal, emitOutput() {}, + }), /No active request owns/); +}); diff --git a/apps/desktop/src/main/desktop-assistant-state.ts b/apps/desktop/src/main/desktop-assistant-state.ts new file mode 100644 index 0000000000..82239f5919 --- /dev/null +++ b/apps/desktop/src/main/desktop-assistant-state.ts @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { z } from 'zod'; +import type { DesktopRuntimeHostClient } from './runtime-host-client.js'; + +const modelSchema = z.object({ connectionId: z.string(), connectionSlug: z.string(), model: z.string() }); +const stateSchema = z.object({ + version: z.literal(1), + models: z.record(z.string(), modelSchema), + sessions: z.array(z.object({ hostId: z.string(), sessionId: z.string(), usedAt: z.number() })), +}); +type State = z.infer; +export const ASSISTANT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000; + +/** Owns only assistant preferences and the exact Sessions created by this client. */ +export class DesktopAssistantState { + private readonly loaded: Promise; + private writes = Promise.resolve(); + + constructor(private readonly path: string) { + this.loaded = readFile(path, 'utf8') + .then((text) => stateSchema.parse(JSON.parse(text))) + .catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return { version: 1 as const, models: {}, sessions: [] }; + throw error; + }); + // The first user operation reports an unreadable state file. + void this.loaded.catch(() => undefined); + } + + async model(hostId: string) { return (await this.loaded).models[hostId]; } + + async selectModel(hostId: string, model: z.infer) { + (await this.loaded).models[hostId] = modelSchema.parse(model); + await this.save(); + } + + async touch(hostId: string, sessionId: string, now = Date.now()) { + const state = await this.loaded; + const entry = state.sessions.find((item) => item.hostId === hostId && item.sessionId === sessionId); + if (entry) entry.usedAt = now; + else state.sessions.push({ hostId, sessionId, usedAt: now }); + await this.save(); + } + + async cleanup(client: Pick, protectedSession?: string, now = Date.now()) { + const state = await this.loaded; + const removed: string[] = []; + for (const entry of [...state.sessions]) { + if (entry.hostId !== client.hostId || entry.sessionId === protectedSession || now - entry.usedAt <= ASSISTANT_RETENTION_MS) continue; + const session = await client.getSession(entry.sessionId); + if (session) { + if (!session.labels.includes('mode:desktop_assistant')) continue; + if (session.liveRunState || now - session.activityAt <= ASSISTANT_RETENTION_MS) continue; + const result = await client.removeSession(entry.sessionId); + if (result.disposition !== 'removed') continue; + } + state.sessions = state.sessions.filter((item) => item !== entry); + removed.push(entry.sessionId); + await this.save(); + } + return removed; + } + + private async save() { + const text = JSON.stringify(await this.loaded); + const write = this.writes.catch(() => undefined).then(async () => { + await mkdir(dirname(this.path), { recursive: true }); + const temporary = `${this.path}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, text, { flag: 'wx', mode: 0o600 }); + await rename(temporary, this.path); + } finally { await unlink(temporary).catch(() => undefined); } + }); + this.writes = write; + await write; + } +} diff --git a/apps/desktop/src/main/desktop-assistant-ui.ts b/apps/desktop/src/main/desktop-assistant-ui.ts new file mode 100644 index 0000000000..a301084515 --- /dev/null +++ b/apps/desktop/src/main/desktop-assistant-ui.ts @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Page } from '@jackwener/opencli/browser/page'; +import type { WebContents } from 'electron'; +import type { AppSettings } from '@maka/core/settings'; +import type { DesktopAssistantAction, DesktopAssistantSnapshot } from '../shared/desktop-assistant.js'; + +const attr = 'data-maka-assistant-target'; +const selector = (target: string) => `[${attr}=${JSON.stringify(target)}]`; +const delay = (ms: number, signal: AbortSignal) => new Promise((resolve, reject) => { + signal.throwIfAborted(); + const finish = () => { signal.removeEventListener('abort', abort); resolve(); }; + const timer = setTimeout(finish, ms); + const abort = () => { clearTimeout(timer); reject(signal.reason); }; + signal.addEventListener('abort', abort, { once: true }); +}); + +/** OpenCLI's AX formatter, transported directly to this window; no daemon or navigation. */ +class WindowPage extends Page { + constructor(private readonly contents: WebContents) { super('maka-assistant'); } + override async getCurrentUrl() { return this.contents.getURL(); } + override async cdp(method: string, params: Record = {}): Promise { + if (method === 'Page.getFrameTree') return {}; + if (!this.contents.debugger.isAttached()) this.contents.debugger.attach('1.3'); + if (method !== 'Accessibility.getFullAXTree') return this.contents.debugger.sendCommand(method, params); + const { root } = await this.contents.debugger.sendCommand('DOM.getDocument', { depth: -1 }); + const allowed = new Set(); + const visit = (node: { backendNodeId: number; attributes?: string[]; children?: typeof node[] }, safe = false) => { + const attrs = node.attributes ?? []; + const at = attrs.indexOf(attr); + const target = at < 0 ? '' : attrs[at + 1] ?? ''; + safe ||= target === 'language' || target.startsWith('displayName.') || target.startsWith('theme.') || target.startsWith('settings.'); + if (safe) allowed.add(node.backendNodeId); + for (const child of node.children ?? []) visit(child, safe); + }; + visit(root); + const result = await this.contents.debugger.sendCommand(method, params); + // Only app-owned navigation and preference controls are sent to the model. + // Transcript, credentials, webviews, and the assistant itself are omitted. + const nodes = result.nodes.filter((node: { backendDOMNodeId?: number; ignored?: boolean }) => !node.ignored && allowed.has(node.backendDOMNodeId ?? -1)); + const childIds = new Set(nodes.flatMap((node: { childIds?: string[] }) => node.childIds ?? [])); + return { nodes: [{ nodeId: 'maka-safe-root', role: { value: 'RootWebArea' }, name: { value: 'Maka controls' }, childIds: nodes.filter((node: { nodeId: string }) => !childIds.has(node.nodeId)).map((node: { nodeId: string }) => node.nodeId) }, ...nodes] }; + } +} + +export class DesktopAssistantUi { + constructor( + private readonly window: () => WebContents, + private readonly readSettings: () => Promise, + private readonly update: (patch: Partial) => void, + private readonly readDisplayName: () => Promise, + ) {} + + async begin(signal: AbortSignal) { + signal.throwIfAborted(); + const origin = await this.window().executeJavaScript(`(() => { + const r = document.querySelector('.desktopAssistant')?.getBoundingClientRect(); + return { x: Math.round(r ? r.x + r.width / 2 : innerWidth / 2), y: Math.round(r ? r.y : innerHeight - 80) }; + })()`); + this.update({ cursor: { ...origin, clicking: false } }); + await delay(80, signal); + } + + async observe() { + const wc = this.window(); + const page = new WindowPage(wc); + const accessibility = await page.snapshot({ source: 'ax' }); + const settings = await this.readSettings(); + const section = await wc.executeJavaScript(`document.querySelector('[data-maka-assistant-section]')?.getAttribute('data-maka-assistant-section') ?? null`); + return { section, language: settings.personalization.uiLocale, theme: settings.appearance.theme, accessibility }; + } + + async visual() { + const wc = this.window(); + const rect = await wc.executeJavaScript(`(() => { + const e = document.querySelector('[data-maka-assistant-target="language"]') ?? document.querySelector('[data-maka-assistant-target^="theme."]'); + if (!e) throw new Error('Open language or appearance settings before requesting visual context'); + const r = e.getBoundingClientRect(); + if (r.left < 0 || r.top < 0 || r.right > innerWidth || r.bottom > innerHeight) throw new Error('Preference control is outside the viewport'); + for (const fx of [0.05, 0.5, 0.95]) for (const fy of [0.05, 0.5, 0.95]) { + if (!e.contains(document.elementFromPoint(r.x + r.width * fx, r.y + r.height * fy))) throw new Error('Preference control is covered; visual context is unavailable'); + } + return { x: Math.ceil(r.x), y: Math.ceil(r.y), width: Math.floor(r.width), height: Math.floor(r.height) }; + })()`); + if (rect.width < 1 || rect.height < 1 || rect.x < 0 || rect.y < 0) throw new Error('Preference control is not visible'); + return (await wc.capturePage(rect)).toPNG().toString('base64'); + } + + async execute(action: DesktopAssistantAction, signal: AbortSignal) { + const section = action.kind === 'navigate' ? action.section : action.target === 'theme' ? 'appearance' : 'general'; + const wc = this.window(); + const opened = await wc.executeJavaScript(`!!document.querySelector('[data-maka-assistant-section]')`); + if (!opened) { + if (!await this.point(selector('settings.open'))) { + await this.click('[data-maka-contract="shell-topbar-rail"] button[aria-expanded="false"]', signal); + } + await this.click(selector('settings.open'), signal); + } + await this.click(selector(`settings.${section}`), signal); + await this.waitFor(async () => await wc.executeJavaScript(`document.querySelector('[data-maka-assistant-section]')?.getAttribute('data-maka-assistant-section')`) === section, signal); + if (action.kind === 'navigate') return { verified: true, section }; + const before = await this.readSettings(); + if (action.target === 'displayName') { + const previous = await this.readDisplayName(); + if (previous === action.value) return { verified: true, target: action.target, value: action.value, previous }; + await this.click(selector('displayName.edit'), signal); + await this.type(`${selector('displayName.input')} input`, action.value, signal); + await this.click(selector('displayName.save'), signal); + await this.waitFor(async () => await this.readDisplayName() === action.value, signal); + return { verified: true, target: action.target, value: action.value, previous }; + } + if (action.target === 'theme') await this.click(selector(`theme.${action.value}`), signal); + else { + const trigger = `${selector('language')} [role="combobox"]`; + await this.click(trigger, signal); + const index = ['auto', 'zh-CN', 'zh-TW', 'en'].indexOf(action.value); + // The list belongs to this combobox, not an arbitrary popup elsewhere. + const listId: string = await wc.executeJavaScript(`document.querySelector(${JSON.stringify(trigger)})?.getAttribute('aria-controls')`); + if (!listId) throw new Error('Language control did not open'); + await this.click(`[id=${JSON.stringify(listId)}] [role="option"]:nth-of-type(${index + 1})`, signal); + } + await this.waitFor(async () => { + const saved = await this.readSettings(); + return (action.target === 'language' ? saved.personalization.uiLocale : saved.appearance.theme) === action.value; + }, signal); + return { verified: true, target: action.target, value: action.value, previous: action.target === 'language' ? before.personalization.uiLocale : before.appearance.theme }; + } + + private async type(css: string, text: string, signal: AbortSignal) { + await this.click(css, signal); + const wc = this.window(); + const focused = () => wc.executeJavaScript(`document.activeElement === document.querySelector(${JSON.stringify(css)})`); + if (!await focused()) throw new Error('Text input did not receive focus'); + signal.throwIfAborted(); + wc.selectAll(); + if (text.length === 0) { wc.delete(); await delay(45, signal); } + // insertText uses Chromium's native editing path (including IME text), so + // React receives genuine input events instead of a bypassed value setter. + for (const character of text) { + signal.throwIfAborted(); + if (!await focused()) throw new Error('Text input lost focus; typing stopped'); + await wc.insertText(character); + await delay(45, signal); + } + const value = await wc.executeJavaScript(`document.querySelector(${JSON.stringify(css)})?.value`); + if (value !== text) throw new Error('Text input did not accept the requested value'); + } + + private async waitFor(check: () => Promise, signal: AbortSignal) { + for (let i = 0; i < 30; i++) { signal.throwIfAborted(); if (await check()) return; await delay(100, signal); } + throw new Error('The interface did not confirm the requested change'); + } + + private async point(css: string) { + return this.window().executeJavaScript(`(() => { + const e = document.querySelector(${JSON.stringify(css)}); + if (!e || e.closest('[inert]') || e.matches(':disabled,[aria-disabled="true"]')) return null; + const r = e.getBoundingClientRect(); + const x = Math.round(r.x + r.width / 2), y = Math.round(r.y + r.height / 2); + const hit = document.elementFromPoint(x, y); + if (r.width < 1 || r.height < 1 || !hit || !e.contains(hit)) return null; + return { x, y }; + })()`); + } + + private async click(css: string, signal: AbortSignal) { + let point: { x: number; y: number } | null = null; + await this.waitFor(async () => { point = await this.point(css); return point !== null; }, signal); + this.update({ cursor: { ...point!, clicking: false } }); + await delay(350, signal); + const current = await this.point(css); + if (!current || current.x !== point!.x || current.y !== point!.y) throw new Error('Control moved or is covered; action stopped'); + signal.throwIfAborted(); + this.update({ cursor: { ...current, clicking: true } }); + const wc = this.window(); + // Await the renderer's synchronous ownership marker before Chromium + // delivers native input; IPC send and input delivery have different queues. + await wc.executeJavaScript(`window.dispatchEvent(new CustomEvent('maka-assistant:input', { detail: ${JSON.stringify(current)} }))`); + signal.throwIfAborted(); + wc.sendInputEvent({ type: 'mouseMove', ...current }); + wc.sendInputEvent({ type: 'mouseDown', button: 'left', clickCount: 1, ...current }); + wc.sendInputEvent({ type: 'mouseUp', button: 'left', clickCount: 1, ...current }); + await delay(150, signal); + } +} diff --git a/apps/desktop/src/main/desktop-assistant.ts b/apps/desktop/src/main/desktop-assistant.ts new file mode 100644 index 0000000000..0e44e5115e --- /dev/null +++ b/apps/desktop/src/main/desktop-assistant.ts @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { randomUUID } from 'node:crypto'; +import { EventEmitter } from 'node:events'; +import type { IpcMain, WebContents } from 'electron'; +import { z } from 'zod'; +import { SETTINGS_SECTIONS, type AppSettings } from '@maka/core/settings'; +import type { SessionEvent } from '@maka/core/events'; +import { buildChatModelChoices } from '@maka/core/chat-model-choice'; +import type { WorkspaceTarget } from '@maka/runtime-host/protocol'; +import type { MakaTool } from '@maka/runtime/tool-runtime'; +import { RuntimeHostSessionObserver } from './runtime-host-session-observer.js'; +import type { DesktopRuntimeHostClient } from './runtime-host-client.js'; +import type { DesktopCapabilityGroup } from './runtime-host-native-capabilities.js'; +import { DesktopAssistantUi } from './desktop-assistant-ui.js'; +import { DesktopAssistantState } from './desktop-assistant-state.js'; +import { projectHostConnections } from './runtime-host-connections-ipc-main.js'; +import type { DesktopAssistantAction, DesktopAssistantSnapshot } from '../shared/desktop-assistant.js'; + +/** Product paths are stable; coordinates are resolved afresh for every action. */ +const PRODUCT_MAP = { + settings: SETTINGS_SECTIONS.map((section) => ({ + section, operation: 'navigate', path: ['settings.open', `settings.${section}`], + })), + preferences: [ + { target: 'language', operation: 'set', section: 'general', values: ['auto', 'zh-CN', 'zh-TW', 'en'], description: 'Interface language, 界面语言 / 顯示語言. Stored on this Desktop client.' }, + { target: 'theme', operation: 'set', section: 'appearance', values: ['auto', 'light', 'dark'], description: 'Appearance theme, 主题 / 外觀. Stored on this Desktop client.' }, + { target: 'displayName', operation: 'set', section: 'general', description: 'The name Maka uses for you, 称呼 / 稱呼. Up to 60 characters, stored on the selected Runtime Host. The Desktop opens the editor, types, and saves.' }, + ], +} as const; + +const actionSchema = z.union([ + z.object({ kind: z.literal('navigate'), section: z.enum(SETTINGS_SECTIONS) }).strict(), + z.object({ kind: z.literal('set'), target: z.literal('language'), value: z.enum(['auto', 'zh-CN', 'zh-TW', 'en']) }).strict(), + z.object({ kind: z.literal('set'), target: z.literal('theme'), value: z.enum(['auto', 'light', 'dark']) }).strict(), + z.object({ kind: z.literal('set'), target: z.literal('displayName'), value: z.string().trim().min(1).max(60).refine((value) => !/[\u0000-\u001f\u007f]/.test(value)) }).strict(), +]); +interface AssistantHost { + client: DesktopRuntimeHostClient; + workspace: WorkspaceTarget; + stop(sessionId: string): Promise; +} +interface AssistantDeps { + ipcMain: IpcMain; + statePath: string; + window(): WebContents; + readSettings(): Promise; + host(): Promise; + clients(): readonly DesktopRuntimeHostClient[]; + isCurrent(client: DesktopRuntimeHostClient): boolean; +} + +export function createDesktopAssistant(deps: AssistantDeps) { + const state = new DesktopAssistantState(deps.statePath); + let snapshot: DesktopAssistantSnapshot = { revision: 0, open: false, expanded: true, phase: 'idle', messages: [], canUndo: false }; + let run: AbortController | undefined; + let host: AssistantHost | undefined; + let sessionId: string | undefined; + let observer: RuntimeHostSessionObserver | undefined; + let stopping: Promise | undefined; + let controlBusy = false; + let selectingModel = false; + let actionFailure: string | undefined; + let undo: { action: DesktopAssistantAction; expected: string } | undefined; + const watchedWindows = new WeakSet(); + const update = (patch: Partial) => { + snapshot = { ...snapshot, ...patch, revision: snapshot.revision + 1 }; + let wc: WebContents; + try { wc = deps.window(); } catch { return; } + if (!wc.isDestroyed()) wc.send('desktop-assistant:changed', snapshot); + }; + const ui = new DesktopAssistantUi(deps.window, deps.readSettings, update, async () => { + if (!host || !deps.isCurrent(host.client)) throw new Error('Runtime Host changed'); + return (await host.client.queryRuntimePolicy()).policy.personalization.displayName; + }); + const fail = (error: unknown) => update({ phase: 'error', expanded: true, cursor: undefined, error: error instanceof Error ? error.message : String(error) }); + const stop = async () => { + run?.abort(new Error('User took control')); + run = undefined; + update({ phase: 'paused', expanded: true, cursor: undefined }); + if (host && sessionId) { + stopping ??= host.stop(sessionId).finally(() => { stopping = undefined; }); + await stopping; + } + }; + const refreshModels = async () => { + const current = await deps.host(); + const catalog = await current.client.loadConnectionCatalog(); + const choices = buildChatModelChoices(projectHostConnections(catalog)); + const preferred = await state.model(current.client.hostId); + const selected = preferred + ? choices.find((choice) => choice.connectionId === preferred.connectionId && choice.model === preferred.model) + : choices.find((choice) => choice.connectionId === catalog.defaultTarget?.connectionId && choice.model === catalog.defaultTarget.modelId); + update({ modelChoices: choices, model: selected }); + }; + const cleanup = async () => { + for (const client of deps.clients()) { + const removed = await state.cleanup(client, run ? sessionId : undefined); + if (sessionId && removed.includes(sessionId) && host?.client === client) { + await observer?.close(); + observer = undefined; + sessionId = undefined; + undo = undefined; + update({ messages: [], phase: 'idle', canUndo: false }); + } + } + }; + const cleanupTimer = setInterval(() => { + void cleanup().catch((error) => console.error('[desktop-assistant] retention cleanup failed', error)); + }, 60 * 60 * 1000); + cleanupTimer.unref(); + const onEvent = (event: SessionEvent) => { + if (!run || run.signal.aborted) return; + if (event.type === 'text_delta' || event.type === 'text_complete') { + const messages = [...snapshot.messages]; + const index = messages.findIndex((m) => m.id === event.messageId); + const previous = index < 0 ? '' : messages[index]!.text; + const text = event.type === 'text_complete' ? event.text : previous.slice(0, event.startOffset ?? previous.length) + event.text; + const message = { id: event.messageId, role: 'assistant' as const, text }; + if (index < 0) messages.push(message); else messages[index] = message; + update({ messages }); + } else if (event.type === 'complete' || event.type === 'abort') { + run = undefined; + if (host && sessionId) void state.touch(host.client.hostId, sessionId).catch(fail); + update({ phase: event.type === 'abort' ? 'paused' : actionFailure ? 'error' : 'completed', expanded: true, cursor: undefined, ...(actionFailure ? { error: actionFailure } : {}) }); + } else if (event.type === 'error' && !event.recoverable) { run = undefined; fail(event.message); } + }; + const submit = async (text: string) => { + if (stopping) await stopping; + if (selectingModel) throw new Error('Wait for the model selection to finish'); + if (run) throw new Error('Stop the current request before sending another'); + actionFailure = undefined; + const active = new AbortController(); + run = active; + const window = deps.window(); + if (!watchedWindows.has(window)) { + watchedWindows.add(window); + const interrupt = () => { if (run) void stop().catch(fail); }; + window.once('destroyed', interrupt); + window.on('did-start-navigation', (_event, _url, isInPlace, isMainFrame) => { + if (isMainFrame && !isInPlace) interrupt(); + }); + } + update({ open: true, expanded: true, phase: 'thinking', error: undefined, messages: [...snapshot.messages, { id: randomUUID(), role: 'user', text }] }); + try { + await refreshModels(); + const model = snapshot.model; + if (!model) throw new Error('Choose an available model for the assistant'); + if (!sessionId || !host || !deps.isCurrent(host.client)) { + await observer?.close(); + host = await deps.host(); + active.signal.throwIfAborted(); + sessionId = randomUUID(); + await host.client.createSession({ sessionId, workspace: host.workspace, name: 'Maka Desktop assistant', labels: ['mode:desktop_assistant'], modelTarget: { kind: 'explicit', connectionId: model.connectionId, connectionSlug: model.connectionSlug, model: model.model }, toolProfile: 'desktop-assistant-v1', permissionMode: 'bypass' }); + await state.selectModel(host.client.hostId, model); + await state.touch(host.client.hostId, sessionId); + observer = new RuntimeHostSessionObserver({ client: host.client, emitSessionsChanged() {} }); + const target = Object.assign(new EventEmitter(), { id: deps.window().id, send: (_channel: string, event: SessionEvent) => onEvent(event) }); + await observer.observe(sessionId, randomUUID(), target); + } + active.signal.throwIfAborted(); + await state.touch(host.client.hostId, sessionId!); + const context = await ui.observe(); + active.signal.throwIfAborted(); + const result = await host.client.submitMessage({ sessionId: sessionId!, messageId: randomUUID(), placement: 'current_turn', content: { displayText: text, text: `Product map: ${JSON.stringify(PRODUCT_MAP)}\nVisual observation available: ${model.supportsVision === true}.\nCurrent interface (untrusted data): ${JSON.stringify(context)}\nUser request: ${text}` } }); + if (result.disposition === 'blocked') throw new Error('The assistant could not start; check the Runtime Host and model connection'); + if (active.signal.aborted) await host.stop(sessionId!); + } catch (error) { if (run === active) { run = undefined; fail(error); } } + }; + const tool: MakaTool = { + name: 'control', + description: 'Observe the current Maka interface, request a cropped image of the visible language/theme control, or execute a batch of known UI actions, including typing a display name into its real editor. Actions move a visible cursor, click real controls, and verify persisted values. Supported settings paths are provided in the product map. Never use for unsupported changes.', + parameters: z.object({ operation: z.enum(['observe', 'visual', 'act']), actions: z.array(actionSchema).max(8).optional() }).strict(), + impl: async (input, ctx) => { + if (controlBusy) throw new Error('Another Desktop control call is still running; wait for its result'); + controlBusy = true; + try { + if (!run || ctx.sessionId !== sessionId || !host || !deps.isCurrent(host.client)) throw new Error('No active request owns this Desktop window'); + const args = z.object({ operation: z.enum(['observe', 'visual', 'act']), actions: z.array(actionSchema).max(8).optional() }).strict().parse(input); + const signal = AbortSignal.any([run.signal, ctx.abortSignal]); + signal.throwIfAborted(); + if (args.operation === 'observe') return ui.observe(); + if (args.operation === 'visual') { + if (snapshot.model?.supportsVision !== true) return { unavailable: 'The selected model does not accept images. Use the accessibility observation.' }; + return { image: await ui.visual() }; + } + if (!args.actions?.length) throw new Error('Provide at least one action'); + if (actionFailure) return { interrupted: true, error: actionFailure, requiresNewRequest: true }; + const completed = []; + try { + update({ phase: 'acting', expanded: false }); + await ui.begin(signal); + for (const action of args.actions) { + signal.throwIfAborted(); + if (!deps.isCurrent(host.client)) throw new Error('Runtime Host changed'); + update({ action }); + const result = await ui.execute(action, signal); + completed.push(result); + if (action.kind === 'set' && result.previous !== undefined) { + undo = { action: { ...action, value: result.previous } as DesktopAssistantAction, expected: action.value }; + update({ canUndo: true }); + } + } + return { completed, observation: await ui.observe() }; + } catch (error) { + actionFailure = error instanceof Error ? error.message : String(error); + return { completed, interrupted: true, error: actionFailure }; + } finally { if (!signal.aborted) update({ phase: 'thinking', cursor: undefined }); } + } finally { controlBusy = false; } + }, + toModelOutput: ({ output }) => { + if (typeof output === 'object' && output !== null && 'image' in output && typeof output.image === 'string') return { type: 'content', value: [{ type: 'file', mediaType: 'image/png', data: { type: 'data', data: output.image } }] }; + return { type: 'text', value: JSON.stringify(output) }; + }, + }; + const group: DesktopCapabilityGroup = { offerId: 'desktop_assistant', label: 'Maka assistant', description: 'Operate this Maka Desktop window through verified UI controls.', tools: [tool] }; + deps.ipcMain.handle('desktop-assistant:command', async (event, command: unknown, payload: unknown) => { + if (event.sender !== deps.window() || event.senderFrame !== event.sender.mainFrame) throw new Error('Assistant commands require the main window'); + switch (command) { + case 'snapshot': return snapshot; + case 'open': update({ open: true, expanded: true }); await refreshModels(); await cleanup(); return; + case 'close': if (run) await stop(); update({ open: false }); return; + case 'expand': update({ expanded: true }); return; + case 'stop': await stop(); return; + case 'submit': await submit(z.string().trim().min(1).max(8000).parse(payload)); return; + case 'model': { + if (run || stopping || selectingModel) throw new Error('Stop the current request before changing the model'); + selectingModel = true; + try { + const input = z.object({ connectionId: z.string(), model: z.string() }).strict().parse(payload); + await refreshModels(); + const choice = snapshot.modelChoices?.find((candidate) => candidate.connectionId === input.connectionId && candidate.model === input.model); + if (!choice) throw new Error('This model is no longer available'); + const current = await deps.host(); + if (sessionId && host?.client === current.client) { + await current.client.updateSessionConfiguration(sessionId, { modelTarget: { kind: 'explicit', connectionId: choice.connectionId, connectionSlug: choice.connectionSlug, model: choice.model } }); + } + await state.selectModel(current.client.hostId, choice); + update({ model: choice, error: undefined }); + } finally { selectingModel = false; } + return; + } + case 'undo': { + if (run || stopping || !host || !deps.isCurrent(host.client) || !undo || undo.action.kind !== 'set') throw new Error('No change can be undone now'); + const saved = await deps.readSettings(); + const current = undo.action.target === 'displayName' ? (await host!.client.queryRuntimePolicy()).policy.personalization.displayName : undo.action.target === 'language' ? saved.personalization.uiLocale : saved.appearance.theme; + if (current !== undo.expected) { undo = undefined; update({ canUndo: false }); throw new Error('The preference changed after the assistant; undo is no longer available'); } + const active = new AbortController(); + run = active; + update({ phase: 'acting', expanded: false }); + try { await ui.begin(active.signal); await ui.execute(undo.action, active.signal); undo = undefined; update({ canUndo: false, phase: 'completed', expanded: true }); } + catch (error) { if (!active.signal.aborted) fail(error); } + finally { if (run === active) run = undefined; update({ cursor: undefined }); } + return; + } + default: throw new Error('Unknown assistant command'); + } + }); + return { group, cleanup, close: async () => { clearInterval(cleanupTimer); run?.abort(); await observer?.close(); } }; +} diff --git a/apps/desktop/src/main/runtime-host-boot.ts b/apps/desktop/src/main/runtime-host-boot.ts index 669d05c8d6..7e26ebc808 100644 --- a/apps/desktop/src/main/runtime-host-boot.ts +++ b/apps/desktop/src/main/runtime-host-boot.ts @@ -76,6 +76,7 @@ import { createSettingsStore } from "@maka/storage/settings-store"; import { resolveStorageRoot } from "@maka/storage/root-authority"; import { createMcpOAuthController } from "./mcp-oauth-controller.js"; +import { createDesktopAssistant } from "./desktop-assistant.js"; import { registerAppClientIpc, registerAppIpc } from "./app-ipc-main.js"; import { createAppQuitCoordinator } from "./app-quit-coordinator.js"; import { @@ -866,6 +867,29 @@ const currentDesktopWorkspaceTarget = async ( } return workspace; }; +const desktopAssistant = createDesktopAssistant({ + ipcMain, + statePath: join(userDataDir, 'desktop-assistant.json'), + window: () => { + const window = mainWindowController.browserWindow(); + if (!window) throw new Error('Maka window is unavailable'); + return window.webContents; + }, + readSettings: () => settingsStore.get(), + host: async () => { + const current = runtimeHostManager?.current(); + if (!current?.candidate) throw new Error('Connect to a Runtime Host first'); + const target = runtimePolicyTargetsByEpoch.get(current.epoch); + if (!target) throw new Error('Runtime Host is reconnecting'); + const workspace: WorkspaceTarget = runtimeHostProfileUsesHostWorkspace(target.policy.kind) + ? await currentDesktopWorkspaceTarget(target.policy) + : { kind: 'host_path', path: workspaceRoot }; + return { client: current.candidate.client, workspace, stop: (id) => current.candidate!.stopSession(id) }; + }, + isCurrent: (client) => runtimeHostManager?.current()?.candidate?.client === client, + clients: () => [...runtimePolicyTargetsByEpoch.values()].filter((target) => target.isActive()).map((target) => target.client), +}); +app.on('will-quit', () => { void desktopAssistant.close(); }); const mcpCapabilityPublisher = createCapabilityRevisionPublisher(() => mcpManager.toolSnapshot().revision, ); @@ -1062,6 +1086,7 @@ const startLocalRuntimeHostManager = () => startRuntimeHostDesktopManager( else mcpServers.set(identified.serverId, [identified]); } return [ + desktopAssistant.group, { offerId: "desktop_settings", label: "Client settings", @@ -1335,6 +1360,7 @@ updateDesktopStartupProgress('renderer'); wireLifecycle(); runtimeHostManager.setDefaultProfile(runtimeHostStartup.preferences.defaultProfileId); sessionLocal.wake(); +void desktopAssistant.cleanup().catch((error) => console.error('[desktop-assistant] retention cleanup failed', error)); await guestSessionMountService.start().catch((error: unknown) => { console.error('[runtime-host] shared Sessions could not be restored:', error); }); diff --git a/apps/desktop/src/main/runtime-host-client.ts b/apps/desktop/src/main/runtime-host-client.ts index 966bc58334..6f98758042 100644 --- a/apps/desktop/src/main/runtime-host-client.ts +++ b/apps/desktop/src/main/runtime-host-client.ts @@ -697,7 +697,7 @@ export class DesktopRuntimeHostClient { async listSessions(): Promise { this.#assertOpen(); try { - return (await readRuntimeHostSessions(this.connection)).map(requireSessionProjection); + return (await readRuntimeHostSessions(this.connection)).map(requireSessionProjection).filter((session) => !session.labels.includes('mode:desktop_assistant')); } catch (error) { if (error instanceof DesktopRuntimeHostClientError) throw error; if (!(error instanceof RuntimeHostCatalogReadError)) throw error; diff --git a/apps/desktop/src/preload/bridge-contract.d.ts b/apps/desktop/src/preload/bridge-contract.d.ts index 68465dac92..e72bbfa7c4 100644 --- a/apps/desktop/src/preload/bridge-contract.d.ts +++ b/apps/desktop/src/preload/bridge-contract.d.ts @@ -712,6 +712,7 @@ export interface DesktopSessionUsageSummary extends UsageSummaryV2 { export interface MakaBridge { sessionLocal: import('../shared/session-local-contract.js').DesktopSessionLocalBridge; + desktopAssistant: import('../shared/desktop-assistant.js').DesktopAssistantBridge; sessionCollaboration: { prepareInvitation( sessionId: string, diff --git a/apps/desktop/src/preload/desktop-assistant.ts b/apps/desktop/src/preload/desktop-assistant.ts new file mode 100644 index 0000000000..e5fbd2d684 --- /dev/null +++ b/apps/desktop/src/preload/desktop-assistant.ts @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ipcRenderer } from 'electron'; +import type { DesktopAssistantBridge, DesktopAssistantSnapshot } from '../shared/desktop-assistant.js'; + +const command = (name: string, payload?: unknown) => ipcRenderer.invoke('desktop-assistant:command', name, payload); +export const desktopAssistantBridge: DesktopAssistantBridge = { + getSnapshot: () => command('snapshot'), + open: () => command('open'), + close: () => command('close'), + expand: () => command('expand'), + submit: (text) => command('submit', text), + selectModel: (connectionId, model) => command('model', { connectionId, model }), + stop: () => command('stop'), + undo: () => command('undo'), + subscribe: (handler) => { + const listener = (_event: Electron.IpcRendererEvent, snapshot: DesktopAssistantSnapshot) => handler(snapshot); + ipcRenderer.on('desktop-assistant:changed', listener); + return () => ipcRenderer.removeListener('desktop-assistant:changed', listener); + }, +}; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 23164f1de7..8e125f6b7c 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -18,6 +18,7 @@ */ import { contextBridge, ipcRenderer } from 'electron'; +import { desktopAssistantBridge } from './desktop-assistant.js'; import { isRuntimeHostProfileKind, type RuntimeHostProfileKind, @@ -1368,6 +1369,7 @@ const browserSelection = createBrowserSelectionCoordinator(runtimeHostSessionRef }, browserDocumentId); const makaBridge = { + desktopAssistant: desktopAssistantBridge, runtimeHost, sessionCollaboration: { async prepareInvitation(sessionId, preset, allowInsecure = false) { diff --git a/apps/desktop/src/renderer/app-shell.tsx b/apps/desktop/src/renderer/app-shell.tsx index cdd11d6866..ce1bce61ed 100644 --- a/apps/desktop/src/renderer/app-shell.tsx +++ b/apps/desktop/src/renderer/app-shell.tsx @@ -17,6 +17,7 @@ * under the License. */ +import { DesktopAssistantRoot } from './features/desktop-assistant'; import { useCallback, useEffect, @@ -270,15 +271,11 @@ export function AppShell({ initialOnboardingSnapshot = null }: AppShellProps = { + {(taskEntry) => ( )} diff --git a/apps/desktop/src/renderer/composition/desktop-feature-services.tsx b/apps/desktop/src/renderer/composition/desktop-feature-services.tsx index 70a402938b..8956ebd155 100644 --- a/apps/desktop/src/renderer/composition/desktop-feature-services.tsx +++ b/apps/desktop/src/renderer/composition/desktop-feature-services.tsx @@ -20,6 +20,8 @@ import type { ReactNode } from 'react'; import { ConversationServicesProvider } from '../features/conversation'; import { createDesktopConversationServices } from '../platform/desktop/create-conversation-services'; +import { DesktopAssistantServicesProvider } from '../features/desktop-assistant'; +import { createDesktopAssistantServices } from '../platform/desktop/create-desktop-assistant-services'; import { AppUpdateServicesProvider } from '../features/app-update/index.js'; import { ConnectionSettingsServicesProvider } from '../features/connection-settings'; import { GoalServicesProvider } from '../features/goals'; @@ -47,6 +49,7 @@ export function createDesktopFeatureServices() { return { appUpdate: createDesktopAppUpdateServices(), conversation: createDesktopConversationServices(), + desktopAssistant: createDesktopAssistantServices(), connectionSettings: createDesktopConnectionSettingsServices(), goal: createDesktopGoalServices(), moduleHub: createDesktopModuleHubServices(), @@ -77,7 +80,7 @@ export function DesktopFeatureServicesProvider(props: { - {props.children} + {props.children} diff --git a/apps/desktop/src/renderer/features/desktop-assistant/index.tsx b/apps/desktop/src/renderer/features/desktop-assistant/index.tsx new file mode 100644 index 0000000000..1b4e4eb187 --- /dev/null +++ b/apps/desktop/src/renderer/features/desktop-assistant/index.tsx @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { createContext, useContext, useEffect, useRef, useState, type ReactNode } from 'react'; +import { MarkdownBody, ModelPicker, modelChoiceValue, modelMenuGroups, useUiLocale } from '@maka/ui'; +import { ArrowUp, MessageSquare as MessageCircle, MousePointer2, Square, X, Undo2 } from '@maka/ui/icons'; +import { Button, IconButton } from '@astryxdesign/core'; +import type { UiCatalog } from '@maka/core/ui-locale'; +import type { DesktopAssistantSnapshot, DesktopAssistantBridge } from '../../../shared/desktop-assistant.js'; + +const Services = createContext(null); +export function DesktopAssistantServicesProvider(props: { services: DesktopAssistantBridge; children?: ReactNode }) { + return {props.children}; +} +const copy = { + en: { title: 'Maka assistant', model: 'Assistant model', chooseModel: 'Choose a model', placeholder: 'Ask a question, or change a setting…', hint: 'Try “Switch the interface to English”', stop: 'Stop', close: 'Close', send: 'Send', undo: 'Undo last change', thinking: 'Thinking…', acting: 'Working in your app…', paused: 'Paused · you have control', idle: 'Ready', completed: 'Done', error: 'Could not finish' }, + 'zh-CN': { title: 'Maka 助手', model: '助手模型', chooseModel: '选择模型', placeholder: '问个问题，或让我帮你调整设置…', hint: '试试「把界面语言切换为英文」', stop: '停止', close: '关闭', send: '发送', undo: '撤销上次修改', thinking: '思考中…', acting: '正在操作界面…', paused: '已暂停 · 由你接管', idle: '准备就绪', completed: '已完成', error: '未能完成' }, + 'zh-TW': { title: 'Maka 助手', model: '助手模型', chooseModel: '選擇模型', placeholder: '問個問題，或讓我幫你調整設定…', hint: '試試「把介面語言切換為英文」', stop: '停止', close: '關閉', send: '傳送', undo: '復原上次修改', thinking: '思考中…', acting: '正在操作介面…', paused: '已暫停 · 由你接管', idle: '準備就緒', completed: '已完成', error: '未能完成' }, +} satisfies UiCatalog>; + +export function DesktopAssistantRoot() { + const bridge = useContext(Services); + if (!bridge) throw new Error('Desktop assistant services are required'); + const locale = useUiLocale(); + const t = copy[locale]; + const [snapshot, setSnapshot] = useState({ revision: -1, open: false, expanded: true, phase: 'idle', messages: [], canUndo: false }); + const [text, setText] = useState(''); + const [error, setError] = useState(); + const input = useRef(null); + const panel = useRef(null); + const restoreFocus = useRef(null); + const bottom = useRef(null); + const state = useRef(snapshot); + state.current = snapshot; + const call = (task: Promise) => { void task.catch((reason: unknown) => setError(reason instanceof Error ? reason.message : String(reason))); }; + useEffect(() => { + let mounted = true; + const update = (next: typeof snapshot) => { if (mounted) setSnapshot((previous) => next.revision >= previous.revision ? next : previous); }; + const release = bridge.subscribe(update); + void bridge.getSnapshot().then(update); + return () => { mounted = false; release(); }; + }, [bridge]); + useEffect(() => { + let expected: { x: number; y: number; until: number; moved: boolean } | undefined; + const onInput = (event: Event) => { + if (event instanceof CustomEvent) expected = { ...event.detail, until: performance.now() + 300, moved: false }; + }; + window.addEventListener('maka-assistant:input', onInput); + const open = () => { + restoreFocus.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; + call(bridge.open()); + }; + const key = (event: KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && event.shiftKey && event.key.toLowerCase() === 'm') { event.preventDefault(); open(); return; } + if (event.key === 'Escape' && state.current.open) { event.preventDefault(); call(state.current.phase === 'acting' || state.current.phase === 'thinking' ? bridge.stop() : bridge.close()); return; } + if (state.current.phase === 'acting') call(bridge.stop()); + }; + const takeover = (event: Event) => { + if (state.current.phase !== 'acting' || !event.isTrusted) return; + if (event instanceof PointerEvent && event.type === 'pointermove' && event.movementX === 0 && event.movementY === 0) return; + if (event instanceof PointerEvent && expected && performance.now() <= expected.until && Math.abs(event.clientX - expected.x) <= 1 && Math.abs(event.clientY - expected.y) <= 1) { + if (event.type === 'pointermove' && !expected.moved) { expected.moved = true; return; } + if (event.type === 'pointerdown') { expected = undefined; return; } + } + if (panel.current?.contains(event.target as Node) && event.type === 'pointermove') return; + call(bridge.stop()); + }; + window.addEventListener('keydown', key, true); + for (const type of ['pointerdown', 'pointermove', 'wheel']) window.addEventListener(type, takeover, true); + return () => { + window.removeEventListener('maka-assistant:input', onInput); window.removeEventListener('keydown', key, true); + for (const type of ['pointerdown', 'pointermove', 'wheel']) window.removeEventListener(type, takeover, true); + }; + }, [bridge]); + useEffect(() => { + if (snapshot.open && snapshot.expanded && snapshot.phase !== 'acting') input.current?.focus(); + if (!snapshot.open && restoreFocus.current?.isConnected) restoreFocus.current.focus(); + }, [snapshot.open, snapshot.expanded, snapshot.phase]); + useEffect(() => { bottom.current?.scrollIntoView({ block: 'nearest' }); }, [snapshot.messages]); + const busy = snapshot.phase === 'thinking' || snapshot.phase === 'acting'; + const submit = () => { + if (!text.trim() || busy || !snapshot.model) return; + const value = text.trim(); setText(''); setError(undefined); call(bridge.submit(value)); + }; + return <> + {!snapshot.open &&
} label={`${t.title} (⌘⇧M / Ctrl+Shift+M)`} variant="secondary" onClick={() => { restoreFocus.current = document.activeElement as HTMLElement; call(bridge.open()); }} />
} + {snapshot.open &&
+
+ {t.title} + {t[snapshot.phase]} + {busy && } onClick={() => call(bridge.stop())} />} + {!snapshot.expanded && } onClick={() => call(bridge.expand())} />} + } onClick={() => call(bridge.close())} /> +
+ {snapshot.expanded && <> + {snapshot.messages.length > 0 ?
+ {snapshot.messages.map((message) =>
{message.role === 'user' ? message.text : }
)} +
+
:

{t.hint}

} + {(error || snapshot.error) &&

{error ?? snapshot.error}

} +
{ event.preventDefault(); submit(); }}> +