Skip to content

Commit d021220

Browse files
committed
perf(desktop): observe browser state in action calls
1 parent 9e67bb1 commit d021220

6 files changed

Lines changed: 346 additions & 7 deletions

File tree

‎apps/desktop/e2e/browser-tools.spec.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,6 +761,28 @@ test.describe('browser tools', () => {
761761
expect(await formState()).toMatchObject({ scrollLeft: 0 })
762762
})
763763

764+
test('batches an action with a fresh observation without replaying form fields', async () => {
765+
const ref = await openForm()
766+
const fill = await execute('browser_fill_form', {
767+
fields: [{ elementId: ref('Name'), kind: 'text', text: 'Observed value' }],
768+
observe: { query: 'Name' },
769+
})
770+
expect(fill.ok, fill.error).toBe(true)
771+
expect(fill.result).toMatchObject({
772+
completed: true,
773+
completedCount: 1,
774+
observation: { ok: true, result: { totalMatches: 1 } },
775+
})
776+
expect(await formState()).toMatchObject({ name: 'Observed value' })
777+
const result = fill.result as { observation: { result: { matches: { elementId: number }[] } } }
778+
const freshId = result.observation.result.matches[0].elementId
779+
expect(freshId).not.toBe(ref('Name'))
780+
const clear = await execute('browser_type', { elementId: freshId, text: '', observe: {} })
781+
expect(clear.ok, clear.error).toBe(true)
782+
expect(clear.result).toMatchObject({ observation: { ok: true } })
783+
expect(await formState()).toMatchObject({ name: '' })
784+
})
785+
764786
test('stops after a route change without writing the next field', async () => {
765787
const ref = await openForm()
766788
const fill = await execute('browser_fill_form', {

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import {
6969
setFocusedInputValue,
7070
typeIntoElement,
7171
} from '@/main/browser-agent/page-functions'
72+
import { withPostActionObservation } from '@/main/browser-agent/post-action-observation'
7273
import * as session from '@/main/browser-agent/session'
7374
import { checkAgentUrl } from '@/main/browser-agent/url-guard'
7475
import { clearCredentials, fillCoordinator, initFillCoordinator } from '@/main/browser-credentials'
@@ -4698,13 +4699,28 @@ export async function executeTool(
46984699
throw new ToolError('This browser action expired before it could dispatch input.')
46994700
}
47004701
}
4701-
const execution = executeToolInner(
4702+
const execution = withPostActionObservation(
47024703
tool,
47034704
params,
4704-
assertCurrentExecution,
4705-
executionDeadline,
4706-
invocationEpoch,
4707-
executionController.signal
4705+
(actionParams) =>
4706+
executeToolInner(
4707+
tool,
4708+
actionParams,
4709+
assertCurrentExecution,
4710+
executionDeadline,
4711+
invocationEpoch,
4712+
executionController.signal
4713+
),
4714+
(query) =>
4715+
executeToolInner(
4716+
query === undefined ? 'browser_snapshot' : 'browser_find',
4717+
query === undefined ? {} : { query },
4718+
assertCurrentExecution,
4719+
executionDeadline,
4720+
invocationEpoch,
4721+
executionController.signal
4722+
),
4723+
assertCurrentExecution
47084724
)
47094725
const guardedExecution =
47104726
watchdogMs === null
@@ -4715,7 +4731,8 @@ export async function executeTool(
47154731
if (
47164732
tool === 'browser_snapshot' ||
47174733
tool === 'browser_open_url' ||
4718-
tool === 'browser_find'
4734+
tool === 'browser_find' ||
4735+
params.observe !== undefined
47194736
) {
47204737
invalidateSnapshot(state)
47214738
}
@@ -4754,7 +4771,12 @@ export async function executeTool(
47544771
// The watchdog cannot cancel an in-flight renderer promise. Invalidate its
47554772
// capture token before releasing the queue so a late snapshot cannot
47564773
// overwrite refs belonging to a newer tab or snapshot.
4757-
if (tool === 'browser_snapshot' || tool === 'browser_open_url' || tool === 'browser_find') {
4774+
if (
4775+
tool === 'browser_snapshot' ||
4776+
tool === 'browser_open_url' ||
4777+
tool === 'browser_find' ||
4778+
params.observe !== undefined
4779+
) {
47584780
invalidateSnapshot(state)
47594781
}
47604782
const message = String(sanitizeBrowserResult(getErrorMessage(error), undefined, 0, 'error'))
Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import { describe, expect, it, vi } from 'vitest'
2+
import { withPostActionObservation } from './post-action-observation'
3+
4+
describe('post-action observation', () => {
5+
it('validates observation arguments before dispatching input', async () => {
6+
for (const observe of [
7+
null,
8+
true,
9+
[],
10+
{ query: '' },
11+
{ query: 'x'.repeat(4097) },
12+
{ extra: 1 },
13+
]) {
14+
const action = vi.fn()
15+
await expect(
16+
withPostActionObservation('browser_click', { observe }, action, vi.fn(), vi.fn())
17+
).rejects.toThrow('observe must')
18+
expect(action).not.toHaveBeenCalled()
19+
}
20+
})
21+
22+
it('keeps standalone actions unchanged and does not capture unrequested state', async () => {
23+
const result = { dispatched: true, effectObserved: false }
24+
const observe = vi.fn()
25+
expect(
26+
await withPostActionObservation('browser_click', {}, async () => result, observe, vi.fn())
27+
).toBe(result)
28+
expect(observe).not.toHaveBeenCalled()
29+
})
30+
31+
it('observes after the action, preserving partial form results and replacing refs', async () => {
32+
const order: string[] = []
33+
const result = await withPostActionObservation(
34+
'browser_fill_form',
35+
{ observe: { query: 'Save' } },
36+
async () => {
37+
order.push('action')
38+
return { completed: false, completedCount: 1, doNotRetry: true }
39+
},
40+
async (query) => {
41+
order.push('observe')
42+
expect(query).toBe('Save')
43+
return { matches: [{ elementId: 24, line: 'button Save [ref=24]' }] }
44+
},
45+
vi.fn()
46+
)
47+
expect(order).toEqual(['action', 'observe'])
48+
expect(result).toMatchObject({
49+
completed: false,
50+
completedCount: 1,
51+
doNotRetry: true,
52+
observation: { ok: true, result: { matches: [{ elementId: 24 }] } },
53+
})
54+
})
55+
56+
it('does not repeat or misreport a dispatched action when observation fails', async () => {
57+
const action = vi.fn(async () => ({ dispatched: true, effectObserved: false }))
58+
const result = await withPostActionObservation(
59+
'browser_click',
60+
{ observe: {} },
61+
action,
62+
async () => {
63+
throw new Error('Page changed')
64+
},
65+
vi.fn()
66+
)
67+
expect(action).toHaveBeenCalledTimes(1)
68+
expect(action).toHaveBeenCalledWith({})
69+
expect(result).toMatchObject({
70+
dispatched: true,
71+
effectObserved: false,
72+
observation: { ok: false, error: 'Page changed' },
73+
})
74+
})
75+
76+
it('never observes after failed actions or expired executions', async () => {
77+
const observe = vi.fn()
78+
await expect(
79+
withPostActionObservation(
80+
'browser_type',
81+
{ observe: {} },
82+
async () => {
83+
throw new Error('Ref expired')
84+
},
85+
observe,
86+
vi.fn()
87+
)
88+
).rejects.toThrow('Ref expired')
89+
await expect(
90+
withPostActionObservation(
91+
'browser_type',
92+
{ observe: {} },
93+
async () => ({ typed: true }),
94+
observe,
95+
() => {
96+
throw new Error('Cancelled')
97+
}
98+
)
99+
).rejects.toThrow('Cancelled')
100+
expect(observe).not.toHaveBeenCalled()
101+
})
102+
})
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import type { BrowserToolName } from '@sim/browser-protocol'
2+
import { getErrorMessage } from '@sim/utils/errors'
3+
import { isRecordLike } from '@sim/utils/object'
4+
import { ToolError } from '@/main/browser-agent/errors'
5+
6+
const OBSERVABLE_ACTIONS: ReadonlySet<BrowserToolName> = new Set([
7+
'browser_click',
8+
'browser_type',
9+
'browser_press_key',
10+
'browser_fill_form',
11+
])
12+
13+
/** Validate before dispatch; an observation failure must never invite replay of a completed action. */
14+
export async function withPostActionObservation(
15+
tool: BrowserToolName,
16+
params: Record<string, unknown>,
17+
action: (params: Record<string, unknown>) => Promise<unknown>,
18+
observe: (query: string | undefined) => Promise<unknown>,
19+
assertCurrent: () => void
20+
): Promise<unknown> {
21+
const request = params.observe
22+
if (request === undefined) return action(params)
23+
if (
24+
!OBSERVABLE_ACTIONS.has(tool) ||
25+
!isRecordLike(request) ||
26+
Object.keys(request).some((key) => key !== 'query') ||
27+
(request.query !== undefined &&
28+
(typeof request.query !== 'string' ||
29+
request.query.length === 0 ||
30+
request.query.length > 4096))
31+
) {
32+
throw new ToolError(
33+
'observe must be {} or {query: nonempty text up to 4096 characters} on a supported action.'
34+
)
35+
}
36+
const query = typeof request.query === 'string' ? request.query : undefined
37+
const { observe: _observe, ...actionParams } = params
38+
const result = await action(actionParams)
39+
assertCurrent()
40+
let observation: unknown
41+
try {
42+
observation = { ok: true, result: await observe(query) }
43+
} catch (error) {
44+
assertCurrent()
45+
observation = {
46+
ok: false,
47+
error: getErrorMessage(error),
48+
note: 'The action already ran. Inspect its result; do not repeat it just because observation failed.',
49+
}
50+
}
51+
assertCurrent()
52+
return isRecordLike(result) ? { ...result, observation } : { result, observation }
53+
}

‎apps/sim/lib/mothership/generated/tool-catalog-v1.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export interface ToolCatalogEntry {
66
capabilities?: unknown
77
clientExecutable?: boolean
88
description?:
9+
| 'Create a workspace in the conversation’s organization under the current user’s workspace-creation policy. Returns its ID for subsequent explicitly workspace-scoped commands. Includes a starter workflow unless skipDefaultWorkflow is true.'
910
| 'Discover and configure organization Search sources. list/get return accessible sources and indexing status; providers returns available integration approvals; approve changes a provider approval when authorized; setup returns the existing connection UI for the user to complete. Put the returned setupUrl in a clickable Markdown link at the end of the reply. Setup does not mean connected or indexed. Use search_workspace and read_document to retrieve source content.'
1011
| 'List currently accessible workspaces with roles and explicit capability restrictions. Bulk results report copilotAllowed and deniedCapabilities; exact workspaceId returns the full capability map. Omitted restrictions never authorize an operation.'
1112
| 'Read and manage account, organization, and workspace settings. list finds sections; get returns current values, updateSchema and operation names; describe returns one operation’s exact input schema; update changes narrow preferences; execute performs a listed operation; open returns the existing user setup flow. When user setup is needed, put the returned setupUrl in a clickable Markdown link at the end of the reply. Workspace resources retain their CLI commands. Account is the acting user; organization is the conversation’s organization. Every operation checks current permissions and entitlements.'
@@ -149,6 +150,7 @@ export interface ToolCatalogEntry {
149150
| 'web_scrape'
150151
| 'web_search'
151152
| 'workflow'
153+
| 'workspaces'
152154
internal?: boolean
153155
mode?: 'async' | 'sync'
154156
name?:
@@ -400,6 +402,18 @@ export const BrowserClick: ToolCatalogEntry = {
400402
description:
401403
"The element id to act on (from the current tab's most recent browser_snapshot). Treat refs as invalid across tab switches or later snapshots.",
402404
},
405+
observe: {
406+
type: 'object',
407+
description:
408+
'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.',
409+
properties: {
410+
query: {
411+
type: 'string',
412+
description:
413+
'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.',
414+
},
415+
},
416+
},
403417
},
404418
required: ['elementId'],
405419
},
@@ -892,6 +906,18 @@ export const BrowserFillForm: ToolCatalogEntry = {
892906
minItems: 1,
893907
type: 'array',
894908
},
909+
observe: {
910+
type: 'object',
911+
description:
912+
'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.',
913+
properties: {
914+
query: {
915+
type: 'string',
916+
description:
917+
'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.',
918+
},
919+
},
920+
},
895921
},
896922
required: ['fields'],
897923
type: 'object',
@@ -1358,6 +1384,18 @@ export const BrowserPressKey: ToolCatalogEntry = {
13581384
description:
13591385
"Key or combination. Named keys (case-insensitive): Enter, Escape (Esc), Tab, Backspace, Delete, Space, ArrowUp/ArrowDown/ArrowLeft/ArrowRight (or Up/Down/Left/Right), Home, End, PageUp, PageDown. Any single character also works ('a', '5', '/', ','). Anything else — 'F5', 'Return', 'Insert' — is rejected. Join modifiers with '+'. Use Mod (aliases Primary, ControlOrMeta, CommandOrControl) for the platform primary modifier, e.g. Mod+K or Mod+,. Raw Control/Ctrl and Cmd/Command/Meta remain available; Control is not generally Cmd on macOS. Check effectObserved and primaryModifier in the result.",
13601386
},
1387+
observe: {
1388+
type: 'object',
1389+
description:
1390+
'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.',
1391+
properties: {
1392+
query: {
1393+
type: 'string',
1394+
description:
1395+
'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.',
1396+
},
1397+
},
1398+
},
13611399
},
13621400
required: ['key'],
13631401
},
@@ -1856,6 +1894,18 @@ export const BrowserType: ToolCatalogEntry = {
18561894
description:
18571895
"The element id to act on (from the current tab's most recent browser_snapshot). Treat refs as invalid across tab switches or later snapshots.",
18581896
},
1897+
observe: {
1898+
type: 'object',
1899+
description:
1900+
'Observe immediately after this action in the same call. Use {} for a fresh snapshot or {query: string} for matching element refs only. Returned refs replace prior refs. A failed observation does not mean the action failed; inspect its result before retrying.',
1901+
properties: {
1902+
query: {
1903+
type: 'string',
1904+
description:
1905+
'Case-insensitive text to find in the resulting page. Omit for the full snapshot. Maximum 4096 characters.',
1906+
},
1907+
},
1908+
},
18591909
submit: { type: 'boolean', description: 'Press Enter after typing. Default false.' },
18601910
text: {
18611911
type: 'string',
@@ -7250,6 +7300,24 @@ export const ListWorkspaces: ToolCatalogEntry = {
72507300
},
72517301
}
72527302

7303+
export const Workspaces: ToolCatalogEntry = {
7304+
id: 'workspaces',
7305+
description:
7306+
'Create a workspace in the conversation’s organization under the current user’s workspace-creation policy. Returns its ID for subsequent explicitly workspace-scoped commands. Includes a starter workflow unless skipDefaultWorkflow is true.',
7307+
route: 'sim',
7308+
parameters: {
7309+
$schema: 'http://json-schema.org/draft-07/schema#',
7310+
type: 'object',
7311+
properties: {
7312+
name: { type: 'string', minLength: 1 },
7313+
skipDefaultWorkflow: { default: false, type: 'boolean' },
7314+
action: { type: 'string', const: 'create' },
7315+
},
7316+
required: ['name', 'action'],
7317+
additionalProperties: false,
7318+
},
7319+
}
7320+
72537321
export const Settings: ToolCatalogEntry = {
72547322
id: 'settings',
72557323
description:
@@ -7945,6 +8013,7 @@ export const TOOL_CATALOG: Record<string, ToolCatalogEntry> = {
79458013
[WebSearch.id]: WebSearch,
79468014
[Workflow.id]: Workflow,
79478015
[ListWorkspaces.id]: ListWorkspaces,
8016+
[Workspaces.id]: Workspaces,
79488017
[Settings.id]: Settings,
79498018
[SearchSources.id]: SearchSources,
79508019
}

0 commit comments

Comments
 (0)