Skip to content

Commit 7b7f36e

Browse files
committed
improvement(browser): name the controls a refused click can use next
A click refused because an overlay covers the target now lists the overlay's own snapshot controls, so the agent dismisses it and retries the same id without another snapshot. A click that lands on a nested control names that control's id. Only real overlays (dialogs, modals, fixed or sticky layers) list controls.
1 parent d5e1dea commit 7b7f36e

4 files changed

Lines changed: 144 additions & 6 deletions

File tree

‎apps/desktop/src/main/browser-agent/driver.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3683,6 +3683,33 @@ describe('credential protection', () => {
36833683
expect(retried).toMatchObject({ ok: true, result: { dispatched: true } })
36843684
})
36853685

3686+
it('names the overlay and nested controls a refused click can use next', async () => {
3687+
const contents = await openPage()
3688+
respondWith(contents, {
3689+
clickElement: {
3690+
error: 'obstructed',
3691+
blocker: 'We use cookies',
3692+
blockerControls: [{ id: 4, name: 'Accept all' }],
3693+
},
3694+
})
3695+
const covered = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 })
3696+
respondWith(contents, {
3697+
clickElement: { error: 'nested-control', blocker: 'Delete channel', controlId: 3 },
3698+
})
3699+
const nested = await driver.executeTool('chat-test', 'browser_click', { elementId: 0 })
3700+
3701+
expect(covered).toEqual({
3702+
ok: false,
3703+
error: expect.stringContaining(
3704+
'[ref=4] "Accept all". Dismiss it with one of those, then retry the same id.'
3705+
),
3706+
})
3707+
expect(nested).toEqual({
3708+
ok: false,
3709+
error: expect.stringContaining('Delete channel [ref=3]'),
3710+
})
3711+
})
3712+
36863713
it('invalidates element ids when the active tab changes', async () => {
36873714
await openPage()
36883715
await driver.executeTool('chat-test', 'browser_open_tab', {})

‎apps/desktop/src/main/browser-agent/driver.ts‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import type { BrowserDownloadsState, BrowserToolbarCommand } from '@sim/desktop-
3333
import { createLogger } from '@sim/logger'
3434
import { getErrorMessage } from '@sim/utils/errors'
3535
import { sleep } from '@sim/utils/helpers'
36-
import { isRecordLike, omit, toRecord } from '@sim/utils/object'
36+
import { isRecordLike, omit, toArray, toRecord } from '@sim/utils/object'
3737
import type { BrowserWindow, MenuItemConstructorOptions, WebContents, WebFrameMain } from 'electron'
3838
import { Menu } from 'electron'
3939
import * as cdp from '@/main/browser-agent/cdp'
@@ -1440,14 +1440,23 @@ function unwrapPageResult(result: unknown): unknown {
14401440
}
14411441
if (code === 'obstructed') {
14421442
const blocker = String((result as { blocker?: unknown }).blocker || 'another element')
1443+
const controls = toArray((result as { blockerControls?: unknown }).blockerControls)
1444+
.map(toRecord)
1445+
.filter((control) => typeof control.id === 'number')
1446+
.map((control) => `[ref=${control.id}] "${String(control.name ?? '')}"`)
14431447
throw new ToolError(
1444-
`That element is covered by ${blocker}. Close or move the overlay, then take a fresh browser_snapshot.`
1448+
controls.length > 0
1449+
? `That element is covered by ${blocker}. The overlay's controls in the current snapshot: ${controls.join(', ')}. Dismiss it with one of those, then retry the same id.`
1450+
: `That element is covered by ${blocker}. Close or move the overlay, then take a fresh browser_snapshot.`
14451451
)
14461452
}
14471453
if (code === 'nested-control') {
14481454
const blocker = String((result as { blocker?: unknown }).blocker || 'a nested control')
1455+
const controlId = (result as { controlId?: unknown }).controlId
14491456
throw new ToolError(
1450-
`The point you targeted lands on ${blocker}, which is its own control inside that element — nothing is covering it. Take a fresh browser_snapshot and use the id of the control you actually want.`
1457+
typeof controlId === 'number'
1458+
? `The point you targeted lands on ${blocker} [ref=${controlId}], which is its own control inside that element — nothing is covering it. Use id ${controlId} if that is the control you want; otherwise target the element through a part that is not a separate control.`
1459+
: `The point you targeted lands on ${blocker}, which is its own control inside that element — nothing is covering it. Take a fresh browser_snapshot and use the id of the control you actually want.`
14511460
)
14521461
}
14531462
if (code === 'suggestions-open') {

‎apps/desktop/src/main/browser-agent/page-functions.test.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ describe('combobox typing surfaces', () => {
427427
expect(focusElementForTyping(0)).toEqual({
428428
error: 'obstructed',
429429
blocker: 'Mondu',
430+
blockerControls: [],
430431
})
431432

432433
expect(clickElement(0, false)).toMatchObject({
@@ -448,6 +449,7 @@ describe('combobox typing surfaces', () => {
448449
expect(focusElementForTyping(0)).toEqual({
449450
error: 'obstructed',
450451
blocker: 'Mondu',
452+
blockerControls: [],
451453
})
452454
})
453455

@@ -643,6 +645,28 @@ describe('collectSnapshot', () => {
643645
expect(clicked).toBe(true)
644646
})
645647

648+
it('names the covering dialog controls so the agent can dismiss it and retry', () => {
649+
document.body.innerHTML = `
650+
<button aria-label="Checkout">Checkout</button>
651+
<div role="dialog" aria-label="Cookie notice"><p>We use cookies</p>
652+
<button>Accept all</button><button aria-label="Close notice">×</button>
653+
</div>`
654+
for (const element of Array.from(document.body.querySelectorAll('*'))) visible(element)
655+
const outline = outlineOf(collectSnapshot())
656+
const ref = refFor(outline, 'Checkout')
657+
const notice = document.querySelector('p') as HTMLParagraphElement
658+
Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => notice })
659+
660+
expect(clickElement(ref, false)).toEqual({
661+
error: 'obstructed',
662+
blocker: 'We use cookies',
663+
blockerControls: [
664+
{ id: refFor(outline, 'Accept all'), name: 'Accept all' },
665+
{ id: refFor(outline, 'Close notice'), name: 'Close notice' },
666+
],
667+
})
668+
})
669+
646670
it('refuses a coordinate click when an overlay owns every hit point', () => {
647671
document.body.innerHTML =
648672
'<button aria-label="Delete draft">Delete</button><div aria-label="Confirmation overlay"></div>'
@@ -658,6 +682,7 @@ describe('collectSnapshot', () => {
658682
expect(clickElement(ref, false)).toEqual({
659683
error: 'obstructed',
660684
blocker: 'Confirmation overlay',
685+
blockerControls: [],
661686
})
662687
})
663688

@@ -682,6 +707,7 @@ describe('collectSnapshot', () => {
682707
expect(clickElement(ref, false)).toEqual({
683708
error: 'nested-control',
684709
blocker: 'Delete channel',
710+
controlId: 1,
685711
})
686712
})
687713

‎apps/desktop/src/main/browser-agent/page-functions.ts‎

Lines changed: 79 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ declare global {
4747
__simAgentShownElements?: WeakSet<Element>
4848
/** Installed by {@link installPageHelpers} before every page function; cached per call. */
4949
__simAgentIsExemptModal: (element: Element) => boolean
50+
/** Installed by {@link installPageHelpers}: the element's id in the current snapshot. */
51+
__simAgentRefOf: (element: Element | null) => number | null
52+
/** Installed by {@link installPageHelpers}: the snapshot controls of the overlay a blocker belongs to. */
53+
__simAgentOverlayControls: (blocker: Element | null) => Array<{ id: number; name: string }>
5054
/** Why the last __simAgentResolveElement call returned null — read by the
5155
* shared stale-error producers so a refusal names its cause instead of
5256
* the blanket "the page changed". Cleared on every successful resolve. */
@@ -84,6 +88,62 @@ export function installPageHelpers(): void {
8488
}
8589
return modal === element
8690
}
91+
window.__simAgentRefOf = (element: Element | null): number | null => {
92+
const index = element ? (window.__simAgentElements ?? []).indexOf(element) : -1
93+
return index >= 0 ? index : null
94+
}
95+
/**
96+
* A refusal that names the overlay's own controls lets the agent dismiss it and retry the
97+
* same id without another snapshot. Only a real overlay qualifies — a dialog, a modal, or a
98+
* fixed or sticky layer — so an ordinary element in the way never lists unrelated page controls.
99+
*/
100+
window.__simAgentOverlayControls = (blocker: Element | null) => {
101+
let overlay: Element | null = null
102+
for (let current = blocker; current && !overlay; current = current.parentElement) {
103+
const role = current.getAttribute('role')
104+
const position = current.ownerDocument.defaultView?.getComputedStyle(current).position
105+
if (
106+
role === 'dialog' ||
107+
role === 'alertdialog' ||
108+
current.getAttribute('aria-modal') === 'true' ||
109+
current.tagName === 'DIALOG' ||
110+
position === 'fixed' ||
111+
position === 'sticky'
112+
) {
113+
overlay = current
114+
}
115+
}
116+
if (!overlay) return []
117+
const controls: Array<{ id: number; name: string }> = []
118+
const registry = window.__simAgentElements ?? []
119+
for (let id = 0; id < registry.length && controls.length < 4; id++) {
120+
const element = registry[id]
121+
if (!element || !overlay.contains(element)) continue
122+
const role = element.getAttribute('role')
123+
const tag = element.tagName
124+
if (
125+
tag !== 'BUTTON' &&
126+
tag !== 'A' &&
127+
role !== 'button' &&
128+
role !== 'link' &&
129+
!(tag === 'INPUT' && ['button', 'submit'].includes((element as HTMLInputElement).type))
130+
) {
131+
continue
132+
}
133+
const name = (
134+
element.getAttribute('aria-label') ||
135+
(element as HTMLElement).innerText ||
136+
(element as HTMLInputElement).value ||
137+
element.getAttribute('title') ||
138+
''
139+
)
140+
.replace(/\s+/g, ' ')
141+
.trim()
142+
.slice(0, 60)
143+
controls.push({ id, name })
144+
}
145+
return controls
146+
}
87147
function findExemptModal(doc: Document): Element | null {
88148
const rendered: Array<{ modal: Element; hidden: Element[] }> = []
89149
for (const modal of Array.from(doc.querySelectorAll('[aria-modal="true"], dialog[open]'))) {
@@ -1361,9 +1421,21 @@ export function clickElement(
13611421
}
13621422
}
13631423
if (nested) {
1364-
return { error: 'nested-control', blocker: blockerLabel(blocker) }
1424+
let control = blocker
1425+
while (control && control !== el && !isIndependentInteractive(control)) {
1426+
control = composedParent(control)
1427+
}
1428+
return {
1429+
error: 'nested-control',
1430+
blocker: blockerLabel(blocker),
1431+
controlId: control && control !== el ? window.__simAgentRefOf(control) : null,
1432+
}
1433+
}
1434+
return {
1435+
error: 'obstructed',
1436+
blocker: blockerLabel(blocker),
1437+
blockerControls: window.__simAgentOverlayControls(blocker),
13651438
}
1366-
return { error: 'obstructed', blocker: blockerLabel(blocker) }
13671439
}
13681440

13691441
let pageX = clientX
@@ -1775,7 +1847,11 @@ export function focusElementForTyping(id: number, moveFocus = true): unknown {
17751847
}
17761848
}
17771849
if (!chosenPoint) {
1778-
return { error: 'obstructed', blocker: blockerLabel(firstBlocker) }
1850+
return {
1851+
error: 'obstructed',
1852+
blocker: blockerLabel(firstBlocker),
1853+
blockerControls: window.__simAgentOverlayControls(firstBlocker),
1854+
}
17791855
}
17801856

17811857
return {

0 commit comments

Comments
 (0)