fix(browser): honor cancellation on pointer routes and keep agent tabs tracked across scopes

- Timed drags and hover routes stop as soon as their tool call is cancelled;
  a drag still releases the button through its existing cleanup.
- A coordinate hover refuses durationMs without via points instead of
  silently moving straight to the target.
- A scope migration carries the chat's agent tab registration with it, so the
  migrated chat's old agent tab is released when it switches tabs.
- Overlay controls are found through shadow roots and with any tag casing.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -15,9 +15,11 @@ import {
   captureScreenshot,
   clickAt,
   consumeAgentContextMenu,
+  dragPointer,
   ensureInstrumented,
   evaluateInIsolatedFrame,
   insertText,
+  movePointer,
   PRIMARY_CLICK,
   pointerPathSteps,
   releaseFileInput,
@@ -259,6 +261,67 @@ describe('browser-agent CDP instrumentation', () => {
     ])
   })
 
+  it('stops a pointer route as soon as it is aborted', async () => {
+    const contents = new WebContentsView().webContents
+    const moves = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => String(method).startsWith('Input.dispatchMouseEvent'))
+    const aborted = new AbortController()
+    aborted.abort()
+    await expect(
+      movePointer(contents, { via: [], durationMs: null }, { x: 5, y: 5 }, aborted.signal)
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(moves()).toHaveLength(0)
+
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const route = movePointer(
+        contents,
+        { via: [{ x: 0, y: 0 }], durationMs: 5_000 },
+        { x: 500, y: 0 },
+        controller.signal
+      )
+      const settled = expect(route).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      const sentBeforeAbort = moves().length
+      controller.abort()
+      await settled
+      expect(moves()).toHaveLength(sentBeforeAbort)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
+  it('releases the button when a timed drag is aborted mid-route', async () => {
+    const contents = new WebContentsView().webContents
+    const types = () =>
+      vi
+        .mocked(contents.debugger.sendCommand)
+        .mock.calls.filter(([method]) => method === 'Input.dispatchMouseEvent')
+        .map(([, params]) => toRecord(params).type)
+    vi.useFakeTimers()
+    try {
+      const controller = new AbortController()
+      const drag = dragPointer(
+        contents,
+        { x: 0, y: 0 },
+        { x: 500, y: 0 },
+        { via: [], durationMs: 5_000 },
+        controller.signal
+      )
+      const settled = expect(drag).rejects.toMatchObject({ name: 'AbortError' })
+      await vi.advanceTimersByTimeAsync(100)
+      controller.abort()
+      await settled
+      expect(types().at(-1)).toBe('mouseReleased')
+      expect(types().filter((type) => type === 'mouseMoved').length).toBeLessThan(20)
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('keeps the default drag pace and lands exactly on every via point', () => {
     const direct = pointerPathSteps({ x: 0, y: 0 }, { via: [], durationMs: null }, { x: 120, y: 0 })
     expect(direct.stepDelayMs).toBe(20)

apps/desktop/src/main/browser-agent/cdp.ts

@@ -1018,6 +1018,7 @@ export async function movePointer(
   signal?: AbortSignal
 ): Promise<void> {
   const [start, ...rest] = [...path.via, to]
+  signal?.throwIfAborted()
   await moveMouse(contents, start.x, start.y)
   if (rest.length === 0) return
   const { points, stepDelayMs } = pointerPathSteps(
@@ -1026,8 +1027,8 @@ export async function movePointer(
     to
   )
   for (const point of points) {
-    signal?.throwIfAborted()
     await interruptibleSleep(stepDelayMs, signal)
+    signal?.throwIfAborted()
     await moveMouse(contents, point.x, point.y)
   }
 }
@@ -1161,7 +1162,8 @@ export async function dragPointer(
   contents: WebContents,
   from: ViewportPoint,
   to: ViewportPoint,
-  path: PointerPath = DIRECT_PATH
+  path: PointerPath = DIRECT_PATH,
+  signal?: AbortSignal
 ): Promise<{ nativeDragIntercepted: boolean }> {
   const { points, stepDelayMs } = pointerPathSteps(from, path, to)
   const interception: DragInterception = { intercepted: false, data: null }
@@ -1210,11 +1212,13 @@ export async function dragPointer(
     // register the drag before the pointer sweeps across the page.
     const heading = points[0] ?? to
     await dragMove(from.x + Math.sign(heading.x - from.x || 1) * 4, from.y + 2)
-    await sleep(stepDelayMs)
+    await interruptibleSleep(stepDelayMs, signal)
     for (const point of points) {
+      signal?.throwIfAborted()
       await dragMove(point.x, point.y)
-      await sleep(stepDelayMs)
+      await interruptibleSleep(stepDelayMs, signal)
     }
+    signal?.throwIfAborted()
     // Hold over the target so drop zones running enter/over animations settle
     // before the release lands.
     await sleep(120)

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -4684,6 +4684,11 @@ describe('credential protection', () => {
       elementId: 0,
       via: [{ x: 1, y: 2 }],
     })
+    const unroutedPace = await driver.executeTool('chat-test', 'browser_hover', {
+      x: 10,
+      y: 10,
+      durationMs: 800,
+    })
     const badPoint = await driver.executeTool('chat-test', 'browser_hover', {
       x: 10,
       y: 10,
@@ -4707,6 +4712,10 @@ describe('credential protection', () => {
       ok: false,
       error: expect.stringContaining('coordinate hover'),
     })
+    expect(unroutedPace).toMatchObject({
+      ok: false,
+      error: expect.stringContaining('paces a hover route'),
+    })
     expect(badPoint).toMatchObject({ ok: false, error: expect.stringContaining('via point') })
     expect(tooLong).toMatchObject({ ok: false, error: expect.stringContaining('durationMs') })
     expect(batched).toMatchObject({

apps/desktop/src/main/browser-agent/driver.ts

@@ -4512,6 +4512,11 @@ async function executeToolInner(
         const x = requireNum(params, 'x')
         const y = requireNum(params, 'y')
         const path = pointerPath(params)
+        if (path.via.length === 0 && path.durationMs !== null) {
+          throw new ToolError(
+            'durationMs paces a hover route; pass via points for the pointer to travel through.'
+          )
+        }
         assertCurrentExecution()
         assertActiveContents(contents)
         const pointTarget = unwrapPageResult(
@@ -5000,7 +5005,7 @@ async function executeToolInner(
       assertActiveContents(contents, dragNavigationEpoch)
       let interception: { nativeDragIntercepted: boolean }
       try {
-        interception = await cdp.dragPointer(contents, from, to, path)
+        interception = await cdp.dragPointer(contents, from, to, path, signal)
       } catch (error) {
         throw new ToolError(
           `Native drag dispatch failed (${getErrorMessage(error)}). The pointer may have been mid-drag; take a fresh snapshot to see the page's current state before retrying.`

apps/desktop/src/main/browser-agent/page-functions.test.ts

@@ -667,6 +667,30 @@ describe('collectSnapshot', () => {
     })
   })
 
+  it('names the controls of an overlay built from a web component', () => {
+    document.body.innerHTML = `
+      <button aria-label="Checkout">Checkout</button>
+      <div id="banner" style="position: fixed"></div>`
+    const host = document.getElementById('banner') as HTMLDivElement
+    const shadow = host.attachShadow({ mode: 'open' })
+    shadow.innerHTML = '<p>We use cookies</p><button aria-label="Close banner">×</button>'
+    for (const element of [
+      ...Array.from(document.body.querySelectorAll('*')),
+      ...Array.from(shadow.querySelectorAll('*')),
+    ]) {
+      visible(element)
+    }
+    const outline = outlineOf(collectSnapshot())
+    const ref = refFor(outline, 'Checkout')
+    const notice = shadow.querySelector('p') as HTMLParagraphElement
+    Object.defineProperty(document, 'elementFromPoint', { configurable: true, value: () => notice })
+
+    expect(clickElement(ref, false)).toMatchObject({
+      error: 'obstructed',
+      blockerControls: [{ id: refFor(outline, 'Close banner'), name: 'Close banner' }],
+    })
+  })
+
   it('refuses a coordinate click when an overlay owns every hit point', () => {
     document.body.innerHTML =
       '<button aria-label="Delete draft">Delete</button><div aria-label="Confirmation overlay"></div>'

apps/desktop/src/main/browser-agent/page-functions.ts

@@ -98,15 +98,22 @@ export function installPageHelpers(): void {
    * fixed or sticky layer — so an ordinary element in the way never lists unrelated page controls.
    */
   window.__simAgentOverlayControls = (blocker: Element | null) => {
+    /** The parent across shadow boundaries, so overlays built from web components qualify. */
+    const composedParent = (element: Element): Element | null => {
+      if (element.parentElement) return element.parentElement
+      const root = element.getRootNode()
+      return 'host' in root ? (root.host as Element) : null
+    }
+    const tagOf = (element: Element): string => String(element.tagName).toUpperCase()
     let overlay: Element | null = null
-    for (let current = blocker; current && !overlay; current = current.parentElement) {
+    for (let current = blocker; current && !overlay; current = composedParent(current)) {
       const role = current.getAttribute('role')
       const position = current.ownerDocument.defaultView?.getComputedStyle(current).position
       if (
         role === 'dialog' ||
         role === 'alertdialog' ||
         current.getAttribute('aria-modal') === 'true' ||
-        current.tagName === 'DIALOG' ||
+        tagOf(current) === 'DIALOG' ||
         position === 'fixed' ||
         position === 'sticky'
       ) {
@@ -118,9 +125,17 @@ export function installPageHelpers(): void {
     const registry = window.__simAgentElements ?? []
     for (let id = 0; id < registry.length && controls.length < 4; id++) {
       const element = registry[id]
-      if (!element || !overlay.contains(element)) continue
+      let inOverlay = false
+      for (
+        let current: Element | null = element;
+        current && !inOverlay;
+        current = composedParent(current)
+      ) {
+        inOverlay = current === overlay
+      }
+      if (!inOverlay) continue
       const role = element.getAttribute('role')
-      const tag = element.tagName
+      const tag = tagOf(element)
       if (
         tag !== 'BUTTON' &&
         tag !== 'A' &&

apps/desktop/src/main/browser-agent/panel.ts

@@ -155,6 +155,12 @@ export function activatePanelScope(scopeId: string | null): void {
 
 /** Retags an active pending scope without tearing down the compositor. */
 export function migratePanelScope(fromScopeId: string, toScopeId: string): void {
+  const agentView = agentViews.get(fromScopeId)
+  if (agentView) {
+    agentViews.delete(fromScopeId)
+    if (!agentViews.has(toScopeId)) agentViews.set(toScopeId, agentView)
+    else if (!isAgentView(agentView)) unparkView(agentView)
+  }
   if (activePanelScopeId !== fromScopeId) return
   activePanelScopeId = toScopeId
   panelCaptureGeneration++

apps/desktop/src/main/browser-agent/session.test.ts

@@ -522,6 +522,20 @@ describe('browser-agent session', () => {
     expect(session.migrateBrowserScope('chat-real', 'occupied')).toBe(false)
   })
 
+  it('carries the agent tab registration through a scope migration', () => {
+    const first = session.withBrowserScope('pending:workspace', () => session.ensureTab())
+    expect(session.migrateBrowserScope('pending:workspace', 'chat-real')).toBe(true)
+    const removeChildView = (
+      win as unknown as { contentView: { removeChildView: ReturnType<typeof vi.fn> } }
+    ).contentView.removeChildView
+    removeChildView.mockClear()
+
+    // The migrated chat moves its agent to a new tab: the first stops being an agent tab.
+    session.withBrowserScope('chat-real', () => session.addAutomationTab())
+
+    expect(removeChildView).toHaveBeenCalledWith(first.view)
+  })
+
   it('retains a migrated provisional alias until the durable scope is disposed', () => {
     const tab = session.withBrowserScope('pending:workspace', () => session.ensureTab())
     expect(session.migrateBrowserScope('pending:workspace', 'chat-real')).toBe(true)