feat(browser): route drags and coordinate hovers along timed pointer paths

browser_drag takes optional via points and a durationMs, and browser_hover can target a viewport point with the same route, moving with no button pressed. Pages that react to pointer speed or position (sliders, physics, canvas hover, cursor-following games) can now be driven the way Claude computer use and OpenAI's CUA allow. A plain drag keeps its exact previous pace; batches refuse timed paths so they cannot outlast the batch watchdog.

Author: waleedlatif1

apps/desktop/src/main/browser-agent/cdp.test.ts

@@ -19,6 +19,7 @@ import {
   evaluateInIsolatedFrame,
   insertText,
   PRIMARY_CLICK,
+  pointerPathSteps,
   releaseFileInput,
   resolveFileInput,
   setColorScheme,
@@ -258,6 +259,26 @@ describe('browser-agent CDP instrumentation', () => {
     ])
   })
 
+  it('keeps the default drag pace and lands exactly on every via point', () => {
+    const direct = pointerPathSteps({ x: 0, y: 0 }, { via: [], durationMs: null }, { x: 120, y: 0 })
+    expect(direct.stepDelayMs).toBe(20)
+    expect(direct.points).toHaveLength(12)
+    expect(direct.points[0]).toEqual({ x: 10, y: 0 })
+    expect(direct.points[11]).toEqual({ x: 120, y: 0 })
+
+    const routed = pointerPathSteps(
+      { x: 0, y: 0 },
+      { via: [{ x: 100, y: 0 }], durationMs: 800 },
+      { x: 100, y: 300 }
+    )
+    expect(routed.points).toContainEqual({ x: 100, y: 0 })
+    expect(routed.points[routed.points.length - 1]).toEqual({ x: 100, y: 300 })
+    expect(routed.points.length * routed.stepDelayMs).toBeCloseTo(800)
+    // The longer second segment gets about three times the steps of the first.
+    const corner = routed.points.findIndex((point) => point.x === 100 && point.y === 0)
+    expect(routed.points.length - 1 - corner).toBeGreaterThan(corner * 2)
+  })
+
   it('holds the button down for holdMs before releasing it', async () => {
     const contents = new WebContentsView().webContents
     const types = () =>

apps/desktop/src/main/browser-agent/cdp.ts

@@ -948,6 +948,90 @@ export async function moveMouse(contents: WebContents, x: number, y: number): Pr
   })
 }
 
+/** A point in CSS viewport pixels. */
+export interface ViewportPoint {
+  x: number
+  y: number
+}
+
+/** The points a pointer passes through on its way, and how long the whole movement takes. */
+export interface PointerPath {
+  via: ViewportPoint[]
+  /** Total movement time; null keeps the default brisk pace. */
+  durationMs: number | null
+}
+
+export const DIRECT_PATH: PointerPath = { via: [], durationMs: null }
+
+const DEFAULT_PATH_STEPS = 12
+const DEFAULT_PATH_STEP_MS = 20
+/** One display frame, so a timed movement looks continuous to animation-driven pages. */
+const TIMED_PATH_STEP_MS = 16
+
+/**
+ * The moves from `from` through `path.via` to `to`, and the pause after each. A direct path
+ * keeps the default 12 moves 20 ms apart. A path with via points or a duration moves one frame
+ * at a time, shares the steps across segments by length, and lands exactly on every via point.
+ */
+export function pointerPathSteps(
+  from: ViewportPoint,
+  path: PointerPath,
+  to: ViewportPoint
+): { points: ViewportPoint[]; stepDelayMs: number } {
+  const lerp = (a: ViewportPoint, b: ViewportPoint, t: number): ViewportPoint => ({
+    x: a.x + (b.x - a.x) * t,
+    y: a.y + (b.y - a.y) * t,
+  })
+  if (path.via.length === 0 && path.durationMs === null) {
+    const points: ViewportPoint[] = []
+    for (let step = 1; step <= DEFAULT_PATH_STEPS; step++) {
+      points.push(lerp(from, to, step / DEFAULT_PATH_STEPS))
+    }
+    return { points, stepDelayMs: DEFAULT_PATH_STEP_MS }
+  }
+  const vertices = [from, ...path.via, to]
+  const durationMs = path.durationMs ?? DEFAULT_PATH_STEPS * DEFAULT_PATH_STEP_MS
+  const lengths = vertices
+    .slice(1)
+    .map((vertex, index) => Math.hypot(vertex.x - vertices[index].x, vertex.y - vertices[index].y))
+  const totalLength = lengths.reduce((sum, length) => sum + length, 0)
+  const totalSteps = Math.max(lengths.length, Math.round(durationMs / TIMED_PATH_STEP_MS))
+  const points: ViewportPoint[] = []
+  lengths.forEach((length, index) => {
+    const share = totalLength > 0 ? length / totalLength : 1 / lengths.length
+    const segmentSteps = Math.max(1, Math.round(totalSteps * share))
+    for (let step = 1; step <= segmentSteps; step++) {
+      points.push(lerp(vertices[index], vertices[index + 1], step / segmentSteps))
+    }
+  })
+  return { points, stepDelayMs: durationMs / points.length }
+}
+
+/**
+ * Moves the pointer with no button pressed through `path.via` to `to`, starting from the first
+ * via point (or `to` itself for a direct move), for hover effects that follow the cursor.
+ */
+export async function movePointer(
+  contents: WebContents,
+  path: PointerPath,
+  to: ViewportPoint,
+  signal?: AbortSignal
+): Promise<void> {
+  const [start, ...rest] = [...path.via, to]
+  await moveMouse(contents, start.x, start.y)
+  if (rest.length === 0) return
+  const { points, stepDelayMs } = pointerPathSteps(
+    start,
+    { via: rest.slice(0, -1), durationMs: path.durationMs },
+    to
+  )
+  for (const point of points) {
+    signal?.throwIfAborted()
+    await interruptibleSleep(stepDelayMs, signal)
+    await moveMouse(contents, point.x, point.y)
+  }
+}
+
 /** One trusted click gesture: which button, how many presses, and held modifiers. */
 export interface PointerClick {
   button: 'left' | 'right' | 'middle'
@@ -1075,11 +1159,11 @@ export async function clickAt(
  */
 export async function dragPointer(
   contents: WebContents,
-  from: { x: number; y: number },
-  to: { x: number; y: number },
-  steps = 12,
-  stepDelayMs = 20
+  from: ViewportPoint,
+  to: ViewportPoint,
+  path: PointerPath = DIRECT_PATH
 ): Promise<{ nativeDragIntercepted: boolean }> {
+  const { points, stepDelayMs } = pointerPathSteps(from, path, to)
   const interception: DragInterception = { intercepted: false, data: null }
   dragInterceptionsByContents.set(contents, interception)
   let interceptEnabled = false
@@ -1124,12 +1208,11 @@ export async function dragPointer(
     })
     // Small first nudge so libraries with a start threshold (commonly 3-8px)
     // register the drag before the pointer sweeps across the page.
-    await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2)
+    const heading = points[0] ?? to
+    await dragMove(from.x + Math.sign(heading.x - from.x || 1) * 4, from.y + 2)
     await sleep(stepDelayMs)
-    const stepCount = Math.max(2, steps)
-    for (let step = 1; step <= stepCount; step++) {
-      const progress = step / stepCount
-      await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress)
+    for (const point of points) {
+      await dragMove(point.x, point.y)
       await sleep(stepDelayMs)
     }
     // Hold over the target so drop zones running enter/over animations settle

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -4623,6 +4623,98 @@ describe('credential protection', () => {
     expect(cdpCalls(contents, 'Input.setInterceptDrags').length).toBeGreaterThan(0)
   })
 
+  it('drags through via points at the requested pace', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'div "Card"' },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_drag', {
+      fromX: 40,
+      fromY: 50,
+      toX: 200,
+      toY: 260,
+      via: [{ x: 300, y: 50 }],
+      durationMs: 320,
+    })
+
+    expect(result).toMatchObject({ ok: true, result: { dispatched: true } })
+    const moves = cdpCalls(contents, 'Input.dispatchMouseEvent')
+      .map(([, event]) => event as { type?: string; x?: number; y?: number; buttons?: number })
+      .filter((event) => event.type === 'mouseMoved' && event.buttons === 1)
+    expect(moves.some((event) => event.x === 300 && event.y === 50)).toBe(true)
+    expect(moves.length).toBeGreaterThanOrEqual(20)
+  })
+
+  it('moves the pointer along a coordinate hover path with no button pressed', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'canvas "Field"' },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_hover', {
+      x: 400,
+      y: 300,
+      via: [
+        { x: 100, y: 300 },
+        { x: 250, y: 150 },
+      ],
+    })
+
+    expect(result).toMatchObject({ ok: true, result: { hovered: true, x: 400, y: 300 } })
+    const events = cdpCalls(contents, 'Input.dispatchMouseEvent').map(
+      ([, event]) => event as { type?: string; x?: number; y?: number; button?: string }
+    )
+    expect(events.every((event) => event.type === 'mouseMoved' && event.button === 'none')).toBe(
+      true
+    )
+    expect(events[0]).toMatchObject({ x: 100, y: 300 })
+    expect(events.some((event) => event.x === 250 && event.y === 150)).toBe(true)
+    expect(events[events.length - 1]).toMatchObject({ x: 400, y: 300 })
+  })
+
+  it('rejects pointer paths where they cannot apply', async () => {
+    await openPage()
+
+    const elementHover = await driver.executeTool('chat-test', 'browser_hover', {
+      elementId: 0,
+      via: [{ x: 1, y: 2 }],
+    })
+    const badPoint = await driver.executeTool('chat-test', 'browser_hover', {
+      x: 10,
+      y: 10,
+      via: [{ x: 'left', y: 2 }],
+    })
+    const tooLong = await driver.executeTool('chat-test', 'browser_drag', {
+      fromX: 1,
+      fromY: 1,
+      toX: 50,
+      toY: 50,
+      durationMs: 10_001,
+    })
+    const batched = await driver.executeTool('chat-test', 'browser_batch', {
+      actions: [
+        { tool: 'browser_click', args: { elementId: 0 } },
+        { tool: 'browser_hover', args: { x: 5, y: 5, durationMs: 500 } },
+      ],
+    })
+
+    expect(elementHover).toMatchObject({
+      ok: false,
+      error: expect.stringContaining('coordinate hover'),
+    })
+    expect(badPoint).toMatchObject({ ok: false, error: expect.stringContaining('via point') })
+    expect(tooLong).toMatchObject({ ok: false, error: expect.stringContaining('durationMs') })
+    expect(batched).toMatchObject({
+      ok: false,
+      error: expect.stringContaining('timed pointer path'),
+    })
+  })
+
   it('drags from a snapshot element to a coordinate target', async () => {
     const contents = await openPage()
     respondWith(contents, {