feat(drive): control and visualize real mouse input

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
 
       - uses: oven-sh/setup-bun@v2
         with:
-          bun-version: "1.3.14"
+          bun-version: "1.4.0"
 
       - run: sudo apt-get update && sudo apt-get install -y ffmpeg
       - run: bun install --frozen-lockfile

.github/workflows/deploy-catalog.yml

@@ -31,7 +31,7 @@ jobs:
 
       - uses: oven-sh/setup-bun@v2
         with:
-          bun-version: "1.3.14"
+          bun-version: "1.4.0"
 
       - run: sudo apt-get update && sudo apt-get install --yes ffmpeg
       - run: bun install --frozen-lockfile

.github/workflows/publish.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: oven-sh/setup-bun@v2
         with:
-          bun-version: "1.3.14"
+          bun-version: "1.4.0"
 
       - run: sudo apt-get update && sudo apt-get install -y ffmpeg
       - run: bun install --frozen-lockfile

apps/catalog/package.json

@@ -2,7 +2,7 @@
   "name": "@opencode-drive/catalog",
   "private": true,
   "type": "module",
-  "packageManager": "bun@1.3.14",
+  "packageManager": "bun@1.4.0",
   "scripts": {
     "generate": "bun ./scripts/generate-catalog.ts",
     "capture": "bun ./scripts/capture-opencode-drive.ts",
@@ -28,8 +28,8 @@
     "react-dom": "^19.2.7"
   },
   "devDependencies": {
-    "@napi-rs/canvas": "1.0.2",
-    "@types/bun": "^1.3.14",
+    "@napi-rs/canvas": "1.0.8",
+    "@types/bun": "^1.4.0",
     "@types/react": "^19.2.17",
     "@types/react-dom": "^19.2.3",
     "effect": "4.0.0-rc.112",

package.json

@@ -2,7 +2,7 @@
   "$schema": "https://json.schemastore.org/package.json",
   "name": "opencode-drive-monorepo",
   "private": true,
-  "packageManager": "bun@1.3.14",
+  "packageManager": "bun@1.4.0",
   "workspaces": [
     "packages/*",
     "apps/*"

packages/drive/CHANGELOG.md

@@ -1,5 +1,20 @@
 # opencode-drive
 
+## 2.1.0
+
+### Minor Changes
+
+- e0fb4c7: Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior.
+
+  Animate pointer travel with a critically damped Motion spring and a configurable, bounded arc while preserving exact recorded input positions and times.
+
+  Refresh the native canvas and terminal replay dependencies, align repository and CI Bun versions, and document the distinction between the pinned V2 SDK and the separately installed OpenCode executable.
+
+### Patch Changes
+
+- Pin the V2 client and its matching protocol/schema packages to `0.0.0-dev-19066`, the published OpenCode build containing mouse control and pointer recording support.
+- f6a3f55: Bump the pinned `@opencode-ai/client` to `0.0.0-dev-18911` so the generated SDK exposes `plugin.awaitActivation`, which scripts need before reading agents or models from a cold location on current V2 servers.
+
 ## 2.0.1
 
 ### Patch Changes

packages/drive/README.md

@@ -8,11 +8,18 @@ This project gives your agents control over OpenCode:
 ## Requirements
 
 OpenCode Drive requires [Bun](https://bun.sh/) 1.3.14 or newer. MP4 recording export also requires `ffmpeg` on `PATH`.
+Repository development, CI, and releases use Bun 1.4.0.
 
 Effect programs must use `effect@4.0.0-rc.112`, Drive's exact peer dependency.
-The platform and test packages use the same release as the current V2 client,
-`@opencode-ai/client@0.0.0-dev-18535` and its matching protocol and schema
-packages. Effect's `latest` tag still selects V3; install the exact V4 peer.
+The platform and test packages use the same release as the V2 client.
+The exact `@opencode-ai/client` version is pinned in
+[`package.json`](package.json), which also selects its matching protocol and
+schema packages. Effect's `latest` tag still selects V3; install the exact V4 peer.
+
+Drive does not download or pin an OpenCode executable: it runs `opencode2` from
+`PATH` unless `--dev` or an explicit command selects another target. New mouse
+controls require a target advertising `ui.mouse` and `ui.recording.pointer`;
+upgrading Drive's SDK dependency alone does not upgrade that executable.
 
 Install dependencies with:
 
@@ -210,6 +217,66 @@ Captured frames use the official full Commit Mono v1.143 faces at 16px with bund
 
 ## UI development
 
+### Mouse control and recorded pointer
+
+`ui.mouse` dispatches real mouse input through OpenCode's renderer. Coordinates
+are zero-based terminal cells, absolute within the current viewport. Moves run
+native hover/leave handlers; a move while a button is down drags. Existing
+`ui.click(element, { x, y })` remains target-relative.
+
+```ts
+yield* ui.mouse({ action: "move", x: 20, y: 8 })
+yield* ui.mouse({ action: "down", x: 20, y: 8, button: "left" })
+yield* ui.mouse({ action: "move", x: 30, y: 8 })
+yield* ui.mouse({ action: "up", x: 30, y: 8, button: "left" })
+yield* ui.mouse({ action: "scroll", x: 30, y: 8, direction: "down" })
+```
+
+Buttons are `left` (default), `middle`, or `right`. Mouse modifiers use
+`{ shift, alt, ctrl }`; keyboard modifiers retain their existing `meta` spelling.
+The CLI takes the same protocol parameters with `--command.ui.mouse`.
+`ui.mouse` requires the optional `ui.mouse` endpoint capability. Unsupported
+endpoints fail with `UiCapabilityError` before input is sent; older endpoints
+remain usable for existing operations.
+
+Enable a minimal spring-animated cursor in exported videos:
+
+```ts
+OpenCodeDriver.use({
+  tui: {
+    recording: true,
+    pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 500, curve: 0.06 },
+  },
+}, ({ ui }) => ui.mouse({ action: "move", x: 20, y: 8 }))
+```
+
+`pointerOverlay: true` uses those defaults. Live CLI runs use
+`start --record --pointer-overlay`. This requires an OpenCode build advertising
+`ui.recording.pointer`; Drive does not silently manufacture mouse evidence for
+older endpoints. The overlay is off by default and affects videos, not the
+terminal text caret or raw screenshots.
+
+OpenCode records actual input positions in `*.pointers.jsonl` beside the terminal
+timeline using the same monotonic clock. Keep this sidecar when copying a
+recording. `exportRecording(timeline, output, { pointerOverlay: true })` can
+re-render it, including trimmed, sped-up, or held clips. Holds freeze the whole
+composition. Keypress labels still use their own output-time display duration.
+Visibility windows are bounded by retained recording time: export does not add
+extra pre/post-roll. Clips crop the raw composition, so an approach to an input
+just outside the clip may still be visible.
+
+The pointer appears before input, follows a small arc to the recorded cell with
+a critically damped Motion spring, and lingers after it. It arrives exactly at
+the input time without overshooting; nearby interactions stay connected.
+`curve` controls the arc height as a fraction of the pixel distance (capped at
+24px); set it to `0` for straight travel. Held drags always follow straight
+segments between recorded points. Motion's sampler runs without React or a browser.
+This is presentation-only: enabling it
+does not delay a script or send extra moves. To test hover along a path, send
+real intermediate `ui.mouse` moves. The runnable
+[`test/manual/pointer-demo.ts`](test/manual/pointer-demo.ts) checks production
+hover/leave and click behavior and exports a wide/narrow recording.
+
 If you are doing UI development in OpenCode, you might want to run it in a simulated mode. This allows `opencode-drive` to drive it and always put it into a state that you want to see.
 
 Run it in visible mode:

packages/drive/package.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "opencode-drive",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "Drive real and simulated OpenCode instances",
   "license": "MIT",
   "repository": {
@@ -11,7 +11,7 @@
   "homepage": "https://github.com/anomalyco/opencode-drive#readme",
   "bugs": "https://github.com/anomalyco/opencode-drive/issues",
   "type": "module",
-  "packageManager": "bun@1.3.14",
+  "packageManager": "bun@1.4.0",
   "engines": {
     "bun": ">=1.3.14"
   },
@@ -60,13 +60,13 @@
   "dependencies": {
     "@effect/platform-node": "4.0.0-rc.112",
     "@effect/platform-node-shared": "4.0.0-rc.112",
-    "@napi-rs/canvas": "1.0.2",
-    "@opencode-ai/client": "0.0.0-dev-18911",
-    "@opentui/core": "0.4.5",
-    "@types/bun": "1.3.13",
+    "@napi-rs/canvas": "1.0.8",
+    "@opencode-ai/client": "0.0.0-dev-19066",
+    "@types/bun": "1.4.0",
     "@typescript/native-preview": "7.0.0-dev.20251207.1",
-    "@wterm/core": "0.3.0",
-    "@wterm/ghostty": "0.3.0"
+    "@wterm/core": "0.4.1",
+    "@wterm/ghostty": "0.4.1",
+    "motion": "13.2.0"
   },
   "peerDependencies": {
     "effect": "4.0.0-rc.112"

packages/drive/src/cli/commands.ts

@@ -9,51 +9,22 @@ import * as SimulationConnector from "../simulation/connector.js"
 import type { DriveCommand } from "./types.js"
 import { appendKeypress, formatArrow, formatPress } from "../recording/keypresses.js"
 
-export const commandInfo = {
-  "ui.type": { value: true, description: "Type text using JSON params" },
-  "ui.press": { value: true, description: "Press a key using JSON params" },
-  "ui.enter": { value: false, description: "Press Enter" },
-  "ui.arrow": {
-    value: true,
-    description: "Press an arrow key using JSON params",
-  },
-  "ui.focus": {
-    value: true,
-    description: "Focus an element using JSON params",
-  },
-  "ui.click": { value: true, description: "Click using JSON params" },
-  "ui.resize": {
-    value: true,
-    description: "Resize terminal viewport using JSON params",
-  },
-  "ui.screenshot": {
-    value: "optional",
-    description: "Take a screenshot with optional JSON params and return its path",
-  },
-  "ui.capture": {
-    value: false,
-    description: "Capture the terminal frame as JSON",
-  },
-  "ui.state": {
-    value: false,
-    description: "Return focus, elements, and available UI actions",
-  },
-  "ui.snapshot": {
-    value: false,
-    description: "Return the semantic UI tree as JSON",
-  },
-  "ui.matches": {
-    value: true,
-    description: "Check for literal screen text using JSON params",
-  },
-  "ui.recording.finish": {
-    value: false,
-    description: "Finish recording and return the timeline path",
-  },
-} as const satisfies Record<
-  DriveCommand["operation"],
-  { readonly value: boolean | "optional"; readonly description: string }
->
+const commandInfo = {
+  "ui.type": true,
+  "ui.press": true,
+  "ui.enter": false,
+  "ui.arrow": true,
+  "ui.focus": true,
+  "ui.click": true,
+  "ui.mouse": true,
+  "ui.resize": true,
+  "ui.screenshot": "optional",
+  "ui.capture": false,
+  "ui.state": false,
+  "ui.snapshot": false,
+  "ui.matches": true,
+  "ui.recording.finish": false,
+} as const satisfies Record<DriveCommand["operation"], boolean | "optional">
 
 type CommandName = DriveCommand["operation"]
 
@@ -62,11 +33,7 @@ export function isCommandName(operation: string): operation is CommandName {
 }
 
 export function commandAcceptsValue(operation: CommandName) {
-  return commandInfo[operation].value
-}
-
-export function commandNames() {
-  return Object.keys(commandInfo).sort()
+  return commandInfo[operation]
 }
 
 export class SimulationError extends Error {
@@ -187,7 +154,7 @@ const execute = (
   )
 
 function decodeCommand(command: DriveCommand): Frontend.Request {
-  if (command.value === undefined && commandInfo[command.operation].value === true)
+  if (command.value === undefined && commandInfo[command.operation] === true)
     throw new Error(`${command.operation} requires a value`)
   const operation = command.operation
   if (operation === "ui.screenshot") throw new Error("ui.screenshot must be decoded by Drive")
@@ -229,6 +196,8 @@ function dispatch(
       ),
     )
   switch (request.method) {
+    case "ui.mouse":
+      return OpenCodeUi.make(connection).mouse(request.params)
     case "ui.type":
       return connection.rpc["ui.type"](request.params)
     case "ui.press":

packages/drive/src/cli/index.ts

@@ -126,6 +126,10 @@ const startCommand = Command.make(
       Flag.withDefault(false),
       Flag.withDescription("Show recent key presses in screenshots and recordings"),
     ),
+    pointerOverlay: Flag.boolean("pointer-overlay").pipe(
+      Flag.withDefault(false),
+      Flag.withDescription("Animate real mouse input in recordings"),
+    ),
     dev: Flag.string("dev").pipe(
       Flag.optional,
       Flag.withDescription("Path to an OpenCode development checkout"),
@@ -233,6 +237,7 @@ function toStartOptions(
     readonly visible: boolean
     readonly record: boolean
     readonly keypressOverlay: boolean
+    readonly pointerOverlay: boolean
     readonly dev: Option.Option<string>
   },
   commands: ReadonlyArray<DriveCommand>,
@@ -251,13 +256,16 @@ function toStartOptions(
     visible: config.visible,
     record: config.record,
     keypressOverlay: config.keypressOverlay,
+    pointerOverlay: config.pointerOverlay,
     dev: Option.getOrUndefined(config.dev),
     command: app,
   }
   if (options.dev !== undefined && app.length > 0)
     throw new Error("--dev cannot be combined with a command after --")
   if (options.keypressOverlay && !options.record)
     throw new Error("--keypress-overlay requires --record")
+  if (options.pointerOverlay && !options.record)
+    throw new Error("--pointer-overlay requires --record")
   return options
 }