From e0fb4c7262c3f94e5e9a0f4219ad1c832dc59921 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 21:16:43 -0400 Subject: [PATCH 1/6] feat(drive): control and visualize real mouse input --- .changeset/real-pointer-recording.md | 5 ++ bun.lock | 43 +--------- packages/drive/README.md | 55 ++++++++++++ packages/drive/package.json | 1 - packages/drive/src/cli/commands.ts | 71 +++++---------- packages/drive/src/cli/index.ts | 8 ++ packages/drive/src/cli/start.ts | 17 +++- packages/drive/src/cli/types.ts | 3 +- packages/drive/src/driver/client.ts | 10 ++- packages/drive/src/driver/llm-state.ts | 11 +-- packages/drive/src/driver/shared.ts | 31 ++----- packages/drive/src/driver/ui.ts | 16 +++- packages/drive/src/instance/control.ts | 22 ----- packages/drive/src/recording/edit.ts | 26 ++++-- packages/drive/src/recording/export.ts | 38 ++++++-- packages/drive/src/recording/index.ts | 1 + packages/drive/src/recording/pointer.ts | 82 ++++++++++++++++++ packages/drive/src/recording/render.ts | 31 +++++++ packages/drive/src/recording/replay.ts | 10 ++- packages/drive/src/simulation/connector.ts | 10 +-- packages/drive/src/simulation/protocol.ts | 32 +++++++ packages/drive/src/simulation/rpc.ts | 4 + packages/drive/src/tool/controller.ts | 4 +- packages/drive/test/cli/integration.test.ts | 17 ++++ packages/drive/test/driver/index.test.ts | 11 +++ packages/drive/test/driver/llm-state.test.ts | 10 +++ packages/drive/test/driver/shared.test.ts | 49 ++++++++++- packages/drive/test/driver/ui.test.ts | 27 ++++++ packages/drive/test/fixtures/fake-opencode.ts | 3 +- .../test/fixtures/pointer-options-script.ts | 15 ++++ .../drive/test/fixtures/tui-options-script.ts | 2 +- packages/drive/test/manual/pointer-demo.ts | 86 +++++++++++++++++++ packages/drive/test/recording/edit.test.ts | 13 +++ .../test/recording/export-dedupe.test.ts | 28 ++++++ packages/drive/test/recording/export.test.ts | 77 ++++++++++++++++- packages/drive/test/recording/pointer.test.ts | 64 ++++++++++++++ skills/opencode-drive/SKILL.md | 23 +++++ 37 files changed, 778 insertions(+), 178 deletions(-) create mode 100644 .changeset/real-pointer-recording.md create mode 100644 packages/drive/src/recording/pointer.ts create mode 100644 packages/drive/test/fixtures/pointer-options-script.ts create mode 100644 packages/drive/test/manual/pointer-demo.ts create mode 100644 packages/drive/test/recording/pointer.test.ts diff --git a/.changeset/real-pointer-recording.md b/.changeset/real-pointer-recording.md new file mode 100644 index 0000000..6c1e674 --- /dev/null +++ b/.changeset/real-pointer-recording.md @@ -0,0 +1,5 @@ +--- +"opencode-drive": minor +--- + +Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior. diff --git a/bun.lock b/bun.lock index e9e0b2e..24564b6 100644 --- a/bun.lock +++ b/bun.lock @@ -41,7 +41,6 @@ "@effect/platform-node-shared": "4.0.0-rc.112", "@napi-rs/canvas": "1.0.2", "@opencode-ai/client": "0.0.0-dev-18911", - "@opentui/core": "0.4.5", "@types/bun": "1.3.13", "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.3.0", @@ -303,24 +302,6 @@ "@opencode-drive/catalog": ["@opencode-drive/catalog@workspace:apps/catalog"], - "@opentui/core": ["@opentui/core@0.4.5", "", { "dependencies": { "bun-ffi-structs": "0.2.4", "diff": "9.0.0", "marked": "17.0.1", "string-width": "7.2.0", "strip-ansi": "7.1.2" }, "optionalDependencies": { "@opentui/core-darwin-arm64": "0.4.5", "@opentui/core-darwin-x64": "0.4.5", "@opentui/core-linux-arm64": "0.4.5", "@opentui/core-linux-arm64-musl": "0.4.5", "@opentui/core-linux-x64": "0.4.5", "@opentui/core-linux-x64-musl": "0.4.5", "@opentui/core-win32-arm64": "0.4.5", "@opentui/core-win32-x64": "0.4.5" }, "peerDependencies": { "web-tree-sitter": "0.25.10" } }, "sha512-JsgRTPkA6e+Vxmumxai6SElOSlRQkbzNKHlCfemlArRiLhfC1IZ9RXJo2QH4xSu+uBOWAM90uss73/pPlkdEig=="], - - "@opentui/core-darwin-arm64": ["@opentui/core-darwin-arm64@0.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-8KUG0oRidnR+oW1RSZJ72/PhZLl+qRRMk5U/mieF4c0SJ5V3tYACpBZAKzQfHNd1f7QzD8FHZct1lPpQgtmkWg=="], - - "@opentui/core-darwin-x64": ["@opentui/core-darwin-x64@0.4.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-R2bocsg55gwjOqCp/MWFgFYzRmsduKegB6nzgFAPCvAD/L5Jf30xpWJWFlSg3x8vxe1L9WJ84dfqa4M7mZZ3wA=="], - - "@opentui/core-linux-arm64": ["@opentui/core-linux-arm64@0.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-R4MZ25a4CzOAGVjW9aj1hUfzQGVfCJwrwBDbNs2SXaIvzcZqkxCVtU4FoQ5LsaD0j/BdNQVg2CIfFkFsm1fDuQ=="], - - "@opentui/core-linux-arm64-musl": ["@opentui/core-linux-arm64-musl@0.4.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ieqdyKI6EIYPalYAETB2wsdP83hr5Ifi+dFnBFUmdEEFHsoKwBmn2S7bsTOYlX7Bg03F4/YPIg+IvRpeC+cUJw=="], - - "@opentui/core-linux-x64": ["@opentui/core-linux-x64@0.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-SNyuQoxMKI1vuJhgxSSW96adWM6LqFl2SoS3GM4tGeneGOanVVG2Y06PvlytXvF4cKik97t0rqkVMRetmOs93w=="], - - "@opentui/core-linux-x64-musl": ["@opentui/core-linux-x64-musl@0.4.5", "", { "os": "linux", "cpu": "x64" }, "sha512-mKVKcIcPiSVVZZsdPSBoWwoa2/TCeQAaMDeHF7PFw2kt5bTXZPP7xxWfRQLCNIcA1eaGl59UuwUWHDR2Ve548Q=="], - - "@opentui/core-win32-arm64": ["@opentui/core-win32-arm64@0.4.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-GHTTsqeR45q2Iek9Rb7ty+x/hAKn2jZ1ujlCgPR8LBKyF7h0E1dNFryoZ7ehMc3kJndP1sKn836IemKFqxuDdQ=="], - - "@opentui/core-win32-x64": ["@opentui/core-win32-x64@0.4.5", "", { "os": "win32", "cpu": "x64" }, "sha512-Y8T/yXCDGagRGiQrtmuB6AhRcPucKFs/Dre3v8kJwNYqDccI4FzUPKclZ7djfmRZNjl7JUqPhZZP/PwDpQocMg=="], - "@oxc-project/types": ["@oxc-project/types@0.139.0", "", {}, "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw=="], "@oxlint-tsgolint/darwin-arm64": ["@oxlint-tsgolint/darwin-arm64@0.21.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-P20j3MLqfwIT+94qGU3htC7dWp4pXGZW1p1p7FRUzu1aopq7c9nPCgf0W/WjktqQ57+iuTq9mbSlwWinl6+H1A=="], @@ -523,7 +504,7 @@ "ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="], - "ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], @@ -537,8 +518,6 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "bun-ffi-structs": ["bun-ffi-structs@0.2.4", "", { "peerDependencies": { "typescript": "^5" } }, "sha512-AJzsqoVFs1KBbJbWHIYrVZLDC3NhTqqh25awRXqzoLzmBAKr5oqk6+CwuYHAekKx+VBCYVohBoKuRq40dV+TYg=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], @@ -559,14 +538,10 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - "diff": ["diff@9.0.0", "", {}, "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw=="], - "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], "effect": ["effect@4.0.0-rc.112", "", { "dependencies": { "fast-check": "^4.9.0", "msgpackr": "^2.0.5" } }, "sha512-wXxwuh1Ywnv4cPRM3Wfa0vDwuOHnZ1TsTgHJkG9XgzND6inhBH9n1vBxhg3iIXOia/OrpmvVmd3lrD4vq6bF3A=="], - "emoji-regex": ["emoji-regex@10.6.0", "", {}, "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A=="], - "enquirer": ["enquirer@2.4.1", "", { "dependencies": { "ansi-colors": "^4.1.1", "strip-ansi": "^6.0.1" } }, "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ=="], "error-stack-parser-es": ["error-stack-parser-es@1.0.5", "", {}, "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA=="], @@ -599,8 +574,6 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - "get-east-asian-width": ["get-east-asian-width@1.6.0", "", {}, "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA=="], - "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "globby": ["globby@11.1.0", "", { "dependencies": { "array-union": "^2.1.0", "dir-glob": "^3.0.1", "fast-glob": "^3.2.9", "ignore": "^5.2.0", "merge2": "^1.4.1", "slash": "^3.0.0" } }, "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g=="], @@ -661,8 +634,6 @@ "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - "marked": ["marked@17.0.1", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-boeBdiS0ghpWcSwoNm/jJBwdpFaMnZWRzjA6SkUMYb40SVaN1x7mmfGKp0jvexGcx+7y2La5zRZsYFZI6Qpypg=="], - "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], @@ -773,9 +744,7 @@ "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], - "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], - - "strip-ansi": ["strip-ansi@7.1.2", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA=="], + "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], @@ -809,8 +778,6 @@ "vitest": ["vitest@4.1.10", "", { "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", "@vitest/pretty-format": "4.1.10", "@vitest/runner": "4.1.10", "@vitest/snapshot": "4.1.10", "@vitest/spy": "4.1.10", "@vitest/utils": "4.1.10", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.10", "@vitest/browser-preview": "4.1.10", "@vitest/browser-webdriverio": "4.1.10", "@vitest/coverage-istanbul": "4.1.10", "@vitest/coverage-v8": "4.1.10", "@vitest/ui": "4.1.10", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw=="], - "web-tree-sitter": ["web-tree-sitter@0.25.10", "", { "peerDependencies": { "@types/emscripten": "^1.40.0" }, "optionalPeers": ["@types/emscripten"] }, "sha512-Y09sF44/13XvgVKgO2cNDw5rGk6s26MgoZPXLESvMXeefBf7i6/73eFurre0IsTW6E14Y0ArIzhUMmjoc7xyzA=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], @@ -833,10 +800,6 @@ "@manypkg/get-packages/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="], - "bun-ffi-structs/typescript": ["typescript@5.8.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ=="], - - "enquirer/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], - "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "miniflare/undici": ["undici@7.28.0", "", {}, "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA=="], @@ -849,8 +812,6 @@ "read-yaml-file/js-yaml": ["js-yaml@3.15.0", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog=="], - "enquirer/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], - "opencode-drive/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], diff --git a/packages/drive/README.md b/packages/drive/README.md index 6dcface..0d39d2a 100644 --- a/packages/drive/README.md +++ b/packages/drive/README.md @@ -210,6 +210,61 @@ Captured frames use the official full Commit Mono v1.143 faces at 16px with bund ## UI development +### Mouse control and recorded pointer + +`ui.mouse` dispatches real mouse input through OpenCode's renderer. Coordinates +are zero-based terminal cells, absolute within the current viewport. Moves run +native hover/leave handlers; a move while a button is down drags. Existing +`ui.click(element, { x, y })` remains target-relative. + +```ts +yield* ui.mouse({ action: "move", x: 20, y: 8 }) +yield* ui.mouse({ action: "down", x: 20, y: 8, button: "left" }) +yield* ui.mouse({ action: "move", x: 30, y: 8 }) +yield* ui.mouse({ action: "up", x: 30, y: 8, button: "left" }) +yield* ui.mouse({ action: "scroll", x: 30, y: 8, direction: "down" }) +``` + +Buttons are `left` (default), `middle`, or `right`. Mouse modifiers use +`{ shift, alt, ctrl }`; keyboard modifiers retain their existing `meta` spelling. +The CLI takes the same protocol parameters with `--command.ui.mouse`. +`ui.mouse` requires the optional `ui.mouse` endpoint capability. Unsupported +endpoints fail with `UiCapabilityError` before input is sent; older endpoints +remain usable for existing operations. + +Enable a minimal eased cursor in exported videos: + +```ts +OpenCodeDriver.use({ + tui: { + recording: true, + pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 220 }, + }, +}, ({ ui }) => ui.mouse({ action: "move", x: 20, y: 8 })) +``` + +`pointerOverlay: true` uses those defaults. Live CLI runs use +`start --record --pointer-overlay`. This requires an OpenCode build advertising +`ui.recording.pointer`; Drive does not silently manufacture mouse evidence for +older endpoints. The overlay is off by default and affects videos, not the +terminal text caret or raw screenshots. + +OpenCode records actual input positions in `*.pointers.jsonl` beside the terminal +timeline using the same monotonic clock. Keep this sidecar when copying a +recording. `exportRecording(timeline, output, { pointerOverlay: true })` can +re-render it, including trimmed, sped-up, or held clips. Holds freeze the whole +composition. Keypress labels still use their own output-time display duration. +Visibility windows are bounded by retained recording time: export does not add +extra pre/post-roll. Clips crop the raw composition, so an approach to an input +just outside the clip may still be visible. + +The pointer appears before input, eases to the recorded cell, and lingers after +it; nearby interactions stay connected. This is presentation-only: enabling it +does not delay a script or send extra moves. To test hover along a path, send +real intermediate `ui.mouse` moves. The runnable +[`test/manual/pointer-demo.ts`](test/manual/pointer-demo.ts) checks production +hover/leave and click behavior and exports a wide/narrow recording. + If you are doing UI development in OpenCode, you might want to run it in a simulated mode. This allows `opencode-drive` to drive it and always put it into a state that you want to see. Run it in visible mode: diff --git a/packages/drive/package.json b/packages/drive/package.json index d9925d6..17ea073 100644 --- a/packages/drive/package.json +++ b/packages/drive/package.json @@ -62,7 +62,6 @@ "@effect/platform-node-shared": "4.0.0-rc.112", "@napi-rs/canvas": "1.0.2", "@opencode-ai/client": "0.0.0-dev-18911", - "@opentui/core": "0.4.5", "@types/bun": "1.3.13", "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.3.0", diff --git a/packages/drive/src/cli/commands.ts b/packages/drive/src/cli/commands.ts index 79bc75f..5b0c757 100644 --- a/packages/drive/src/cli/commands.ts +++ b/packages/drive/src/cli/commands.ts @@ -9,51 +9,22 @@ import * as SimulationConnector from "../simulation/connector.js" import type { DriveCommand } from "./types.js" import { appendKeypress, formatArrow, formatPress } from "../recording/keypresses.js" -export const commandInfo = { - "ui.type": { value: true, description: "Type text using JSON params" }, - "ui.press": { value: true, description: "Press a key using JSON params" }, - "ui.enter": { value: false, description: "Press Enter" }, - "ui.arrow": { - value: true, - description: "Press an arrow key using JSON params", - }, - "ui.focus": { - value: true, - description: "Focus an element using JSON params", - }, - "ui.click": { value: true, description: "Click using JSON params" }, - "ui.resize": { - value: true, - description: "Resize terminal viewport using JSON params", - }, - "ui.screenshot": { - value: "optional", - description: "Take a screenshot with optional JSON params and return its path", - }, - "ui.capture": { - value: false, - description: "Capture the terminal frame as JSON", - }, - "ui.state": { - value: false, - description: "Return focus, elements, and available UI actions", - }, - "ui.snapshot": { - value: false, - description: "Return the semantic UI tree as JSON", - }, - "ui.matches": { - value: true, - description: "Check for literal screen text using JSON params", - }, - "ui.recording.finish": { - value: false, - description: "Finish recording and return the timeline path", - }, -} as const satisfies Record< - DriveCommand["operation"], - { readonly value: boolean | "optional"; readonly description: string } -> +const commandInfo = { + "ui.type": true, + "ui.press": true, + "ui.enter": false, + "ui.arrow": true, + "ui.focus": true, + "ui.click": true, + "ui.mouse": true, + "ui.resize": true, + "ui.screenshot": "optional", + "ui.capture": false, + "ui.state": false, + "ui.snapshot": false, + "ui.matches": true, + "ui.recording.finish": false, +} as const satisfies Record type CommandName = DriveCommand["operation"] @@ -62,11 +33,7 @@ export function isCommandName(operation: string): operation is CommandName { } export function commandAcceptsValue(operation: CommandName) { - return commandInfo[operation].value -} - -export function commandNames() { - return Object.keys(commandInfo).sort() + return commandInfo[operation] } export class SimulationError extends Error { @@ -187,7 +154,7 @@ const execute = ( ) function decodeCommand(command: DriveCommand): Frontend.Request { - if (command.value === undefined && commandInfo[command.operation].value === true) + if (command.value === undefined && commandInfo[command.operation] === true) throw new Error(`${command.operation} requires a value`) const operation = command.operation if (operation === "ui.screenshot") throw new Error("ui.screenshot must be decoded by Drive") @@ -229,6 +196,8 @@ function dispatch( ), ) switch (request.method) { + case "ui.mouse": + return OpenCodeUi.make(connection).mouse(request.params) case "ui.type": return connection.rpc["ui.type"](request.params) case "ui.press": diff --git a/packages/drive/src/cli/index.ts b/packages/drive/src/cli/index.ts index c1080fc..ece9c38 100644 --- a/packages/drive/src/cli/index.ts +++ b/packages/drive/src/cli/index.ts @@ -126,6 +126,10 @@ const startCommand = Command.make( Flag.withDefault(false), Flag.withDescription("Show recent key presses in screenshots and recordings"), ), + pointerOverlay: Flag.boolean("pointer-overlay").pipe( + Flag.withDefault(false), + Flag.withDescription("Animate real mouse input in recordings"), + ), dev: Flag.string("dev").pipe( Flag.optional, Flag.withDescription("Path to an OpenCode development checkout"), @@ -233,6 +237,7 @@ function toStartOptions( readonly visible: boolean readonly record: boolean readonly keypressOverlay: boolean + readonly pointerOverlay: boolean readonly dev: Option.Option }, commands: ReadonlyArray, @@ -251,6 +256,7 @@ function toStartOptions( visible: config.visible, record: config.record, keypressOverlay: config.keypressOverlay, + pointerOverlay: config.pointerOverlay, dev: Option.getOrUndefined(config.dev), command: app, } @@ -258,6 +264,8 @@ function toStartOptions( throw new Error("--dev cannot be combined with a command after --") if (options.keypressOverlay && !options.record) throw new Error("--keypress-overlay requires --record") + if (options.pointerOverlay && !options.record) + throw new Error("--pointer-overlay requires --record") return options } diff --git a/packages/drive/src/cli/start.ts b/packages/drive/src/cli/start.ts index c44e6f7..02822cb 100644 --- a/packages/drive/src/cli/start.ts +++ b/packages/drive/src/cli/start.ts @@ -68,12 +68,13 @@ const startScoped = Effect.fn("DriveCli.startScoped")(function* (options: StartO Effect.tap(() => Effect.sync(() => logSuccess(`loading script ${scriptModule}`))), ) : undefined - const script = loadedScript && options.keypressOverlay + const script = loadedScript && (options.keypressOverlay || options.pointerOverlay) ? { ...loadedScript, tui: { ...loadedScript.tui, - keypressOverlay: true, + ...(options.keypressOverlay ? { keypressOverlay: true } : {}), + ...(options.pointerOverlay ? { pointerOverlay: true } : {}), }, } : loadedScript @@ -169,7 +170,7 @@ async function runLifecycle( options.script !== undefined ) return Promise.resolve(undefined) - recording ??= finishRecording(instance, onProgress) + recording ??= finishRecording(instance, onProgress, options.pointerOverlay) return recording } const interrupt = () => { @@ -390,6 +391,7 @@ function shouldCleanArtifacts( async function finishRecording( instance: OpenCodeInstance.Instance, onProgress?: (percent: number) => void, + pointerOverlay?: boolean, ) { const expected = await runEffect(instance.recording) if (!expected) throw new Error("recording was not enabled for this instance") @@ -415,7 +417,7 @@ async function finishRecording( ), ) } - return finalizeRecording(timeline, expected, { onProgress }) + return finalizeRecording(timeline, expected, { onProgress, pointerOverlay }) } const startDetached = Effect.fn("DriveCli.startDetached")(function* ( @@ -437,6 +439,7 @@ const startDetached = Effect.fn("DriveCli.startDetached")(function* ( ...(options.dev ? ["--dev", options.dev] : []), ...(options.record ? ["--record"] : []), ...(options.keypressOverlay ? ["--keypress-overlay"] : []), + ...(options.pointerOverlay ? ["--pointer-overlay"] : []), ...(options.command.length ? ["--", ...options.command] : []), ], { cwd: process.cwd(), @@ -492,6 +495,12 @@ function run( if (!driveScript) { log("waiting for OpenCode") await runEffect(instance.waitForDrive("both")) + if (options.pointerOverlay) + await runEffect(Effect.scoped(Effect.gen(function* () { + const connection = yield* SimulationConnector.ui(instance.endpoints.ui) + if (!SimulationConnector.supportsCapability(connection.compatibility, "ui.recording.pointer")) + yield* Effect.fail(new Error("--pointer-overlay requires an OpenCode endpoint with ui.recording.pointer")) + }))) log("OpenCode ready") } if (driveScript) { diff --git a/packages/drive/src/cli/types.ts b/packages/drive/src/cli/types.ts index 9e5e795..8d6e03a 100644 --- a/packages/drive/src/cli/types.ts +++ b/packages/drive/src/cli/types.ts @@ -1,7 +1,7 @@ import type { Frontend } from "../client/index.js" export interface DriveCommand { - readonly operation: Exclude | "ui.screenshot" + readonly operation: Exclude | "ui.screenshot" readonly value?: string } @@ -13,6 +13,7 @@ export interface StartOptions { readonly visible: boolean readonly record: boolean readonly keypressOverlay: boolean + readonly pointerOverlay?: boolean readonly dev?: string readonly command: ReadonlyArray } diff --git a/packages/drive/src/driver/client.ts b/packages/drive/src/driver/client.ts index 4724d29..acefd42 100644 --- a/packages/drive/src/driver/client.ts +++ b/packages/drive/src/driver/client.ts @@ -6,9 +6,11 @@ import * as Scope from "effect/Scope" import * as Deferred from "effect/Deferred" import type * as OpenCodeInstance from "../instance/runtime.js" import type * as SimulationConnector from "../simulation/connector.js" +import { supportsCapability } from "../simulation/connector.js" import type { Frontend } from "../client/protocol.js" import { finalizeRecording } from "../recording/finalize.js" import { appendMark } from "../recording/marks.js" +import type { PointerOverlayOptions } from "../recording/pointer.js" import { error, type OpenCodeDriverError } from "./error.js" import * as OpenCodeUi from "./ui.js" import * as SharedEffect from "./shared.js" @@ -17,6 +19,8 @@ export interface TuiOptions { readonly recording?: boolean /** Show recent semantic key presses in screenshots and exported recordings. */ readonly keypressOverlay?: boolean + /** Animate real mouse input in exported videos. Requires recording and a pointer-capable endpoint. */ + readonly pointerOverlay?: boolean | PointerOverlayOptions readonly viewport?: Frontend.ResizeParams } @@ -90,6 +94,10 @@ export const make = Effect.fn("OpenCodeTui.make")(function* ( error("tui.launch", "keypressOverlay requires recording"), ) const connection = yield* connector.ui(launched.endpoint, { compatibility }) + if (options.pointerOverlay && launched.recording === undefined) + return yield* Effect.fail(error("tui.launch", "pointerOverlay requires recording")) + if (options.pointerOverlay && !supportsCapability(connection.compatibility, "ui.recording.pointer")) + return yield* Effect.fail(error("tui.launch", "pointerOverlay requires an OpenCode endpoint with ui.recording.pointer")) const ui = OpenCodeUi.make(connection, { screenshotDirectory: launched.media, ...(options.keypressOverlay && launched.recording @@ -120,7 +128,7 @@ export const make = Effect.fn("OpenCodeTui.make")(function* ( const exportFinishedRecording = yield* SharedEffect.make( Effect.flatMap(finishTimeline, (timeline) => Effect.tryPromise({ - try: (signal) => finalizeRecording(timeline, recording, { signal }), + try: (signal) => finalizeRecording(timeline, recording, { signal, pointerOverlay: options.pointerOverlay }), catch: (cause) => error("recording.export", cause), }), ), diff --git a/packages/drive/src/driver/llm-state.ts b/packages/drive/src/driver/llm-state.ts index 8de277e..b67f487 100644 --- a/packages/drive/src/driver/llm-state.ts +++ b/packages/drive/src/driver/llm-state.ts @@ -53,8 +53,7 @@ export const Mode = Data.taggedEnum() export interface State { readonly mode: Mode - readonly titleHandler: TitleHandler - readonly titleConfigured: boolean + readonly titleHandler: TitleHandler | undefined readonly requests: ReadonlyArray readonly responses: ReadonlyArray readonly activeNormal: ReadonlyArray @@ -69,8 +68,7 @@ export interface State { export const initial: State = { mode: Mode.Unset(), - titleHandler: () => Effect.succeed("OpenCode Drive"), - titleConfigured: false, + titleHandler: undefined, requests: [], responses: [], activeNormal: [], @@ -121,7 +119,7 @@ export const rejectServe = (state: State): RejectionError | undefined => { /** Why a title call must be rejected, or undefined to proceed. */ export const rejectTitle = (state: State): RejectionError | undefined => { if (state.failure !== undefined) return state.failure - if (state.titleConfigured) + if (state.titleHandler !== undefined) return new LlmModeError({ operation: "title", message: "llm.title may only be configured once", @@ -148,7 +146,6 @@ export const serve = (state: State, handler: ServeHandler): State => ({ export const configureTitle = (state: State, handler: TitleHandler): State => ({ ...state, - titleConfigured: true, titleHandler: handler, }) @@ -267,7 +264,7 @@ export const startTitle = ( titleIndex: state.titleIndex + 1, }, { - handler: state.titleHandler, + handler: state.titleHandler ?? (() => Effect.succeed("OpenCode Drive")), index: state.titleIndex, awaiting: state.activeNormal, }, diff --git a/packages/drive/src/driver/shared.ts b/packages/drive/src/driver/shared.ts index fbb3abc..9337029 100644 --- a/packages/drive/src/driver/shared.ts +++ b/packages/drive/src/driver/shared.ts @@ -1,31 +1,16 @@ -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Ref from "effect/Ref" -import * as Scope from "effect/Scope" -import * as Semaphore from "effect/Semaphore" +import { Effect, Fiber, Scope } from "effect" /** Starts a terminal operation once in its owner's scope, independently of callers. */ export const make = Effect.fn("SharedEffect.make")(function* ( effect: Effect.Effect, ) { const scope = yield* Scope.Scope - const result = yield* Deferred.make() - const started = yield* Ref.make(false) - const lock = yield* Semaphore.make(1) - - return Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - yield* lock.withPermit( - Effect.gen(function* () { - if (yield* Ref.get(started)) return - yield* Ref.set(started, true) - yield* Deferred.complete(result, effect).pipe( - Effect.asVoid, - Effect.forkIn(scope, { uninterruptible: true }), - ) - }), - ) - return yield* restore(Deferred.await(result)) - }), + const start = yield* Effect.cached( + Effect.forkIn(Effect.exit(effect), scope, { uninterruptible: true }), + ) + return start.pipe( + Effect.uninterruptible, + Effect.flatMap(Fiber.join), + Effect.flatten, ) }) diff --git a/packages/drive/src/driver/ui.ts b/packages/drive/src/driver/ui.ts index 8c8dfb7..013fd69 100644 --- a/packages/drive/src/driver/ui.ts +++ b/packages/drive/src/driver/ui.ts @@ -96,7 +96,7 @@ export class UiNodeAmbiguousError extends Schema.TaggedError()( "UiCapabilityError", { - capability: Schema.Literals(["ui.snapshot", "ui.click.semantic"]), + capability: Schema.Literals(["ui.snapshot", "ui.click.semantic", "ui.mouse"]), message: Schema.String, }, ) {} @@ -163,6 +163,10 @@ export interface Ui { readonly focus: ( target: number | Frontend.Element, ) => Effect.Effect + /** Send real terminal mouse input at zero-based absolute cell coordinates. */ + readonly mouse: ( + params: Frontend.MouseParams, + ) => Effect.Effect readonly click: ( target: number | Frontend.Element | Frontend.SemanticNode, position?: Position, @@ -219,6 +223,7 @@ export const transform = (ui: Ui, apply: Transform): Ui => ({ enter: () => apply(ui.enter()), arrow: (direction) => apply(ui.arrow(direction)), focus: (target) => apply(ui.focus(target)), + mouse: (params) => apply(ui.mouse(params)), click: (target, position) => apply(ui.click(target, position)), resize: (viewport) => apply(ui.resize(viewport)), submit: (text) => apply(ui.submit(text)), @@ -318,6 +323,14 @@ export const make = (connection: UiConnection, options?: Options): Control => { const resize = Effect.fn("Ui.resize")((viewport: Frontend.ResizeParams) => call("resize", rpc["ui.resize"](viewport)), ) + const mouse = Effect.fn("Ui.mouse")(function* (params: Frontend.MouseParams) { + if (!supportsCapability(connection.compatibility, "ui.mouse")) + return yield* Effect.fail(new UiCapabilityError({ + capability: "ui.mouse", + message: "ui.mouse is not available on this OpenCode endpoint", + })) + return yield* call("mouse", rpc["ui.mouse"](params)) + }) const submit = Effect.fn("Ui.submit")(function* (text: string) { yield* type(text) return yield* enter() @@ -511,6 +524,7 @@ export const make = (connection: UiConnection, options?: Options): Control => { enter, arrow, focus, + mouse, click, resize, submit, diff --git a/packages/drive/src/instance/control.ts b/packages/drive/src/instance/control.ts index 433ae35..0eb770c 100644 --- a/packages/drive/src/instance/control.ts +++ b/packages/drive/src/instance/control.ts @@ -91,20 +91,6 @@ export async function requestStop( return value } -export async function requestResponses(path: string, input: ResponseUpdate) { - const response = await send( - path, - Object.keys(input).length === 0 - ? "responses" - : `responses ${JSON.stringify(input)}`, - ) - if (!response.startsWith("success ")) throw responseError(response) - const value: unknown = JSON.parse(response.slice("success ".length)) - if (!isResponseConfiguration(value)) - throw new Error("instance returned an invalid response configuration") - return value -} - function send(path: string, command: string, onProgress?: (percent: number) => void) { return new Promise((resolve, reject) => { const socket = connect(path) @@ -159,14 +145,6 @@ function stringArray(value: unknown) { return value } -function isResponseConfiguration( - value: unknown, -): value is ResponseConfiguration { - if (typeof value !== "object" || value === null) return false - if (!("types" in value) || !stringArrayValue(value.types)) return false - return "tools" in value && stringArrayValue(value.tools) -} - function isStopResult(value: unknown): value is StopResult { if (typeof value !== "object" || value === null) return false if (!("screenshots" in value) || !stringArrayValue(value.screenshots)) diff --git a/packages/drive/src/recording/edit.ts b/packages/drive/src/recording/edit.ts index fa8b4c5..8e5e869 100644 --- a/packages/drive/src/recording/edit.ts +++ b/packages/drive/src/recording/edit.ts @@ -25,8 +25,12 @@ export interface RecordingClip { export interface EditedSample extends SampledFrame { readonly label?: string + /** Raw playback time; sourceAtMs still identifies the borrowed terminal snapshot. */ + readonly playbackAtMs?: number } +type PlaybackSample = EditedSample & { readonly playbackAtMs: number } + /** The annotation label active at `atMs` on the raw timeline, if any. */ export function labelAt(annotations: ReadonlyArray, atMs: number) { let active: string | undefined @@ -50,9 +54,9 @@ export function labelAt(annotations: ReadonlyArray, atMs: n export function applyClips( samples: ReadonlyArray, clips: ReadonlyArray, -): EditedSample[] { +): PlaybackSample[] { if (clips.length === 0) throw new Error("clips must not be empty") - const edited: EditedSample[] = [] + const edited: PlaybackSample[] = [] let offset = 0 for (const clip of clips) { const speed = clip.speed ?? 1 @@ -64,14 +68,22 @@ export function applyClips( const inside = samples.filter((sample) => sample.sourceAtMs > clip.fromMs && sample.sourceAtMs <= clip.toMs) const kept = opener ? [opener, ...inside] : inside for (const sample of kept) { - edited.push({ ...sample, atMs: scale(sample.sourceAtMs), label: clip.label }) + edited.push({ + ...sample, + atMs: scale(sample.sourceAtMs), + playbackAtMs: Math.max(sample.sourceAtMs, clip.fromMs), + label: clip.label, + }) } const clipEnd = offset + (clip.toMs - clip.fromMs) / speed const last = edited.at(-1) - // Preserve quiet time up to the clip end and any hold after it by - // re-emitting the final frame. - if (last && (clip.holdMs ?? 0) >= 0 && last.atMs < clipEnd + (clip.holdMs ?? 0)) { - edited.push({ ...last, atMs: clipEnd + (clip.holdMs ?? 0) }) + // Borrow terminal pixels, not their timestamp. Animation reaches the exact + // clip boundary, then the whole composition freezes during a hold. + if (last && last.atMs < clipEnd) { + edited.push({ ...last, atMs: clipEnd, playbackAtMs: clip.toMs }) + } + if (last && (clip.holdMs ?? 0) > 0) { + edited.push({ ...last, atMs: clipEnd + (clip.holdMs ?? 0), playbackAtMs: clip.toMs }) } offset = clipEnd + (clip.holdMs ?? 0) } diff --git a/packages/drive/src/recording/export.ts b/packages/drive/src/recording/export.ts index 4c63667..b87550a 100644 --- a/packages/drive/src/recording/export.ts +++ b/packages/drive/src/recording/export.ts @@ -6,6 +6,7 @@ import { encodeFrames } from "./encode.js" import { progressReporter } from "./frame-rate.js" import { replayRecording, type ReplayOptions } from "./replay.js" import { CellHeight, CellWidth, FooterHeight, formatTimecode, renderFrame } from "./render.js" +import { loadPointers, pointerAt, PointerOverlayOptions } from "./pointer.js" import { activeKeypresses, injectKeypressSamples, @@ -27,6 +28,8 @@ export interface ExportRecordingOptions extends ReplayOptions { clips?: ReadonlyArray /** Semantic key presses to display briefly over the terminal. */ keypresses?: ReadonlyArray + /** Animate actual mouse input from the recording sidecar. Off by default. */ + pointerOverlay?: boolean | PointerOverlayOptions onProgress?: (percent: number) => void signal?: AbortSignal } @@ -48,21 +51,41 @@ export async function exportRecording( options.signal?.throwIfAborted() const annotations = options.annotations ?? [] const keypresses = options.keypresses ?? [] + const pointerOptions = options.pointerOverlay + ? PointerOverlayOptions.make(options.pointerOverlay === true ? {} : options.pointerOverlay) + : undefined + const pointers = pointerOptions ? await loadPointers(timelinePath) : [] const edited = options.clips && options.clips.length > 0 ? applyClips(replayed, options.clips) - : replayed.map((sample) => ({ ...sample, label: undefined })) - const outputKeypresses = mapKeypresses(keypresses, options.clips) - const samples = injectKeypressSamples(edited, outputKeypresses) + : replayed.map((sample) => ({ ...sample, playbackAtMs: sample.sourceAtMs, label: undefined })) + // Even an unedited video has a raw-time origin: replay trims blank startup + // or starts at startAtMs. Overlay times must use that same retained range. + const outputKeypresses = mapKeypresses(keypresses, options.clips?.length ? options.clips : [{ + fromMs: replayed[0]!.sourceAtMs, + toMs: replayed.at(-1)!.sourceAtMs, + }]) + const originalSamples = new Set(edited) + const samples = injectKeypressSamples(edited, outputKeypresses).map((sample) => { + if (originalSamples.has(sample)) return sample + const previous = edited.findLast((frame) => frame.atMs <= sample.atMs) + const next = edited.find((frame) => frame.atMs > sample.atMs) + if (!previous || !next || sample.atMs === previous.atMs) return sample + return { + ...sample, + playbackAtMs: previous.playbackAtMs + (next.playbackAtMs - previous.playbackAtMs) * + (sample.atMs - previous.atMs) / (next.atMs - previous.atMs), + } + }) const footerEnabled = options.footer === false ? false : options.footer !== undefined || annotations.length > 0 || (options.clips?.length ?? 0) > 0 const brand = typeof options.footer === "object" ? options.footer.brand : undefined - const footer = (sample: { atMs: number; sourceAtMs: number; label?: string }) => + const footer = (sample: { atMs: number; playbackAtMs: number; label?: string }) => footerEnabled ? { - label: sample.label ?? labelAt(annotations, sample.sourceAtMs), + label: sample.label ?? labelAt(annotations, sample.playbackAtMs), timecodeMs: sample.atMs, brand, } @@ -89,6 +112,7 @@ export async function exportRecording( header: header(final.atMs), footer: footer(final), keys: activeKeypresses(outputKeypresses, final.atMs), + pointer: pointerOptions ? pointerAt(pointers, final.playbackAtMs, pointerOptions) : undefined, }), { signal: options.signal }, ) @@ -100,6 +124,7 @@ export async function exportRecording( const label = header(sample.atMs) const overlay = footer(sample) const keys = activeKeypresses(outputKeypresses, sample.atMs) + const pointer = pointerOptions ? pointerAt(pointers, sample.playbackAtMs, pointerOptions) : undefined let frameKey = frameKeys.get(sample.frame) if (frameKey === undefined) { frameKey = createHash("sha256").update(JSON.stringify(sample.frame)).digest("hex") @@ -112,8 +137,9 @@ export async function exportRecording( label, overlay ? [overlay.label, overlay.brand, formatTimecode(overlay.timecodeMs)] : undefined, keys, + pointer, ]), - render: () => renderFrame(sample.frame, { cols, rows, header: label, footer: overlay, keys }), + render: () => renderFrame(sample.frame, { cols, rows, header: label, footer: overlay, keys, pointer }), } }), outputPath, diff --git a/packages/drive/src/recording/index.ts b/packages/drive/src/recording/index.ts index 7187d6e..a31b80d 100644 --- a/packages/drive/src/recording/index.ts +++ b/packages/drive/src/recording/index.ts @@ -1,4 +1,5 @@ export { decodeTimeline } from "./decode.js" +export { loadPointers, pointerAt, pointersPath, PointerOverlayOptions, RecordingPointer, type PointerFrame } from "./pointer.js" export { applyClips, labelAt, diff --git a/packages/drive/src/recording/pointer.ts b/packages/drive/src/recording/pointer.ts new file mode 100644 index 0000000..cfd1532 --- /dev/null +++ b/packages/drive/src/recording/pointer.ts @@ -0,0 +1,82 @@ +import { Schema } from "effect" + +/** Actual simulation input, timestamped on the terminal recording's clock. */ +export const RecordingPointer = Schema.Struct({ + atMs: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)), + action: Schema.Literals(["move", "down", "up", "click", "scroll"]), + x: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)), + y: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)), +}) +export interface RecordingPointer extends Schema.Schema.Type {} + +const Milliseconds = Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)) +export const PointerOverlayOptions = Schema.Struct({ + /** Reveal the pointer this long before the next input. Default: 180ms. */ + leadMs: Schema.optionalKey(Milliseconds), + /** Keep it visible after input, merging nearby interactions. Default: 700ms. */ + lingerMs: Schema.optionalKey(Milliseconds), + /** Ease between nearby positions, arriving at the input instant. Default: 220ms. */ + motionMs: Schema.optionalKey(Milliseconds), +}) +export interface PointerOverlayOptions extends Schema.Schema.Type {} + +export interface PointerFrame { + /** Zero-based terminal cell coordinates, not pixels. */ + readonly x: number + readonly y: number + readonly opacity: number + readonly pressed: boolean +} + +export function pointersPath(timeline: string) { + return `${timeline.replace(/\.jsonl$/, "")}.pointers.jsonl` +} + +export async function loadPointers(timeline: string): Promise> { + const file = Bun.file(pointersPath(timeline)) + if (!(await file.exists())) return [] + const decode = Schema.decodeUnknownSync(Schema.fromJsonString(RecordingPointer)) + const events = (await file.text()).split("\n").filter((line) => line.trim() !== "").map((line) => decode(line)) + if (events.some((event, index) => event.atMs < (events[index - 1]?.atMs ?? 0))) + throw new Error("Recording pointer timestamps must be nondecreasing") + return events +} + +/** Pure raw-timeline interpolation. Never sends input or changes script timing. */ +export function pointerAt( + events: ReadonlyArray, + atMs: number, + options: PointerOverlayOptions = {}, +): PointerFrame | undefined { + const lead = options.leadMs ?? 180 + const linger = options.lingerMs ?? 700 + const motion = options.motionMs ?? 220 + const index = events.findLastIndex((event) => event.atMs <= atMs) + const previous = events[index] + const next = events[index + 1] + const button = events.findLast((event) => event.atMs <= atMs && + (event.action === "down" || event.action === "up" || event.action === "click")) + const held = button?.action === "down" + const fadingOut = previous ? 1 - ease((atMs - previous.atMs - linger + Math.min(120, linger)) / Math.min(120, linger)) : 0 + const fadingIn = next ? ease((atMs - next.atMs + lead) / Math.min(120, lead)) : 0 + const opacity = held ? 1 : Math.max(fadingOut, fadingIn) + if (opacity <= 0) return undefined + const destination = next && atMs >= next.atMs - Math.max(lead, motion) ? next : previous + if (!destination) return undefined + const connected = previous && (held || destination.atMs - previous.atMs <= linger + lead) + const start = previous ? Math.max(previous.atMs, destination.atMs - motion) : destination.atMs + const amount = connected ? ease((atMs - start) / (destination.atMs - start)) : 1 + const origin = connected ? previous : destination + return { + x: origin.x + (destination.x - origin.x) * amount, + y: origin.y + (destination.y - origin.y) * amount, + opacity, + pressed: held || (button?.action === "click" && atMs - button.atMs < 120), + } +} + +function ease(value: number) { + // A zero-duration interval is an immediate transition, including 0 / 0. + const progress = Number.isNaN(value) ? 1 : Math.max(0, Math.min(1, value)) + return progress * progress * (3 - 2 * progress) +} diff --git a/packages/drive/src/recording/render.ts b/packages/drive/src/recording/render.ts index 823c9d6..0971cef 100644 --- a/packages/drive/src/recording/render.ts +++ b/packages/drive/src/recording/render.ts @@ -13,6 +13,7 @@ import { } from "../frame/index.js" import type { Frontend } from "../client/protocol.js" import type { CapturedFrame } from "./types.js" +import type { PointerFrame } from "./pointer.js" export { CellHeight, CellWidth } from "../frame/index.js" @@ -73,6 +74,7 @@ export interface RenderFrameOptions { readonly footer?: RenderFrameFooter /** Recent semantic key presses, rendered as a KeyCastr-style overlay. */ readonly keys?: ReadonlyArray + readonly pointer?: PointerFrame } export const FooterHeight = 40 @@ -229,6 +231,35 @@ export function renderFrame(frame: CapturedFrame | Frontend.CapturedFrame, optio x += pill.width + gap } } + if (options.pointer) { + const pointer = options.pointer + context.save() + context.beginPath() + context.rect(0, headerHeight, frame.cols * CellWidth, frame.rows * CellHeight) + context.clip() + context.translate((pointer.x + 0.5) * CellWidth, headerHeight + (pointer.y + 0.5) * CellHeight) + context.globalAlpha = pointer.opacity + context.scale(pointer.pressed ? 0.88 : 1, pointer.pressed ? 0.88 : 1) + context.shadowColor = "rgba(0, 0, 0, 0.45)" + context.shadowBlur = 3 + context.shadowOffsetY = 1 + context.beginPath() + context.moveTo(0, 0) + context.lineTo(1, 20) + context.lineTo(6, 15) + context.lineTo(10, 23) + context.lineTo(14, 21) + context.lineTo(10, 13) + context.lineTo(17, 13) + context.closePath() + context.fillStyle = "#f7f7f7" + context.strokeStyle = "#202020" + context.lineWidth = 1.5 + context.lineJoin = "round" + context.fill() + context.stroke() + context.restore() + } return canvas.toBuffer("image/png") } diff --git a/packages/drive/src/recording/replay.ts b/packages/drive/src/recording/replay.ts index 45cfb9a..8013938 100644 --- a/packages/drive/src/recording/replay.ts +++ b/packages/drive/src/recording/replay.ts @@ -1,7 +1,7 @@ import { decodeTimeline } from "./decode.js" import { resolveFps } from "./frame-rate.js" import { createTerminalParser, type TerminalParserFactory } from "./terminal.js" -import type { SampledFrame, TimelineHeader } from "./types.js" +import type { SampledFrame, TimelineHeader, TimelineRecord } from "./types.js" export interface ReplayOptions { fps?: number @@ -26,6 +26,14 @@ export async function replay(path: string, options: InternalReplayOptions = {}): if (options.durationMs !== undefined && (!Number.isFinite(options.durationMs) || options.durationMs < 0)) throw new Error("durationMs must be a non-negative finite number") const records = decodeTimeline(path)[Symbol.asyncIterator]() + try { + return await replayFrames(records, options, interval) + } finally { + await records.return(undefined) + } +} + +async function replayFrames(records: AsyncGenerator, options: InternalReplayOptions, interval: number) { const first = await records.next() options.signal?.throwIfAborted() if (first.done || first.value.type !== "header") throw new Error("Recording timeline is missing its header") diff --git a/packages/drive/src/simulation/connector.ts b/packages/drive/src/simulation/connector.ts index 8680ede..6d269aa 100644 --- a/packages/drive/src/simulation/connector.ts +++ b/packages/drive/src/simulation/connector.ts @@ -170,10 +170,10 @@ export const ui = Effect.fn("SimulationConnector.ui")(function* ( const rpc = yield* RpcClient.make(UiRpcs).pipe( Effect.provideService(RpcClient.Protocol, protocol), ) - const required = FrontendProtocol.Capabilities.filter( - (capability) => - capability !== "ui.snapshot" && capability !== "ui.click.semantic", - ) + const optional: ReadonlyArray = [ + "ui.snapshot", "ui.click.semantic", "ui.mouse", "ui.recording.pointer", + ] + const required = FrontendProtocol.Capabilities.filter((capability) => !optional.includes(capability)) const compatibility = yield* negotiate( endpoint, "ui", @@ -183,7 +183,7 @@ export const ui = Effect.fn("SimulationConnector.ui")(function* ( expectedRole: "ui", offeredVersions: [1], requiredCapabilities: required, - optionalCapabilities: ["ui.snapshot", "ui.click.semantic"], + optionalCapabilities: optional, }), options?.compatibility, ) diff --git a/packages/drive/src/simulation/protocol.ts b/packages/drive/src/simulation/protocol.ts index 5624239..9dcca6f 100644 --- a/packages/drive/src/simulation/protocol.ts +++ b/packages/drive/src/simulation/protocol.ts @@ -116,6 +116,8 @@ export namespace Frontend { "ui.focus", "ui.click", "ui.click.semantic", + "ui.mouse", + "ui.recording.pointer", "ui.resize", "ui.matches", "ui.state", @@ -143,6 +145,30 @@ export namespace Frontend { export interface SemanticClickTarget extends Schema.Schema.Type {} + const MousePosition = { + x: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + y: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + modifiers: Schema.optionalKey(Schema.Struct({ + shift: Schema.optionalKey(Schema.Boolean), + alt: Schema.optionalKey(Schema.Boolean), + ctrl: Schema.optionalKey(Schema.Boolean), + })), + } + export const MouseParams = Schema.Union([ + Schema.Struct({ ...MousePosition, action: Schema.Literal("move") }), + Schema.Struct({ + ...MousePosition, + action: Schema.Literals(["down", "up"]), + button: Schema.optionalKey(Schema.Literals(["left", "middle", "right"])), + }), + Schema.Struct({ + ...MousePosition, + action: Schema.Literal("scroll"), + direction: Schema.Literals(["up", "down", "left", "right"]), + }), + ]) + export type MouseParams = Schema.Schema.Type + export const Action = Schema.Union([ Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }), Schema.Struct({ @@ -156,6 +182,7 @@ export namespace Frontend { direction: Schema.Literals(["up", "down", "left", "right"]), }), Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }), + Schema.Struct({ type: Schema.Literal("ui.mouse"), params: MouseParams }), Schema.Struct({ type: Schema.Literal("ui.click"), target: Schema.Number, @@ -340,6 +367,11 @@ export namespace Frontend { method: Schema.Literal("ui.click"), params: ClickParams, }), + Schema.Struct({ + ...JsonRpc.RequestFields, + method: Schema.Literal("ui.mouse"), + params: MouseParams, + }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.resize"), diff --git a/packages/drive/src/simulation/rpc.ts b/packages/drive/src/simulation/rpc.ts index 023351b..66e400f 100644 --- a/packages/drive/src/simulation/rpc.ts +++ b/packages/drive/src/simulation/rpc.ts @@ -77,6 +77,10 @@ export const UiRpcs = RpcGroup.make( payload: Frontend.ClickParams, success: Frontend.State, }), + request("ui.mouse", { + payload: Frontend.MouseParams, + success: Frontend.State, + }), request("ui.resize", { payload: Frontend.ResizeParams, success: Frontend.State, diff --git a/packages/drive/src/tool/controller.ts b/packages/drive/src/tool/controller.ts index c57850e..8862123 100644 --- a/packages/drive/src/tool/controller.ts +++ b/packages/drive/src/tool/controller.ts @@ -269,7 +269,6 @@ export const make = Effect.fn("ToolController.make")(function* (configuration?: const indexes = new Map() const active = new Map>() const background = new Map() - let closing = false const server = yield* Effect.acquireRelease( Effect.sync(() => Bun.serve({ @@ -279,7 +278,7 @@ export const make = Effect.fn("ToolController.make")(function* (configuration?: fetch(request) { if (request.headers.get("authorization") !== `Bearer ${token}`) return new Response("Unauthorized", { status: 401 }) - if (closing && request.method === "POST") + if (closed && request.method === "POST") return new Response("Tool controller is stopping", { status: 503 }) const pathname = new URL(request.url).pathname const shellID = pathname.match(/^\/background\/([^/]+)$/)?.[1] @@ -309,7 +308,6 @@ export const make = Effect.fn("ToolController.make")(function* (configuration?: ), (server) => Effect.gen(function* () { - closing = true closed = true yield* Effect.all(closeControls, { discard: true }) yield* Effect.sync(() => { diff --git a/packages/drive/test/cli/integration.test.ts b/packages/drive/test/cli/integration.test.ts index 1575c33..382eb45 100644 --- a/packages/drive/test/cli/integration.test.ts +++ b/packages/drive/test/cli/integration.test.ts @@ -1211,6 +1211,23 @@ describe("opencode-drive", () => { }) }) + test("inherits --record for pointer-enabled scripts without leaking recording to additional TUIs", async () => { + const root = await temporary() + const child = spawn([ + "start", "--name", "pointer-options-test", "--record", "--pointer-overlay", + "--script", fixture("pointer-options-script.ts"), + "--", process.execPath, fixture("fake-opencode.ts"), + ], root) + const stderr = new Response(child.stderr).text() + expect(await child.exited).toBe(0) + const artifacts = artifactPath(await stderr) + expect(await Bun.file(join(artifacts, "pointer-options.json")).json()).toEqual({ + primaryRecording: true, + secondaryRecording: false, + rejected: "pointerOverlay requires recording", + }) + }, 60_000) + test.each(["setup", "run"] as const)("rejects a Promise-returning script %s callback", async (callback) => { const root = await temporary() const script = join(root, `promise-${callback}-script.js`) diff --git a/packages/drive/test/driver/index.test.ts b/packages/drive/test/driver/index.test.ts index 2a6b325..5b78b9e 100644 --- a/packages/drive/test/driver/index.test.ts +++ b/packages/drive/test/driver/index.test.ts @@ -160,6 +160,17 @@ it.live("rejects recording for a visible TUI", () => }), ) +it.live("rejects pointer overlays on endpoints without recorded input evidence", () => Effect.gen(function* () { + const failure = yield* OpenCodeDriver.make({ + tui: { recording: true, pointerOverlay: true }, + opencode: { command: [...fakeOpenCode, "omit-pointer-capability"] }, + }).pipe(Effect.scoped, Effect.flip) + expect(failure).toMatchObject({ + _tag: "OpenCodeDriverError", operation: "tui.launch", + message: "pointerOverlay requires an OpenCode endpoint with ui.recording.pointer", + }) +})) + it.live("supports explicit terminal settlement with make", () => Effect.gen(function* () { let artifacts = "" diff --git a/packages/drive/test/driver/llm-state.test.ts b/packages/drive/test/driver/llm-state.test.ts index f1015f0..217ffff 100644 --- a/packages/drive/test/driver/llm-state.test.ts +++ b/packages/drive/test/driver/llm-state.test.ts @@ -17,6 +17,16 @@ const request = (id: string): LlmState.AttachedRequest => const handler: LlmState.ServeHandler = () => Stream.empty describe("LlmState", () => { + it("allows configuring a title handler after a default title, but only once", () => { + const [state, first] = LlmState.startTitle(LlmState.initial, completion()) + expect(Effect.runSync(first.handler(request("title").request, first.index))).toBe("OpenCode Drive") + expect(LlmState.rejectTitle(state)).toBeUndefined() + const configured = LlmState.configureTitle(state, () => Effect.succeed("Configured title")) + expect(LlmState.rejectTitle(configured)).toMatchObject({ _tag: "LlmModeError", operation: "title" }) + const [, next] = LlmState.startTitle(configured, completion()) + expect(next.index).toBe(1) + expect(Effect.runSync(next.handler(request("title").request, next.index))).toBe("Configured title") + }) it("rejects queue/send after serve mode is selected", () => { const state = LlmState.serve(LlmState.initial, handler) expect(LlmState.rejectEnqueue(state, "queue")).toMatchObject({ diff --git a/packages/drive/test/driver/shared.test.ts b/packages/drive/test/driver/shared.test.ts index 76e1013..56e8b5d 100644 --- a/packages/drive/test/driver/shared.test.ts +++ b/packages/drive/test/driver/shared.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "@effect/vitest" -import { Deferred, Effect, Fiber, Ref } from "effect" +import { Deferred, Effect, Exit, Fiber, Latch, Ref, Scope } from "effect" import * as SharedEffect from "../../src/driver/shared.js" it.effect("does not let an interrupted caller poison terminal work", () => @@ -21,3 +21,50 @@ it.effect("does not let an interrupted caller poison terminal work", () => expect(yield* Ref.get(runs)).toBe(1) }), ) + +it.effect("shares one lazy result after every original waiter leaves", () => Effect.gen(function* () { + const entered = yield* Latch.make() + const gate = yield* Latch.make() + const runs = yield* Ref.make(0) + const shared = yield* SharedEffect.make(Effect.gen(function* () { + yield* Ref.update(runs, (count) => count + 1) + yield* entered.open + yield* gate.await + return "done" + })) + expect(yield* Ref.get(runs)).toBe(0) + const first = yield* Effect.forkChild(shared) + const second = yield* Effect.forkChild(shared) + yield* entered.await + yield* Fiber.interrupt(first) + yield* Fiber.interrupt(second) + yield* gate.open + expect(yield* shared).toBe("done") + expect(yield* shared).toBe("done") + expect(yield* Ref.get(runs)).toBe(1) +})) + +it.effect("replays failures and defects without rerunning", () => Effect.gen(function* () { + for (const failure of [Effect.fail("failed"), Effect.die("defect")]) { + const runs = yield* Ref.make(0) + const shared = yield* SharedEffect.make(Ref.update(runs, (count) => count + 1).pipe(Effect.andThen(failure))) + const first = yield* Effect.exit(shared) + const second = yield* Effect.exit(shared) + expect(first).toEqual(second) + expect(Exit.isFailure(first)).toBe(true) + expect(yield* Ref.get(runs)).toBe(1) + } +})) + +it.effect("owner shutdown joins terminal work, including a first start from its finalizer", () => Effect.gen(function* () { + const scope = yield* Scope.make() + const entered = yield* Latch.make() + const gate = yield* Latch.make() + const shared = yield* SharedEffect.make(entered.open.pipe(Effect.andThen(gate.await))).pipe(Scope.provide(scope)) + yield* Scope.addFinalizer(scope, shared) + const close = yield* Effect.forkChild(Scope.close(scope, Exit.void)) + yield* entered.await + expect(close.pollUnsafe()).toBeUndefined() + yield* gate.open + yield* Fiber.join(close) +})) diff --git a/packages/drive/test/driver/ui.test.ts b/packages/drive/test/driver/ui.test.ts index c318310..c5a381d 100644 --- a/packages/drive/test/driver/ui.test.ts +++ b/packages/drive/test/driver/ui.test.ts @@ -69,6 +69,33 @@ void operationErrorExcludesScreenshotError void screenshotErrorIsSpecific describe("OpenCodeUi", () => { + it.live("dispatches real mouse params unchanged and keeps transforms around mouse operations", () => Effect.gen(function* () { + const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, state)) + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const operations: string[] = [] + const ui = OpenCodeUi.transform(OpenCodeUi.make(connection), (effect) => effect.pipe( + Effect.tap(() => Effect.sync(() => { operations.push("mouse") })), + )) + const params = { action: "move", x: 23, y: 11, modifiers: { shift: true } } as const + expect(yield* ui.mouse(params)).toEqual(state) + expect(peer.received[0]?.request).toMatchObject({ method: "ui.mouse", params }) + expect(operations).toEqual(["mouse"]) + })) + + it.live("keeps older endpoints usable and rejects unsupported mouse before sending input", () => Effect.gen(function* () { + const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, state), { + capabilities: (capabilities) => capabilities.filter((capability) => capability !== "ui.mouse"), + }) + yield* Effect.addFinalizer(() => Effect.promise(() => peer.stop())) + const connection = yield* SimulationConnector.ui(peer.url) + const ui = OpenCodeUi.make(connection) + expect(yield* ui.mouse({ action: "move", x: 1, y: 2 }).pipe(Effect.flip)).toMatchObject({ + _tag: "UiCapabilityError", capability: "ui.mouse", + }) + expect(peer.received).toEqual([]) + expect(yield* ui.state()).toEqual(state) + })) it.live("captures a normalized terminal frame", () => { const peer = startTransportPeer(({ request, socket }) => sendResult(socket, request, frame)) diff --git a/packages/drive/test/fixtures/fake-opencode.ts b/packages/drive/test/fixtures/fake-opencode.ts index 415263b..96ba518 100644 --- a/packages/drive/test/fixtures/fake-opencode.ts +++ b/packages/drive/test/fixtures/fake-opencode.ts @@ -395,7 +395,8 @@ function handshake(params: unknown, role: "ui" | "backend") { protocolVersion: 1, role, server: { name: "opencode", version: "test" }, - capabilities: [...required, ...optional], + capabilities: [...required, ...optional].filter((capability) => + !process.argv.includes("omit-pointer-capability") || capability !== "ui.recording.pointer"), } } diff --git a/packages/drive/test/fixtures/pointer-options-script.ts b/packages/drive/test/fixtures/pointer-options-script.ts new file mode 100644 index 0000000..0113ecb --- /dev/null +++ b/packages/drive/test/fixtures/pointer-options-script.ts @@ -0,0 +1,15 @@ +import { defineScript } from "opencode-drive" +import { Effect } from "effect" + +// Recording is deliberately inherited from CLI --record, not declared here. +export default defineScript({ + run: ({ artifacts, tui, tuis }) => Effect.gen(function* () { + const secondary = yield* tuis.launch() + const rejected = yield* tuis.launch({ pointerOverlay: true }).pipe(Effect.flip) + yield* Effect.tryPromise(() => Bun.write(`${artifacts}/pointer-options.json`, JSON.stringify({ + primaryRecording: tui.recording !== undefined, + secondaryRecording: secondary.recording !== undefined, + rejected: rejected.message, + }))) + }), +}) diff --git a/packages/drive/test/fixtures/tui-options-script.ts b/packages/drive/test/fixtures/tui-options-script.ts index a2f44e0..8ca9805 100644 --- a/packages/drive/test/fixtures/tui-options-script.ts +++ b/packages/drive/test/fixtures/tui-options-script.ts @@ -2,7 +2,7 @@ import { defineScript } from "opencode-drive" import * as Effect from "effect/Effect" export default defineScript({ - tui: { recording: true, viewport: { cols: 90, rows: 30 } }, + tui: { recording: true, pointerOverlay: true, viewport: { cols: 90, rows: 30 } }, run: ({ artifacts, tui, tuis }) => Effect.gen(function* () { const secondary = yield* tuis.launch() diff --git a/packages/drive/test/manual/pointer-demo.ts b/packages/drive/test/manual/pointer-demo.ts new file mode 100644 index 0000000..f2240e9 --- /dev/null +++ b/packages/drive/test/manual/pointer-demo.ts @@ -0,0 +1,86 @@ +import { Effect } from "effect" +import { Frontend, OpenCodeDriver } from "opencode-drive" + +// OPENCODE_DEV=/path/to/opencode OPENCODE_DRIVE_MEDIA_DIR=$PWD/.drive-output \ +// bun run drive run test/manual/pointer-demo.ts +// Real production devtools controls; no simulated model calls or visual hover substitutes. +export default OpenCodeDriver.use({ + opencode: { dev: process.env.OPENCODE_DEV, compatibility: "required" }, + tui: { recording: true, pointerOverlay: true, viewport: { cols: 100, rows: 30 } }, + tuiConfig: { theme: { name: "opencode", mode: "dark" } }, + keepArtifacts: true, +}, ({ ui, tui, artifacts }) => Effect.gen(function* () { + console.log("artifacts:", artifacts) + const recording = tui.recording + if (!recording) return yield* Effect.fail(new Error("recording required")) + yield* ui.waitFor("Simulated Model") + yield* ui.mouse({ action: "move", x: 80, y: 14 }) + yield* recording.mark("Target-relative click opens the production Theme panel") + yield* Effect.sleep(800) + yield* ui.mouse({ action: "move", x: 16, y: 29 }) + yield* Effect.sleep(500) + const theme = (yield* ui.state()).elements.find((element) => element.x === 16 && element.y === 29 && element.width === 7) + if (!theme) return yield* Effect.fail(new Error("Theme toolbar element unavailable")) + yield* recording.mark("Target-relative click opens Theme; pointer lands on the actual cell") + yield* ui.click(theme, { x: 3, y: 0 }) + yield* ui.waitFor("Switch to light") + const before = yield* ui.capture() + const light = locate(before, "Switch to light") + yield* Effect.sleep(350) + yield* ui.mouse({ action: "move", ...light }) + yield* recording.mark("Real hover highlights the action without clicking") + yield* ui.waitFor(() => ui.capture().pipe(Effect.map((hover) => + background(before, light.x, light.y) !== background(hover, light.x, light.y))), { timeout: 2_000 }) + console.log("hover screenshot:", yield* ui.screenshot("pointer-hover")) + yield* Effect.sleep(700) + yield* recording.mark("Mouse leaves the action; native hover clears without clicking") + const selected = yield* ui.capture() + yield* ui.mouse({ action: "move", x: 80, y: 14 }) + yield* ui.waitFor(() => ui.capture().pipe(Effect.map((outside) => + background(selected, light.x, light.y) !== background(outside, light.x, light.y))), { timeout: 2_000 }) + console.log("leave screenshot:", yield* ui.screenshot("pointer-leave")) + yield* Effect.sleep(800) + yield* ui.mouse({ action: "move", ...light }) + yield* Effect.sleep(350) + yield* recording.mark("Native down / up — nearby clicks keep the cursor visible") + yield* ui.mouse({ action: "down", ...light }) + yield* ui.mouse({ action: "up", ...light }) + yield* ui.waitFor("Switch to dark") + yield* Effect.sleep(750) + const dark = locate(yield* ui.capture(), "Switch to dark") + yield* ui.mouse({ action: "down", ...dark }) + yield* ui.mouse({ action: "up", ...dark }) + yield* ui.waitFor("Switch to light") + yield* Effect.sleep(1_000) + yield* recording.mark("The recorded pointer fades; real input timing is unchanged") + yield* Effect.sleep(1_200) + yield* ui.mouse({ action: "down", x: 19, y: 29 }) + yield* ui.mouse({ action: "up", x: 19, y: 29 }) + yield* recording.mark("Narrow viewport — input and pointer use the resized terminal") + yield* ui.resize({ cols: 70, rows: 24 }) + yield* Effect.sleep(500) + yield* ui.mouse({ action: "move", x: 19, y: 23 }) + yield* Effect.sleep(350) + yield* ui.mouse({ action: "down", x: 19, y: 23 }) + yield* ui.mouse({ action: "up", x: 19, y: 23 }) + yield* ui.waitFor("Switch to light") + yield* Effect.sleep(1_400) + console.log("video:", yield* recording.finish()) +})) + +function locate(frame: Frontend.CapturedFrame, text: string) { + const row = frame.lines.findIndex((line) => line.spans.map((span) => span.text).join("").includes(text)) + if (row < 0) throw new Error(`missing text: ${text}`) + const line = frame.lines[row]?.spans.map((span) => span.text).join("") ?? "" + // Click the action's blank area rather than beginning native text selection. + return { x: line.indexOf(text) + text.length + 2, y: row } +} + +function background(frame: Frontend.CapturedFrame, x: number, y: number) { + let column = 0 + for (const span of frame.lines[y]?.spans ?? []) { + column += span.width + if (column > x) return JSON.stringify(span.bg) + } + throw new Error("cell outside frame") +} diff --git a/packages/drive/test/recording/edit.test.ts b/packages/drive/test/recording/edit.test.ts index 85a5588..1eb43ad 100644 --- a/packages/drive/test/recording/edit.test.ts +++ b/packages/drive/test/recording/edit.test.ts @@ -57,6 +57,19 @@ describe("applyClips", () => { const edited = applyClips(samples, [{ fromMs: 150, toMs: 300 }]) expect(edited.map((entry) => entry.sourceAtMs)).toEqual([100, 200, 300]) expect(edited[0]!.atMs).toBe(0) + expect(edited[0]!.playbackAtMs).toBe(150) + expect(edited[0]!.frame).toBe(samples[1]!.frame) + }) + + test("separates off-grid playback boundaries from borrowed pixels through reordered clips and holds", () => { + const edited = applyClips(samples, [ + { fromMs: 310, toMs: 390, speed: 2, holdMs: 30 }, + { fromMs: 110, toMs: 190 }, + ]) + expect(edited.map((entry) => [entry.atMs, entry.sourceAtMs, entry.playbackAtMs])).toEqual([ + [0, 300, 310], [40, 300, 390], [70, 300, 390], + [70, 100, 110], [150, 100, 190], + ]) }) test("speed rescales the clip and holds freeze the last frame", () => { diff --git a/packages/drive/test/recording/export-dedupe.test.ts b/packages/drive/test/recording/export-dedupe.test.ts index 23bafa8..297848d 100644 --- a/packages/drive/test/recording/export-dedupe.test.ts +++ b/packages/drive/test/recording/export-dedupe.test.ts @@ -44,3 +44,31 @@ test("deduplicates equal snapshots with distinct identities", async () => { expect(encoded.frames).toHaveLength(31) expect(new Set(encoded.frames.map((frame) => frame.key))).toHaveLength(1) }) + +test("synthetic keypress samples use current playback time for animated pointers", async () => { + const directory = await mkdtemp(join(tmpdir(), "drive-overlay-clock-test-")) + directories.push(directory) + const timeline = join(directory, "timeline.jsonl") + await writeFile(timeline, [ + { type: "header", version: 1, cols: 40, rows: 10, encoding: "base64" }, + { type: "output", at_ms: 1_000, data: Buffer.from("ready").toString("base64") }, + { type: "output", at_ms: 1_200, data: "" }, + ].map((event) => JSON.stringify(event)).join("\n") + "\n") + await writeFile(join(directory, "timeline.pointers.jsonl"), [ + { action: "move", atMs: 900, x: 1, y: 2 }, + { action: "move", atMs: 1_050, x: 20, y: 2 }, + ].map((event) => JSON.stringify(event)).join("\n") + "\n") + await exportRecording(timeline, join(directory, "video.mp4"), { + fps: 10, startAtMs: 1_000, durationMs: 200, + pointerOverlay: true, + keypresses: [{ atMs: 1_050, label: "Enter" }], + }) + const middle = encoded.frames.find((frame) => frame.atMs === 50) + expect(middle).toBeDefined() + const identity: unknown = JSON.parse(middle!.key) + expect(identity).toEqual(expect.arrayContaining([ + ["Enter"], { x: 20, y: 2, opacity: 1, pressed: false }, + ])) + expect(encoded.frames.at(-1)?.atMs).toBe(200) + expect(new Set(encoded.frames.map((frame) => frame.key)).size).toBeGreaterThan(2) +}) diff --git a/packages/drive/test/recording/export.test.ts b/packages/drive/test/recording/export.test.ts index db5e114..08bc0cc 100644 --- a/packages/drive/test/recording/export.test.ts +++ b/packages/drive/test/recording/export.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { fileURLToPath } from "node:url" import { createCanvas, loadImage } from "@napi-rs/canvas" -import { encodeFrames, exportRecording, joinFrames, renderFrame } from "../../src/recording/index.js" +import { encodeFrames, exportRecording, joinFrames, pointerAt, renderFrame, replayRecording } from "../../src/recording/index.js" const directories: string[] = [] @@ -28,6 +28,81 @@ test("exports the final frame as a PNG and creates its parent", async () => { expect((await stat(output)).size).toBeGreaterThan(100) }) +test.each([undefined, 2_000])("aligns keypresses with the replay origin (%s), without extending the recording", async (startAtMs) => { + const directory = await mkdtemp(join(tmpdir(), "drive-export-key-origin-")) + directories.push(directory) + const timeline = join(directory, "timeline.jsonl") + await writeFile(timeline, [ + { type: "header", version: 1, cols: 40, rows: 10, encoding: "base64" }, + { type: "output", at_ms: 2_000, data: Buffer.from("ready").toString("base64") }, + { type: "output", at_ms: 2_400, data: "" }, + ].map((event) => JSON.stringify(event)).join("\n") + "\n") + const options = { startAtMs, durationMs: startAtMs === undefined ? undefined : 350 } + const plain = join(directory, "plain.png") + const overlaid = join(directory, "keys.png") + const baseline = await exportRecording(timeline, plain, options) + const result = await exportRecording(timeline, overlaid, { + ...options, + // This key expires at the retained end; test a point before expiry. + keypresses: [{ atMs: 2_100, label: "Enter" }], + }) + expect(result.durationMs).toBe(baseline.durationMs) + expect(result.durationMs).toBeLessThanOrEqual(400) +}) + +test("renders an opt-in pointer at raw time through trims, speed changes and holds", async () => { + const directory = await mkdtemp(join(tmpdir(), "drive-export-pointer-")) + directories.push(directory) + const timeline = join(directory, "timeline.jsonl") + await writeFile(timeline, [ + { type: "header", version: 1, cols: 40, rows: 10, encoding: "base64" }, + { type: "output", at_ms: 2_000, data: Buffer.from("ready").toString("base64") }, + { type: "output", at_ms: 2_400, data: "" }, + ].map((event) => JSON.stringify(event)).join("\n") + "\n") + await writeFile(join(directory, "timeline.pointers.jsonl"), JSON.stringify({ action: "click", atMs: 2_350, x: 5, y: 2 }) + "\n") + const plain = join(directory, "plain.png") + const overlaid = join(directory, "pointer.png") + const options = { clips: [{ fromMs: 2_000, toMs: 2_400, speed: 2, holdMs: 500 }], header: "Pointer" } + const baseline = await exportRecording(timeline, plain, options) + const result = await exportRecording(timeline, overlaid, { ...options, pointerOverlay: true }) + expect(result).toEqual(baseline) + expect(result.durationMs).toBe(700) + expect(await readFile(overlaid)).not.toEqual(await readFile(plain)) + const image = await loadImage(await readFile(overlaid)) + const canvas = createCanvas(image.width, image.height) + const context = canvas.getContext("2d") + context.drawImage(image, 0, 0) + // Icon origin is centered in (5,2), with the 40px header offset applied. + const ink = context.getImageData(55, 90, 20, 24).data + expect(ink.some((channel, index) => index % 4 !== 3 && channel > 200)).toBe(true) +}) + +test("off-grid clips sample pointer playback time rather than the older terminal snapshot", async () => { + const directory = await mkdtemp(join(tmpdir(), "drive-pointer-off-grid-")) + directories.push(directory) + const timeline = join(directory, "timeline.jsonl") + await writeFile(timeline, [ + { type: "header", version: 1, cols: 40, rows: 10, encoding: "base64" }, + { type: "output", at_ms: 0, data: Buffer.from("ready").toString("base64") }, + { type: "output", at_ms: 1_200, data: "" }, + ].map((event) => JSON.stringify(event)).join("\n") + "\n") + const pointers = [ + { action: "move", atMs: 900, x: 1, y: 2 }, + { action: "move", atMs: 1_050, x: 20, y: 2 }, + ] as const + await writeFile(join(directory, "timeline.pointers.jsonl"), pointers.map((event) => JSON.stringify(event)).join("\n") + "\n") + const output = join(directory, "pointer.png") + await exportRecording(timeline, output, { + fps: 10, + clips: [{ fromMs: 1_010, toMs: 1_090, speed: 2, holdMs: 500 }], + footer: false, + pointerOverlay: true, + }) + const samples = await replayRecording(timeline, { fps: 10 }) + expect(await readFile(output)).toEqual(renderFrame(samples[0]!.frame, { pointer: pointerAt(pointers, 1_090) })) + expect(await readFile(output)).not.toEqual(renderFrame(samples[0]!.frame, { pointer: pointerAt(pointers, 1_000) })) +}) + test("rejects invalid encoding frame rates", async () => { await expect( encodeFrames( diff --git a/packages/drive/test/recording/pointer.test.ts b/packages/drive/test/recording/pointer.test.ts new file mode 100644 index 0000000..0d4a0b8 --- /dev/null +++ b/packages/drive/test/recording/pointer.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "@effect/vitest" +import { Effect, Schema } from "effect" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { loadPointers, pointerAt, pointersPath, PointerOverlayOptions, type RecordingPointer } from "../../src/recording/pointer.js" + +const click: RecordingPointer = { action: "click", atMs: 1_000, x: 10, y: 5 } + +describe("recorded pointer", () => { + it.live("loads portable raw timestamps and rejects malformed or out-of-order sidecars", () => Effect.gen(function* () { + const directory = yield* Effect.acquireRelease( + Effect.promise(() => mkdtemp(join(tmpdir(), "drive-pointer-"))), + (path) => Effect.promise(() => rm(path, { recursive: true, force: true })), + ) + const timeline = join(directory, "timeline.jsonl") + expect(yield* Effect.promise(() => loadPointers(timeline))).toEqual([]) + yield* Effect.promise(() => Bun.write(pointersPath(timeline), JSON.stringify(click) + "\n")) + expect(yield* Effect.promise(() => loadPointers(timeline))).toEqual([click]) + yield* Effect.promise(() => Bun.write(pointersPath(timeline), JSON.stringify({ ...click, x: -1 }) + "\n")) + expect(yield* Effect.tryPromise(() => loadPointers(timeline)).pipe(Effect.isFailure)).toBe(true) + yield* Effect.promise(() => Bun.write(pointersPath(timeline), [click, { ...click, atMs: 1 }].map((event) => JSON.stringify(event)).join("\n"))) + expect(yield* Effect.tryPromise(() => loadPointers(timeline)).pipe(Effect.isFailure)).toBe(true) + })) + it("appears before input, lands at the real cell and fades after it", () => { + expect(pointerAt([click], 800)).toBeUndefined() + expect(pointerAt([click], 900)).toMatchObject({ x: 10, y: 5, pressed: false }) + expect(pointerAt([click], 1_000)).toEqual({ x: 10, y: 5, opacity: 1, pressed: true }) + expect(pointerAt([click], 1_400)?.opacity).toBe(1) + expect(pointerAt([click], 1_650)?.opacity).toBeLessThan(1) + expect(pointerAt([click], 1_700)).toBeUndefined() + }) + + it("keeps nearby clicks connected with smooth arrival and no early jump", () => { + const second = { ...click, atMs: 1_500, x: 30 } + expect(pointerAt([click, second], 1_200)).toMatchObject({ x: 10, opacity: 1 }) + expect(pointerAt([click, second], 1_390)).toMatchObject({ x: 20, opacity: 1 }) + expect(pointerAt([click, second], 1_500)).toMatchObject({ x: 30, opacity: 1, pressed: true }) + }) + + it("does not fly across idle gaps and stays visible during a held drag", () => { + const later = { ...click, atMs: 5_000, x: 80 } + expect(pointerAt([click, later], 1_300)).toMatchObject({ x: 10 }) + expect(pointerAt([click, later], 3_000)).toBeUndefined() + expect(pointerAt([click, later], 4_900)).toMatchObject({ x: 80 }) + const drag: ReadonlyArray = [ + { ...click, action: "down" }, + { action: "move", atMs: 1_500, x: 20, y: 5 }, + { action: "up", atMs: 5_000, x: 30, y: 5 }, + ] + expect(pointerAt(drag, 3_000)).toMatchObject({ x: 20, opacity: 1, pressed: true }) + expect(pointerAt(drag, 5_000)?.pressed).toBe(false) + expect(pointerAt(drag, 5_700)).toBeUndefined() + }) + + it("supports simultaneous events, disabled motion and zero visibility windows", () => { + expect(pointerAt([click, { ...click, action: "up" }], 1_000)).toMatchObject({ x: 10, pressed: false }) + expect(pointerAt([click], 900, { leadMs: 0 })).toBeUndefined() + expect(pointerAt([click], 1_001, { lingerMs: 0 })).toBeUndefined() + expect(pointerAt([click, { ...click, atMs: 1_500, x: 30 }], 1_450, { motionMs: 0 })?.x).toBe(10) + expect(pointerAt([], 0)).toBeUndefined() + expect(() => Schema.decodeUnknownSync(PointerOverlayOptions)({ leadMs: -1 })).toThrow() + }) +}) diff --git a/skills/opencode-drive/SKILL.md b/skills/opencode-drive/SKILL.md index fc0dd04..54c82b5 100644 --- a/skills/opencode-drive/SKILL.md +++ b/skills/opencode-drive/SKILL.md @@ -140,6 +140,11 @@ UI operations are Effects: - `ui.getElement(query, options?)`, `ui.focus(...)`, and `ui.click(...)` target interactive elements. - `ui.screenshot(name?)` asks OpenCode for the raw frame, saves it as a PNG inside Drive, and returns its absolute path. It requires no media-directory configuration. - `ui.resize({ cols, rows })`, `ui.press(...)`, and `ui.arrow(...)` control the TUI. +- `ui.mouse({ action: "move", x, y })` sends real mouse movement at absolute, + zero-based terminal cell coordinates. `down`/`up` accept `button` + (`left` by default, `middle`, `right`); moves with a held button drag. + `scroll` requires `direction`. Mouse modifiers are `{ shift, alt, ctrl }`. + Older endpoints without `ui.mouse` fail with `UiCapabilityError`. Build deterministic simulated responses with the `Llm` namespace and schedule them through the driver's `llm` controller: @@ -183,6 +188,20 @@ yield* secondary.ui.screenshot("secondary") ### Annotated Recordings +Set `tui: { recording: true, pointerOverlay: true }` for a minimal animated mouse +cursor in exported videos. A configuration object accepts `leadMs` (180), +`lingerMs` (700), and `motionMs` (220). Nearby inputs stay connected. This needs +an OpenCode build advertising `ui.recording.pointer`; input timestamps and actual +click coordinates come from its recording clock, not Drive-side estimates. +Keep the `*.pointers.jsonl` sidecar with the terminal recording when copying it. +`exportRecording` accepts the same `pointerOverlay` option. + +The cursor animation never sends input or delays the script. To exercise hover +along a path, send real intermediate `ui.mouse` moves. Use state-based waits for +the hover result. Clips crop the already-composited raw animation (including +approaches to later inputs); holds freeze both terminal pixels and the pointer. +Visibility windows are bounded by retained recording time, not extra video time. + With `tui: { recording: true }`, label moments during the run and the exported MP4 gets a burned-in footer (segment label bottom-left, elapsed timecode and "drive" branding bottom-right): @@ -506,6 +525,7 @@ Drive chooses the PNG directory. Do not configure a media directory or expect th - `--command.ui.arrow '{"direction":"down"}'` - `--command.ui.focus '{"target":12}'` - `--command.ui.click '{"target":12,"x":4,"y":1}'` +- `--command.ui.mouse '{"action":"move","x":20,"y":8}'` - `--command.ui.resize '{"cols":120,"rows":40}'` - `--command.ui.screenshot` or `--command.ui.screenshot '{"name":"home"}'` - `--command.ui.state` @@ -523,6 +543,9 @@ Add `--keypress-overlay` when the video should show KeyCastr-style pills for the agent's semantic hotkeys, Enter presses, and arrow navigation. It requires `--record`; batched typed text is deliberately omitted. +Add `--pointer-overlay` for real mouse input with an eased cursor in the video. +It also requires `--record` and a pointer-capable OpenCode build. + ```bash opencode-drive start --name demo --record --keypress-overlay opencode-drive stop --name demo From 844e088c96b84aafd4866e1ff4d4e9a5c5448fb4 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 21:29:35 -0400 Subject: [PATCH 2/6] feat(drive): animate pointer travel with Motion springs --- .changeset/real-pointer-recording.md | 2 + bun.lock | 9 ++++ packages/drive/README.md | 13 ++++-- packages/drive/package.json | 3 +- packages/drive/src/recording/pointer.ts | 31 +++++++++++--- packages/drive/test/recording/pointer.test.ts | 41 +++++++++++++++++-- skills/opencode-drive/SKILL.md | 5 ++- 7 files changed, 89 insertions(+), 15 deletions(-) diff --git a/.changeset/real-pointer-recording.md b/.changeset/real-pointer-recording.md index 6c1e674..19a5507 100644 --- a/.changeset/real-pointer-recording.md +++ b/.changeset/real-pointer-recording.md @@ -3,3 +3,5 @@ --- Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior. + +Animate pointer travel with a critically damped Motion spring and a configurable, bounded arc while preserving exact recorded input positions and times. diff --git a/bun.lock b/bun.lock index 24564b6..1bdcc96 100644 --- a/bun.lock +++ b/bun.lock @@ -45,6 +45,7 @@ "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.3.0", "@wterm/ghostty": "0.3.0", + "motion": "13.2.0", }, "devDependencies": { "@effect/vitest": "4.0.0-rc.112", @@ -570,6 +571,8 @@ "find-up": ["find-up@4.1.0", "", { "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" } }, "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw=="], + "framer-motion": ["framer-motion@13.2.0", "", { "dependencies": { "motion-dom": "^13.2.0", "motion-utils": "^13.0.0", "tslib": "^2.4.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-9E33ebgMaO33w1nN/jEdW8z3/GO483fMi4rqbMG9rt83XgW9QLKRe4NcmJ8s+fQ3O34++UHrIQwlIWGIWTITjA=="], + "fs-extra": ["fs-extra@7.0.1", "", { "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw=="], "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], @@ -642,6 +645,12 @@ "miniflare": ["miniflare@4.20260708.1", "", { "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.34.5", "undici": "7.28.0", "workerd": "1.20260708.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, "bin": { "miniflare": "bootstrap.js" } }, "sha512-c94O9zRDISdqO18EHt6l0iF/fWgWt8p18PJvRsA/L/NJZ9Cfke3s/F5Blg1XXF7WDutVRzWVWy8Vy4LaT5ifsA=="], + "motion": ["motion@13.2.0", "", { "dependencies": { "framer-motion": "^13.2.0", "tslib": "^2.4.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-4Hrb5vD6HhjFstLUiCmWvtpsw+WTpP4R+QXfSDYZBz7+uxE/LrRg3aV0ReJxHrTRffHhbIE6svEqnotngXvesQ=="], + + "motion-dom": ["motion-dom@13.2.0", "", { "dependencies": { "motion-utils": "^13.0.0" } }, "sha512-N6gdSoWRDk0Rh/fVtlqUtLs+fEN3ELFZI3cn3IQE9Mnf3E+Mh8wjO6MstzCOPFh4Yf0L1as5m2eUyYWj8ylVSQ=="], + + "motion-utils": ["motion-utils@13.0.0", "", {}, "sha512-7DnN7TmbLcYXcG4RVadXIihWlyuM9afoUww8Y5Agg431kGKiuL2/OMyP4mJ5wLz+pvN3t5ySClLOaVXJ+wekRQ=="], + "mri": ["mri@1.2.0", "", {}, "sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA=="], "msgpackr": ["msgpackr@2.1.0", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.4" } }, "sha512-p/pBCVO63CsvvpkomUnNNag6+n38rULuDA6HHe70o2gtC8ODI52foF/4ko2qQcp6OiErJXTmrZeXmsGGHsIQNQ=="], diff --git a/packages/drive/README.md b/packages/drive/README.md index 0d39d2a..2a71100 100644 --- a/packages/drive/README.md +++ b/packages/drive/README.md @@ -232,13 +232,13 @@ The CLI takes the same protocol parameters with `--command.ui.mouse`. endpoints fail with `UiCapabilityError` before input is sent; older endpoints remain usable for existing operations. -Enable a minimal eased cursor in exported videos: +Enable a minimal spring-animated cursor in exported videos: ```ts OpenCodeDriver.use({ tui: { recording: true, - pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 220 }, + pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 220, curve: 0.12 }, }, }, ({ ui }) => ui.mouse({ action: "move", x: 20, y: 8 })) ``` @@ -258,8 +258,13 @@ Visibility windows are bounded by retained recording time: export does not add extra pre/post-roll. Clips crop the raw composition, so an approach to an input just outside the clip may still be visible. -The pointer appears before input, eases to the recorded cell, and lingers after -it; nearby interactions stay connected. This is presentation-only: enabling it +The pointer appears before input, follows a small arc to the recorded cell with +a critically damped Motion spring, and lingers after it. It arrives exactly at +the input time without overshooting; nearby interactions stay connected. +`curve` controls the arc height as a fraction of the pixel distance (capped at +40px); set it to `0` for straight travel. Held drags always follow straight +segments between recorded points. Motion's sampler runs without React or a browser. +This is presentation-only: enabling it does not delay a script or send extra moves. To test hover along a path, send real intermediate `ui.mouse` moves. The runnable [`test/manual/pointer-demo.ts`](test/manual/pointer-demo.ts) checks production diff --git a/packages/drive/package.json b/packages/drive/package.json index 17ea073..4c3a14d 100644 --- a/packages/drive/package.json +++ b/packages/drive/package.json @@ -65,7 +65,8 @@ "@types/bun": "1.3.13", "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.3.0", - "@wterm/ghostty": "0.3.0" + "@wterm/ghostty": "0.3.0", + "motion": "13.2.0" }, "peerDependencies": { "effect": "4.0.0-rc.112" diff --git a/packages/drive/src/recording/pointer.ts b/packages/drive/src/recording/pointer.ts index cfd1532..db146cc 100644 --- a/packages/drive/src/recording/pointer.ts +++ b/packages/drive/src/recording/pointer.ts @@ -1,4 +1,10 @@ import { Schema } from "effect" +import { spring } from "motion" +import { CellHeight, CellWidth } from "../frame/index.js" + +// Motion's duration-based bounce: 0 resolves to a critically damped spring. +// Sample normalized time so any configured travel window still ends at real input. +const travel = spring({ keyframes: [0, 1], duration: 1_000, bounce: 0 }) /** Actual simulation input, timestamped on the terminal recording's clock. */ export const RecordingPointer = Schema.Struct({ @@ -15,8 +21,10 @@ export const PointerOverlayOptions = Schema.Struct({ leadMs: Schema.optionalKey(Milliseconds), /** Keep it visible after input, merging nearby interactions. Default: 700ms. */ lingerMs: Schema.optionalKey(Milliseconds), - /** Ease between nearby positions, arriving at the input instant. Default: 220ms. */ + /** Spring between nearby positions, arriving at the input instant. Default: 220ms. */ motionMs: Schema.optionalKey(Milliseconds), + /** Arc height as a fraction of travel distance, capped at 40px. Default: 0.12; 0 is straight. */ + curve: Schema.optionalKey(Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 1 }))), }) export interface PointerOverlayOptions extends Schema.Schema.Type {} @@ -65,18 +73,29 @@ export function pointerAt( if (!destination) return undefined const connected = previous && (held || destination.atMs - previous.atMs <= linger + lead) const start = previous ? Math.max(previous.atMs, destination.atMs - motion) : destination.atMs - const amount = connected ? ease((atMs - start) / (destination.atMs - start)) : 1 + const amount = connected ? travel.next(progress((atMs - start) / (destination.atMs - start)) * 1_000).value : 1 const origin = connected ? previous : destination + // Use pixels for the arc: terminal cells are twice as tall as they are wide. + // A held drag follows the recorded segments, never a decorative detour. + const dx = (destination.x - origin.x) * CellWidth + const dy = (destination.y - origin.y) * CellHeight + const distance = Math.hypot(dx, dy) + const bend = held || distance === 0 ? 0 : + Math.min(40, distance * (options.curve ?? 0.12)) * 4 * amount * (1 - amount) / distance return { - x: origin.x + (destination.x - origin.x) * amount, - y: origin.y + (destination.y - origin.y) * amount, + x: origin.x + (dx * amount + dy * bend) / CellWidth, + y: origin.y + (dy * amount - dx * bend) / CellHeight, opacity, pressed: held || (button?.action === "click" && atMs - button.atMs < 120), } } function ease(value: number) { + const amount = progress(value) + return amount * amount * (3 - 2 * amount) +} + +function progress(value: number) { // A zero-duration interval is an immediate transition, including 0 / 0. - const progress = Number.isNaN(value) ? 1 : Math.max(0, Math.min(1, value)) - return progress * progress * (3 - 2 * progress) + return Number.isNaN(value) ? 1 : Math.max(0, Math.min(1, value)) } diff --git a/packages/drive/test/recording/pointer.test.ts b/packages/drive/test/recording/pointer.test.ts index 0d4a0b8..8a7d284 100644 --- a/packages/drive/test/recording/pointer.test.ts +++ b/packages/drive/test/recording/pointer.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from "@effect/vitest" +import assert from "node:assert/strict" import { Effect, Schema } from "effect" import { mkdtemp, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join } from "node:path" +import { spring } from "motion" import { loadPointers, pointerAt, pointersPath, PointerOverlayOptions, type RecordingPointer } from "../../src/recording/pointer.js" const click: RecordingPointer = { action: "click", atMs: 1_000, x: 10, y: 5 } @@ -31,11 +33,35 @@ describe("recorded pointer", () => { expect(pointerAt([click], 1_700)).toBeUndefined() }) - it("keeps nearby clicks connected with smooth arrival and no early jump", () => { + it("keeps nearby clicks connected with a critically damped spring and exact arrival", () => { const second = { ...click, atMs: 1_500, x: 30 } expect(pointerAt([click, second], 1_200)).toMatchObject({ x: 10, opacity: 1 }) - expect(pointerAt([click, second], 1_390)).toMatchObject({ x: 20, opacity: 1 }) - expect(pointerAt([click, second], 1_500)).toMatchObject({ x: 30, opacity: 1, pressed: true }) + const expected = spring({ keyframes: [10, 30], duration: 1_000, bounce: 0 }) + expect(pointerAt([click, second], 1_390)?.x).toBeCloseTo(expected.next(500).value) + expect(pointerAt([click, second], 1_390)?.y).toBeLessThan(5) + expect(pointerAt([click, second], 1_500)).toMatchObject({ x: 30, y: 5, opacity: 1, pressed: true }) + const positions = Array.from({ length: 221 }, (_, index) => visible([click, second], 1_280 + index).x) + expect(positions.every((x) => x >= 10 && x <= 30)).toBe(true) + expect(positions.every((x, index) => x >= (positions[index - 1] ?? 10))).toBe(true) + }) + + it("can disable the arc without losing the spring and caps long-distance bends", () => { + const second = { ...click, atMs: 1_500, x: 100 } + expect(pointerAt([click, second], 1_335, { curve: 0 })).toMatchObject({ y: 5 }) + const positions = Array.from({ length: 221 }, (_, index) => visible([click, second], 1_280 + index)) + expect(positions.some((point) => point.y < 5)).toBe(true) + expect(positions.every((point) => (5 - point.y) * 20 <= 40)).toBe(true) + const vertical = visible([click, { ...second, x: 10, y: 50 }], 1_335) + expect(vertical.x).toBeGreaterThan(10) + expect((vertical.x - 10) * 10).toBeLessThanOrEqual(40) + }) + + it("samples identically when clips request frames in reverse or repeated order", () => { + const events = [click, { ...click, atMs: 1_500, x: 30 }] + const times = [1_280, 1_310, 1_390, 1_450, 1_500] + const forward = times.map((time) => pointerAt(events, time)) + expect(times.toReversed().map((time) => pointerAt(events, time))).toEqual(forward.toReversed()) + expect(times.map((time) => pointerAt(events, time))).toEqual(forward) }) it("does not fly across idle gaps and stays visible during a held drag", () => { @@ -49,6 +75,7 @@ describe("recorded pointer", () => { { action: "up", atMs: 5_000, x: 30, y: 5 }, ] expect(pointerAt(drag, 3_000)).toMatchObject({ x: 20, opacity: 1, pressed: true }) + expect(pointerAt(drag, 1_390, { curve: 1 })).toMatchObject({ y: 5, pressed: true }) expect(pointerAt(drag, 5_000)?.pressed).toBe(false) expect(pointerAt(drag, 5_700)).toBeUndefined() }) @@ -60,5 +87,13 @@ describe("recorded pointer", () => { expect(pointerAt([click, { ...click, atMs: 1_500, x: 30 }], 1_450, { motionMs: 0 })?.x).toBe(10) expect(pointerAt([], 0)).toBeUndefined() expect(() => Schema.decodeUnknownSync(PointerOverlayOptions)({ leadMs: -1 })).toThrow() + expect(() => Schema.decodeUnknownSync(PointerOverlayOptions)({ curve: -0.1 })).toThrow() + expect(() => Schema.decodeUnknownSync(PointerOverlayOptions)({ curve: 1.1 })).toThrow() }) }) + +function visible(events: ReadonlyArray, atMs: number) { + const frame = pointerAt(events, atMs) + assert(frame, `Expected visible pointer at ${atMs}ms`) + return frame +} diff --git a/skills/opencode-drive/SKILL.md b/skills/opencode-drive/SKILL.md index 54c82b5..2c1acd2 100644 --- a/skills/opencode-drive/SKILL.md +++ b/skills/opencode-drive/SKILL.md @@ -190,7 +190,10 @@ yield* secondary.ui.screenshot("secondary") Set `tui: { recording: true, pointerOverlay: true }` for a minimal animated mouse cursor in exported videos. A configuration object accepts `leadMs` (180), -`lingerMs` (700), and `motionMs` (220). Nearby inputs stay connected. This needs +`lingerMs` (700), `motionMs` (220), and `curve` (0.12). Travel uses a critically +damped Motion spring, arriving exactly at input time along a bounded arc. Use +`curve: 0` for straight travel; held drags never add decorative curvature. +Nearby inputs stay connected. This needs an OpenCode build advertising `ui.recording.pointer`; input timestamps and actual click coordinates come from its recording clock, not Drive-side estimates. Keep the `*.pointers.jsonl` sidecar with the terminal recording when copying it. From 961e75b290f794b9627a88a35dede34a917c2b9e Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 21:55:50 -0400 Subject: [PATCH 3/6] fix(drive): soften recorded pointer travel --- packages/drive/README.md | 4 +- packages/drive/src/recording/pointer.ts | 13 +++--- packages/drive/test/recording/pointer.test.ts | 42 +++++++++++++------ skills/opencode-drive/SKILL.md | 2 +- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/packages/drive/README.md b/packages/drive/README.md index 2a71100..7fbc345 100644 --- a/packages/drive/README.md +++ b/packages/drive/README.md @@ -238,7 +238,7 @@ Enable a minimal spring-animated cursor in exported videos: OpenCodeDriver.use({ tui: { recording: true, - pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 220, curve: 0.12 }, + pointerOverlay: { leadMs: 180, lingerMs: 700, motionMs: 500, curve: 0.06 }, }, }, ({ ui }) => ui.mouse({ action: "move", x: 20, y: 8 })) ``` @@ -262,7 +262,7 @@ The pointer appears before input, follows a small arc to the recorded cell with a critically damped Motion spring, and lingers after it. It arrives exactly at the input time without overshooting; nearby interactions stay connected. `curve` controls the arc height as a fraction of the pixel distance (capped at -40px); set it to `0` for straight travel. Held drags always follow straight +24px); set it to `0` for straight travel. Held drags always follow straight segments between recorded points. Motion's sampler runs without React or a browser. This is presentation-only: enabling it does not delay a script or send extra moves. To test hover along a path, send diff --git a/packages/drive/src/recording/pointer.ts b/packages/drive/src/recording/pointer.ts index db146cc..0372296 100644 --- a/packages/drive/src/recording/pointer.ts +++ b/packages/drive/src/recording/pointer.ts @@ -21,9 +21,9 @@ export const PointerOverlayOptions = Schema.Struct({ leadMs: Schema.optionalKey(Milliseconds), /** Keep it visible after input, merging nearby interactions. Default: 700ms. */ lingerMs: Schema.optionalKey(Milliseconds), - /** Spring between nearby positions, arriving at the input instant. Default: 220ms. */ + /** Spring between nearby positions, arriving at the input instant. Default: 500ms. */ motionMs: Schema.optionalKey(Milliseconds), - /** Arc height as a fraction of travel distance, capped at 40px. Default: 0.12; 0 is straight. */ + /** Arc height as a fraction of travel distance, capped at 24px. Default: 0.06; 0 is straight. */ curve: Schema.optionalKey(Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 1 }))), }) export interface PointerOverlayOptions extends Schema.Schema.Type {} @@ -58,7 +58,7 @@ export function pointerAt( ): PointerFrame | undefined { const lead = options.leadMs ?? 180 const linger = options.lingerMs ?? 700 - const motion = options.motionMs ?? 220 + const motion = options.motionMs ?? 500 const index = events.findLastIndex((event) => event.atMs <= atMs) const previous = events[index] const next = events[index + 1] @@ -69,9 +69,10 @@ export function pointerAt( const fadingIn = next ? ease((atMs - next.atMs + lead) / Math.min(120, lead)) : 0 const opacity = held ? 1 : Math.max(fadingOut, fadingIn) if (opacity <= 0) return undefined - const destination = next && atMs >= next.atMs - Math.max(lead, motion) ? next : previous + const connected = previous && next && (held || next.atMs - previous.atMs <= linger + lead) + // A longer travel window must not reveal disconnected input during the old pointer's fade-out. + const destination = next && atMs >= next.atMs - Math.max(lead, connected ? motion : 0) ? next : previous if (!destination) return undefined - const connected = previous && (held || destination.atMs - previous.atMs <= linger + lead) const start = previous ? Math.max(previous.atMs, destination.atMs - motion) : destination.atMs const amount = connected ? travel.next(progress((atMs - start) / (destination.atMs - start)) * 1_000).value : 1 const origin = connected ? previous : destination @@ -81,7 +82,7 @@ export function pointerAt( const dy = (destination.y - origin.y) * CellHeight const distance = Math.hypot(dx, dy) const bend = held || distance === 0 ? 0 : - Math.min(40, distance * (options.curve ?? 0.12)) * 4 * amount * (1 - amount) / distance + Math.min(24, distance * (options.curve ?? 0.06)) * 4 * amount * (1 - amount) / distance return { x: origin.x + (dx * amount + dy * bend) / CellWidth, y: origin.y + (dy * amount - dx * bend) / CellHeight, diff --git a/packages/drive/test/recording/pointer.test.ts b/packages/drive/test/recording/pointer.test.ts index 8a7d284..e83dc13 100644 --- a/packages/drive/test/recording/pointer.test.ts +++ b/packages/drive/test/recording/pointer.test.ts @@ -34,26 +34,42 @@ describe("recorded pointer", () => { }) it("keeps nearby clicks connected with a critically damped spring and exact arrival", () => { - const second = { ...click, atMs: 1_500, x: 30 } + const second = { ...click, atMs: 1_800, x: 30 } expect(pointerAt([click, second], 1_200)).toMatchObject({ x: 10, opacity: 1 }) const expected = spring({ keyframes: [10, 30], duration: 1_000, bounce: 0 }) - expect(pointerAt([click, second], 1_390)?.x).toBeCloseTo(expected.next(500).value) - expect(pointerAt([click, second], 1_390)?.y).toBeLessThan(5) - expect(pointerAt([click, second], 1_500)).toMatchObject({ x: 30, y: 5, opacity: 1, pressed: true }) - const positions = Array.from({ length: 221 }, (_, index) => visible([click, second], 1_280 + index).x) + expect(pointerAt([click, second], 1_550)?.x).toBeCloseTo(expected.next(500).value) + expect(pointerAt([click, second], 1_550)?.y).toBeLessThan(5) + expect(pointerAt([click, second], 1_800)).toMatchObject({ x: 30, y: 5, opacity: 1, pressed: true }) + const positions = Array.from({ length: 501 }, (_, index) => visible([click, second], 1_300 + index).x) expect(positions.every((x) => x >= 10 && x <= 30)).toBe(true) expect(positions.every((x, index) => x >= (positions[index - 1] ?? 10))).toBe(true) }) it("can disable the arc without losing the spring and caps long-distance bends", () => { - const second = { ...click, atMs: 1_500, x: 100 } - expect(pointerAt([click, second], 1_335, { curve: 0 })).toMatchObject({ y: 5 }) - const positions = Array.from({ length: 221 }, (_, index) => visible([click, second], 1_280 + index)) + const second = { ...click, atMs: 1_800, x: 100 } + expect(pointerAt([click, second], 1_425, { curve: 0 })).toMatchObject({ y: 5 }) + const positions = Array.from({ length: 501 }, (_, index) => visible([click, second], 1_300 + index)) expect(positions.some((point) => point.y < 5)).toBe(true) - expect(positions.every((point) => (5 - point.y) * 20 <= 40)).toBe(true) - const vertical = visible([click, { ...second, x: 10, y: 50 }], 1_335) + expect(positions.every((point) => (5 - point.y) * 20 <= 24)).toBe(true) + const vertical = visible([click, { ...second, x: 10, y: 50 }], 1_425) expect(vertical.x).toBeGreaterThan(10) - expect((vertical.x - 10) * 10).toBeLessThanOrEqual(40) + expect((vertical.x - 10) * 10).toBeLessThanOrEqual(24) + }) + + it("more than halves peak travel speed compared with the previous 220ms window", () => { + const events = [click, { ...click, atMs: 1_800, x: 100 }] + const peak = (options: PointerOverlayOptions) => { + const positions = Array.from({ length: 501 }, (_, index) => visible(events, 1_300 + index, options).x) + return Math.max(...positions.map((x, index) => x - (positions[index - 1] ?? x))) + } + expect(peak({})).toBeLessThan(peak({ motionMs: 220 }) * 0.5) + }) + + it("does not borrow an old fade-out to jump early across a disconnected gap", () => { + const later = { ...click, atMs: 2_000, x: 80 } + expect(pointerAt([click, later], 1_600)).toMatchObject({ x: 10, y: 5 }) + expect(pointerAt([click, later], 1_750)).toBeUndefined() + expect(pointerAt([click, later], 1_850)).toMatchObject({ x: 80, y: 5 }) }) it("samples identically when clips request frames in reverse or repeated order", () => { @@ -92,8 +108,8 @@ describe("recorded pointer", () => { }) }) -function visible(events: ReadonlyArray, atMs: number) { - const frame = pointerAt(events, atMs) +function visible(events: ReadonlyArray, atMs: number, options?: PointerOverlayOptions) { + const frame = pointerAt(events, atMs, options) assert(frame, `Expected visible pointer at ${atMs}ms`) return frame } diff --git a/skills/opencode-drive/SKILL.md b/skills/opencode-drive/SKILL.md index 2c1acd2..bb8c624 100644 --- a/skills/opencode-drive/SKILL.md +++ b/skills/opencode-drive/SKILL.md @@ -190,7 +190,7 @@ yield* secondary.ui.screenshot("secondary") Set `tui: { recording: true, pointerOverlay: true }` for a minimal animated mouse cursor in exported videos. A configuration object accepts `leadMs` (180), -`lingerMs` (700), `motionMs` (220), and `curve` (0.12). Travel uses a critically +`lingerMs` (700), `motionMs` (500), and `curve` (0.06). Travel uses a critically damped Motion spring, arriving exactly at input time along a bounded arc. Use `curve: 0` for straight travel; held drags never add decorative curvature. Nearby inputs stay connected. This needs From 61e444eb28e6ef0edd697a71fc38d872bdfbfc14 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 22:10:06 -0400 Subject: [PATCH 4/6] chore(drive): refresh runtime and build pins --- .changeset/real-pointer-recording.md | 2 ++ .github/workflows/ci.yml | 2 +- .github/workflows/deploy-catalog.yml | 2 +- .github/workflows/publish.yml | 2 +- apps/catalog/package.json | 6 ++-- bun.lock | 48 +++++++++++++--------------- package.json | 2 +- packages/drive/README.md | 13 ++++++-- packages/drive/package.json | 10 +++--- 9 files changed, 46 insertions(+), 41 deletions(-) diff --git a/.changeset/real-pointer-recording.md b/.changeset/real-pointer-recording.md index 19a5507..e1e3c2f 100644 --- a/.changeset/real-pointer-recording.md +++ b/.changeset/real-pointer-recording.md @@ -5,3 +5,5 @@ Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior. Animate pointer travel with a critically damped Motion spring and a configurable, bounded arc while preserving exact recorded input positions and times. + +Refresh the native canvas and terminal replay dependencies, align repository and CI Bun versions, and document the distinction between the pinned V2 SDK and the separately installed OpenCode executable. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af22fda..e25a8c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: oven-sh/setup-bun@v2 with: - bun-version: "1.3.14" + bun-version: "1.4.0" - run: sudo apt-get update && sudo apt-get install -y ffmpeg - run: bun install --frozen-lockfile diff --git a/.github/workflows/deploy-catalog.yml b/.github/workflows/deploy-catalog.yml index 1a366ad..385ec9e 100644 --- a/.github/workflows/deploy-catalog.yml +++ b/.github/workflows/deploy-catalog.yml @@ -31,7 +31,7 @@ jobs: - uses: oven-sh/setup-bun@v2 with: - bun-version: "1.3.14" + bun-version: "1.4.0" - run: sudo apt-get update && sudo apt-get install --yes ffmpeg - run: bun install --frozen-lockfile diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a7506b2..e160975 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: - uses: oven-sh/setup-bun@v2 with: - bun-version: "1.3.14" + bun-version: "1.4.0" - run: sudo apt-get update && sudo apt-get install -y ffmpeg - run: bun install --frozen-lockfile diff --git a/apps/catalog/package.json b/apps/catalog/package.json index e6519b4..e83c22e 100644 --- a/apps/catalog/package.json +++ b/apps/catalog/package.json @@ -2,7 +2,7 @@ "name": "@opencode-drive/catalog", "private": true, "type": "module", - "packageManager": "bun@1.3.14", + "packageManager": "bun@1.4.0", "scripts": { "generate": "bun ./scripts/generate-catalog.ts", "capture": "bun ./scripts/capture-opencode-drive.ts", @@ -28,8 +28,8 @@ "react-dom": "^19.2.7" }, "devDependencies": { - "@napi-rs/canvas": "1.0.2", - "@types/bun": "^1.3.14", + "@napi-rs/canvas": "1.0.8", + "@types/bun": "^1.4.0", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", "effect": "4.0.0-rc.112", diff --git a/bun.lock b/bun.lock index 1bdcc96..113f3f4 100644 --- a/bun.lock +++ b/bun.lock @@ -19,8 +19,8 @@ "react-dom": "^19.2.7", }, "devDependencies": { - "@napi-rs/canvas": "1.0.2", - "@types/bun": "^1.3.14", + "@napi-rs/canvas": "1.0.8", + "@types/bun": "^1.4.0", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", "effect": "4.0.0-rc.112", @@ -39,12 +39,12 @@ "dependencies": { "@effect/platform-node": "4.0.0-rc.112", "@effect/platform-node-shared": "4.0.0-rc.112", - "@napi-rs/canvas": "1.0.2", + "@napi-rs/canvas": "1.0.8", "@opencode-ai/client": "0.0.0-dev-18911", - "@types/bun": "1.3.13", + "@types/bun": "1.4.0", "@typescript/native-preview": "7.0.0-dev.20251207.1", - "@wterm/core": "0.3.0", - "@wterm/ghostty": "0.3.0", + "@wterm/core": "0.4.1", + "@wterm/ghostty": "0.4.1", "motion": "13.2.0", }, "devDependencies": { @@ -263,29 +263,29 @@ "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="], - "@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="], + "@napi-rs/canvas": ["@napi-rs/canvas@1.0.8", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.8", "@napi-rs/canvas-darwin-arm64": "1.0.8", "@napi-rs/canvas-darwin-x64": "1.0.8", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8", "@napi-rs/canvas-linux-arm64-gnu": "1.0.8", "@napi-rs/canvas-linux-arm64-musl": "1.0.8", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.8", "@napi-rs/canvas-linux-x64-gnu": "1.0.8", "@napi-rs/canvas-linux-x64-musl": "1.0.8", "@napi-rs/canvas-win32-arm64-msvc": "1.0.8", "@napi-rs/canvas-win32-x64-msvc": "1.0.8" } }, "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g=="], - "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="], + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.8", "", { "os": "android", "cpu": "arm64" }, "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw=="], - "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="], + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.8", "", { "os": "darwin", "cpu": "arm64" }, "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg=="], - "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="], + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.8", "", { "os": "darwin", "cpu": "x64" }, "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg=="], - "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="], + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.8", "", { "os": "linux", "cpu": "arm" }, "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ=="], - "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="], + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA=="], - "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="], + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg=="], - "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="], + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.8", "", { "os": "linux", "cpu": "none" }, "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg=="], - "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="], + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.8", "", { "os": "linux", "cpu": "x64" }, "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw=="], - "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="], + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.8", "", { "os": "linux", "cpu": "x64" }, "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg=="], - "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="], + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.8", "", { "os": "win32", "cpu": "arm64" }, "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ=="], - "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="], + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.8", "", { "os": "win32", "cpu": "x64" }, "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ=="], "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.6", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg=="], @@ -413,7 +413,7 @@ "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/bun": ["@types/bun@1.4.0", "", { "dependencies": { "bun-types": "1.4.0" } }, "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ=="], "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], @@ -499,9 +499,9 @@ "@vitest/utils": ["@vitest/utils@4.1.10", "", { "dependencies": { "@vitest/pretty-format": "4.1.10", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA=="], - "@wterm/core": ["@wterm/core@0.3.0", "", {}, "sha512-aQ73QBP+eyA3kcn31mA7DmAi7qmEsQijZdmvgwxtSjCi2248EAOZgtkGDpjlspfrxyVxQbUl/IF+gCVcXt4nZQ=="], + "@wterm/core": ["@wterm/core@0.4.1", "", {}, "sha512-YypeqYJPrx98MoQODPnj/QVKqCQoO8d9f0oosaQ4GfFukvR3Q09U2LTRGLgVmAjPWa0D6MQHQNcMQJBsPn6i+g=="], - "@wterm/ghostty": ["@wterm/ghostty@0.3.0", "", { "dependencies": { "@wterm/core": "0.3.0" } }, "sha512-TDjEmUSRD7IaFxq31P7+I3Inkp5mHCfBfO6LYj3/vbLbWOkOZOKPhjAcZ36AYFHe9Q0HLCFvLWmrxIO3aGCjDA=="], + "@wterm/ghostty": ["@wterm/ghostty@0.4.1", "", { "dependencies": { "@wterm/core": "0.4.1" } }, "sha512-HqG/03oTXME302x+BwOaWDSi2b1Txgd28rbIHhRcTIlY4R4+vqafMmXDrKWEmNsMuoqbTc4prAr/MmVQvm/TmQ=="], "ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="], @@ -519,7 +519,7 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "bun-types": ["bun-types@1.4.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-iIKw23BspnQQYd3prITOBxeUsxBHnwzX6YJfGMuNOZzeNcMmVqzIIVGRm1l69ogaPQmb4wB6BN8mA5bE9YuC5Q=="], "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], @@ -815,14 +815,10 @@ "miniflare/ws": ["ws@8.21.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g=="], - "opencode-drive/@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], - "opencode-drive/typescript": ["typescript@5.8.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ=="], "read-yaml-file/js-yaml": ["js-yaml@3.15.0", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog=="], - "opencode-drive/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], - "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], } } diff --git a/package.json b/package.json index 2e062a9..e76f133 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/package.json", "name": "opencode-drive-monorepo", "private": true, - "packageManager": "bun@1.3.14", + "packageManager": "bun@1.4.0", "workspaces": [ "packages/*", "apps/*" diff --git a/packages/drive/README.md b/packages/drive/README.md index 7fbc345..df83491 100644 --- a/packages/drive/README.md +++ b/packages/drive/README.md @@ -8,11 +8,18 @@ This project gives your agents control over OpenCode: ## Requirements OpenCode Drive requires [Bun](https://bun.sh/) 1.3.14 or newer. MP4 recording export also requires `ffmpeg` on `PATH`. +Repository development, CI, and releases use Bun 1.4.0. Effect programs must use `effect@4.0.0-rc.112`, Drive's exact peer dependency. -The platform and test packages use the same release as the current V2 client, -`@opencode-ai/client@0.0.0-dev-18535` and its matching protocol and schema -packages. Effect's `latest` tag still selects V3; install the exact V4 peer. +The platform and test packages use the same release as the V2 client. +The exact `@opencode-ai/client` version is pinned in +[`package.json`](package.json), which also selects its matching protocol and +schema packages. Effect's `latest` tag still selects V3; install the exact V4 peer. + +Drive does not download or pin an OpenCode executable: it runs `opencode2` from +`PATH` unless `--dev` or an explicit command selects another target. New mouse +controls require a target advertising `ui.mouse` and `ui.recording.pointer`; +upgrading Drive's SDK dependency alone does not upgrade that executable. Install dependencies with: diff --git a/packages/drive/package.json b/packages/drive/package.json index 4c3a14d..254b43a 100644 --- a/packages/drive/package.json +++ b/packages/drive/package.json @@ -11,7 +11,7 @@ "homepage": "https://github.com/anomalyco/opencode-drive#readme", "bugs": "https://github.com/anomalyco/opencode-drive/issues", "type": "module", - "packageManager": "bun@1.3.14", + "packageManager": "bun@1.4.0", "engines": { "bun": ">=1.3.14" }, @@ -60,12 +60,12 @@ "dependencies": { "@effect/platform-node": "4.0.0-rc.112", "@effect/platform-node-shared": "4.0.0-rc.112", - "@napi-rs/canvas": "1.0.2", + "@napi-rs/canvas": "1.0.8", "@opencode-ai/client": "0.0.0-dev-18911", - "@types/bun": "1.3.13", + "@types/bun": "1.4.0", "@typescript/native-preview": "7.0.0-dev.20251207.1", - "@wterm/core": "0.3.0", - "@wterm/ghostty": "0.3.0", + "@wterm/core": "0.4.1", + "@wterm/ghostty": "0.4.1", "motion": "13.2.0" }, "peerDependencies": { From 4d8f082251e64583da4562e9d5b015c4673fb7e7 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 22:13:37 -0400 Subject: [PATCH 5/6] chore(drive): prepare 2.1.0 release --- .changeset/await-plugin-activation.md | 5 ----- .changeset/real-pointer-recording.md | 9 --------- packages/drive/CHANGELOG.md | 14 ++++++++++++++ packages/drive/package.json | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 .changeset/await-plugin-activation.md delete mode 100644 .changeset/real-pointer-recording.md diff --git a/.changeset/await-plugin-activation.md b/.changeset/await-plugin-activation.md deleted file mode 100644 index 41409ab..0000000 --- a/.changeset/await-plugin-activation.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"opencode-drive": patch ---- - -Bump the pinned `@opencode-ai/client` to `0.0.0-dev-18911` so the generated SDK exposes `plugin.awaitActivation`, which scripts need before reading agents or models from a cold location on current V2 servers. diff --git a/.changeset/real-pointer-recording.md b/.changeset/real-pointer-recording.md deleted file mode 100644 index e1e3c2f..0000000 --- a/.changeset/real-pointer-recording.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -"opencode-drive": minor ---- - -Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior. - -Animate pointer travel with a critically damped Motion spring and a configurable, bounded arc while preserving exact recorded input positions and times. - -Refresh the native canvas and terminal replay dependencies, align repository and CI Bun versions, and document the distinction between the pinned V2 SDK and the separately installed OpenCode executable. diff --git a/packages/drive/CHANGELOG.md b/packages/drive/CHANGELOG.md index 8cf19df..4551af7 100644 --- a/packages/drive/CHANGELOG.md +++ b/packages/drive/CHANGELOG.md @@ -1,5 +1,19 @@ # opencode-drive +## 2.1.0 + +### Minor Changes + +- e0fb4c7: Add real terminal mouse move, button, and scroll control through `ui.mouse`, plus an opt-in animated pointer in recordings. Mouse controls and pointer recording negotiate capabilities with OpenCode; older endpoints remain usable for existing operations. Fix keypress placement after recording trims and simplify shared terminal-operation ownership without changing settlement behavior. + + Animate pointer travel with a critically damped Motion spring and a configurable, bounded arc while preserving exact recorded input positions and times. + + Refresh the native canvas and terminal replay dependencies, align repository and CI Bun versions, and document the distinction between the pinned V2 SDK and the separately installed OpenCode executable. + +### Patch Changes + +- f6a3f55: Bump the pinned `@opencode-ai/client` to `0.0.0-dev-18911` so the generated SDK exposes `plugin.awaitActivation`, which scripts need before reading agents or models from a cold location on current V2 servers. + ## 2.0.1 ### Patch Changes diff --git a/packages/drive/package.json b/packages/drive/package.json index 254b43a..1fd4b16 100644 --- a/packages/drive/package.json +++ b/packages/drive/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "opencode-drive", - "version": "2.0.1", + "version": "2.1.0", "description": "Drive real and simulated OpenCode instances", "license": "MIT", "repository": { From dcce0cd5fbd87260f20d770f615f927a8e65c962 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 22:48:55 -0400 Subject: [PATCH 6/6] chore(drive): pin released mouse-capable V2 client --- bun.lock | 10 +++++----- packages/drive/CHANGELOG.md | 1 + packages/drive/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/bun.lock b/bun.lock index 113f3f4..e37e133 100644 --- a/bun.lock +++ b/bun.lock @@ -32,7 +32,7 @@ }, "packages/drive": { "name": "opencode-drive", - "version": "2.0.1", + "version": "2.1.0", "bin": { "opencode-drive": "bin/opencode-drive", }, @@ -40,7 +40,7 @@ "@effect/platform-node": "4.0.0-rc.112", "@effect/platform-node-shared": "4.0.0-rc.112", "@napi-rs/canvas": "1.0.8", - "@opencode-ai/client": "0.0.0-dev-18911", + "@opencode-ai/client": "0.0.0-dev-19066", "@types/bun": "1.4.0", "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.4.1", @@ -295,11 +295,11 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - "@opencode-ai/client": ["@opencode-ai/client@0.0.0-dev-18911", "", { "dependencies": { "@opencode-ai/protocol": "0.0.0-dev-18911", "@opencode-ai/schema": "0.0.0-dev-18911" }, "peerDependencies": { "effect": "4.0.0-rc.112", "solid-js": ">=1.9.0" }, "optionalPeers": ["effect", "solid-js"] }, "sha512-Gq951CE1ue4MF11+4qJfdxoWD5TxdBHe47RIPmlaAvYXPtAXZbdRY9XHQzBFSQEnYWW22QQEG39GuIS0ITpHHA=="], + "@opencode-ai/client": ["@opencode-ai/client@0.0.0-dev-19066", "", { "dependencies": { "@opencode-ai/protocol": "0.0.0-dev-19066", "@opencode-ai/schema": "0.0.0-dev-19066" }, "peerDependencies": { "effect": "4.0.0-rc.112", "solid-js": ">=1.9.0" }, "optionalPeers": ["effect", "solid-js"] }, "sha512-Cx6f1O2FZnhjdMT1WRfoIbmQRQC2AiMReFYBjYSrdQo/+y/0pgo7odMULYEYUfGgM6qeNTHGdPxHqBcgqxhcmg=="], - "@opencode-ai/protocol": ["@opencode-ai/protocol@0.0.0-dev-18911", "", { "dependencies": { "@opencode-ai/schema": "0.0.0-dev-18911", "effect": "4.0.0-rc.112" } }, "sha512-fWup6XvEEK9WuA87+Q46OnAtwyFKti/nYstZN3A6HxQdOtAre/mvLb/SVnnrBNaDIjt9yD+3HsY6wbhtn717KA=="], + "@opencode-ai/protocol": ["@opencode-ai/protocol@0.0.0-dev-19066", "", { "dependencies": { "@opencode-ai/schema": "0.0.0-dev-19066", "effect": "4.0.0-rc.112" } }, "sha512-kY9+qKQGp5W59pW+eDhOOydomeD0q7LaFu3P+38MD0iRIOZ55dCJVR3Ao77DvVuZd8dM2oLw8o/HuUto7kF5Qg=="], - "@opencode-ai/schema": ["@opencode-ai/schema@0.0.0-dev-18911", "", { "dependencies": { "@standard-schema/spec": "1.1.0", "effect": "4.0.0-rc.112" } }, "sha512-8xJlzsJizyt/e0NJFZxqYk4ugyX9ziVZ03WJBJriPsZxhW+EbQHdyPRGhxwswGdV6mvW3e8JBnHP4plUlGd5rw=="], + "@opencode-ai/schema": ["@opencode-ai/schema@0.0.0-dev-19066", "", { "dependencies": { "@standard-schema/spec": "1.1.0", "effect": "4.0.0-rc.112" } }, "sha512-+QnYPonCepmAqViSL5LNmyE8YmTkvXLdH2aXk8ge6/o+bDbC+EI9QM3aSj/vHxeLA4bBD4yX0dYMZ6DC2YYhSA=="], "@opencode-drive/catalog": ["@opencode-drive/catalog@workspace:apps/catalog"], diff --git a/packages/drive/CHANGELOG.md b/packages/drive/CHANGELOG.md index 4551af7..9a2f8f9 100644 --- a/packages/drive/CHANGELOG.md +++ b/packages/drive/CHANGELOG.md @@ -12,6 +12,7 @@ ### Patch Changes +- Pin the V2 client and its matching protocol/schema packages to `0.0.0-dev-19066`, the published OpenCode build containing mouse control and pointer recording support. - f6a3f55: Bump the pinned `@opencode-ai/client` to `0.0.0-dev-18911` so the generated SDK exposes `plugin.awaitActivation`, which scripts need before reading agents or models from a cold location on current V2 servers. ## 2.0.1 diff --git a/packages/drive/package.json b/packages/drive/package.json index 1fd4b16..490cdb6 100644 --- a/packages/drive/package.json +++ b/packages/drive/package.json @@ -61,7 +61,7 @@ "@effect/platform-node": "4.0.0-rc.112", "@effect/platform-node-shared": "4.0.0-rc.112", "@napi-rs/canvas": "1.0.8", - "@opencode-ai/client": "0.0.0-dev-18911", + "@opencode-ai/client": "0.0.0-dev-19066", "@types/bun": "1.4.0", "@typescript/native-preview": "7.0.0-dev.20251207.1", "@wterm/core": "0.4.1",