diff --git a/README.md b/README.md index 37dbe9c..eb4646b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # bb plugins -Seven bb plugins I use for product design work, kept together with the few build and repository tools they share. [![CI](https://github.com/brsbl/bb-plugins/actions/workflows/ci.yml/badge.svg)](https://github.com/brsbl/bb-plugins/actions/workflows/ci.yml) +Eight bb plugins I use for product design work, kept together with the few build and repository tools they share. [![CI](https://github.com/brsbl/bb-plugins/actions/workflows/ci.yml/badge.svg)](https://github.com/brsbl/bb-plugins/actions/workflows/ci.yml) [bb](https://getbb.app) is an agentic IDE for running coding agents across projects, threads, and environments. Its plugins can add UI, commands, skills, and server capabilities; this repository is where I build and maintain mine. @@ -8,6 +8,16 @@ Seven bb plugins I use for product design work, kept together with the few build Each plugin has its own workspace under `plugins/` and a short README with the story behind it. +### Browser Context + +Selects and annotates elements or regions in BB's Browser, stages inspectable context mentions, and gives agents an explicit user-visible control mode for the exact tab. + +![Browser Context selection and agent-control actions in bb](plugins/browser-context/docs/screenshot.png) + +[Source](plugins/browser-context) · [README](plugins/browser-context/README.md) + +Install: `bb plugin install git:https://github.com/brsbl/bb-plugins.git@plugin/browser-context --yes` + ### Design Doctrine Turns recurring product-design feedback into a searchable rule library that agents can apply while designing, building, and critiquing. Its maintenance workflow keeps the rules grounded in real review evidence. diff --git a/package-lock.json b/package-lock.json index 1dcdb47..8af4ec8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2958,6 +2958,10 @@ ], "license": "MIT" }, + "node_modules/bb-plugin-browser-context": { + "resolved": "plugins/browser-context", + "link": true + }, "node_modules/bb-plugin-design-doctrine": { "resolved": "plugins/design-doctrine", "link": true @@ -5039,6 +5043,83 @@ "vitest": "^4.0.0" } }, + "plugins/browser-context": { + "name": "bb-plugin-browser-context", + "version": "0.1.0", + "license": "UNLICENSED", + "dependencies": { + "@hugeicons/core-free-icons": "^4.1.3", + "@hugeicons/react": "^1.1.6", + "zod": "^4.3.6" + }, + "devDependencies": { + "@get-bb/plugin-sdk": "file:./vendor/get-bb-plugin-sdk-0.4.9.tgz", + "@radix-ui/react-tooltip": "^1.2.12", + "@testing-library/react": "^16.3.2", + "@types/better-sqlite3": "^7.6.12", + "@types/node": "^22.0.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", + "better-sqlite3": "^12.10.0", + "cron-parser": "^5.5.0", + "hono": "^4.11.9", + "jsdom": "^29.0.1", + "react": "^19.2.7", + "react-dom": "^19.2.7", + "typescript": "^5.7.0", + "vitest": "^4.1.8" + }, + "engines": { + "bb": ">=0.0.34", + "bbPluginSdk": "^0.4.9" + } + }, + "plugins/browser-context/node_modules/@get-bb/plugin-sdk": { + "version": "0.4.9", + "resolved": "file:plugins/browser-context/vendor/get-bb-plugin-sdk-0.4.9.tgz", + "integrity": "sha512-kHHoV2IxNC7Inl02B4OtYILtyMPzQ7i2LKXvRl7XUtkcmDCEZfZE3FW9H1DFoHZW8cQf7g82s/Se73AvJzojcw==", + "dev": true, + "peerDependencies": { + "@testing-library/react": "^16.3.2", + "@types/better-sqlite3": "^7.6.12", + "@types/react": "^19.0.0", + "better-sqlite3": ">=12", + "cron-parser": "^5.5.0", + "hono": "^4.11.9", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "zod": "^4.3.6" + }, + "peerDependenciesMeta": { + "@testing-library/react": { + "optional": true + }, + "@types/better-sqlite3": { + "optional": true + }, + "@types/react": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "cron-parser": { + "optional": true + }, + "hono": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "plugins/design-doctrine": { "name": "bb-plugin-design-doctrine", "version": "0.3.0", diff --git a/plugins/browser-context/README.md b/plugins/browser-context/README.md new file mode 100644 index 0000000..714f20d --- /dev/null +++ b/plugins/browser-context/README.md @@ -0,0 +1,27 @@ +# Browser Context + +Select elements or drag over regions in BB's Browser, annotate the numbered captures, then add the batch to the current thread composer. A separate agent-control action lets the user or agent visibly enable control of one exact Browser tab. + +![Inspectable Browser Context mention in bb](docs/screenshot.png) + +## Install + +```bash +bb plugin install git:https://github.com/brsbl/bb-plugins.git@plugin/browser-context --yes +``` + +## Use + +Open a Browser tab and click the Browser Context action to enter selection mode. Click an element or drag over a region, add an optional comment, and click the same Browser toolbar action again to build a numbered batch. Hover or focus a staged row to return its target to the live preview, edit any comment in place, or remove one selection without affecting the rest. Choose **Send to agent** once to preserve the existing draft and append one inspectable mention pill followed by its ordinary editable comment for every selection. Activate a pill to reopen its capture preview and exact metadata. Screenshots remain preview-only and are never sent; immutable DOM, React, accessibility, and geometry metadata resolves into hidden agent context at send time. The standard Send or Queue flow remains in control. + +Click the AI Browser action to let the current thread's agent control that exact client, window, tab, and navigation epoch. The visible Browser frame marks the active mode; clicking the action again exits it. Navigation changes the target epoch, so stale control requests cannot silently move to a different page. + +## Develop + +From the monorepo root: + +```bash +npm ci +npm run check --workspace=bb-plugin-browser-context +bb plugin install "path:$PWD/plugins/browser-context" --yes +``` diff --git a/plugins/browser-context/app-registration.test.ts b/plugins/browser-context/app-registration.test.ts new file mode 100644 index 0000000..c5f015e --- /dev/null +++ b/plugins/browser-context/app-registration.test.ts @@ -0,0 +1,30 @@ +// @vitest-environment jsdom + +import { installTestPluginRuntime } from "@get-bb/plugin-sdk/testing/app"; +import { describe, expect, it, vi } from "vitest"; + +describe("Browser Context registration", () => { + it("registers on current hosts and remains inert on older hosts", async () => { + installTestPluginRuntime(); + const { registerBrowserContextApp } = await import("./app.js"); + const register = vi.fn(); + + registerBrowserContextApp({ experimental_browserAction: register }); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ + id: "agent-control", + title: "Agent control", + component: expect.any(Function), + }), + ); + expect(register).toHaveBeenCalledWith( + expect.objectContaining({ + id: "capture", + title: "Select page context", + component: expect.any(Function), + }), + ); + expect(register).toHaveBeenCalledTimes(2); + expect(() => registerBrowserContextApp({})).not.toThrow(); + }); +}); diff --git a/plugins/browser-context/app.css b/plugins/browser-context/app.css new file mode 100644 index 0000000..60d1876 --- /dev/null +++ b/plugins/browser-context/app.css @@ -0,0 +1,597 @@ +@scope ([data-bb-plugin="browser-context"]) { + .bb-browser-context-action { + display: inline-flex; + width: 28px; + height: 28px; + align-items: center; + justify-content: center; + border: 0; + border-radius: 7px; + background: transparent; + color: var(--muted-foreground, #667085); + cursor: pointer; + transition: + background-color 140ms ease, + box-shadow 140ms ease, + color 140ms ease; + } + + .bb-browser-context-action-trigger { + display: inline-flex; + width: 28px; + height: 28px; + } + + .bb-browser-context-action:hover:not(:disabled), + .bb-browser-context-action[aria-pressed="true"] { + background: var(--state-hover, rgb(0 0 0 / 6%)); + color: var(--foreground, #101828); + } + + .bb-browser-context-action[aria-pressed="true"] { + box-shadow: inset 0 0 0 1px + color-mix(in srgb, var(--ring, #4c76e8) 68%, transparent); + color: var(--ring, #4c76e8); + } + + .bb-browser-context-action:disabled { + cursor: not-allowed; + opacity: 0.42; + } + + .bb-browser-context-action:focus-visible { + outline: 2px solid var(--ring, #4c76e8); + outline-offset: 2px; + } + + .bb-browser-context-action svg { + width: 17px; + height: 17px; + } + + .bb-browser-context-action[aria-pressed="true"] svg { + animation: bb-browser-context-activate 220ms ease-out; + transform-origin: center; + } + + .bb-browser-context-status { + position: fixed; + z-index: 2147483102; + right: 12px; + top: 92px; + max-width: 320px; + padding: 9px 11px; + border: 1px solid var(--destructive, #b42318); + border-radius: 9px; + background: var(--popover, #fff); + box-shadow: 0 8px 24px rgb(16 24 40 / 14%); + color: var(--destructive, #b42318); + font: + 500 12px/1.4 Inter, + system-ui, + sans-serif; + } + + .bb-browser-context-review { + position: absolute; + inset: 0; + overflow: hidden; + pointer-events: auto; + background: var(--background, #fff); + color: var(--foreground, #101828); + font: + 400 13px/1.45 Inter, + system-ui, + sans-serif; + } + + .bb-browser-context-canvas { + position: absolute; + inset: 0; + overflow: hidden; + background: #f2f4f7; + } + + .bb-browser-context-canvas > img { + display: block; + width: 100%; + height: 100%; + object-fit: fill; + user-select: none; + } + + .bb-browser-context-target { + position: absolute; + min-width: 8px; + min-height: 8px; + border: 2px solid #3867e8; + border-radius: 3px; + background: rgb(56 103 232 / 12%); + box-shadow: + 0 0 0 1px rgb(255 255 255 / 78%), + 0 3px 12px rgb(16 24 40 / 24%); + cursor: help; + outline: none; + opacity: 0.68; + transition: + background-color 120ms ease, + border-color 120ms ease, + box-shadow 120ms ease, + opacity 120ms ease; + } + + .bb-browser-context-target:hover, + .bb-browser-context-target:focus-visible, + .bb-browser-context-target[data-active="true"] { + border-color: #1747c8; + background: rgb(56 103 232 / 23%); + box-shadow: + 0 0 0 2px rgb(255 255 255 / 92%), + 0 4px 16px rgb(16 24 40 / 30%); + opacity: 1; + } + + .bb-browser-context-target-badge, + .bb-browser-context-comment-index { + display: inline-flex; + width: 20px; + height: 20px; + align-items: center; + justify-content: center; + border-radius: 999px; + background: #3867e8; + color: #fff; + font-size: 11px; + font-weight: 700; + } + + .bb-browser-context-target-badge { + position: absolute; + top: -12px; + left: -12px; + box-shadow: 0 1px 4px rgb(16 24 40 / 24%); + } + + .bb-browser-context-target-tooltip { + position: absolute; + left: 0; + top: calc(100% + 8px); + z-index: 2; + display: grid; + width: max-content; + max-width: 280px; + gap: 2px; + padding: 6px 12px; + border: 0; + border-radius: 6px; + background: var(--primary, #101828); + color: var(--primary-foreground, #fff); + text-align: left; + white-space: normal; + } + + .bb-browser-context-target-tooltip strong { + font-size: 11px; + font-weight: 650; + } + + .bb-browser-context-target-tooltip span { + color: color-mix(in srgb, var(--primary-foreground, #fff) 78%, transparent); + font-size: 11px; + } + + .bb-browser-context-comment-card { + position: absolute; + z-index: 3; + top: 10px; + right: 10px; + display: grid; + width: min(300px, calc(100% - 20px)); + gap: 7px; + padding: 10px; + border: 1px solid var(--border, #d0d5dd); + border-radius: 12px; + background: color-mix(in srgb, var(--popover, #fff) 96%, transparent); + box-shadow: 0 14px 36px rgb(16 24 40 / 20%); + backdrop-filter: blur(12px); + } + + .bb-browser-context-comment-card-left { + right: auto; + left: 10px; + } + + .bb-browser-context-comment-toolbar { + display: flex; + min-width: 0; + align-items: flex-start; + justify-content: space-between; + gap: 6px; + } + + .bb-browser-context-comment-heading { + display: flex; + flex: 1; + min-width: 0; + align-items: center; + gap: 7px; + cursor: grab; + touch-action: none; + user-select: none; + } + + .bb-browser-context-comment-heading:active { + cursor: grabbing; + } + + .bb-browser-context-comment-heading > svg { + width: 12px; + height: 16px; + flex: none; + color: var(--muted-foreground, #667085); + } + + .bb-browser-context-comment-heading > span:last-child { + display: grid; + min-width: 0; + } + + .bb-browser-context-comment-heading strong { + font-size: 12px; + font-weight: 650; + } + + .bb-browser-context-comment-heading small { + overflow: hidden; + color: var(--muted-foreground, #667085); + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; + } + + .bb-browser-context-comment-utilities { + display: flex; + flex: none; + align-items: center; + gap: 2px; + } + + .bb-browser-context-utility { + display: inline-flex; + width: 26px; + height: 26px; + flex: none; + align-items: center; + justify-content: center; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--muted-foreground, #667085); + cursor: pointer; + } + + .bb-browser-context-utility:hover:not(:disabled) { + background: var(--state-hover, #f2f4f7); + color: var(--foreground, #101828); + } + + .bb-browser-context-utility:focus-visible { + outline: 2px solid var(--ring, #4c76e8); + outline-offset: 1px; + } + + .bb-browser-context-utility:disabled { + cursor: wait; + opacity: 0.5; + } + + .bb-browser-context-utility svg { + width: 14px; + height: 14px; + } + + .bb-browser-context-comment-card textarea { + min-height: 72px; + resize: vertical; + padding: 9px 10px; + border: 1px solid var(--border, #d0d5dd); + border-radius: 8px; + background: var(--background, #fff); + color: var(--foreground, #101828); + font: inherit; + outline: none; + } + + .bb-browser-context-selection-list { + display: grid; + max-height: 148px; + gap: 3px; + overflow-y: auto; + margin: 0; + padding: 2px 0; + list-style: none; + } + + .bb-browser-context-selection-list li { + display: flex; + min-width: 0; + align-items: center; + border-radius: 7px; + } + + .bb-browser-context-selection-list li[data-active="true"] { + background: var(--state-hover, rgb(0 0 0 / 6%)); + } + + .bb-browser-context-selection-row { + display: flex; + min-width: 0; + flex: 1; + align-items: center; + gap: 7px; + padding: 5px 4px 5px 6px; + border: 0; + background: transparent; + color: var(--foreground, #101828); + cursor: pointer; + text-align: left; + } + + .bb-browser-context-selection-row > span:last-child { + display: grid; + min-width: 0; + } + + .bb-browser-context-selection-row strong, + .bb-browser-context-selection-row small { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .bb-browser-context-selection-row strong { + font-size: 11px; + font-weight: 650; + } + + .bb-browser-context-selection-row small { + color: var(--muted-foreground, #667085); + font-size: 10px; + } + + .bb-browser-context-selection-row:focus-visible, + .bb-browser-context-remove:focus-visible { + outline: 2px solid var(--ring, #4c76e8); + outline-offset: -1px; + } + + .bb-browser-context-remove { + display: inline-flex; + width: 25px; + height: 25px; + flex: none; + align-items: center; + justify-content: center; + margin-right: 2px; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--muted-foreground, #667085); + cursor: pointer; + } + + .bb-browser-context-remove:hover:not(:disabled) { + background: color-mix( + in srgb, + var(--destructive, #b42318) 10%, + transparent + ); + color: var(--destructive, #b42318); + } + + .bb-browser-context-remove:disabled { + cursor: wait; + opacity: 0.5; + } + + .bb-browser-context-remove svg { + width: 13px; + height: 13px; + } + + .bb-browser-context-comment-card textarea:focus-visible { + border-color: var(--ring, #4c76e8); + box-shadow: 0 0 0 2px + color-mix(in srgb, var(--ring, #4c76e8) 24%, transparent); + } + + .bb-browser-context-comment-submit:disabled { + cursor: default; + } + + .bb-browser-context-comment-submit { + flex: 0 0 auto; + min-width: 30px; + padding: 0 8px; + } + + .bb-browser-context-comment-submit svg { + width: 16px; + height: 16px; + } + + .bb-browser-context-comment-submit kbd { + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 18px; + padding: 2px 6px; + border-radius: 4px; + background: var(--state-hover, #f2f4f7); + color: var(--subtle-foreground, var(--muted-foreground, #667085)); + font: + 500 10px/1 Inter, + system-ui, + sans-serif; + white-space: nowrap; + opacity: 0.72; + } + + .bb-browser-context-submitted-comments { + display: grid; + max-height: 116px; + gap: 4px; + overflow-y: auto; + margin: 0; + padding: 0; + list-style: none; + } + + .bb-browser-context-submitted-comments li { + display: grid; + grid-template-columns: 18px minmax(0, 1fr); + align-items: start; + gap: 6px; + padding: 6px 8px; + border-radius: 7px; + background: color-mix( + in srgb, + var(--muted, var(--state-hover, #f2f4f7)) 72%, + transparent + ); + } + + .bb-browser-context-submitted-comments li > span { + display: inline-flex; + width: 18px; + height: 18px; + align-items: center; + justify-content: center; + border-radius: 999px; + background: var(--state-hover, #f2f4f7); + color: var(--muted-foreground, #667085); + font-size: 10px; + font-weight: 650; + } + + .bb-browser-context-submitted-comments p { + margin: 1px 0 0; + color: var(--foreground, #101828); + font-size: 11px; + line-height: 1.35; + overflow-wrap: anywhere; + white-space: pre-wrap; + } + + .bb-browser-context-error { + margin: 0; + font-size: 11px; + } + + .bb-browser-context-error { + color: var(--destructive, #b42318); + } + + .bb-browser-context-review-actions { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 6px; + margin-top: 2px; + } + + .bb-browser-context-review-actions button { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 5px; + height: 30px; + padding: 0 10px; + border: 1px solid var(--border, #d0d5dd); + border-radius: 7px; + background: var(--background, #fff); + color: var(--foreground, #101828); + cursor: pointer; + font: + 600 11px/1 Inter, + system-ui, + sans-serif; + } + + .bb-browser-context-review-actions button:hover:not(:disabled) { + background: var(--state-hover, #f2f4f7); + } + + .bb-browser-context-review-actions button:focus-visible { + outline: 2px solid var(--ring, #4c76e8); + outline-offset: 2px; + } + + .bb-browser-context-review-actions button:disabled { + cursor: wait; + opacity: 0.55; + } + + .bb-browser-context-review-actions .bb-browser-context-primary { + border-color: var(--primary, #101828); + background: var(--primary, #101828); + color: var(--primary-foreground, #fff); + } + + .bb-browser-context-review-actions + .bb-browser-context-primary:hover:not(:disabled) { + background: color-mix(in srgb, var(--primary, #101828) 88%, transparent); + } +} + +[data-bb-plugin="browser-context"].bb-browser-context-action-tooltip { + z-index: 2147483103; + max-width: min(20rem, var(--radix-tooltip-content-available-width)); + overflow: hidden; + padding: 6px 12px; + border-radius: 6px; + background: var(--primary, #101828); + color: var(--primary-foreground, #fff); + font: + 500 12px/1.25 Inter, + system-ui, + sans-serif; + overflow-wrap: anywhere; + animation: bb-browser-context-tooltip-in 120ms ease-out; +} + +@keyframes bb-browser-context-activate { + 0% { + opacity: 0.62; + transform: scale(0.88); + } + + 100% { + opacity: 1; + transform: scale(1); + } +} + +@keyframes bb-browser-context-tooltip-in { + from { + opacity: 0; + transform: scale(0.95); + } + + to { + opacity: 1; + transform: scale(1); + } +} + +@media (prefers-reduced-motion: reduce) { + [data-bb-plugin="browser-context"] + .bb-browser-context-action[aria-pressed="true"] + svg { + animation: none; + } + + [data-bb-plugin="browser-context"].bb-browser-context-action-tooltip { + animation: none; + } +} diff --git a/plugins/browser-context/app.test.tsx b/plugins/browser-context/app.test.tsx new file mode 100644 index 0000000..4a4c132 --- /dev/null +++ b/plugins/browser-context/app.test.tsx @@ -0,0 +1,1043 @@ +// @vitest-environment jsdom + +import { cleanup, fireEvent, screen, waitFor } from "@testing-library/react"; +import type { JsonValue, PluginBrowserActionProps } from "@get-bb/plugin-sdk/app"; +import { + loadPluginApp, + renderSlot, + type PluginRpcTestHandlers, +} from "@get-bb/plugin-sdk/testing/app"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { rpcContract } from "./server.js"; +import type { BrowserSelectionCapture } from "./lib/browser-selection.js"; + +const capture: BrowserSelectionCapture = { + version: 2, + kind: "element", + page: { + url: "https://example.com/settings", + title: "Settings", + viewport: { width: 800, height: 600 }, + scroll: { x: 0, y: 0 }, + }, + rect: { x: 40, y: 50, width: 180, height: 36 }, + element: { + selector: "button#save", + tag: "button", + id: "save", + classNames: ["primary"], + rect: { x: 40, y: 50, width: 180, height: 36 }, + dom: '', + text: "Save", + styles: { display: "inline-flex" }, + accessibility: { + source: "dom-hint", + roleHint: "button", + nameHint: "Save", + attributes: {}, + }, + reactComponentStack: ["SaveButton"], + }, + region: null, + screenshot: { + dataUrl: "data:image/png;base64,aQ==", + pixelSize: { width: 1600, height: 1200 }, + deviceScaleFactor: 2, + pageZoom: 1, + cssToImageScale: { x: 2, y: 2 }, + }, +}; + +const regionCapture: BrowserSelectionCapture = { + ...capture, + kind: "region", + rect: { x: 280, y: 160, width: 360, height: 220 }, + element: null, + region: { + commonAncestor: { + kind: "element", + absoluteLocator: { selectors: ["main > section.settings"] }, + }, + targets: [ + { + absoluteLocator: { + selectors: ["main > section.settings > button#save"], + }, + relativeLocator: { selectors: [":scope > button#save"] }, + text: "Save", + rect: { x: 300, y: 180, width: 180, height: 36 }, + accessibility: { + source: "dom-hint", + roleHint: "button", + nameHint: "Save", + attributes: {}, + }, + }, + ], + groups: [], + omittedTargetCount: 0, + omittedGroupCount: 0, + scanTruncated: false, + }, +}; + +const browserTarget = { + clientId: "client:one", + windowId: "window:one", + tabId: "browser:one", + navigationEpoch: 2, +}; + +function preparedBatch(count: number) { + return { + mentions: Array.from({ length: count }, (_, index) => ({ + id: `00000000-0000-4000-8000-${String(index + 1).padStart(12, "0")}`, + label: "Save", + preview: index === 0 ? "Button on Settings" : "Page · Settings", + })), + }; +} + +type CreateCaptureMentionsHandler = PluginRpcTestHandlers< + typeof rpcContract +>["createCaptureMentions"]; + +function rpcHandlers( + createCaptureMentions: CreateCaptureMentionsHandler = vi.fn(async () => + preparedBatch(1), + ), + overrides: Partial> = {}, +): PluginRpcTestHandlers { + let agentControlEnabled = false; + return { + getAgentControlMode: vi.fn(async () => ({ + enabled: agentControlEnabled, + })), + setAgentControlMode: vi.fn(async ({ enabled }) => { + agentControlEnabled = enabled; + return { enabled }; + }), + prepareCapture: vi.fn(async () => ({ + promptText: "> Legacy context", + })), + prepareCaptures: vi.fn(async () => ({ promptText: "> Legacy context" })), + createCaptureMentions, + ...overrides, + }; +} + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +async function loadAction() { + const app = await loadPluginApp(() => import("./app.js")); + const registration = app.browserActions.find(({ id }) => id === "capture"); + if (registration === undefined) throw new Error("Browser action missing"); + return registration; +} + +async function loadAgentControlAction() { + const app = await loadPluginApp(() => import("./app.js")); + const registration = app.browserActions.find( + ({ id }) => id === "agent-control", + ); + if (registration === undefined) { + throw new Error("Agent-control Browser action missing"); + } + return registration; +} + +type BrowserActionTestProps = PluginBrowserActionProps & { + experimental_overlayRoot: HTMLElement | null; +}; + +function pageSelectionResult(value: BrowserSelectionCapture): JsonValue { + const { screenshot, ...selection } = value; + return { + ...selection, + deviceScaleFactor: screenshot.deviceScaleFactor, + } as unknown as JsonValue; +} + +function selectionRunner(...values: Array) { + let index = 0; + return vi.fn( + async (request) => { + if (request.world === "main") { + return { + navigationEpoch: 2, + value: { element: null, targets: [] } as JsonValue, + }; + } + if (request.source.includes("pageSelectionOverlayCleanup")) { + return { + navigationEpoch: 2, + value: { removed: 1 } as JsonValue, + }; + } + const value = values[index++]; + if (value instanceof Error) throw value; + if (value === undefined) throw new Error("No selection result queued"); + return { + navigationEpoch: 2, + value: pageSelectionResult(value), + }; + }, + ); +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function actionProps( + overrides: Partial = {}, +): BrowserActionTestProps { + return { + tabId: "browser:one", + threadId: "thr_1", + projectId: "prj_1", + url: capture.page.url, + experimental_browserTarget: browserTarget, + experimental_pageContentScriptsAvailable: true, + experimental_runPageContentScript: selectionRunner(capture), + experimental_capturePage: vi.fn(async () => ({ + navigationEpoch: 2, + dataUrl: capture.screenshot.dataUrl, + pixelSize: capture.screenshot.pixelSize, + })), + experimental_setOverlayOpen: vi.fn(), + experimental_overlayRoot: document.body, + ...overrides, + }; +} + +describe("Browser agent-control action", () => { + it("enters from the toolbar, shows the active state, and exits from the same control", async () => { + const registration = await loadAgentControlAction(); + const runPageContentScript = vi.fn( + async ( + request: Parameters< + PluginBrowserActionProps["experimental_runPageContentScript"] + >[0], + ) => ({ + navigationEpoch: 2, + value: request.input ?? null, + }), + ); + const setAgentControlMode: PluginRpcTestHandlers< + typeof rpcContract + >["setAgentControlMode"] = vi.fn(async ({ enabled }) => ({ enabled })); + const slot = renderSlot( + registration, + actionProps({ experimental_runPageContentScript: runPageContentScript }), + { + rpc: rpcHandlers(undefined, { setAgentControlMode }), + }, + ); + + const enable = await screen.findByRole("button", { + name: "Enable agent control", + }); + await waitFor(() => + expect(enable.getAttribute("aria-pressed")).toBe("false"), + ); + runPageContentScript.mockClear(); + + fireEvent.click(enable); + const exit = await screen.findByRole("button", { + name: "Exit agent control", + }); + expect(exit.getAttribute("aria-pressed")).toBe("true"); + expect(exit.querySelector('[data-icon="AiBrowser"]')).not.toBeNull(); + expect(setAgentControlMode).toHaveBeenCalledWith({ + threadId: "thr_1", + projectId: "prj_1", + target: browserTarget, + enabled: true, + }); + await waitFor(() => + expect(runPageContentScript).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ enabled: true }), + timeoutMs: 2_000, + }), + { signal: expect.any(AbortSignal) }, + ), + ); + + fireEvent.focus(exit); + expect((await screen.findByRole("tooltip")).textContent).toBe( + "Exit agent control", + ); + fireEvent.click(exit); + await screen.findByRole("button", { name: "Enable agent control" }); + expect(setAgentControlMode).toHaveBeenLastCalledWith( + expect.objectContaining({ enabled: false }), + ); + await waitFor(() => + expect(runPageContentScript).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ enabled: false }), + }), + { signal: expect.any(AbortSignal) }, + ), + ); + + slot.lifecycle.unmount(); + }); + + it("reflects agent-entered mode through realtime and keeps unrelated tabs unchanged", async () => { + const registration = await loadAgentControlAction(); + const runPageContentScript = vi.fn( + async ( + request: Parameters< + PluginBrowserActionProps["experimental_runPageContentScript"] + >[0], + ) => ({ navigationEpoch: 2, value: request.input ?? null }), + ); + const slot = renderSlot( + registration, + actionProps({ experimental_runPageContentScript: runPageContentScript }), + { rpc: rpcHandlers() }, + ); + await screen.findByRole("button", { name: "Enable agent control" }); + runPageContentScript.mockClear(); + + await slot.behavior.emitRealtime("agent-control-mode", { + enabled: true, + projectId: "prj_1", + source: "agent", + target: { ...browserTarget, windowId: "window:other" }, + threadId: "thr_1", + }); + expect( + screen.getByRole("button", { name: "Enable agent control" }), + ).toBeDefined(); + + await slot.behavior.emitRealtime("agent-control-mode", { + enabled: true, + projectId: "prj_1", + source: "agent", + target: browserTarget, + threadId: "thr_1", + }); + await screen.findByRole("button", { name: "Exit agent control" }); + await waitFor(() => + expect(runPageContentScript).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ enabled: true }), + }), + { signal: expect.any(AbortSignal) }, + ), + ); + + slot.lifecycle.unmount(); + }); + + it("does not let stale hydration or toggle responses overwrite realtime state", async () => { + const registration = await loadAgentControlAction(); + const hydration = deferred<{ enabled: boolean }>(); + const toggle = deferred<{ enabled: boolean }>(); + const getAgentControlMode: PluginRpcTestHandlers< + typeof rpcContract + >["getAgentControlMode"] = vi.fn(() => hydration.promise); + const setAgentControlMode: PluginRpcTestHandlers< + typeof rpcContract + >["setAgentControlMode"] = vi.fn(() => toggle.promise); + const slot = renderSlot(registration, actionProps(), { + rpc: rpcHandlers(undefined, { + getAgentControlMode, + setAgentControlMode, + }), + }); + await waitFor(() => expect(getAgentControlMode).toHaveBeenCalledOnce()); + + await slot.behavior.emitRealtime("agent-control-mode", { + enabled: true, + projectId: "prj_1", + source: "agent", + target: browserTarget, + threadId: "thr_1", + }); + await screen.findByRole("button", { name: "Exit agent control" }); + hydration.resolve({ enabled: false }); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Exit agent control" }), + ).toBeDefined(), + ); + + fireEvent.click(screen.getByRole("button", { name: "Exit agent control" })); + await waitFor(() => expect(setAgentControlMode).toHaveBeenCalledOnce()); + await slot.behavior.emitRealtime("agent-control-mode", { + enabled: true, + projectId: "prj_1", + source: "agent", + target: browserTarget, + threadId: "thr_1", + }); + toggle.resolve({ enabled: false }); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Exit agent control" }), + ).toBeDefined(), + ); + slot.lifecycle.unmount(); + }); + + it("reconciles missed agent state after realtime reconnects", async () => { + const registration = await loadAgentControlAction(); + const getAgentControlMode: PluginRpcTestHandlers< + typeof rpcContract + >["getAgentControlMode"] = vi + .fn() + .mockResolvedValueOnce({ enabled: false }) + .mockResolvedValueOnce({ enabled: true }); + const slot = renderSlot(registration, actionProps(), { + realtimeConnectionState: "connected", + rpc: rpcHandlers(undefined, { getAgentControlMode }), + }); + await screen.findByRole("button", { name: "Enable agent control" }); + await waitFor(() => expect(getAgentControlMode).toHaveBeenCalledOnce()); + + await slot.behavior.setRealtimeConnectionState("reconnecting"); + await slot.behavior.setRealtimeConnectionState("connected"); + + await waitFor(() => expect(getAgentControlMode).toHaveBeenCalledTimes(2)); + await screen.findByRole("button", { name: "Exit agent control" }); + slot.lifecycle.unmount(); + }); + + it("does not surface inactive-frame errors for blank tabs or navigation invalidation", async () => { + const registration = await loadAgentControlAction(); + const blankRunner = vi.fn< + PluginBrowserActionProps["experimental_runPageContentScript"] + >(); + const blankSlot = renderSlot( + registration, + actionProps({ + url: "", + experimental_runPageContentScript: blankRunner, + }), + { rpc: rpcHandlers() }, + ); + await screen.findByRole("button", { name: "Enable agent control" }); + await waitFor(() => expect(blankRunner).not.toHaveBeenCalled()); + expect(screen.queryByRole("status")).toBeNull(); + blankSlot.lifecycle.unmount(); + + const navigationRunner = vi.fn< + PluginBrowserActionProps["experimental_runPageContentScript"] + >(async () => { + throw new Error("The Browser page changed before the script ran"); + }); + const navigationSlot = renderSlot( + registration, + actionProps({ experimental_runPageContentScript: navigationRunner }), + { rpc: rpcHandlers() }, + ); + await waitFor(() => expect(navigationRunner).toHaveBeenCalledOnce()); + expect(screen.queryByRole("status")).toBeNull(); + navigationSlot.lifecycle.unmount(); + }); +}); + +describe("Browser Context action", () => { + it("uses the magic-selection glyph for the Browser toolbar action", async () => { + const registration = await loadAction(); + const slot = renderSlot(registration, actionProps(), { + rpc: rpcHandlers(), + }); + + const action = screen.getByRole("button", { + name: "Select page context", + }); + expect( + action.querySelector('[data-icon="CursorMagicSelection04"]'), + ).not.toBeNull(); + expect(action.getAttribute("title")).toBeNull(); + fireEvent.pointerMove(action); + expect((await screen.findByRole("tooltip")).textContent).toBe( + "Select page context", + ); + + slot.lifecycle.unmount(); + }, 15_000); + + it("quietly exits selection when the page runtime inactivity limit expires", async () => { + const registration = await loadAction(); + const timedOut = selectionRunner( + new Error( + "Error invoking remote method 'bb-desktop:browser:experimental-run-page-script': Error: Browser page script timed out", + ), + ); + renderSlot( + registration, + actionProps({ experimental_runPageContentScript: timedOut }), + { rpc: rpcHandlers() }, + ); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + expect( + screen.getByRole("button", { name: "Cancel page selection" }), + ).toBeDefined(); + await waitFor(() => + expect( + screen.getByRole("button", { name: "Select page context" }), + ).toBeDefined(), + ); + expect(screen.queryByRole("status")).toBeNull(); + expect( + screen.queryByRole("region", { name: "Browser context preview" }), + ).toBeNull(); + }); + + it("stages one clicked element while preserving the editable composer draft", async () => { + const registration = await loadAction(); + const props = actionProps(); + const prepared = preparedBatch(1); + const createCaptureMentions = vi.fn(async () => prepared); + const slot = renderSlot(registration, props, { + rpc: rpcHandlers(createCaptureMentions), + composer: { text: "Make this clearer" }, + }); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + expect(props.experimental_runPageContentScript).toHaveBeenCalledWith( + expect.objectContaining({ + source: expect.stringContaining("pageSelectionController"), + input: { overlayId: expect.stringMatching(/^capture-/u) }, + timeoutMs: 60_000, + }), + { signal: expect.any(AbortSignal) }, + ); + + await screen.findByRole("region", { name: "Browser context preview" }); + expect(props.experimental_capturePage).toHaveBeenCalledWith({ + format: "png", + expectedNavigationEpoch: 2, + }); + const capturePage = vi.mocked(props.experimental_capturePage); + const runPageContentScript = vi.mocked( + props.experimental_runPageContentScript, + ); + const captureOrder = capturePage.mock.invocationCallOrder[0]; + const cleanupCall = runPageContentScript.mock.calls.findIndex(([request]) => + request.source.includes("pageSelectionOverlayCleanup"), + ); + expect(cleanupCall).toBeGreaterThanOrEqual(0); + expect( + runPageContentScript.mock.invocationCallOrder[cleanupCall], + ).toBeGreaterThan(captureOrder ?? 0); + expect(screen.queryByRole("button", { name: "Retake" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Add selection" })).toBeNull(); + const addSelection = screen.getByRole("button", { + name: "Add page selection", + }); + expect( + addSelection.querySelector('[data-icon="CursorMagicSelection04"]'), + ).not.toBeNull(); + expect(addSelection.getAttribute("aria-pressed")).toBe("false"); + const cancel = screen.getByRole("button", { name: "Cancel annotation" }); + expect(cancel.getAttribute("title")).toBe("Cancel annotation"); + expect(props.experimental_setOverlayOpen).toHaveBeenCalledWith(true); + + const emptySubmit = screen.getByRole("button", { + name: "Submit comment", + }); + expect(emptySubmit.textContent).toBe("Enter"); + expect(emptySubmit.querySelector("kbd")?.textContent).toBe("Enter"); + expect((emptySubmit as HTMLButtonElement).disabled).toBe(true); + + fireEvent.change(screen.getByLabelText("Comment for selection 1"), { + target: { value: "Make this action more prominent" }, + }); + const submit = screen.getByRole("button", { name: "Submit comment" }); + expect(submit.textContent).toBe("Enter"); + expect(submit.querySelector("kbd")?.textContent).toBe("Enter"); + expect((submit as HTMLButtonElement).disabled).toBe(false); + expect(submit.getAttribute("title")).toBeNull(); + expect(submit.querySelector('[data-icon="UploadCircle01"]')).not.toBeNull(); + const actionRow = submit.parentElement; + expect(actionRow?.className).toContain("bb-browser-context-review-actions"); + expect( + actionRow?.querySelector('button[aria-label="Add selection"]'), + ).toBeNull(); + expect( + actionRow?.querySelector(".bb-browser-context-primary")?.textContent, + ).toContain("Send to agent"); + const comment = screen.getByLabelText("Comment for selection 1"); + submit.focus(); + fireEvent.click(submit); + await waitFor(() => expect(document.activeElement).toBe(comment)); + expect((comment as HTMLTextAreaElement).value).toBe(""); + const freshSubmit = screen.getByRole("button", { + name: "Submit comment", + }); + expect( + freshSubmit.querySelector('[data-icon="UploadCircle01"]'), + ).not.toBeNull(); + expect(freshSubmit.textContent).toBe("Enter"); + expect((freshSubmit as HTMLButtonElement).disabled).toBe(true); + expect( + screen.getByRole("list", { + name: "Submitted comments for selection 1", + }).textContent, + ).toContain("Make this action more prominent"); + expect( + screen.queryByRole("button", { name: "Comment submitted" }), + ).toBeNull(); + fireEvent.change(comment, { + target: { value: "Keep the label concise" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Submit comment" })); + expect( + screen + .getByRole("list", { + name: "Submitted comments for selection 1", + }) + .querySelectorAll("li"), + ).toHaveLength(2); + expect(createCaptureMentions).not.toHaveBeenCalled(); + expect(slot.inspection.composer.text).toBe("Make this clearer"); + const target = screen.getByRole("button", { + name: "Selection 1: button#save", + }); + fireEvent.mouseEnter(target); + expect(target.textContent).toContain("2 comments · Keep the label concise"); + + fireEvent.click(screen.getByRole("button", { name: "Send to agent" })); + await waitFor(() => expect(createCaptureMentions).toHaveBeenCalledOnce()); + expect(createCaptureMentions).toHaveBeenCalledWith({ + threadId: "thr_1", + projectId: "prj_1", + annotations: [ + { + comment: "Make this action more prominent\nKeep the label concise", + comments: [ + "Make this action more prominent", + "Keep the label concise", + ], + capture, + }, + ], + }); + expect(slot.inspection.composer.text).toBe( + "Make this clearer\n\nSave Make this action more prominent\nKeep the label concise", + ); + expect(slot.inspection.composer.mentions).toEqual([ + { + provider: "captures", + id: prepared.mentions[0]!.id, + label: prepared.mentions[0]!.label, + experimental_preview: prepared.mentions[0]!.preview, + experimental_inspectable: true, + }, + ]); + expect(slot.inspection.composer.attachmentCount).toBe(0); + expect(slot.inspection.composer.focusCount).toBe(2); + expect( + screen.queryByRole("region", { name: "Browser context preview" }), + ).toBeNull(); + + await slot.behavior.setComposerText("Make this clearer and more compact"); + expect(slot.inspection.composer.text).toBe( + "Make this clearer and more compact", + ); + expect(slot.inspection.composer.attachmentCount).toBe(0); + }); + + it("keeps a fresh composer on the same selection after repeated Enter submissions until dismissed", async () => { + const registration = await loadAction(); + const createCaptureMentions = vi.fn(async () => preparedBatch(1)); + const slot = renderSlot(registration, actionProps(), { + rpc: rpcHandlers(createCaptureMentions), + }); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + const comment = await screen.findByLabelText("Comment for selection 1"); + fireEvent.change(comment, { target: { value: "Tighten this action" } }); + fireEvent.keyDown(comment, { key: "Enter" }); + + expect( + screen.getByRole("button", { name: "Submit comment" }), + ).toHaveProperty("disabled", true); + expect((comment as HTMLTextAreaElement).value).toBe(""); + expect( + screen.getByRole("list", { + name: "Submitted comments for selection 1", + }).textContent, + ).toContain("Tighten this action"); + expect(createCaptureMentions).not.toHaveBeenCalled(); + expect(slot.inspection.composer.text).toBe(""); + expect( + screen.getByRole("region", { name: "Browser context preview" }), + ).toBeDefined(); + + fireEvent.change(comment, { target: { value: "Keep the label" } }); + expect( + screen.getByRole("button", { name: "Submit comment" }), + ).toBeDefined(); + fireEvent.keyDown(comment, { key: "Enter", shiftKey: true }); + expect((comment as HTMLTextAreaElement).value).toBe("Keep the label"); + fireEvent.keyDown(comment, { key: "Enter" }); + expect((comment as HTMLTextAreaElement).value).toBe(""); + const submittedComments = screen.getByRole("list", { + name: "Submitted comments for selection 1", + }); + expect(submittedComments.querySelectorAll("li")).toHaveLength(2); + expect(submittedComments.textContent).toContain("Tighten this action"); + expect(submittedComments.textContent).toContain("Keep the label"); + expect(createCaptureMentions).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole("button", { name: "Cancel annotation" })); + expect( + screen.queryByRole("region", { name: "Browser context preview" }), + ).toBeNull(); + }); + + it("keeps click and drag comments attached across numbered multi-selection staging", async () => { + const registration = await loadAction(); + const inspect = selectionRunner(capture, regionCapture); + const props = actionProps({ experimental_runPageContentScript: inspect }); + const prepared = preparedBatch(2); + const createCaptureMentions = vi.fn(async () => prepared); + const slot = renderSlot(registration, props, { + rpc: rpcHandlers(createCaptureMentions), + }); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("button", { name: "Selection 1: button#save" }); + fireEvent.change(screen.getByLabelText("Comment for selection 1"), { + target: { value: "Make the save action prominent" }, + }); + + const addSelection = screen.getByRole("button", { + name: "Add page selection", + }); + expect(screen.queryByRole("button", { name: "Add selection" })).toBeNull(); + expect( + addSelection.querySelector('[data-icon="CursorMagicSelection04"]'), + ).not.toBeNull(); + fireEvent.pointerMove(addSelection); + expect((await screen.findByRole("tooltip")).textContent).toBe( + "Add page selection", + ); + fireEvent.click(addSelection); + await waitFor(() => + expect( + inspect.mock.calls.filter(([request]) => + request.source.includes("pageSelectionController"), + ), + ).toHaveLength(2), + ); + await screen.findByRole("button", { + name: "Selection 2: 1 target in region", + }); + fireEvent.change(screen.getByLabelText("Comment for selection 2"), { + target: { value: "Reduce spacing in this group" }, + }); + + const firstRow = screen.getByRole("button", { + name: "Edit selection 1: button#save", + }); + fireEvent.mouseEnter(firstRow); + expect(screen.getByLabelText("Comment for selection 1")).toHaveProperty( + "value", + "Make the save action prominent", + ); + expect( + screen + .getByRole("button", { name: "Selection 1: button#save" }) + .getAttribute("data-active"), + ).toBe("true"); + + fireEvent.click(screen.getByRole("button", { name: "Send to agent" })); + await waitFor(() => expect(createCaptureMentions).toHaveBeenCalledOnce()); + expect(createCaptureMentions).toHaveBeenCalledWith({ + threadId: "thr_1", + projectId: "prj_1", + annotations: [ + { + comment: "Make the save action prominent", + comments: ["Make the save action prominent"], + capture, + }, + { + comment: "Reduce spacing in this group", + comments: ["Reduce spacing in this group"], + capture: regionCapture, + }, + ], + }); + expect(slot.inspection.composer.attachmentCount).toBe(0); + expect(slot.inspection.composer.text).toBe( + "Save Make the save action prominent\nSave Reduce spacing in this group", + ); + expect(slot.inspection.composer.mentions).toHaveLength(2); + expect(slot.inspection.composer.mentions[0]?.id).not.toBe( + slot.inspection.composer.mentions[1]?.id, + ); + }); + + it("restores the existing annotation batch when selecting another target fails", async () => { + const registration = await loadAction(); + const inspect = selectionRunner(capture, new Error("Selection failed")); + const props = actionProps({ experimental_runPageContentScript: inspect }); + renderSlot(registration, props, { rpc: rpcHandlers() }); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("button", { name: "Selection 1: button#save" }); + fireEvent.change(screen.getByLabelText("Comment for selection 1"), { + target: { value: "Keep this comment" }, + }); + + fireEvent.click(screen.getByRole("button", { name: "Add page selection" })); + await screen.findByText("Selection failed"); + + expect( + screen.getByRole("region", { name: "Browser context preview" }), + ).toBeDefined(); + expect(screen.getByLabelText("Comment for selection 1")).toHaveProperty( + "value", + "Keep this comment", + ); + expect(props.experimental_setOverlayOpen).toHaveBeenLastCalledWith(true); + }); + + it("removes one staged selection without disturbing the remaining annotation", async () => { + const registration = await loadAction(); + const inspect = selectionRunner(capture, regionCapture); + const createCaptureMentions = vi.fn(async () => preparedBatch(1)); + renderSlot( + registration, + actionProps({ experimental_runPageContentScript: inspect }), + { rpc: rpcHandlers(createCaptureMentions) }, + ); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("button", { name: "Selection 1: button#save" }); + fireEvent.change(screen.getByLabelText("Comment for selection 1"), { + target: { value: "Keep this comment" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Add page selection" })); + await screen.findByRole("button", { + name: "Selection 2: 1 target in region", + }); + + fireEvent.click(screen.getByRole("button", { name: "Remove selection 2" })); + expect( + screen.queryByRole("button", { name: "Remove selection 2" }), + ).toBeNull(); + expect(screen.getByLabelText("Comment for selection 1")).toHaveProperty( + "value", + "Keep this comment", + ); + fireEvent.click(screen.getByRole("button", { name: "Send to agent" })); + await waitFor(() => expect(createCaptureMentions).toHaveBeenCalledOnce()); + expect(createCaptureMentions).toHaveBeenCalledWith({ + threadId: "thr_1", + projectId: "prj_1", + annotations: [ + { + comment: "Keep this comment", + comments: ["Keep this comment"], + capture, + }, + ], + }); + }); + + it("exits active selection and the completed annotation session without staging", async () => { + const registration = await loadAction(); + let observedSignal: AbortSignal | null = null; + const inspect = vi.fn( + async (_request: unknown, options: { signal: AbortSignal }) => { + observedSignal = options.signal; + return await new Promise<{ navigationEpoch: number; value: null }>( + (resolve) => { + options.signal.addEventListener( + "abort", + () => resolve({ navigationEpoch: 2, value: null }), + { once: true }, + ); + }, + ); + }, + ); + const selectingSlot = renderSlot( + registration, + actionProps({ experimental_runPageContentScript: inspect }), + { rpc: rpcHandlers() }, + ); + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + const cancelSelection = screen.getByRole("button", { + name: "Cancel page selection", + }); + expect(cancelSelection.getAttribute("aria-pressed")).toBe("true"); + expect( + cancelSelection.querySelector('[data-icon="CursorMagicSelection04"]'), + ).not.toBeNull(); + fireEvent.click(cancelSelection); + await waitFor(() => expect(observedSignal?.aborted).toBe(true)); + selectingSlot.lifecycle.unmount(); + + const previewProps = actionProps(); + const previewSlot = renderSlot(registration, previewProps, { + rpc: rpcHandlers(), + }); + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("region", { name: "Browser context preview" }); + fireEvent.click(screen.getByRole("button", { name: "Cancel annotation" })); + expect(previewProps.experimental_setOverlayOpen).toHaveBeenLastCalledWith( + false, + ); + expect(previewSlot.inspection.composer.attachmentCount).toBe(0); + }); + + it("moves the compact annotation panel without changing the selection", async () => { + const registration = await loadAction(); + renderSlot(registration, actionProps(), { rpc: rpcHandlers() }); + + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + const target = await screen.findByRole("button", { + name: "Selection 1: button#save", + }); + const review = screen.getByRole("region", { + name: "Browser context preview", + }); + const panel = review.querySelector("aside"); + if (!(panel instanceof HTMLElement)) throw new Error("Panel missing"); + vi.spyOn(review, "getBoundingClientRect").mockReturnValue({ + bottom: 600, + height: 600, + left: 0, + right: 800, + top: 0, + width: 800, + x: 0, + y: 0, + toJSON: () => ({}), + }); + vi.spyOn(panel, "getBoundingClientRect").mockReturnValue({ + bottom: 210, + height: 200, + left: 490, + right: 790, + top: 10, + width: 300, + x: 490, + y: 10, + toJSON: () => ({}), + }); + Object.defineProperties(panel, { + offsetHeight: { configurable: true, value: 200 }, + offsetWidth: { configurable: true, value: 300 }, + }); + const handle = screen.getByTitle("Drag annotation"); + fireEvent.pointerDown(handle, { + button: 0, + clientX: 510, + clientY: 30, + pointerId: 7, + }); + fireEvent.pointerMove(handle, { + clientX: 220, + clientY: 240, + pointerId: 7, + }); + fireEvent.pointerUp(handle, { pointerId: 7 }); + + expect(panel.style.left).toBe("200px"); + expect(panel.style.top).toBe("220px"); + expect(target).toBeDefined(); + }); + + it("does not stage a batch after the action unmounts during preparation", async () => { + const registration = await loadAction(); + const prepared = preparedBatch(1); + let finish: ((value: typeof prepared) => void) | undefined; + const slot = renderSlot(registration, actionProps(), { + rpc: rpcHandlers( + () => + new Promise((resolve) => { + finish = resolve; + }), + ), + }); + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("region", { name: "Browser context preview" }); + fireEvent.click(screen.getByRole("button", { name: "Send to agent" })); + await waitFor(() => expect(finish).toBeDefined()); + slot.lifecycle.unmount(); + finish?.(prepared); + await Promise.resolve(); + + expect(slot.inspection.composer.attachmentCount).toBe(0); + expect(slot.inspection.composer.text).toBe(""); + }); + + it("shows preparation errors in place and disables unsupported hosts", async () => { + const registration = await loadAction(); + const slot = renderSlot(registration, actionProps(), { + rpc: rpcHandlers(async () => { + throw new Error("Thread is temporarily unavailable"); + }), + }); + fireEvent.click( + screen.getByRole("button", { name: "Select page context" }), + ); + await screen.findByRole("region", { name: "Browser context preview" }); + fireEvent.click(screen.getByRole("button", { name: "Send to agent" })); + expect((await screen.findByRole("alert")).textContent).toContain( + "Thread is temporarily unavailable", + ); + expect(slot.inspection.composer.attachmentCount).toBe(0); + slot.lifecycle.unmount(); + + renderSlot(registration, actionProps({ experimental_overlayRoot: null }), { + rpc: rpcHandlers(), + }); + const unsupported = screen.getByRole("button", { + name: "Select page context", + }); + expect((unsupported as HTMLButtonElement).disabled).toBe(true); + expect(unsupported.getAttribute("title")).toBeNull(); + fireEvent.focus( + screen.getByLabelText( + "Browser annotations require a newer BB desktop app.", + ), + ); + expect((await screen.findByRole("tooltip")).textContent).toBe( + "Browser annotations require a newer BB desktop app.", + ); + }); +}); diff --git a/plugins/browser-context/app.tsx b/plugins/browser-context/app.tsx new file mode 100644 index 0000000..39dd4f5 --- /dev/null +++ b/plugins/browser-context/app.tsx @@ -0,0 +1,1187 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type CSSProperties, + type KeyboardEvent as ReactKeyboardEvent, + type PointerEvent as ReactPointerEvent, +} from "react"; +import { createPortal } from "react-dom"; +import { + AiBrowserIcon, + CursorMagicSelection04Icon, + UploadCircle01Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import * as TooltipPrimitive from "@radix-ui/react-tooltip"; +import { + definePluginApp, + useComposer, + useRealtime, + useRealtimeConnectionState, + useRpc, + type PluginAppSlots, + type PluginBrowserActionProps, +} from "@get-bb/plugin-sdk/app"; + +import type { rpcContract } from "./server.js"; +import { + selectBrowserContext, + type BrowserSelectionCapture, +} from "./lib/browser-selection.js"; +import { setBrowserAgentControlFrame } from "./lib/agent-control-mode.js"; +import "./app.css"; + +interface BrowserContextCompatibleSlots { + experimental_browserAction?: PluginAppSlots["experimental_browserAction"]; +} + +const AGENT_CONTROL_MODE_CHANNEL = "agent-control-mode"; + +type AgentControlTarget = + PluginBrowserActionProps["experimental_browserTarget"]; + +interface AgentControlModeSignal { + enabled: boolean; + projectId: string; + source: "agent" | "user"; + target: AgentControlTarget; + threadId: string; +} + +function agentControlTarget(value: unknown): AgentControlTarget | null { + if (typeof value !== "object" || value === null) return null; + const target = value as Record; + if ( + typeof target.clientId !== "string" || + typeof target.windowId !== "string" || + typeof target.tabId !== "string" || + typeof target.navigationEpoch !== "number" || + !Number.isInteger(target.navigationEpoch) || + target.navigationEpoch < 0 + ) { + return null; + } + return target as unknown as AgentControlTarget; +} + +function sameAgentControlTarget( + left: AgentControlTarget, + right: AgentControlTarget, +): boolean { + return ( + left.clientId === right.clientId && + left.windowId === right.windowId && + left.tabId === right.tabId && + left.navigationEpoch === right.navigationEpoch + ); +} + +function agentControlModeSignal( + payload: unknown, +): AgentControlModeSignal | null { + if (typeof payload !== "object" || payload === null) return null; + const value = payload as Record; + const target = agentControlTarget(value.target); + if ( + typeof value.enabled !== "boolean" || + typeof value.projectId !== "string" || + (value.source !== "agent" && value.source !== "user") || + typeof value.threadId !== "string" || + target === null + ) { + return null; + } + return { ...value, target } as AgentControlModeSignal; +} + +function errorMessage(error: unknown): string { + return error instanceof Error + ? error.message + : "Browser Context could not complete that request."; +} + +function isSelectionInactivityTimeout(error: unknown): boolean { + return /browser page script timed out/iu.test(errorMessage(error)); +} + +function CloseIcon() { + return ( + + ); +} + +function DragHandleIcon() { + return ( + + ); +} + +function RemoveIcon() { + return ( + + ); +} + +function cloneCapture(capture: BrowserSelectionCapture) { + const cloneLocator = (locator: { selectors: readonly string[] }) => ({ + selectors: [...locator.selectors], + }); + return { + ...capture, + element: + capture.element === null + ? null + : { + ...capture.element, + classNames: [...capture.element.classNames], + reactComponentStack: + capture.element.reactComponentStack === null + ? null + : [...capture.element.reactComponentStack], + }, + region: + capture.region === null + ? null + : { + commonAncestor: + capture.region.commonAncestor === null + ? null + : { + ...capture.region.commonAncestor, + absoluteLocator: cloneLocator( + capture.region.commonAncestor.absoluteLocator, + ), + }, + targets: capture.region.targets.map((target) => ({ + absoluteLocator: cloneLocator(target.absoluteLocator), + relativeLocator: cloneLocator(target.relativeLocator), + text: target.text, + rect: { ...target.rect }, + ...(target.accessibility === undefined + ? {} + : { + accessibility: { + ...target.accessibility, + attributes: { ...target.accessibility.attributes }, + }, + }), + ...(target.react === undefined + ? {} + : { + react: { + componentStack: [...target.react.componentStack], + ...(target.react.source === undefined + ? {} + : { source: { ...target.react.source } }), + }, + }), + })), + groups: capture.region.groups.map((group) => ({ + absoluteLocator: cloneLocator(group.absoluteLocator), + relativeLocator: cloneLocator(group.relativeLocator), + count: group.count, + rect: { ...group.rect }, + })), + omittedTargetCount: capture.region.omittedTargetCount, + omittedGroupCount: capture.region.omittedGroupCount, + scanTruncated: capture.region.scanTruncated, + }, + }; +} + +function percent(value: number, total: number): string { + if (total <= 0) return "0%"; + return `${Math.max(0, Math.min(100, (value / total) * 100))}%`; +} + +interface PendingAnnotation { + id: number; + capture: BrowserSelectionCapture; + comment: string; + submittedComments: string[]; +} + +const MAX_ANNOTATION_COMMENT_LENGTH = 4_000; + +function annotationComment(annotation: PendingAnnotation): string { + return annotationComments(annotation).join("\n"); +} + +function annotationComments(annotation: PendingAnnotation): string[] { + return [...annotation.submittedComments, annotation.comment.trim()].filter( + (comment) => comment.length > 0, + ); +} + +function annotationCommentSummary(annotation: PendingAnnotation): string { + const draft = annotation.comment.trim(); + if (draft.length > 0) return draft; + const latest = annotation.submittedComments.at(-1); + if (latest === undefined) return "No comment yet"; + return annotation.submittedComments.length === 1 + ? latest + : `${annotation.submittedComments.length} comments · ${latest}`; +} + +function remainingCommentLength(annotation: PendingAnnotation): number { + const submittedLength = annotation.submittedComments.join("\n").length; + const separatorLength = annotation.submittedComments.length > 0 ? 1 : 0; + return Math.max( + 0, + MAX_ANNOTATION_COMMENT_LENGTH - submittedLength - separatorLength, + ); +} + +function captureTargetLabel(capture: BrowserSelectionCapture): string { + const regionTargetCount = + capture.region === null + ? 0 + : capture.region.targets.length + capture.region.omittedTargetCount; + if (capture.kind === "element") { + return `${capture.element?.tag ?? "element"}${capture.element?.id ? `#${capture.element.id}` : ""}`; + } + if (regionTargetCount === 0) return "empty region"; + return `${regionTargetCount} target${regionTargetCount === 1 ? "" : "s"} in region`; +} + +function sharesPreview( + left: BrowserSelectionCapture, + right: BrowserSelectionCapture, +): boolean { + return ( + left.page.url === right.page.url && + left.page.viewport.width === right.page.viewport.width && + left.page.viewport.height === right.page.viewport.height && + left.page.scroll.x === right.page.scroll.x && + left.page.scroll.y === right.page.scroll.y + ); +} + +interface CaptureReviewProps { + annotations: readonly PendingAnnotation[]; + activeId: number; + error: string | null; + staging: boolean; + onAddToPrompt(): void; + onCancel(): void; + onCommentChange(id: number, comment: string): void; + onCommentSubmit(id: number): void; + onRemove(id: number): void; + onSelect(id: number): void; +} + +function CaptureReview({ + annotations, + activeId, + error, + staging, + onAddToPrompt, + onCancel, + onCommentChange, + onCommentSubmit, + onRemove, + onSelect, +}: CaptureReviewProps) { + const [hoveringTargetId, setHoveringTargetId] = useState(null); + const [panelPosition, setPanelPosition] = useState<{ + x: number; + y: number; + } | null>(null); + const reviewRef = useRef(null); + const panelRef = useRef(null); + const commentRef = useRef(null); + const dragRef = useRef<{ + pointerId: number; + offsetX: number; + offsetY: number; + } | null>(null); + const activeIndex = Math.max( + 0, + annotations.findIndex((annotation) => annotation.id === activeId), + ); + const active = annotations[activeIndex] ?? annotations[0]!; + const capture = active.capture; + const viewport = capture.page.viewport; + const targetLabel = captureTargetLabel(capture); + const previewAnnotations = annotations.filter((annotation) => + sharesPreview(annotation.capture, capture), + ); + const commentCardClassName = + capture.rect.x + capture.rect.width / 2 > viewport.width / 2 + ? "bb-browser-context-comment-card bb-browser-context-comment-card-left" + : "bb-browser-context-comment-card"; + const commentCardStyle: CSSProperties | undefined = + panelPosition === null + ? undefined + : { + left: `${panelPosition.x}px`, + right: "auto", + top: `${panelPosition.y}px`, + }; + + const beginPanelDrag = (event: ReactPointerEvent) => { + if (event.button !== 0 || staging) return; + const panel = panelRef.current; + const review = reviewRef.current; + if (panel === null || review === null) return; + const panelRect = panel.getBoundingClientRect(); + dragRef.current = { + pointerId: event.pointerId, + offsetX: event.clientX - panelRect.left, + offsetY: event.clientY - panelRect.top, + }; + event.currentTarget.setPointerCapture?.(event.pointerId); + event.preventDefault(); + }; + + const movePanel = (event: ReactPointerEvent) => { + const drag = dragRef.current; + const panel = panelRef.current; + const review = reviewRef.current; + if ( + drag === null || + drag.pointerId !== event.pointerId || + panel === null || + review === null + ) { + return; + } + const reviewRect = review.getBoundingClientRect(); + const inset = 8; + const maxX = Math.max(inset, reviewRect.width - panel.offsetWidth - inset); + const maxY = Math.max( + inset, + reviewRect.height - panel.offsetHeight - inset, + ); + setPanelPosition({ + x: Math.max( + inset, + Math.min(maxX, event.clientX - reviewRect.left - drag.offsetX), + ), + y: Math.max( + inset, + Math.min(maxY, event.clientY - reviewRect.top - drag.offsetY), + ), + }); + }; + + const endPanelDrag = (event: ReactPointerEvent) => { + if (dragRef.current?.pointerId !== event.pointerId) return; + dragRef.current = null; + if (event.currentTarget.hasPointerCapture?.(event.pointerId) === true) { + event.currentTarget.releasePointerCapture?.(event.pointerId); + } + }; + + const submitActiveComment = () => { + if (staging || active.comment.trim().length === 0) return; + onCommentSubmit(active.id); + queueMicrotask(() => commentRef.current?.focus()); + }; + + const handleCommentKeyDown = ( + event: ReactKeyboardEvent, + ) => { + if ( + event.key !== "Enter" || + event.shiftKey || + event.nativeEvent.isComposing + ) { + return; + } + event.preventDefault(); + submitActiveComment(); + }; + + return ( +
+
+ {`Captured + {previewAnnotations.map((annotation) => { + const annotationIndex = annotations.indexOf(annotation); + const annotationLabel = captureTargetLabel(annotation.capture); + const isActive = annotation.id === active.id; + const rectStyle = { + left: percent(annotation.capture.rect.x, viewport.width), + top: percent(annotation.capture.rect.y, viewport.height), + width: percent(annotation.capture.rect.width, viewport.width), + height: percent(annotation.capture.rect.height, viewport.height), + }; + return ( + + ); + })} +
+ +