diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..607fb21b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +site/public/media/*.mp4 filter=lfs diff=lfs merge=lfs -text +site/public/media/*.webm filter=lfs diff=lfs merge=lfs -text +site/public/media/*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92cf3985..e480e5ec 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,9 +67,11 @@ jobs: - uses: actions/setup-node@v6 with: node-version: '22' + # web/ pins its own pnpm via packageManager; the repo-root package.json + # pins a different pnpm for Vercel, so point the action at web/ explicitly. - uses: pnpm/action-setup@v6 with: - version: 10.34.5 + package_json_file: web/package.json run_install: false - run: pnpm install --frozen-lockfile working-directory: web @@ -88,9 +90,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v6 + # fmt runs `pnpm dlx` at the repo root; take the version from the root + # package.json's packageManager pin instead of double-specifying it. - uses: pnpm/action-setup@v6 with: - version: 10.34.5 run_install: false - uses: actions/setup-go@v7 with: diff --git a/showcase/.gitignore b/showcase/.gitignore index c9680c6e..af5f612b 100644 --- a/showcase/.gitignore +++ b/showcase/.gitignore @@ -1,5 +1,6 @@ # Dependencies node_modules/ +.env.local # All capture/render scratch (raw recordings, screenshots, transcodes). # Published media lives under site/public/media/ (copied there by hand) and diff --git a/showcase/AGENTS.md b/showcase/AGENTS.md index 938f7da3..5e2f3a2a 100644 --- a/showcase/AGENTS.md +++ b/showcase/AGENTS.md @@ -37,7 +37,9 @@ and `go test ./pkg/gen//` from the repo root. 3. `pnpm slacksim:build && pnpm exec vite preview --config vite.slacksim.config.ts --port 5178 --strictPort &` 4. Render: `SIM_PORT=5178 node engine/capture/clip.mjs out/clips/ dark 1600x900` - (and/or `still.mjs …` for a caption-free still). + (and/or `still.mjs …` for a caption-free still). For spoken + narration, put `ELEVENLABS_API_KEY=...` in ignored `.env.local` and run + `node --env-file=.env.local engine/capture/clip.mjs ...`. 5. Copy to `site/public/media/`, add a `site/content/_manifest.json` entry, reference it from a guide (see [`site/AGENTS.md`](../site/AGENTS.md)). diff --git a/showcase/README.md b/showcase/README.md index e4689789..8d42a5bd 100644 --- a/showcase/README.md +++ b/showcase/README.md @@ -26,7 +26,7 @@ showcase/ │ ├── capture/ Playwright drivers: shot.mjs (still), still.mjs (beat still), │ │ clip.mjs (narrated webm+mp4+poster), webshot.mjs (real product login) │ ├── mocks/ authored HTML mocks for surfaces we can't capture (artifact viewer) -│ └── tts/ pluggable TTS seam (stubbed; clips are caption-driven for now) +│ └── tts/ ElevenLabs/OpenAI narration, per-scene cache, and audio/video muxing └── out/ gitignored render scratch (raw videos, throwaway shots) ``` @@ -72,9 +72,15 @@ The engine reads `SIM_PORT` (default 5178; set `SIM_PORT=5180` for consolesim). # a still of a scenario played to beat N (caption-free) SIM_PORT=5178 node engine/capture/still.mjs out/x.png dark 1600x900 -# a narrated clip: plays every beat, records webm+mp4+poster +# a captioned clip: plays every beat, records webm+mp4+poster SIM_PORT=5180 node engine/capture/clip.mjs out/clips/ dark 1120x780 +# with ElevenLabs narration (put ELEVENLABS_API_KEY=... in ignored .env.local) +SIM_PORT=5178 node --env-file=.env.local engine/capture/clip.mjs out/clips/ dark 1280x720 + +# narrate the existing Agent Builder web UI recording without a new live session +node --env-file=.env.local engine/capture/narrate-builder.mjs out/clips/builder-first-agent + # a one-off still of any URL (5th arg is JS run against window.__showcase) node engine/capture/shot.mjs "http://localhost:5178/?scenario=…" out/x.png 1600x900 ``` @@ -82,7 +88,10 @@ node engine/capture/shot.mjs "http://localhost:5178/?scenario=…" out/x.png 160 Then copy the output into `site/public/media/`, add an entry to `site/content/_manifest.json`, and reference it from a page with `` or `` — see -[`site/AGENTS.md`](../site/AGENTS.md). +[`site/AGENTS.md`](../site/AGENTS.md). MP4, WebM, and PNG files in +`site/public/media/` use Git LFS. Run `git lfs pull` after checkout before +building or publishing the site so those media files contain playable assets +rather than LFS pointers. ## Generated content (run from the repo root) diff --git a/showcase/demos/slacksim/src/scenarios/hubspot-companies.ts b/showcase/demos/slacksim/src/scenarios/hubspot-companies.ts new file mode 100644 index 00000000..c4abcb30 --- /dev/null +++ b/showcase/demos/slacksim/src/scenarios/hubspot-companies.ts @@ -0,0 +1,93 @@ +import { newScenario } from "../store/scenario"; +import type { Story } from "../runtime/story"; +import type { Block } from "../blockkit/types"; +import contactShare from "../fixtures/blockkit/contact-share.json"; +import contactShareResolved from "../fixtures/blockkit/contact-share-resolved.json"; + +// Scripted HubSpot example. Company names, people and scores are fabricated; +// the owner-approval card is captured from OAP's real Slack renderer. +const SALES = "C1"; +const DM = "D1"; +const REQUEST = "1756700000.000000"; +const CARD = "1756700300.000000"; +const OWNER = "U_OWNER"; +const SAM = "U6"; +const BOT = "U7"; + +export function hubspotCompanies(): Story { + const s = newScenario() + .workspace("Acme Robotics", { glyph: "A", accent: "#4a154b" }) + .at("2026-08-31T09:00:00Z") + .user(OWNER, "jordan") + .user(SAM, "sam") + .bot(BOT, "hubspot-companies", { badge: "AGENT" }) + .me(OWNER) + .channel(SALES, "sales", { starred: true }) + .dm(DM, [BOT]); + return { + scenario: s.open(SALES).build(), + intro: + "HubSpot CRM: a scheduled company digest, with owner approval for private contacts.", + beats: [ + { + id: "digest", + caption: + "Each week, the agent finds newly created companies with strong fit scores and tags their owners.", + hold: 1000, + run: (c) => + c.postMessage(SALES, BOT, { + text: "*New companies this week*\n• *Circldot* — fit score 91, managed by <@U_OWNER>\n• *Northstar Labs* — fit score 85, managed by <@U_OWNER>\n2 companies met the reporting threshold.", + }), + }, + { + id: "request", + caption: + "Sam asks for Circldot's contacts. Those records belong to Jordan, so the agent pauses.", + hold: 900, + run: (c) => { + c.postMessage(SALES, SAM, { + text: "<@U7> can you pull Circldot's contacts for the QBR?", + ts: REQUEST, + }); + c.openThread(REQUEST); + c.postReply(SALES, REQUEST, BOT, { + text: "Circldot's contacts need approval from <@U_OWNER>, the account owner. I've asked Jordan now.", + }); + c.setStatus(REQUEST, "waiting for Circldot's owner"); + }, + }, + { + id: "owner-approval", + caption: + "The approval goes to Jordan in a DM, with the exact company and data being shared.", + hold: 1100, + run: (c) => { + c.postMessage(DM, BOT, { blocks: contactShare as Block[], ts: CARD }); + c.switchChannel(DM); + }, + }, + { + id: "approve", + caption: + "Jordan approves. The card resolves in place before any contacts appear in the sales thread.", + hold: 900, + run: (c) => + c.editMessage(DM, CARD, { blocks: contactShareResolved as Block[] }), + }, + { + id: "contacts", + caption: + "Now the agent shares Circldot's contacts with Sam, and records whose approval allowed it.", + hold: 1100, + run: (c) => { + c.switchChannel(SALES); + c.openThread(REQUEST); + c.setStatus(REQUEST, undefined); + c.postReply(SALES, REQUEST, BOT, { + text: "Circldot contacts for <@U6>, approved by <@U_OWNER>:\n• Dana Okafor — VP Engineering\n• Priya Raman — Head of Data", + }); + }, + }, + ], + }; +} diff --git a/showcase/demos/slacksim/src/scenarios/index.ts b/showcase/demos/slacksim/src/scenarios/index.ts index b29c9abc..a21ddd3c 100644 --- a/showcase/demos/slacksim/src/scenarios/index.ts +++ b/showcase/demos/slacksim/src/scenarios/index.ts @@ -13,6 +13,9 @@ import { infoLeakageDemo } from "./info-leakage"; import { queuedReply } from "./queued-reply"; import { artifactRevise } from "./artifact-revise"; import { threadAdoption } from "./thread-adoption"; +import { hubspotCompanies } from "./hubspot-companies"; +import { reviewbotDemo } from "./reviewbot-demo"; +import { pmAgentDemo } from "./pm-agent-demo"; // The registry of built-in slacksim stories. The capture engine and the dev // server resolve a story by name (?scenario=). A story is a scenario plus @@ -32,6 +35,9 @@ export const STORIES: Record Story> = { "queued-reply": queuedReply, "artifact-revise": artifactRevise, "thread-adoption": threadAdoption, + "hubspot-companies": hubspotCompanies, + "reviewbot-demo": reviewbotDemo, + "pm-agent-demo": pmAgentDemo, }; export const DEFAULT_SCENARIO = "reviewbot-pr"; diff --git a/showcase/demos/slacksim/src/scenarios/pm-agent-demo.ts b/showcase/demos/slacksim/src/scenarios/pm-agent-demo.ts new file mode 100644 index 00000000..e23c522a --- /dev/null +++ b/showcase/demos/slacksim/src/scenarios/pm-agent-demo.ts @@ -0,0 +1,53 @@ +import { newScenario } from "../store/scenario"; +import type { Story } from "../runtime/story"; + +// Scripted product-manager example. The agent reads fabricated GitHub and +// Linear records, then reports alignment without writing to either service. +const PRODUCT = "C1"; +const BOT = "U2"; +const ROOT = "1756900000.000000"; + +export function pmAgentDemo(): Story { + const s = newScenario() + .workspace("Acme Robotics", { glyph: "A", accent: "#4a154b" }) + .at("2026-09-02T15:00:00Z") + .user("U1", "jordan") + .bot(BOT, "pm-bot", { badge: "AGENT" }) + .me("U1") + .channel(PRODUCT, "product", { starred: true }); + s.message(PRODUCT, "U1", { + text: "<@U2> how do our open PRs and Linear issues align with our product goals this week?", + ts: ROOT, + }); + + return { + scenario: s.open(PRODUCT).openThread(ROOT).build(), + intro: "pm-bot: a read-only view of work in flight against product goals.", + beats: [ + { + id: "read", + caption: + "pm-bot reads only the GitHub repos it was granted and the linked Linear issues.", + hold: 1000, + run: (c) => { + c.setStatus(ROOT, "reading GitHub and Linear"); + c.postReply(PRODUCT, ROOT, BOT, { + text: "Reading `acme/web` and `acme/api`, then matching open PRs to Linear issues. I won't change either system.", + }); + }, + }, + { + id: "summary", + caption: + "It groups work by goal, showing what is on track and what needs attention.", + hold: 1300, + run: (c) => { + c.setStatus(ROOT, undefined); + c.postReply(PRODUCT, ROOT, BOT, { + text: "*Weekly product alignment*\n✅ *Faster checkout:* `acme/web` PR #18 and LIN-241 address the payment retry flow.\n⚠️ *Self-serve onboarding:* LIN-255 is blocked; no open PR covers the first-run guide.\n➡️ Suggested next step: assign LIN-255 and link its implementation PR.\nNo changes were made in GitHub or Linear.", + }); + }, + }, + ], + }; +} diff --git a/showcase/demos/slacksim/src/scenarios/reviewbot-demo.ts b/showcase/demos/slacksim/src/scenarios/reviewbot-demo.ts new file mode 100644 index 00000000..069d53cd --- /dev/null +++ b/showcase/demos/slacksim/src/scenarios/reviewbot-demo.ts @@ -0,0 +1,78 @@ +import { newScenario } from "../store/scenario"; +import type { Story } from "../runtime/story"; + +// Scripted example of examples/reviewbot. GitHub and Slack accounts are fake; +// the sequence follows the bundle's pull-request webhook → read-only review → Slack → Check Run flow. +const REVIEWS = "C1"; +const BOT = "U2"; +const ROOT = "1756800000.000000"; + +export function reviewbotDemo(): Story { + const s = newScenario() + .workspace("Acme Robotics", { glyph: "A", accent: "#4a154b" }) + .at("2026-09-01T15:00:00Z") + .user("U1", "jordan") + .bot(BOT, "reviewbot", { badge: "AGENT" }) + .me("U1") + .channel(REVIEWS, "auto-code-reviews", { starred: true }); + s.message(REVIEWS, BOT, { + text: "GitHub pull request updated: `acme/widget` PR #42, head `8f27ca1`. Starting a review.", + ts: ROOT, + }); + + return { + scenario: s.open(REVIEWS).openThread(ROOT).build(), + intro: "reviewbot: a pull-request event starts a read-only review.", + beats: [ + { + id: "read-diff", + caption: + "A verified pull-request event starts the session. reviewbot checks out the diff read-only and selects its review skill.", + hold: 1200, + run: (c) => { + c.setStatus(ROOT, "reading PR #42 diff"); + c.postReply(REVIEWS, ROOT, BOT, { + text: "Reviewing 3 changed files with the staged `review-pr` skill. Repository access is read-only.", + }); + }, + }, + { + id: "findings", + caption: + "The review finds a missing permission check and shows a report reference beside the Slack summary.", + hold: 1300, + run: (c) => { + c.postReply(REVIEWS, ROOT, BOT, { + text: "*PR #42 review — do not merge yet*\nHigh: `payments/handler.go` skips the account-owner check on retries. The new test covers the happy path but not a retry by a different user.", + attachments: [ + { + color: "#e01e5a", + blocks: [ + { + type: "section", + text: { + type: "mrkdwn", + text: ":page_facing_up: *pr42-review.html* — report reference", + }, + }, + ], + }, + ], + }); + }, + }, + { + id: "check-run", + caption: + "Only after delivery does reviewbot conclude the GitHub Check Run on the exact head commit.", + hold: 1200, + run: (c) => { + c.setStatus(ROOT, undefined); + c.postReply(REVIEWS, ROOT, BOT, { + text: "Check Run concluded: action required on `8f27ca1`. The review did not modify the repository or comment on the PR.", + }); + }, + }, + ], + }; +} diff --git a/showcase/demos/slacksim/src/store/scenario.test.ts b/showcase/demos/slacksim/src/store/scenario.test.ts index bd33bf80..75496a7f 100644 --- a/showcase/demos/slacksim/src/store/scenario.test.ts +++ b/showcase/demos/slacksim/src/store/scenario.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; import { newScenario } from "./scenario"; import { SimStore } from "./simstore"; +import { loadStory } from "../scenarios"; describe("scenario builder determinism", () => { it("produces byte-identical scenarios across two builds", () => { @@ -36,6 +37,62 @@ describe("scenario builder determinism", () => { }); }); +describe("example-agent demo stories", () => { + it("shows the weekly HubSpot digest before Sam asks for contacts", () => { + const story = loadStory("hubspot-companies"); + const store = new SimStore(story.scenario); + story.beats?.[0].run(store as never); + expect( + store + .getSnapshot() + .messages.some((message) => + message.text?.includes("New companies this week"), + ), + ).toBe(true); + expect( + store + .getSnapshot() + .messages.some((message) => + message.text?.includes("can you pull Circldot"), + ), + ).toBe(false); + story.beats?.[1].run(store as never); + expect( + store + .getSnapshot() + .messages.some((message) => + message.text?.includes("can you pull Circldot"), + ), + ).toBe(true); + }); + + it("shows a reviewbot report reference alongside the summary", () => { + const story = loadStory("reviewbot-demo"); + const store = new SimStore(story.scenario); + for (const beat of story.beats ?? []) beat.run(store as never); + const summary = store + .getSnapshot() + .messages.find((message) => message.text?.includes("do not merge yet")); + expect(summary).toBeDefined(); + expect(JSON.stringify(summary?.attachments)).toContain("pr42-review.html"); + }); + + it.each([ + ["hubspot-companies", "Circldot", "approved by"], + ["reviewbot-demo", "PR #42", "Check Run"], + ["pm-agent-demo", "product goals", "No changes were made"], + ])("shows the request and outcome for %s", (name, request, outcome) => { + const story = loadStory(name); + const store = new SimStore(story.scenario); + const initial = JSON.stringify(store.getSnapshot().messages); + for (const beat of story.beats ?? []) beat.run(store as never); + const finished = JSON.stringify(store.getSnapshot().messages); + expect(finished).toContain(request); + expect(finished).toContain(outcome); + expect(finished).not.toBe(initial); + }); +}); + describe("thread rollups derived, not stored", () => { it("computes replyCount, unique replyUserIds, and lastReplyTs on the parent", () => { const s = newScenario() diff --git a/showcase/engine/capture/clip.mjs b/showcase/engine/capture/clip.mjs index 3ae966c3..28739f31 100644 --- a/showcase/engine/capture/clip.mjs +++ b/showcase/engine/capture/clip.mjs @@ -1,23 +1,23 @@ -// Clip capture: play a Story's beats in a real browser, recording a narrated -// (caption-driven) webm, then transcode to mp4 + extract a poster with ffmpeg. +// Clip capture: play a Story's beats in a real browser, then render captioned, +// voiced webm/mp4 clips and a poster. // // The Story's beats + captions ARE the script: each beat sets its caption on the // in-page overlay, optionally animates the cursor to click a target, mutates the -// sim, then holds for a reading-time-derived dwell. A pluggable TTS layer can -// later replace captions with narration audio; the timing model is the same. +// sim, then holds long enough for the narration to finish. // // Usage: node engine/capture/clip.mjs [theme] [WxH] import { chromium } from 'playwright' -import { spawnSync } from 'node:child_process' import { mkdirSync } from 'node:fs' import path from 'node:path' +import { narrate } from '../tts/narrate.mjs' +import { encodeClip, remainingSceneHoldMs } from '../tts/mux.mjs' const scenario = process.argv[2] || 'multiplayer-approval' const outBase = process.argv[3] || `out/clips/${scenario}` const theme = process.argv[4] || 'dark' const [W, H] = (process.argv[5] || '1600x900').split('x').map(Number) const PORT = process.env.SIM_PORT || process.env.SLACKSIM_PORT || '5178' -const url = `http://localhost:${PORT}/?scenario=${scenario}&theme=${theme}` +const url = `http://127.0.0.1:${PORT}/?scenario=${scenario}&theme=${theme}` const outDir = path.dirname(outBase) mkdirSync(outDir, { recursive: true }) @@ -28,14 +28,35 @@ const rawDir = path.join('out', '_raw') mkdirSync(rawDir, { recursive: true }) const easeInOutCubic = (t) => (t < 0.5 ? 4 * t * t * t : 1 - Math.pow(-2 * t + 2, 3) / 2) -const readingMs = (text) => Math.min(6500, 1500 + (text ? text.length : 0) * 42) - const browser = await chromium.launch() +// Synthesize before the recorded context starts: an API call can take seconds, +// and that wait must never become dead air at the front of the finished clip. +const probeContext = await browser.newContext({ viewport: { width: W, height: H } }) +const probePage = await probeContext.newPage() +await probePage.goto(url, { waitUntil: 'networkidle' }) +await probePage.waitForFunction(() => !!window.__showcaseStory && !!window.__showcase) +const meta = await probePage.evaluate(() => ({ + count: window.__showcaseStory.count, + intro: window.__showcaseStory.intro, + captions: window.__showcaseStory.captions, + holds: window.__showcaseStory.holds, + clicks: window.__showcaseStory.clicks, + ids: window.__showcaseStory.ids, +})) +await probeContext.close() +const introText = meta.intro || 'A demo of an OAP agent in Slack.' +const voice = await narrate([ + { id: 'intro', text: introText }, + ...meta.ids.map((id, i) => ({ id, text: meta.captions[i] })), +], path.join('out', 'audio', scenario)) +const scenes = [] + const context = await browser.newContext({ viewport: { width: W, height: H }, deviceScaleFactor: 1, recordVideo: { dir: rawDir, size: { width: W, height: H } }, }) +const T0 = Date.now() const page = await context.newPage() await page.goto(url, { waitUntil: 'networkidle' }) await page.waitForFunction(() => !!window.__showcaseStory && !!window.__showcase) @@ -73,18 +94,10 @@ async function clickHere() { await page.waitForTimeout(120) } -const meta = await page.evaluate(() => ({ - count: window.__showcaseStory.count, - intro: window.__showcaseStory.intro, - captions: window.__showcaseStory.captions, - holds: window.__showcaseStory.holds, - clicks: window.__showcaseStory.clicks, - ids: window.__showcaseStory.ids, -})) - // Intro title card — the story's own caption, or a generic fallback. -await caption(meta.intro || 'A demo of an OAP agent in Slack.') -await page.waitForTimeout(2800) +await caption(introText) +scenes.push({ path: voice[0].path, startMs: Date.now() - T0 }) +await page.waitForTimeout(Math.max(2800, voice[0].durationMs + 500)) for (let i = 0; i < meta.count; i++) { const cap = meta.captions[i] @@ -107,7 +120,11 @@ for (let i = 0; i < meta.count; i++) { } await runBeat(i) - await page.waitForTimeout(Math.max(meta.holds[i] ?? 0, readingMs(cap))) + const startMs = Date.now() - T0 + scenes.push({ path: voice[i + 1].path, startMs }) + await page.waitForTimeout(remainingSceneHoldMs({ + caption: cap, audioDurationMs: voice[i + 1].durationMs, holdMs: meta.holds[i] ?? 0, + })) } await caption(null) @@ -118,17 +135,5 @@ await context.close() // flushes the webm const rawPath = await video.path() await browser.close() -// ffmpeg post: a clean webm (VP9), an mp4 (H.264) for broad compat, and a poster. -const webmOut = `${outBase}.webm` -const mp4Out = `${outBase}.mp4` -const posterOut = `${outBase}.png` - -function ff(args, label) { - const r = spawnSync('ffmpeg', ['-y', '-loglevel', 'error', ...args], { stdio: 'inherit' }) - if (r.status !== 0) throw new Error(`ffmpeg ${label} failed (${r.status})`) -} -ff(['-i', rawPath, '-c:v', 'libvpx-vp9', '-b:v', '0', '-crf', '32', '-an', webmOut], 'webm') -ff(['-i', rawPath, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '20', '-movflags', '+faststart', '-an', mp4Out], 'mp4') -ff(['-ss', '3', '-i', rawPath, '-frames:v', '1', posterOut], 'poster') - -console.log(`wrote:\n ${webmOut}\n ${mp4Out}\n ${posterOut}\n(raw: ${rawPath})`) +encodeClip(rawPath, outBase, scenes) +console.log(`wrote:\n ${outBase}.webm\n ${outBase}.mp4\n ${outBase}.png\n(raw: ${rawPath}; voice: ${voice[0].provider})`) diff --git a/showcase/engine/capture/narrate-builder.mjs b/showcase/engine/capture/narrate-builder.mjs new file mode 100644 index 00000000..ce5bebb7 --- /dev/null +++ b/showcase/engine/capture/narrate-builder.mjs @@ -0,0 +1,38 @@ +// Add ElevenLabs voiceover to the existing real-product Agent Builder capture. +// The source is the web UI clip already published in docs/public/media. Keeping +// the source intact makes the voice pass repeatable without starting a new +// workshop or installing another agent in a cluster. +// Usage: ELEVENLABS_API_KEY=... node engine/capture/narrate-builder.mjs [outBase] +import { mkdirSync } from 'node:fs' +import path from 'node:path' +import { narrate } from '../tts/narrate.mjs' +import { encodeClip } from '../tts/mux.mjs' + +if (!process.env.ELEVENLABS_API_KEY) throw new Error('set ELEVENLABS_API_KEY to narrate the builder clip') + +const source = path.join('docs', 'public', 'media', 'builder-first-agent.mp4') +const outBase = process.argv[2] || path.join('out', 'clips', 'builder-first-agent') +mkdirSync(path.dirname(outBase), { recursive: true }) + +// Starts track the visible stages of the existing 88-second web UI footage. +// Lines stay short enough to finish before the next stage begins. +const lines = [ + { id: 'choose', startMs: 1000, text: 'Meet the Agent Builder. Start a session and choose Agent Builder from the list.' }, + { id: 'describe', startMs: 10500, text: 'Describe your agent in plain language. Here, we ask for a friendly limerick bot with no tools.' }, + { id: 'questions', startMs: 20500, text: 'The builder restates the goal, records decisions, and asks about anything still unclear.' }, + { id: 'approve', startMs: 30500, text: 'Once you answer, it moves through tools and permissions. You approve a stage before it changes the draft.' }, + { id: 'build', startMs: 40500, text: 'It builds the agent and checks the draft until the platform reports it valid.' }, + { id: 'test', startMs: 50500, text: 'Next, test the new agent right here in the web UI, using a real chat session.' }, + { id: 'deliver', startMs: 60500, text: 'The builder reads the test result and saves a portable copy of exactly what you tried.' }, + { id: 'install', startMs: 70500, text: 'Installing is a separate decision. An admin chooses the namespace and approves the request.' }, + { id: 'finished', startMs: 80500, text: 'Now the new agent is installed.' }, +] + +const audio = await narrate(lines, path.join('out', 'audio', 'builder-first-agent')) +for (let i = 0; i < lines.length - 1; i++) { + if (lines[i].startMs + audio[i].durationMs > lines[i + 1].startMs) { + throw new Error(`voice line ${lines[i].id} overlaps ${lines[i + 1].id}; shorten the script or adjust its cue`) + } +} +encodeClip(source, outBase, lines.map((line, i) => ({ path: audio[i].path, startMs: line.startMs }))) +console.log(`wrote ${outBase}.webm, ${outBase}.mp4, ${outBase}.png`) diff --git a/showcase/engine/capture/still.mjs b/showcase/engine/capture/still.mjs index 9873cf4f..47776bcc 100644 --- a/showcase/engine/capture/still.mjs +++ b/showcase/engine/capture/still.mjs @@ -15,7 +15,7 @@ const PORT = process.env.SIM_PORT || process.env.SLACKSIM_PORT || '5178' const browser = await chromium.launch() const page = await browser.newPage({ viewport: { width: W, height: H }, deviceScaleFactor: 2 }) -await page.goto(`http://localhost:${PORT}/?scenario=${scenario}&theme=${theme}`, { waitUntil: 'networkidle' }) +await page.goto(`http://127.0.0.1:${PORT}/?scenario=${scenario}&theme=${theme}`, { waitUntil: 'networkidle' }) await page.waitForFunction(() => !!window.__showcaseStory && !!window.__showcase) const count = await page.evaluate(() => window.__showcaseStory.count) for (let i = 0; i <= Math.min(beatIndex, count - 1); i++) { diff --git a/showcase/engine/tts/mux.mjs b/showcase/engine/tts/mux.mjs new file mode 100644 index 00000000..4b0071f2 --- /dev/null +++ b/showcase/engine/tts/mux.mjs @@ -0,0 +1,38 @@ +import { spawnSync } from 'node:child_process' + +const run = (bin, args, label) => { + const result = spawnSync(bin, args, { encoding: 'utf8' }) + if (result.status !== 0) throw new Error(`${label}: ${result.stderr || result.error?.message || result.status}`) + return result.stdout +} + +export function remainingSceneHoldMs({ caption, audioDurationMs, elapsedMs = 0, holdMs = 0 }) { + const readingMs = Math.min(6500, 1500 + (caption?.length || 0) * 42) + return Math.max(0, Math.max(holdMs, readingMs, audioDurationMs ? audioDurationMs + 500 : 0) - elapsedMs) +} + +// Scenes are { path, startMs } on the *finished* video timeline. The caller +// chooses those starts after its page has reached the state being described. +export function encodeClip(videoPath, outBase, scenes = []) { + const voiced = scenes.filter((scene) => scene.path) + const duration = Number(run('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nk=1:nw=1', videoPath], 'video duration').trim()) + if (!Number.isFinite(duration) || duration <= 0) throw new Error(`invalid video duration: ${videoPath}`) + + const inputs = ['-i', videoPath, ...voiced.flatMap((scene) => ['-i', scene.path])] + const audioFilters = voiced.map((scene, i) => + `[${i + 1}:a]adelay=${Math.max(0, Math.round(scene.startMs))}:all=1[a${i}]`) + const mixed = voiced.length === 1 + ? `[a0]apad,atrim=duration=${duration}[voice]` + : `[${voiced.map((_, i) => `a${i}`).join('][')}]amix=inputs=${voiced.length}:duration=longest:dropout_transition=0,apad,atrim=duration=${duration}[voice]` + const audio = voiced.length ? ['-filter_complex', [...audioFilters, mixed].join(';'), '-map', '[voice]'] : ['-an'] + + for (const [ext, videoCodec, audioCodec, extra] of [ + ['webm', ['-c:v', 'libvpx-vp9', '-b:v', '0', '-crf', '32'], ['-c:a', 'libopus'], []], + ['mp4', ['-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-crf', '20'], ['-c:a', 'aac', '-b:a', '160k'], ['-movflags', '+faststart']], + ]) { + run('ffmpeg', ['-y', '-loglevel', 'error', ...inputs, ...audio, '-map', '0:v:0', ...videoCodec, + ...(voiced.length ? audioCodec : []), ...extra, '-t', String(duration), `${outBase}.${ext}`], `${ext} render`) + } + run('ffmpeg', ['-y', '-loglevel', 'error', '-ss', String(Math.min(3, duration / 2)), '-i', videoPath, + '-frames:v', '1', `${outBase}.png`], 'poster') +} diff --git a/showcase/engine/tts/mux.test.mjs b/showcase/engine/tts/mux.test.mjs new file mode 100644 index 00000000..7294ff95 --- /dev/null +++ b/showcase/engine/tts/mux.test.mjs @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' +import { encodeClip, remainingSceneHoldMs } from './mux.mjs' + +const run = (bin, args) => { + const result = spawnSync(bin, args, { encoding: 'utf8' }) + if (result.status !== 0) throw new Error(`${bin}: ${result.stderr}`) + return result.stdout +} + +describe('narrated video output', () => { + it('holds a scene long enough for speech after its action has run', () => { + expect(remainingSceneHoldMs({ caption: '', audioDurationMs: 2400, elapsedMs: 700, holdMs: 1000 })).toBe(2200) + }) + + it('keeps a delayed narration track in both published video formats', () => { + const dir = mkdtempSync(path.join(tmpdir(), 'oap-voice-')) + const video = path.join(dir, 'source.webm') + const voice = path.join(dir, 'voice.mp3') + run('ffmpeg', ['-y', '-loglevel', 'error', '-f', 'lavfi', '-i', 'color=c=black:s=320x180:r=10:d=2', '-c:v', 'libvpx-vp9', video]) + run('ffmpeg', ['-y', '-loglevel', 'error', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=0.4', '-q:a', '5', voice]) + + const base = path.join(dir, 'clip') + encodeClip(video, base, [{ path: voice, startMs: 700 }]) + for (const file of [`${base}.webm`, `${base}.mp4`]) { + const probe = JSON.parse(run('ffprobe', ['-v', 'error', '-show_streams', '-show_format', '-of', 'json', file])) + expect(probe.streams.map((stream) => stream.codec_type)).toContain('audio') + expect(Number(probe.format.duration)).toBeGreaterThan(1.9) + expect(Number(probe.format.duration)).toBeLessThan(2.2) + } + }) +}) diff --git a/site/components/media.test.tsx b/site/components/media.test.tsx new file mode 100644 index 00000000..e0b59e33 --- /dev/null +++ b/site/components/media.test.tsx @@ -0,0 +1,15 @@ +// @vitest-environment jsdom +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { Clip } from "./media"; + +describe("Clip", () => { + it("lets a viewer hear a narrated clip with its controls", () => { + render(); + const video = screen.getByText(/codebot proposes a plan/i) + .previousElementSibling as HTMLVideoElement; + expect(video.tagName).toBe("VIDEO"); + expect(video.controls).toBe(true); + expect(video.muted).toBe(false); + }); +}); diff --git a/site/components/media.tsx b/site/components/media.tsx index c70b3ba3..e6350f9b 100644 --- a/site/components/media.tsx +++ b/site/components/media.tsx @@ -91,8 +91,7 @@ function Lightbox({ ); } -// An autoplay-muted-loop clip with a poster and click-to-expand. This is how a -// guide embeds a scenario clip captured by the engine. +// A playable clip with a poster and controls. Narrated clips must start unmuted. export function Clip({ name, caption }: { name: string; caption?: string }) { const asset = getAsset(name); if (!asset || (!asset.webm && !asset.mp4)) return ; @@ -101,8 +100,6 @@ export function Clip({ name, caption }: { name: string; caption?: string }) {