From 7e1af842d8a5c6e0564fac6bb7d81a6b9df66acd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 13:49:14 -0700 Subject: [PATCH 1/8] Claim isolated Opus target quest Co-Authored-By: GPT-6 From efb1c86c141c035f66b64064458f3280a11e1e2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:02:34 -0700 Subject: [PATCH 2/8] Capture isolated Opus timing on the ARM nightly runner Temporary branch-only diagnostic workflow; restore Nightly before the final quest PR. Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 334 +----------------------- test/audio-quality/clients/js/driver.ts | 41 +++ 2 files changed, 47 insertions(+), 328 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c39b95e165b..74147c410f3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,179 +1,13 @@ name: Nightly - -# The gates that don't belong on a pull request. -# -# `audit` is time-based, not diff-based: an advisory is published without this -# repo changing, so a per-PR run both fires on PRs that can't affect it and -# stays silent on the days nobody pushes. `features` is diff-independent for a -# different reason: it compiles the workspace at feature sets that share almost -# no artifacts with the default one, which is several minutes of duplicate work -# that a PR gate can't justify for how rarely it catches something. `obs` is -# diff-independent because its PR-time counterpart can't be: obs.yml has to -# name the paths that can break an external link to a Rust staticlib, and no -# list of paths covers every one of them (see the comment there). -# -# The `release-*` jobs are diff-independent because the release workflows only -# ever run on a tag: a change that compiles under plain cargo can still break a -# release artifact (crane vendors and stubs sources in ways cargo never does, -# the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages -# link the FFI on every platform) and nothing notices until the tag is pushed. -# -# `audio-quality` is here because of what it measures. It plays twenty-four rows of -# audio in real time, a minute each, which no merge gate should carry, and every -# number it grades is a timing one that moves with whatever else the runner is -# doing. A per-PR arm would fail on unrelated load often enough to be ignored; -# nightly it competes with nothing, and a regression surfaces a day late against -# a budget that still means something. The recorded-trace replays are the -# exception: deterministic and seconds long, so audio-quality.yml also gates -# PRs on them. -# -# A failure here lands on main rather than being caught in review, which is the -# accepted trade. Anything that must block a merge belongs in check.yml. - +# Temporary quest-branch diagnostic; restore the full workflow before the final PR. permissions: contents: read - on: workflow_dispatch: - schedule: - - cron: "0 7 * * *" - concurrency: - group: nightly-${{ github.ref }} - cancel-in-progress: true - + group: opus-target-diagnostic-${{ github.ref }} + cancel-in-progress: false jobs: - x11: - uses: ./.github/workflows/x11.yml - - # Each independent workload gets its own hour. Rust and JS benchmark groups - # keep one build environment each, so splitting the feature compile does not - # turn every benchmark into a fresh compile of the same dependencies. - nightly: - name: Nightly (${{ matrix.name }}) - runs-on: ubuntu-24.04-arm - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # Unset keys default to the plain dev shell, the Rust cache, no perf - # sampling, and nextest's default profile. - include: - - name: Features - recipe: rs features - profile: ci - - name: Go publisher runtime benchmark - recipe: go bench -benchtime=1x - - name: Audio grouping relay benchmark - recipe: bench-audio - # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. - - name: Relay lock-wait profile - recipe: bench-lock fanout-heavy 4 - perf: true - - name: Rust benchmarks - recipe: gh nightly rust - - name: JS benchmarks - recipe: gh nightly js - uncached: true - - name: BBB rendition alignment - recipe: pub check-bbb - uncached: true - - name: Audit - recipe: rs audit - uncached: true - # OBS (over moq-ffi) and moq-c's C fixtures link a Rust staticlib from - # outside cargo, so a stale `rs/moq-c/native-libs/*.txt` breaks them and - # no cargo build. obs.yml's path filter can't list every such input; - # this diff-independent run catches what it misses, a day late. - - name: OBS - recipe: obs ci - shell: .#obs - - name: moq-c C fixtures - recipe: rs c-tests - - steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main - with: - tool-cache: false - - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - # Trust the flake's cachix substituter, as check.yml does, so the dev - # shell is substituted rather than built from source. - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - # Restore only, like `tests` below. Saving its own store cost 3.6 GB of the - # repository's 10 GB budget and evicted the one every pull request restores. - - name: Rust cache - if: ${{ !matrix.uncached }} - uses: ./.github/actions/rust-cache - - # perf samples worker switch-outs in kernel context into a large buffer; bench-lock refuses without it. - - name: Allow perf kernel sampling - if: matrix.perf - run: sudo sysctl -w kernel.perf_event_paranoid=-1 - - - name: Run workload - run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} - env: - NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} - - tests: - name: Tests (${{ matrix.recipe }}) - runs-on: ubuntu-24.04-arm - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # `rs uring` is a separate feature compile (off the default set), and - # MOQ_STRICT fails it below the 6.12 kernel floor. The embedding tests for - # io_uring live there; uring.yml also runs it on affected PRs. - # `test drain` stands up two relays and drains one under a live JS - # viewer: diff-independent, since a break can arrive through the relay, - # js/net, or the wire between them. - # `gh test` exercises workflow and release tooling, which no pull request runs. - recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "gh test", "test json-properties"] - steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main - with: - tool-cache: false - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - # Trust the flake's cachix substituter, as check.yml does, so the dev - # shell is substituted rather than built from source. - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - name: Rust cache - uses: ./.github/actions/rust-cache - - name: Test - run: nix develop --command just ${{ matrix.recipe }} - env: - NEXTEST_PROFILE: ci - MOQ_STRICT: 1 - - # Full platform coverage without rebuilding after every main queue merge. - platform: - uses: ./.github/workflows/platform.yml - - # Plays the matrix in headless Chromium through moq-shaper and grades what the - # listener heard against test/audio-quality/budgets.json. ARM64 like wasm.yml, - # the only architecture cache.yml warms a Rust store for. audio-quality: name: Audio quality runs-on: ubuntu-24.04-arm @@ -215,7 +49,9 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - name: Audio quality if: ${{ !cancelled() }} - run: nix develop --command just test audio-quality --enforce --capture + run: | + nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated,plain --duration 60 --capture + nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings plain,isolated --duration 60 --capture shell: bash -leo pipefail {0} env: # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. @@ -232,161 +68,3 @@ jobs: path: ${{ runner.temp }}/moq-test-runs/ retention-days: 7 if-no-files-found: ignore - - # Every tag-driven release workflow, run against main with publishing skipped: - # the workflows gate their publish jobs on `github.ref_type == 'tag'`, and no - # secrets are passed, so a dry-run cannot publish even by accident. - # - # A caller has to grant every permission a nested job declares, skipped or - # not, so these carry the release-time grants. Nothing that runs here uses - # them. - release-py-ffi: - name: Release Py FFI - uses: ./.github/workflows/release-py-ffi.yml - permissions: - contents: read - id-token: write - - release-swift-lib: - name: Release Swift Lib - uses: ./.github/workflows/release-swift-lib.yml - # Its publisher check reads upstream run jobs; a called workflow can't - # exceed the permissions granted here. - permissions: - contents: read - actions: read - - release-shared-ffi: - name: Release FFI - uses: ./.github/workflows/release-shared-ffi.yml - permissions: - contents: write - - release-dart-ffi: - name: Release Dart FFI - uses: ./.github/workflows/release-dart-ffi.yml - permissions: - contents: write - id-token: write - - release-moq-c: - name: moq-c - uses: ./.github/workflows/moq-c.yml - permissions: - contents: write - - release-cpp: - name: Release Cpp - uses: ./.github/workflows/release-cpp.yml - permissions: - contents: write - - release-moq-relay: - name: moq-relay - uses: ./.github/workflows/moq-relay.yml - permissions: - contents: write - actions: write - - release-moq-cli: - name: moq-cli - uses: ./.github/workflows/moq-cli.yml - permissions: - contents: write - actions: write - - release-moq-gst: - name: moq-gst - uses: ./.github/workflows/moq-gst.yml - permissions: - contents: write - actions: write - - # The three Docker images share one Dockerfile, so one package covers it. - release-docker: - name: Docker - uses: ./.github/workflows/docker.yml - with: - package: moq-relay - permissions: - contents: read - - sizes: - name: Shipping sizes - # Measures what these release jobs upload rather than rebuilding it. - needs: [release-py-ffi, release-shared-ffi, release-moq-c, release-moq-relay, release-moq-cli] - runs-on: ubuntu-24.04-arm - timeout-minutes: 120 - permissions: - contents: read - actions: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - uses: ./.github/actions/rust-cache - - name: Collect and compare sizes - run: nix develop --command just gh size-nightly size-report "$GITHUB_RUN_ID" - env: - GH_TOKEN: ${{ github.token }} - - name: Preserve sizes and attribution - if: ${{ !cancelled() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: nightly-size-history - path: | - size-report/history.json - size-report/sizes.json - size-report/summary.md - size-report/ffi-bloat.txt - size-report/js/bytes.json - size-report/js/*.metafile.json - retention-days: 45 - if-no-files-found: error - - fuzz: - name: Fuzz (${{ matrix.target }}) - runs-on: ubuntu-24.04-arm - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - target: [lite, announce, ietf, varint, path, pattern] - steps: - - name: Free disk space - uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main - with: - tool-cache: false - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - # Trust the flake's cachix substituter, as check.yml does, so the dev - # shell is substituted rather than built from source. - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - name: Install nightly Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - toolchain: nightly - - name: Install cargo-fuzz - run: cargo install --locked cargo-fuzz - - name: Fuzz - run: nix develop --command just rs fuzz ${{ matrix.target }} -- -max_total_time=300 - - name: Preserve failure inputs - if: ${{ failure() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: fuzz-${{ matrix.target }} - path: rs/moq-net/fuzz/artifacts/ - if-no-files-found: ignore diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index 99b89d843be..a124bd02b23 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -129,8 +129,34 @@ const browser = await launch(["--autoplay-policy=no-user-gesture-required"]); let environment: Environment | undefined; let notes: string[] = []; let failed = false; +let finishTrace: (() => Promise) | undefined; try { const page = await browser.newPage(); + if (values.capture) { + await page.addInitScript(() => { + const tasks: { at: number; duration: number }[] = []; + Object.defineProperty(globalThis, "audioQualityLongTasks", { value: tasks }); + new PerformanceObserver((entries) => { + for (const task of entries.getEntries()) tasks.push({ at: task.startTime, duration: task.duration }); + }).observe({ type: "longtask", buffered: true }); + }); + const session = await page.context().newCDPSession(page); + const events: unknown[] = []; + session.on("Tracing.dataCollected", (event: { value: unknown[] }) => events.push(...event.value)); + const complete = new Promise((resolve) => session.once("Tracing.tracingComplete", () => resolve())); + await session.send("Tracing.start", { + categories: "v8,disabled-by-default-v8.gc,devtools.timeline,blink.user_timing", + transferMode: "ReportEvents", + }); + finishTrace = async () => { + finishTrace = undefined; + await session.send("Tracing.end"); + await complete; + await Bun.write(join(out, `${tag}.trace.json`), JSON.stringify({ traceEvents: events })); + await session.detach(); + }; + } + page.on("pageerror", (error) => console.error(`[page error] ${error.message}`)); await page.route(new URL("/certificate.sha256", url).href, (route) => route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), @@ -162,11 +188,26 @@ try { // The context rate is only known once the graph exists, which is after the catalog. environment = (await page.evaluate(() => globalThis.audioQuality.environment())) ?? environment; notes = await page.evaluate(() => globalThis.audioQuality.notes()); + if (values.capture) { + const tasks = await page.evaluate( + () => + (globalThis as typeof globalThis & { audioQualityLongTasks: { at: number; duration: number }[] }) + .audioQualityLongTasks, + ); + await Bun.write(join(out, `${tag}.longtasks.json`), JSON.stringify(tasks)); + } + await finishTrace?.(); await page.close(); } catch (err) { failed = true; refuse("driver", err instanceof Error ? err.message : String(err)); } finally { + try { + await finishTrace?.(); + } catch (error) { + failed = true; + refuse("trace", error instanceof Error ? error.message : String(error)); + } await browser.close().catch(() => {}); server.stop(true); } From 87fd8dc6bd51b232d8ca910486a60b85e4063426 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:08:05 -0700 Subject: [PATCH 3/8] Separate audio quality controls from optional GC tracing Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 1 + test/audio-quality/clients/js/driver.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 74147c410f3..54c21d8e357 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -52,6 +52,7 @@ jobs: run: | nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated,plain --duration 60 --capture nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings plain,isolated --duration 60 --capture + nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated --duration 60 --capture --trace shell: bash -leo pipefail {0} env: # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index a124bd02b23..c8f6461848e 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -36,6 +36,7 @@ const { values } = parseArgs({ tag: { type: "string" }, out: { type: "string" }, capture: { type: "boolean", default: false }, + trace: { type: "boolean", default: false }, }, }); @@ -51,10 +52,11 @@ if ( !out || !Number.isFinite(durationMs) || durationMs <= 0 || - (ring !== "plain" && ring !== "isolated") + (ring !== "plain" && ring !== "isolated") || + (values.trace && !values.capture) ) { console.error( - "usage: driver.ts --url U --fingerprint F --broadcast B --page DIR --ring plain|isolated --tag T --out DIR [--delay auto] [--duration S>0] [--capture]", + "usage: driver.ts --url U --fingerprint F --broadcast B --page DIR --ring plain|isolated --tag T --out DIR [--delay auto] [--duration S>0] [--capture] [--trace]", ); process.exit(2); } @@ -140,12 +142,14 @@ try { for (const task of entries.getEntries()) tasks.push({ at: task.startTime, duration: task.duration }); }).observe({ type: "longtask", buffered: true }); }); + } + if (values.trace) { const session = await page.context().newCDPSession(page); const events: unknown[] = []; session.on("Tracing.dataCollected", (event: { value: unknown[] }) => events.push(...event.value)); const complete = new Promise((resolve) => session.once("Tracing.tracingComplete", () => resolve())); await session.send("Tracing.start", { - categories: "v8,disabled-by-default-v8.gc,devtools.timeline,blink.user_timing", + categories: "disabled-by-default-v8.gc,blink.user_timing", transferMode: "ReportEvents", }); finishTrace = async () => { From 3a21f5bdbebdb239b856e77385a39df5fd952b2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:15:08 -0700 Subject: [PATCH 4/8] Forward optional tracing through the audio quality wrapper Co-Authored-By: GPT-6 --- test/audio-quality/README.md | 5 ++++- test/audio-quality/clients/js/cli.test.ts | 19 +++++++++++++++++++ test/audio-quality/run.sh | 7 +++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 test/audio-quality/clients/js/cli.test.ts diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index ac5525516e3..a9046df8ac9 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -85,7 +85,10 @@ asks, not the target), so it adds no upstream subscription demand and still sees the target. Groups drain concurrently, so an earlier stalled group cannot hide a later arrival. Capture errors void the row. Capture adds local decoding and recording work; it is off by default and -on in nightly so a run preserves the inputs for diagnosis. +on in nightly so a run preserves the inputs for diagnosis. It also records native main-thread +long tasks in `.longtasks.json`, on the same viewer clock. `--trace` implies capture and adds a +Chromium GC timeline in `.trace.json`. Tracing can perturb playback; compare it with a separate +untraced control and never use traced rows to calibrate quality budgets. To replay a capture, put its arrivals and final environment configuration into the version 2 `Trace` below, subtract the first arrival's `at` from every arrival and from the last sample's diff --git a/test/audio-quality/clients/js/cli.test.ts b/test/audio-quality/clients/js/cli.test.ts new file mode 100644 index 00000000000..78bb0e758f0 --- /dev/null +++ b/test/audio-quality/clients/js/cli.test.ts @@ -0,0 +1,19 @@ +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +test("trace is accepted through the audio-quality wrapper", () => { + const result = Bun.spawnSync([ + "bash", + fileURLToPath(new URL("../../run.sh", import.meta.url)), + "--list", + "--profiles", + "near-zero", + "--codecs", + "opus", + "--rings", + "isolated", + "--trace", + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString().trim()).toBe("chromium-opus-48000-near-zero-isolated"); +}); diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index e7377edf725..3bd6e349bc3 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -34,6 +34,7 @@ OUT="" LIST=0 ENFORCE=0 CAPTURE=0 +TRACE=0 PROFILE="${AQ_PROFILE:-debug}" split() { @@ -97,6 +98,11 @@ while [[ $# -gt 0 ]]; do CAPTURE=1 shift ;; + --trace) + TRACE=1 + CAPTURE=1 + shift + ;; --enforce) ENFORCE=1 shift @@ -306,6 +312,7 @@ if [[ ${#ROWS[@]} -gt 0 ]]; then status=0 capture_args=() [[ $CAPTURE -eq 0 ]] || capture_args=(--capture) + [[ $TRACE -eq 0 ]] || capture_args+=(--trace) harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ --url "http://127.0.0.1:$SHAPER_PORT" \ --fingerprint "$RELAY_URL/certificate.sha256" \ From ee897d7bee3f633f1aaf48c6742e200ad03c1f87 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:25:25 -0700 Subject: [PATCH 5/8] Profile the isolated-first ARM audio row Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 54c21d8e357..88160c792d1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -35,24 +35,9 @@ jobs: - name: Playwright Chromium run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" - - name: WebSocket fallback startup benchmark - run: nix develop --command just js qmux-bench - - - name: Browser publisher fairness benchmark - run: nix develop --command just js serve-bench - shell: bash -leo pipefail {0} - - name: Native headless playback - if: ${{ !cancelled() }} - run: nix develop --command just test native-play - shell: bash -leo pipefail {0} - env: - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - name: Audio quality if: ${{ !cancelled() }} - run: | - nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated,plain --duration 60 --capture - nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings plain,isolated --duration 60 --capture - nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated --duration 60 --capture --trace + run: nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated --duration 60 --trace shell: bash -leo pipefail {0} env: # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. From d71d37706232583aff7060793d3dc5c1feffcb7a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:35:10 -0700 Subject: [PATCH 6/8] Capture sampled CPU profiles for audio diagnosis Co-Authored-By: GPT-6 --- test/audio-quality/README.md | 3 ++- test/audio-quality/clients/js/driver.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index a9046df8ac9..029fdc2dcfe 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -87,7 +87,8 @@ Groups drain concurrently, so an earlier stalled group cannot hide a later arriv errors void the row. Capture adds local decoding and recording work; it is off by default and on in nightly so a run preserves the inputs for diagnosis. It also records native main-thread long tasks in `.longtasks.json`, on the same viewer clock. `--trace` implies capture and adds a -Chromium GC timeline in `.trace.json`. Tracing can perturb playback; compare it with a separate +Chromium GC timeline in `.trace.json` and sampled CPU profile in `.profile.json`. +Tracing can perturb playback; compare it with a separate untraced control and never use traced rows to calibrate quality budgets. To replay a capture, put its arrivals and final environment configuration into the version 2 diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index c8f6461848e..3c5e3802621 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -145,6 +145,8 @@ try { } if (values.trace) { const session = await page.context().newCDPSession(page); + await session.send("Profiler.enable"); + await session.send("Profiler.start"); const events: unknown[] = []; session.on("Tracing.dataCollected", (event: { value: unknown[] }) => events.push(...event.value)); const complete = new Promise((resolve) => session.once("Tracing.tracingComplete", () => resolve())); @@ -156,6 +158,8 @@ try { finishTrace = undefined; await session.send("Tracing.end"); await complete; + const { profile } = await session.send("Profiler.stop"); + await Bun.write(join(out, `${tag}.profile.json`), JSON.stringify(profile)); await Bun.write(join(out, `${tag}.trace.json`), JSON.stringify({ traceEvents: events })); await session.detach(); }; From 794047c4bbb3b210c0e94a759b84c1225ca0123b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:47:59 -0700 Subject: [PATCH 7/8] Retain audio stall evidence and restore Nightly jobs Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 344 +++++++++++++++++- .../opus-isolated-target.md | 24 ++ 2 files changed, 364 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 88160c792d1..c39b95e165b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,13 +1,179 @@ name: Nightly -# Temporary quest-branch diagnostic; restore the full workflow before the final PR. + +# The gates that don't belong on a pull request. +# +# `audit` is time-based, not diff-based: an advisory is published without this +# repo changing, so a per-PR run both fires on PRs that can't affect it and +# stays silent on the days nobody pushes. `features` is diff-independent for a +# different reason: it compiles the workspace at feature sets that share almost +# no artifacts with the default one, which is several minutes of duplicate work +# that a PR gate can't justify for how rarely it catches something. `obs` is +# diff-independent because its PR-time counterpart can't be: obs.yml has to +# name the paths that can break an external link to a Rust staticlib, and no +# list of paths covers every one of them (see the comment there). +# +# The `release-*` jobs are diff-independent because the release workflows only +# ever run on a tag: a change that compiles under plain cargo can still break a +# release artifact (crane vendors and stubs sources in ways cargo never does, +# the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages +# link the FFI on every platform) and nothing notices until the tag is pushed. +# +# `audio-quality` is here because of what it measures. It plays twenty-four rows of +# audio in real time, a minute each, which no merge gate should carry, and every +# number it grades is a timing one that moves with whatever else the runner is +# doing. A per-PR arm would fail on unrelated load often enough to be ignored; +# nightly it competes with nothing, and a regression surfaces a day late against +# a budget that still means something. The recorded-trace replays are the +# exception: deterministic and seconds long, so audio-quality.yml also gates +# PRs on them. +# +# A failure here lands on main rather than being caught in review, which is the +# accepted trade. Anything that must block a merge belongs in check.yml. + permissions: contents: read + on: workflow_dispatch: + schedule: + - cron: "0 7 * * *" + concurrency: - group: opus-target-diagnostic-${{ github.ref }} - cancel-in-progress: false + group: nightly-${{ github.ref }} + cancel-in-progress: true + jobs: + x11: + uses: ./.github/workflows/x11.yml + + # Each independent workload gets its own hour. Rust and JS benchmark groups + # keep one build environment each, so splitting the feature compile does not + # turn every benchmark into a fresh compile of the same dependencies. + nightly: + name: Nightly (${{ matrix.name }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # Unset keys default to the plain dev shell, the Rust cache, no perf + # sampling, and nextest's default profile. + include: + - name: Features + recipe: rs features + profile: ci + - name: Go publisher runtime benchmark + recipe: go bench -benchtime=1x + - name: Audio grouping relay benchmark + recipe: bench-audio + # Smoke-tests the capture path: exact off-CPU waits, resolved worker stacks and CPU. + - name: Relay lock-wait profile + recipe: bench-lock fanout-heavy 4 + perf: true + - name: Rust benchmarks + recipe: gh nightly rust + - name: JS benchmarks + recipe: gh nightly js + uncached: true + - name: BBB rendition alignment + recipe: pub check-bbb + uncached: true + - name: Audit + recipe: rs audit + uncached: true + # OBS (over moq-ffi) and moq-c's C fixtures link a Rust staticlib from + # outside cargo, so a stale `rs/moq-c/native-libs/*.txt` breaks them and + # no cargo build. obs.yml's path filter can't list every such input; + # this diff-independent run catches what it misses, a day late. + - name: OBS + recipe: obs ci + shell: .#obs + - name: moq-c C fixtures + recipe: rs c-tests + + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, as check.yml does, so the dev + # shell is substituted rather than built from source. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + # Restore only, like `tests` below. Saving its own store cost 3.6 GB of the + # repository's 10 GB budget and evicted the one every pull request restores. + - name: Rust cache + if: ${{ !matrix.uncached }} + uses: ./.github/actions/rust-cache + + # perf samples worker switch-outs in kernel context into a large buffer; bench-lock refuses without it. + - name: Allow perf kernel sampling + if: matrix.perf + run: sudo sysctl -w kernel.perf_event_paranoid=-1 + + - name: Run workload + run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} + env: + NEXTEST_PROFILE: ${{ matrix.profile || 'default' }} + + tests: + name: Tests (${{ matrix.recipe }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # `rs uring` is a separate feature compile (off the default set), and + # MOQ_STRICT fails it below the 6.12 kernel floor. The embedding tests for + # io_uring live there; uring.yml also runs it on affected PRs. + # `test drain` stands up two relays and drains one under a live JS + # viewer: diff-independent, since a break can arrive through the relay, + # js/net, or the wire between them. + # `gh test` exercises workflow and release tooling, which no pull request runs. + recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "gh test", "test json-properties"] + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, as check.yml does, so the dev + # shell is substituted rather than built from source. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: ./.github/actions/rust-cache + - name: Test + run: nix develop --command just ${{ matrix.recipe }} + env: + NEXTEST_PROFILE: ci + MOQ_STRICT: 1 + + # Full platform coverage without rebuilding after every main queue merge. + platform: + uses: ./.github/workflows/platform.yml + + # Plays the matrix in headless Chromium through moq-shaper and grades what the + # listener heard against test/audio-quality/budgets.json. ARM64 like wasm.yml, + # the only architecture cache.yml warms a Rust store for. audio-quality: name: Audio quality runs-on: ubuntu-24.04-arm @@ -35,9 +201,21 @@ jobs: - name: Playwright Chromium run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" + - name: WebSocket fallback startup benchmark + run: nix develop --command just js qmux-bench + + - name: Browser publisher fairness benchmark + run: nix develop --command just js serve-bench + shell: bash -leo pipefail {0} + - name: Native headless playback + if: ${{ !cancelled() }} + run: nix develop --command just test native-play + shell: bash -leo pipefail {0} + env: + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs - name: Audio quality if: ${{ !cancelled() }} - run: nix develop --command just test audio-quality --profiles near-zero --codecs opus --rings isolated --duration 60 --trace + run: nix develop --command just test audio-quality --enforce --capture shell: bash -leo pipefail {0} env: # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. @@ -54,3 +232,161 @@ jobs: path: ${{ runner.temp }}/moq-test-runs/ retention-days: 7 if-no-files-found: ignore + + # Every tag-driven release workflow, run against main with publishing skipped: + # the workflows gate their publish jobs on `github.ref_type == 'tag'`, and no + # secrets are passed, so a dry-run cannot publish even by accident. + # + # A caller has to grant every permission a nested job declares, skipped or + # not, so these carry the release-time grants. Nothing that runs here uses + # them. + release-py-ffi: + name: Release Py FFI + uses: ./.github/workflows/release-py-ffi.yml + permissions: + contents: read + id-token: write + + release-swift-lib: + name: Release Swift Lib + uses: ./.github/workflows/release-swift-lib.yml + # Its publisher check reads upstream run jobs; a called workflow can't + # exceed the permissions granted here. + permissions: + contents: read + actions: read + + release-shared-ffi: + name: Release FFI + uses: ./.github/workflows/release-shared-ffi.yml + permissions: + contents: write + + release-dart-ffi: + name: Release Dart FFI + uses: ./.github/workflows/release-dart-ffi.yml + permissions: + contents: write + id-token: write + + release-moq-c: + name: moq-c + uses: ./.github/workflows/moq-c.yml + permissions: + contents: write + + release-cpp: + name: Release Cpp + uses: ./.github/workflows/release-cpp.yml + permissions: + contents: write + + release-moq-relay: + name: moq-relay + uses: ./.github/workflows/moq-relay.yml + permissions: + contents: write + actions: write + + release-moq-cli: + name: moq-cli + uses: ./.github/workflows/moq-cli.yml + permissions: + contents: write + actions: write + + release-moq-gst: + name: moq-gst + uses: ./.github/workflows/moq-gst.yml + permissions: + contents: write + actions: write + + # The three Docker images share one Dockerfile, so one package covers it. + release-docker: + name: Docker + uses: ./.github/workflows/docker.yml + with: + package: moq-relay + permissions: + contents: read + + sizes: + name: Shipping sizes + # Measures what these release jobs upload rather than rebuilding it. + needs: [release-py-ffi, release-shared-ffi, release-moq-c, release-moq-relay, release-moq-cli] + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - uses: ./.github/actions/rust-cache + - name: Collect and compare sizes + run: nix develop --command just gh size-nightly size-report "$GITHUB_RUN_ID" + env: + GH_TOKEN: ${{ github.token }} + - name: Preserve sizes and attribution + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: nightly-size-history + path: | + size-report/history.json + size-report/sizes.json + size-report/summary.md + size-report/ffi-bloat.txt + size-report/js/bytes.json + size-report/js/*.metafile.json + retention-days: 45 + if-no-files-found: error + + fuzz: + name: Fuzz (${{ matrix.target }}) + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: [lite, announce, ietf, varint, path, pattern] + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, as check.yml does, so the dev + # shell is substituted rather than built from source. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Install nightly Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + toolchain: nightly + - name: Install cargo-fuzz + run: cargo install --locked cargo-fuzz + - name: Fuzz + run: nix develop --command just rs fuzz ${{ matrix.target }} -- -max_total_time=300 + - name: Preserve failure inputs + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: fuzz-${{ matrix.target }} + path: rs/moq-net/fuzz/artifacts/ + if-no-files-found: ignore diff --git a/quest/m1/audio-jitter-target/opus-isolated-target.md b/quest/m1/audio-jitter-target/opus-isolated-target.md index 429fa3d1a1f..58745220b24 100644 --- a/quest/m1/audio-jitter-target/opus-isolated-target.md +++ b/quest/m1/audio-jitter-target/opus-isolated-target.md @@ -35,6 +35,30 @@ ceilings in `test/audio-quality/budgets.json` from re-measured nightly runs. Public API: none expected. Wire: none. +## Investigation + +The 2026-10-15 ARM comparison still reproduces the defect with Opus warmup +#5569 included. [The untraced controls](https://github.com/moq-dev/moq/actions/runs/38807783281) +run `isolated,plain` and then `plain,isolated`, each in a fresh harness run. +Isolated-first reaches a 560 ms target with two underrun episodes; the other +three rows hold 60 ms with no episodes. Configuration is identical within +each pair, including 80 ms codec warmup. Row order remains a confound. + +The failing row's arrival gaps coincide with 227–320 ms main-thread long +tasks. A [GC profile](https://github.com/moq-dev/moq/actions/runs/38809803022) +reproduces the pauses with no recorded GC work overlapping the longest tasks. +A [sampled CPU profile](https://github.com/moq-dev/moq/actions/runs/38811094714) +has 100–315 ms sampling gaps across the pauses, with boundary samples in +`Stream.Reader` awaiting native `ReadableStreamDefaultReader.read()`. This +locates the browser read/scheduling boundary but does not establish its cause. +Recorded arrival replay produces the same target with either ring. + +Use `--capture` to preserve arrivals and native long tasks; `--trace` also +saves GC events and a sampled CPU profile. Tracing can perturb playback, so +compare separate untraced controls and never calibrate budgets from traced +rows. The original fix and budget-tightening goals remain open; the diagnostic +change does not alter the estimator, ring, or ceilings. + ## Related - [Watch](/quest/m1/audio-jitter-target/watch.md) - the manual browser proof that follows this fix From 8be4594ae906f9004a65529ab17a7bb50741f578 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 14:48:41 -0700 Subject: [PATCH 8/8] Keep investigation evidence within the quest plan Co-Authored-By: GPT-6 --- quest/m1/audio-jitter-target/opus-isolated-target.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/quest/m1/audio-jitter-target/opus-isolated-target.md b/quest/m1/audio-jitter-target/opus-isolated-target.md index 58745220b24..3f075623ae1 100644 --- a/quest/m1/audio-jitter-target/opus-isolated-target.md +++ b/quest/m1/audio-jitter-target/opus-isolated-target.md @@ -35,8 +35,6 @@ ceilings in `test/audio-quality/budgets.json` from re-measured nightly runs. Public API: none expected. Wire: none. -## Investigation - The 2026-10-15 ARM comparison still reproduces the defect with Opus warmup #5569 included. [The untraced controls](https://github.com/moq-dev/moq/actions/runs/38807783281) run `isolated,plain` and then `plain,isolated`, each in a fresh harness run.