From 23772e8bdc4a5e2cd3b49b99d4a9c8c0174e0fec Mon Sep 17 00:00:00 2001 From: zackees Date: Sat, 25 Apr 2026 03:04:28 -0700 Subject: [PATCH] ci(library-selection): #205 acceptance + bench workflows + bench/ stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the gates we already shipped into CI and stubs the warm-cache benchmark home that #205 calls for. No runtime changes. ## What ships ### CI workflows - `.github/workflows/acceptance-205.yml` — runs the `#[ignore]`'d Phase 6 gates from PR #208 (`teensylc_acceptance` + `stm32_acceptance`) on a manual trigger, nightly schedule, and PRs that touch resolver-affecting paths. 45-minute timeout per job; `fail-fast: false` so a teensy regression doesn't mask an stm32 one. - `.github/workflows/bench-205.yml` — runs the criterion benches from PR #210 (`scan_throughput` + `resolve_cold`) and uploads the HTML reports as 14-day artifacts. Manual + scoped-PR triggers; no CI threshold gates yet (P-02 / P-03 numbers captured for review until runner variance is characterized). ### `bench/fastled-examples/` stub - `bench/README.md` — top-level orientation. - `bench/fastled-examples/README.md` — placeholder for the warm-cache matrix benchmark called for by #205 P-01 / AC#5. Marked as awaiting Phase 4 (zccache K/V cache, gated on zackees/zccache#130) since there's no warm path to measure today. - `docs/INDEX.md` row added. ### Issue housekeeping - Closed #204 with a comment referencing PR #207 (foundation) + PR #208 (acceptance gate at `tests/teensylc_acceptance.rs`). - Commented on already-closed #202 noting the same resolution path for STM32 SPI auto-discovery. ## Verification - YAML files parse via `pyyaml.safe_load`. - Workflow paths-filters point at real files (`crates/fbuild-build/tests/{teensylc,stm32}_acceptance.rs`, the two `benches/*.rs` files). - `[[bench]]` names match `Cargo.toml` (`scan_throughput`, `resolve_cold`). ## Out of scope (still tracked) - Phase 4 — zccache K/V memoization (zackees/zccache#130). - P-01 (warm matrix) and P-04 (cache-hit) perf gates — depend on Phase 4. - CI threshold gates on P-02 / P-03 — wait for runner-variance data. - Phase 8.b — final `framework_libs.rs` cleanup once Phase 4 lands. - Baseline numeric capture (`tasks/baseline-205.md` placeholder; needs the new acceptance workflow to finish a successful nightly run). Refs: FastLED/fbuild#202 (commented), FastLED/fbuild#204 (closed), FastLED/fbuild#205, zackees/zccache#130 Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/acceptance-205.yml | 55 +++++++++++++++++++++++ .github/workflows/bench-205.yml | 54 +++++++++++++++++++++++ bench/README.md | 28 ++++++++++++ bench/fastled-examples/README.md | 66 ++++++++++++++++++++++++++++ docs/INDEX.md | 1 + 5 files changed, 204 insertions(+) create mode 100644 .github/workflows/acceptance-205.yml create mode 100644 .github/workflows/bench-205.yml create mode 100644 bench/README.md create mode 100644 bench/fastled-examples/README.md diff --git a/.github/workflows/acceptance-205.yml b/.github/workflows/acceptance-205.yml new file mode 100644 index 00000000..d715e2d8 --- /dev/null +++ b/.github/workflows/acceptance-205.yml @@ -0,0 +1,55 @@ +name: Acceptance gates (#205) + +on: + # Don't run on every PR -- these tests take 10+ minutes each because they + # download Teensyduino + arm-gcc + STM32duino. Trigger explicitly. + workflow_dispatch: + schedule: + # Nightly at 04:00 UTC. Catches drift in Teensyduino / STM32duino + # framework downloads and fbuild's resolver behaviour. + - cron: '0 4 * * *' + pull_request: + paths: + # Run when something that could affect resolver behaviour changes. + - 'crates/fbuild-header-scan/**' + - 'crates/fbuild-library-select/**' + - 'crates/fbuild-build/src/framework_libs.rs' + - 'crates/fbuild-build/src/teensy/**' + - 'crates/fbuild-build/src/stm32/**' + - 'crates/fbuild-build/tests/teensylc_acceptance.rs' + - 'crates/fbuild-build/tests/stm32_acceptance.rs' + - '.github/workflows/acceptance-205.yml' + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +jobs: + acceptance: + name: Acceptance (${{ matrix.gate }}) + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - gate: teensyLC + test_bin: teensylc_acceptance + test_fn: teensylc_blink_meets_205_acceptance_criteria + - gate: stm32f103c8 SPI + test_bin: stm32_acceptance + test_fn: stm32f103c8_blink_with_spi_auto_discovers_library_205_ac4 + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v3 + - name: Setup soldr + uses: zackees/setup-soldr@v0 + with: + cache: true + build-cache: true + target-cache: true + + - name: Run ${{ matrix.gate }} acceptance gate + run: | + soldr cargo test -p fbuild-build --test ${{ matrix.test_bin }} -- --ignored --exact ${{ matrix.test_fn }} --nocapture --test-threads=1 diff --git a/.github/workflows/bench-205.yml b/.github/workflows/bench-205.yml new file mode 100644 index 00000000..1abc1cc6 --- /dev/null +++ b/.github/workflows/bench-205.yml @@ -0,0 +1,54 @@ +name: Perf benches (#205) + +on: + workflow_dispatch: + pull_request: + paths: + - 'crates/fbuild-header-scan/**' + - 'crates/fbuild-library-select/**' + - 'crates/fbuild-test-support/src/mini_framework.rs' + - '.github/workflows/bench-205.yml' + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +jobs: + bench: + name: Bench (${{ matrix.bench }}) + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - crate: fbuild-header-scan + bench: scan_throughput + target: P-03 (>= 50 MB/s single-thread) + - crate: fbuild-library-select + bench: resolve_cold + target: P-02 (<= 200 ms cold for typical project) + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v3 + - name: Setup soldr + uses: zackees/setup-soldr@v0 + with: + cache: true + build-cache: true + target-cache: true + + - name: "Run ${{ matrix.bench }} (target: ${{ matrix.target }})" + run: | + soldr cargo bench -p ${{ matrix.crate }} --bench ${{ matrix.bench }} -- --output-format bencher | tee bench-${{ matrix.bench }}.txt + + - name: Upload criterion HTML report + if: always() + uses: actions/upload-artifact@v4 + with: + name: criterion-${{ matrix.bench }} + path: | + target/criterion/** + bench-${{ matrix.bench }}.txt + if-no-files-found: warn + retention-days: 14 diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..c6a75f9f --- /dev/null +++ b/bench/README.md @@ -0,0 +1,28 @@ +# bench + +End-to-end performance benchmarks that drive the `fbuild` CLI against real +sketches and frameworks, as opposed to per-crate micro-benchmarks. + +Per-crate criterion benches live alongside their crate, e.g.: + +- `crates/fbuild-header-scan/benches/scan_throughput.rs` +- `crates/fbuild-library-select/benches/resolve_cold.rs` + +Run those with: + +```bash +uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold +uv run soldr cargo bench -p fbuild-header-scan --bench scan_throughput +``` + +## Subdirectories + +- [`fastled-examples/`](fastled-examples/README.md) — placeholder for the + warm-cache library-selection harness across the FastLED examples matrix + (`FastLED/fbuild#205` AC#5, P-01). Awaits Phase 4 zccache K/V memoization + before there's a warm path to measure. + +Other end-to-end matrices (whole-build wall-clock, deploy+flash latency, +emulator boot) may join this directory in the future. Each subdirectory +must carry its own `README.md` explaining what it measures, how to run it, +and which CI gate (if any) it feeds. diff --git a/bench/fastled-examples/README.md b/bench/fastled-examples/README.md new file mode 100644 index 00000000..a7216cea --- /dev/null +++ b/bench/fastled-examples/README.md @@ -0,0 +1,66 @@ +# bench/fastled-examples + +Warm-cache library-selection benchmarks across the FastLED examples matrix. +This is the harness referenced by `FastLED/fbuild#205` for acceptance +criterion **AC#5 / P-01**: + +> Warm library-selection on FastLED examples matrix `<= current fbuild +> + 50 ms`. + +## Status: empty placeholder + +There is no harness here yet, and on purpose. P-01 measures the **warm** +path through the resolver, which depends on the zccache K/V memoization +delivered by `#205` Phase 4 (gated on `zackees/zccache#130`). Until that +lands, every `resolve()` call is cold and there is no warm number to +gate against. Adding a "warm-ish" harness today would just measure cold +work twice and produce a misleading baseline. + +When Phase 4 ships and a `zccache` release is cut, a follow-up PR drops +the actual harness into this directory and wires it into CI. + +## The plan once Phase 4 lands + +1. Iterate the FastLED examples tree (`~/dev/fastled/examples/**`) under + each supported board: at minimum `teensyLC`, `teensy30`, `teensy41`, + `stm32f103c8`, `esp32-s3`, `uno`, `ws2812`. The matrix expands with + board coverage. +2. For each `(example, board)` pair, run the resolver twice: + - **Cold pass.** Empty `~/.zccache/`. Captures the K/V miss path and + the underlying scan + walk + LDF cost. This is the P-02 lane. + - **Warm pass.** Populated `~/.zccache/`. Captures the K/V hit path, + where the only work should be cache lookup + result deserialization. + This is the P-01 lane. +3. Diff the warm scan time against a captured baseline checked into + `tasks/baseline-205.md`. CI fails the job if any `(example, board)` + regresses the warm path by more than 50 ms vs. that baseline (the + `#205` AC#5 threshold). +4. Emit a structured JSON report (`bench/fastled-examples/report.json`) + that future PR comments can diff. Format TBD with the harness. + +## Running a partial version today + +The closest signal available right now is the per-crate cold-resolve +criterion bench: + +```bash +uv run soldr cargo bench -p fbuild-library-select --bench resolve_cold +``` + +That bench drives a synthetic ~30-library Teensyduino-class tree built +from `fbuild-test-support`'s `MiniFramework` rather than real FastLED +sketches, and it measures the cold path only. It is a useful regression +guard for the resolver itself, but it does **not** satisfy AC#5. + +## Cross-links + +- Issue: `FastLED/fbuild#205` +- Phase 4 design note (the prerequisite for this directory): + [`../../tasks/zccache-kv-design.md`](../../tasks/zccache-kv-design.md) +- Subsystem architecture: + [`../../docs/architecture/library-selection.md`](../../docs/architecture/library-selection.md) +- Foundation baseline that the warm threshold compares against: + [`../../tasks/baseline-205.md`](../../tasks/baseline-205.md) +- Per-crate cold benches (different scope, same subsystem): + [`../../crates/fbuild-library-select/benches/README.md`](../../crates/fbuild-library-select/benches/README.md), + [`../../crates/fbuild-header-scan/benches/README.md`](../../crates/fbuild-header-scan/benches/README.md) diff --git a/docs/INDEX.md b/docs/INDEX.md index b62c9aa3..62ad4572 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -32,6 +32,7 @@ A grep-friendly FAQ that maps common questions to the file that answers them. Bo | Why does warm-pass build take ~30 s per sketch? (#91) | [PERF_WARM_BUILD.md](PERF_WARM_BUILD.md) | | What does `FBUILD_PERF_LOG=1` do? | [PERF_WARM_BUILD.md](PERF_WARM_BUILD.md#instrumentation) | | How fast is `soldr` when building fbuild itself? | [SOLDR_BUILD_PERF.md](SOLDR_BUILD_PERF.md) | +| Where do end-to-end perf benchmarks live (FastLED matrix, P-01)? | [../bench/fastled-examples/README.md](../bench/fastled-examples/README.md) | | What architecture docs should I read for a given crate? | [CLAUDE.md](CLAUDE.md) | ## Conventions