diff --git a/.config/hakari.toml b/.config/hakari.toml index 1cf0c2ee789..71289bde5b2 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -22,7 +22,7 @@ platforms = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-pc-win # exact-versions = true [traversal-excludes] -workspace-members = ["moq-net-fuzz", "moq-quic-fuzz"] +workspace-members = ["moq-mux-fuzz", "moq-net-fuzz", "moq-quic-fuzz", "moq-rtmp-fuzz"] [final-excludes] workspace-members = ["kio-macros"] diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2b374b37d0d..cbec471b82d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -679,7 +679,7 @@ jobs: strategy: fail-fast: false matrix: - target: [lite, announce, ietf, varint, path, pattern, ingest, request, packet, params, streamid, streams, frame, connection] + target: [lite, announce, ietf, varint, path, pattern, ingest, request, packet, params, streamid, streams, frame, connection, fmp4, mkv, ts, flv, cmaf, annexb, catalog, amf0] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/Cargo.lock b/Cargo.lock index f334502c713..0b3e66c54e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5348,6 +5348,14 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "moq-mux-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "moq-mux", +] + [[package]] name = "moq-native" version = "0.20.0" @@ -5604,6 +5612,14 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "moq-rtmp-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "moq-rtmp", +] + [[package]] name = "moq-rtsp" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 9068e382a7e..fd42fd2a7ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] # Keep source-only members discoverable in Maturin's dependency-pruned archive, # including the separately packaged UniFFI generator. -members = ["demo/carla/bridge", "rs/*", "rs/kio/macros", "rs/moq-net/fuzz", "rs/moq-quic/fuzz"] +members = ["demo/carla/bridge", "rs/*", "rs/kio/macros", "rs/moq-mux/fuzz", "rs/moq-net/fuzz", "rs/moq-quic/fuzz", "rs/moq-rtmp/fuzz"] default-members = [ "rs/bench-harness", "rs/kio", diff --git a/quest/m1/README.md b/quest/m1/README.md index 4ac7931ae00..c2eeb35dc63 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -136,6 +136,8 @@ per-frame CPU growth since #5926. - [@moq/net group IDs past 2^53](/quest/m1/js-group-u62.md) - JS carries every 62-bit group ID instead of refusing those past 2^53 - [Pre-cut audit findings](/quest/m1/cut-audit/README.md) - fix the crashes, hostile-input costs, and correctness bugs the 2026-10-24 audit of main found - [Vendored code has every upstream fix it needs](/quest/m1/vendor-sync/README.md) - quinn, libv4l-rs, nvidia-video-codec-sdk, rust-media-libs, cros-codecs, and UniFFI generator fixes are ported or planned, once per release cycle +- [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the nightly media fuzz targets run clean: mp4-atom, MKV block, TS step, SPS overflow, and catalog encoder bugs they found are fixed +- [fMP4 sample plans, trun counts, and trak counts are bounded](/quest/m1/fmp4-metadata-limits.md) - a `moof` or `moov` within the importer's bounds cannot plan hundreds of megabytes or take quadratic time; 116 bytes cost 584 MB today - [Mesh dials recover from a stuck flow](/quest/m1/mesh-fresh-port.md) - a relay redials a failing cluster peer from a fresh UDP port, so one dropped 5-tuple can't strand a link - [Cluster routing](/quest/m1/cluster-routing/README.md) - prove shared routing contracts, implement per-origin routes, and connect outbound local clusters - [Routing guide](/quest/m1/routing-guide.md) - concepts and practical outbound federation setup diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index 2da59567d3b..5df75dc4e79 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -57,7 +57,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts-js.md). ## Related -- [Untrusted media parsers have fuzz targets](/quest/m2/media-fuzz.md) - would have caught the media parser findings here - [Announce interest streams share one route table](/quest/m2/announce-interest-memory.md) - the audit's inherited announce memory finding - [A parked read of a resumed group reads its budget without re-syncing](/quest/m2/recover-budget.md) - measure-first resume cost - [Transcode rungs scale off the runtime](/quest/m2/transcode-rungs.md) - the audit's transcode findings diff --git a/quest/m1/fmp4-metadata-limits.md b/quest/m1/fmp4-metadata-limits.md new file mode 100644 index 00000000000..fa4497653c7 --- /dev/null +++ b/quest/m1/fmp4-metadata-limits.md @@ -0,0 +1,66 @@ +# [M] fMP4 sample plans, trun counts, and trak counts are bounded + +## Goal + +An fMP4 `moov` or `moof` that fits the importer's bounds +(`fmp4::MAX_HELD_BYTES`, `fmp4::MAX_FRAGMENT_SAMPLES`) parses and plans in +memory near `MAX_HELD_BYTES` and in time linear in its size, rather than +hundreds of megabytes or minutes of CPU: a `moof`'s sample plan is bounded +in bytes, a `trun`'s sample count is refused before mp4-atom expands it, and +a `moov`'s tracks are bounded and cost linear time. Each case below has a +test that asserts its bound. + +## Plan + +Decided 2026-11-03 (maintainer, on +[moq-dev/moq#7313](https://github.com/moq-dev/moq/pull/7313)): the fixes land +here, in m1, not in #7313, because 116 bytes cost 584 MB on any fMP4 or HLS +ingest. + +- ✅ Land #7313 as-is, with the fixes in an m1 quest +- Land #7313 as-is, with the fixes staying in m2 +- Fix all three in #7313 +- Accept the numbers + +Earlier, decided 2026-10-31 (maintainer, from +[moq-dev/moq#7232](https://github.com/moq-dev/moq/pull/7232)): measure the +parse amplification #7232's bounds allow, and fix each case or accept it with +the number. [moq-dev/moq#7313](https://github.com/moq-dev/moq/pull/7313) +added the media fuzz targets but cannot measure this: libFuzzer inputs stay a +few KiB, far below the 32 MiB bound. It measured the cases below by hand +instead, through `Import::decode` with moq-mux's counting allocator, as peak +live heap. Every one needs a fix: + +| Input | Size | Peak | Note | +|---|---|---|---| +| `moof` with field-less `trun`s declaring 2^21 samples (`MAX_FRAGMENT_SAMPLES`), then an `mdat` header | 116 B | 584 MB | the plan alone, before any `mdat` byte | +| `moof` of 2^21 4-byte `trun` entries (`sample_size` only), then an `mdat` header | 8 MiB | 667 MB (80x) | 684 MB with passthrough | +| `moof` of 4-byte entries filling `MAX_HELD_BYTES` (8.4M entries) | 32 MiB | 369 MB (11x) | refused as `TooManySamples` only after mp4-atom parses it | +| `moov` of repeated audio `trak`s (bbb.mp4's, 477 B each) | 0.24 / 0.48 / 0.95 MB | 35x | CPU quadratic: 0.25 s, 0.9 s, 3.9 s; allocations 96 MB, 360 MB, 1.4 GB | + +- Planning costs about 280 B per sample: a `Sample` in each `Run`, a + `Chunk` in the `mdat`'s queue, and the growth of both unsized collections. + Bound the plan's memory, not just the sample count: for example lower + `MAX_FRAGMENT_SAMPLES` until the plan fits near `MAX_HELD_BYTES`, + presize the collections, and shrink `Chunk` (its `Text` variant sets its + size). +- mp4-atom expands each explicit `trun` entry to a 40-byte `TrunEntry` + before the sample count is checked. Refuse the count before parsing it: + read the `trun` sample counts from the held body, or bound a `moof` body + separately from a `moov` one. +- Each `trak` added to a `moov` costs time and allocation in proportion to + the tracks already added. Extrapolated to 32 MiB (about 70,000 `trak`s), + that is over an hour of CPU and a peak above 1 GB. Find the step that + revisits every earlier track (catalog renditions, track creation, or + enrollment), make it linear, and bound the tracks a `moov` declares. + +The bounds are moq-mux's public consts. This quest sets their values; +[one policy for oversized container metadata](/quest/m2/import-metadata-policy.md) +documents them later with MKV's. + +Public API: possibly new or changed `pub const` bounds in moq-mux. Wire: none. + +## Related + +- [One policy for oversized container metadata](/quest/m2/import-metadata-policy.md) - documents the importer bounds this quest sets +- [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the bugs the media fuzz targets found diff --git a/quest/m1/fuzz-findings/README.md b/quest/m1/fuzz-findings/README.md new file mode 100644 index 00000000000..7992ba83af4 --- /dev/null +++ b/quest/m1/fuzz-findings/README.md @@ -0,0 +1,43 @@ +# Media fuzz findings + +## Goal + +Every media fuzz target in `nightly.yml` (`rs/moq-mux/fuzz` and +`rs/moq-rtmp/fuzz`) runs a full nightly session without a crash or a +bounded-work violation, so the jobs find new bugs instead of the same ones. +Each fixed finding's minimized input is committed under the target's +`regressions/`, which the merge gate replays. + +## Plan + +[moq-dev/moq#7313](https://github.com/moq-dev/moq/pull/7313) added the +targets and landed with six nightly jobs red (decided 2026-10-30, +maintainer). Its short local runs found the bugs the children fix; the +minimized inputs are in [its PR comment](https://github.com/moq-dev/moq/pull/7313#issuecomment-6520535160). + +Each child commits its input under `regressions/` with its fix, since an +input committed before the fix would fail the merge gate. After the last +child, run every target nightly for a few sessions; a new finding becomes a +child here. + +For the crates outside the `kixelated` and `moq-dev` orgs, moq-mux fixes +the bug on its own side and the bug is reported upstream (decided +2026-10-30, maintainer): [austinleroy/webm-iterable#11](https://github.com/austinleroy/webm-iterable/issues/11), +[therishidesai/h264-parser#9](https://github.com/therishidesai/h264-parser/issues/9), +[ScuffleCloud/scuffle#653](https://github.com/ScuffleCloud/scuffle/issues/653) +(scuffle-h265), and [ScuffleCloud/scuffle#654](https://github.com/ScuffleCloud/scuffle/issues/654) +(scuffle-expgolomb). Posting anything further on those repos still needs the +maintainer's approval. + +[moq-dev/moq#7234](https://github.com/moq-dev/moq/pull/7234) replaced +webm-iterable's block parsing with moq-mux's own block-header reader, which +fixed the MKV block-header panic; its input is +`rs/moq-mux/fuzz/regressions/mkv/short-block-header`. + +## Required + +- [moq-mux on a fixed mp4-atom](/quest/m1/fuzz-findings/mp4-atom-bump.md) - waiting on a 0.17.x release containing kixelated/mp4-atom#264, then moq-mux bumps to it and commits the `sgpd` and `iref` regressions +- [TS step scan stays ordered](/quest/m1/fuzz-findings/ts-step-scan.md) - the step detector's look-ahead never subtracts past its start +- [SPS fields cannot overflow](/quest/m1/fuzz-findings/sps-overflow.md) - hostile H.264 and H.265 SPS crop and size fields are refused instead of wrapping +- [mp4-atom iloc is bounded](/quest/m1/fuzz-findings/mp4-atom-iloc.md) - zero-size `iloc` extents cannot allocate megabytes from a few bytes +- [A catalog update reuses its encoder](/quest/m1/fuzz-findings/catalog-encoder.md) - republishing a compressed catalog or HLS group stops allocating a fresh DEFLATE encoder (about 200 KB) each time diff --git a/quest/m1/fuzz-findings/catalog-encoder.md b/quest/m1/fuzz-findings/catalog-encoder.md new file mode 100644 index 00000000000..3c7cb1a32de --- /dev/null +++ b/quest/m1/fuzz-findings/catalog-encoder.md @@ -0,0 +1,30 @@ +# [M] A catalog update reuses its encoder + +## Goal + +Republishing a compressed track's latest value, as every hang catalog and +HLS update does, no longer allocates a fresh DEFLATE encoder (about 200 KB) per +group. In the `annexb` target, the 202-byte input that caused 24 encoders +and about 4.9 MB of allocation stays inside the fuzz bound, and is committed +under `rs/moq-mux/fuzz/regressions/`. + +## Plan + +Decided 2026-10-30 (maintainer): the encoder per update is a bug, not an +accepted cost, so the fuzz bound gets no per-update allowance. + +Each compressed group is its own DEFLATE stream (draft-lcurley-moq-flate), +so `Flate::deflate` in `rs/moq-net/src/model/group.rs` creates a +`flate::Encoder` on a group's first write and drops it at `finish`. A +publisher that writes one frame per group, as a catalog does, pays the +allocation on every update. + +Recommended: keep a finished group's encoder, reset with `Compress::reset`, +on the track's cache account (`cache::Track`) and hand it to the next group +the same track opens, billed to the pool while parked so the cache budget +still counts it. At most one spare per track, so concurrent groups still +allocate their own. The spare lives as long as the track; say in the PR +whether idle compressed tracks holding it is acceptable. The wire format +does not change: a reset encoder starts a cold window, as a new one does. + +Public API: none. Wire: none. diff --git a/quest/m1/fuzz-findings/mp4-atom-bump.md b/quest/m1/fuzz-findings/mp4-atom-bump.md new file mode 100644 index 00000000000..5f57fb053a0 --- /dev/null +++ b/quest/m1/fuzz-findings/mp4-atom-bump.md @@ -0,0 +1,28 @@ +# [XS] moq-mux on a fixed mp4-atom + +## Goal + +moq-mux depends on an mp4-atom release with the `sgpd` entry-count bound +from [kixelated/mp4-atom#264](https://github.com/kixelated/mp4-atom/pull/264), +so the 22-byte init that grows a Vec past 3 GB in 0.17.0 fails to decode. +The `cmaf` and `fmp4` targets' `sgpd` and `iref` inputs are committed under +`rs/moq-mux/fuzz/regressions/`. + +## Plan + +#264 merged on 2026-10-30 as +kixelated/mp4-atom@06a532aabe47ea108343557fec4cf6c646c2a980, after 0.17.0 +was published, so 0.17.0 lacks it. + +moq-mux is already on 0.17.0 +([moq-dev/moq#7308](https://github.com/moq-dev/moq/pull/7308)), which fixes +the `meta/iref.rs` subtract overflow (kixelated/mp4-atom#219), so what remains +is a move within 0.17 and the regression inputs. If the +[iloc fix](/quest/m1/fuzz-findings/mp4-atom-iloc.md) merges before the +release, the same bump covers it. + +Public API: none expected in moq-mux. Wire: none. + +## Waiting + +- Maintainer: cut an mp4-atom 0.17.x release that contains kixelated/mp4-atom@06a532aabe47ea108343557fec4cf6c646c2a980 (agents do not cut it); check that crates.io's latest mp4-atom is above 0.17.0 and that its tag contains that commit diff --git a/quest/m1/fuzz-findings/mp4-atom-iloc.md b/quest/m1/fuzz-findings/mp4-atom-iloc.md new file mode 100644 index 00000000000..a91e56f1183 --- /dev/null +++ b/quest/m1/fuzz-findings/mp4-atom-iloc.md @@ -0,0 +1,27 @@ +# [S] mp4-atom iloc is bounded + +## Goal + +An `iloc` box costs mp4-atom heap proportional to its size: in the `cmaf` +target, 31 bytes of `iloc` no longer allocate about 6 MB. + +## Plan + +`src/meta/iloc.rs` decodes `extent_count` extents per item. When the +offset, length and index sizes are all 0, each extent consumes no input, so +every 6-byte item can push 65,535 extents (about 1.5 MB), and the item count +repeats that. + +Fix it in [kixelated/mp4-atom](https://github.com/kixelated/mp4-atom), as +[#264](https://github.com/kixelated/mp4-atom/pull/264) did for `sgpd`. A +plain count-versus-remaining-bytes check, which `sgpd` and `evte` use, would +refuse a valid single zero-size extent at the end of the box. Instead, decode +fails on a zero-size extent (one that takes no bytes of the box) after an +item's first extent, so a valid single zero-size extent still decodes +(decided 2026-10-30, maintainer). The fix does not cut a release. + +Then bump moq-mux with the +[mp4-atom bump](/quest/m1/fuzz-findings/mp4-atom-bump.md), or its own bump if +that has landed, and commit the input under `rs/moq-mux/fuzz/regressions/`. + +Public API: none expected in moq-mux. Wire: none. diff --git a/quest/m1/fuzz-findings/sps-overflow.md b/quest/m1/fuzz-findings/sps-overflow.md new file mode 100644 index 00000000000..6d0e9710175 --- /dev/null +++ b/quest/m1/fuzz-findings/sps-overflow.md @@ -0,0 +1,45 @@ +# [S] SPS fields cannot overflow + +## Goal + +A hostile H.264 or H.265 SPS reaching moq-mux (fMP4 `avcC`/`hvcC`, Annex-B, +or MPEG-TS) is refused or ignored without integer overflow, so debug builds +do not panic and release builds do not wrap into wrong dimensions. The +`fmp4`, `annexb`, and `ts` targets' SPS inputs are committed under +`rs/moq-mux/fuzz/regressions/` and pass. + +## Plan + +The fuzzers found: + +- h264-parser 0.4.2 `sps.rs:186,197,204`: `(pic_width_in_mbs_minus1 + 1) * 16` + and the crop subtraction overflow on large exp-Golomb values. Since + [moq-dev/moq#7398](https://github.com/moq-dev/moq/pull/7398), moq-mux parses + the H.264 SPS with `h264-reader` and no longer uses h264-parser; + [rechecking the SPS target on h264-reader](/quest/m2/sps-fuzz-h264-reader.md) + decides whether the H.264 half of this quest remains. +- scuffle-h265 0.2.2 `sps/mod.rs:303,305` (`ctb_log2_size_y - min_tb_log2_size_y` + in a `range_check!`) and `sps/scaling_list.rs:90` + (`matrix_id - scaling_list_pred_matrix_id_delta`): subtract overflow. The + scaling-list one also panics in release builds, on the wrapped index at + `:92`. +- scuffle-expgolomb 0.1.5 `read_exp_golomb`: 64 or more leading zeros + underflow `result - 1` (release returns `u64::MAX`). + +Validate the SPS fields in moq-mux before calling the parser, refusing an +SPS that would reach those lines (decided 2026-10-30, maintainer). moq-mux +already reads SPS bits itself with `h264_reader::rbsp::BitReader` (the H.265 +head in `codec/h265/sps.rs`, and `codec/annexb.rs`). SPS parsing stays on the +crates moq-mux uses now, h264-reader for H.264 and scuffle-h265 plus +`codec/h265/sps.rs` for H.265; it does not move to mp4-atom, which stays +box-level. + +Reported upstream, without waiting on a fix: +[therishidesai/h264-parser#9](https://github.com/therishidesai/h264-parser/issues/9) +(filed 2026-10-25 by another moq-mux user, covering the crop and width +overflows), [ScuffleCloud/scuffle#653](https://github.com/ScuffleCloud/scuffle/issues/653) +(scuffle-h265), and [ScuffleCloud/scuffle#654](https://github.com/ScuffleCloud/scuffle/issues/654) +(scuffle-expgolomb). Each has a minimal reproducer. scuffle-h265 was last +released in 2025-05. + +Public API: none. Wire: none. diff --git a/quest/m1/fuzz-findings/ts-step-scan.md b/quest/m1/fuzz-findings/ts-step-scan.md new file mode 100644 index 00000000000..a249bf7006d --- /dev/null +++ b/quest/m1/fuzz-findings/ts-step-scan.md @@ -0,0 +1,19 @@ +# [S] TS step scan stays ordered + +## Goal + +The MPEG-TS step detector's look-ahead never reads with `scan < at`, so its +`self.scan - self.at` staging check (`rs/moq-mux/src/container/ts/step.rs`, +`Ahead::scan`) neither overflows in debug nor wraps in release, which cut +the look-ahead short. The `ts` target's 567-byte input is committed under +`rs/moq-mux/fuzz/regressions/` and passes. + +## Plan + +Find how an `Ahead` gets a `scan` cursor behind its `at` (the input offset of +the packet that showed the step) from the input in the +[findings comment](https://github.com/moq-dev/moq/pull/7313#issuecomment-6520535160), +and fix the cause rather than saturating the subtraction. A unit test in +`step.rs` reproduces it with the module's `pes` helper. + +Public API: none. Wire: none. diff --git a/quest/m2/README.md b/quest/m2/README.md index 741ee1a6dc6..8db07a57cf3 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -26,7 +26,6 @@ lite-07; bitrate enforcement is not part of this release. - [Mesh libraries](/quest/m2/mesh/README.md) - an `Endpoint` reaches any enrolled node by endpoint ID over relays or a sparse direct overlay, with address hints and a `.status` track; the mesh binary and UI live in Kixel - [Byte streams](/quest/m2/stream/README.md) - TCP connections ride MoQ losslessly through relays, proxied per hop rather than as IP packets - [Release execution](/quest/m2/release/README.md) - publication, installed-artifact proof, cleanup, and downstream adoption after m1 -- [Untrusted media parsers have fuzz targets](/quest/m2/media-fuzz.md) - fMP4, MKV, FLV, TS, Annex-B, catalog, and AMF0 parsing fuzzed with a bounded-work invariant - [One policy for oversized container metadata](/quest/m2/import-metadata-policy.md) - fMP4 and MKV refuse or stream huge metadata by one documented rule - [Wire fuzz coverage](/quest/m2/wire-fuzz-coverage.md) - the cmsg walks in moq-sock and moq-uring have invariant fuzz targets, and the unsafe peer input reaches runs under Miri - [VA-API renderer tests](/quest/m2/vaapi-render-tests.md) - the VA-API renderer, decode, and transcode tests run in `just rs gpu` on AMD and Intel and fail instead of skipping, and hosted CI reports them ignored diff --git a/quest/m2/import-metadata-policy.md b/quest/m2/import-metadata-policy.md index 7bd58751495..6ca5f60f316 100644 --- a/quest/m2/import-metadata-policy.md +++ b/quest/m2/import-metadata-policy.md @@ -22,4 +22,4 @@ Public API: possibly new `pub const` bounds and a shared refusal error in moq-mu ## Related -- [Untrusted media parsers have fuzz targets](/quest/m2/media-fuzz.md) - would measure parsed-state amplification under either policy +- [fMP4 sample plans, trun counts, and trak counts are bounded](/quest/m1/fmp4-metadata-limits.md) - measured what fMP4's bounds let a `moof` or `moov` build, and sets the bounds this policy documents diff --git a/quest/m2/media-fuzz.md b/quest/m2/media-fuzz.md deleted file mode 100644 index 64248d4af3c..00000000000 --- a/quest/m2/media-fuzz.md +++ /dev/null @@ -1,38 +0,0 @@ -# [M] Untrusted media parsers have fuzz targets - -## Goal - -The media parsers reachable from untrusted publishers (fMP4 import and CMAF -decode, the MKV, FLV, and TS importers, the Annex-B splitters, and hang -catalog parsing) run under nightly fuzz targets that assert bounded work -and allocation per input byte, not just "no panic". - -## Plan - -From the 2026-10-24 pre-cut audit at 151a529670. `rs/moq-mux` and `rs/hang` -have no fuzz targets; only `rs/moq-net/fuzz` and `rs/moq-quic/fuzz` exist, -and [wire fuzz coverage](/quest/m2/wire-fuzz-coverage.md) puts media -importers out of scope. The audit's -[zero-size sample](https://github.com/moq-dev/moq/pull/6864) and -malformed avc1 codec-string findings are the kind -`cargo fuzz` finds in minutes; they are reachable through RTMP, SRT, HLS, -and fMP4 ingest and through catalog subscription. Include RTMP's AMF0 -decoder too; its borrowed receive views should stay allocation-free. - -Add a fuzz crate per package, or one shared media target crate, seeded from -the existing fixtures. Assert a work and allocation bound proportional to -input size. Wire the targets into the nightly gate next to moq-net's, and -replay committed regressions in the merge gate. - -Measure parse amplification too (decided 2026-10-31, maintainer, from -[moq-dev/moq#7232](https://github.com/moq-dev/moq/pull/7232)). That PR bounds -a held `moov` or `moof` at 32 MiB (`fmp4::MAX_HELD_BYTES`) and a `moof` at -2^21 samples (`fmp4::MAX_FRAGMENT_SAMPLES`), but a body at the bound -decodes into larger structures: an explicit `trun` entry costs 4 to 16 -input bytes and about 40 parsed, so 2.5x to 10x. A `trun` without -per-sample fields parses to no entries, but planning still resolves every -sample it declares, up to the 2^21 bound. Record the peak allocation for a -maximal `moov`, a maximal `moof` of 4-byte entries, and a field-less `moof` -at the sample bound, and fix each or accept it with the number. - -Public API: none. Wire: none. diff --git a/quest/m2/sps-fuzz-h264-reader.md b/quest/m2/sps-fuzz-h264-reader.md index ee6e864b085..145d32144d5 100644 --- a/quest/m2/sps-fuzz-h264-reader.md +++ b/quest/m2/sps-fuzz-h264-reader.md @@ -9,7 +9,3 @@ The H.264 SPS fuzz target runs against moq-mux's parser after [moq-dev/moq#7398] Decided 2026-10-30 (maintainer, follow-up planning): [moq-dev/moq#7313](https://github.com/moq-dev/moq/pull/7313) found the crop and width overflows in `h264-parser` ([therishidesai/h264-parser#9](https://github.com/therishidesai/h264-parser/issues/9)). #7398 replaces that crate for H.264, so the finding has to be rechecked on the new parser. Run the target long enough to cover the old reproducer and a fresh corpus. If `h264-reader` is clean, narrow the SPS quest to H.265. Public API: none. Wire: none. - -## Required - -- [Media parsers have fuzz targets](/quest/m2/media-fuzz.md) - its PR, #7313, adds the SPS target and the fuzz-findings epic diff --git a/quest/m2/wire-fuzz-coverage.md b/quest/m2/wire-fuzz-coverage.md index 0da12bd6d36..38c17b971c0 100644 --- a/quest/m2/wire-fuzz-coverage.md +++ b/quest/m2/wire-fuzz-coverage.md @@ -29,4 +29,4 @@ Public API: none. Wire: none. - [CI cost and latency](/quest/m0/ci/README.md) - where the nightly budget lives - [Bound cmsg payload reads](https://github.com/moq-dev/moq/pull/7394) - fixes the moq-sock overread a cmsg target would hit -- [Untrusted media parsers have fuzz targets](/quest/m2/media-fuzz.md) - the media parser targets beside these +- [Media fuzz findings](/quest/m1/fuzz-findings/README.md) - the media parser targets beside these, and the bugs they found diff --git a/rs/justfile b/rs/justfile index 0c0cf620a67..47bb11e7027 100644 --- a/rs/justfile +++ b/rs/justfile @@ -13,7 +13,7 @@ set working-directory := '..' # needs a nightly toolchain and sanitizer flags, so `just rs fuzz` is the one recipe # that may compile it. Every `--workspace` below therefore carries this, as does # sh/rs/select.sh, since it cannot read this. -no_fuzz := "--exclude moq-net-fuzz --exclude moq-quic-fuzz" +no_fuzz := "--exclude moq-net-fuzz --exclude moq-quic-fuzz --exclude moq-mux-fuzz --exclude moq-rtmp-fuzz" default: just check @@ -396,9 +396,9 @@ audio-devices: just rs test -p moq-audio --features aec,pulseaudio --run-ignored only -E 'not test(/benchmark_/)' # Needs nightly and cargo-fuzz; CI replays the corpus on stable through each -# crate's own tests instead. See rs/moq-net/fuzz/README.md and rs/moq-quic/fuzz/README.md. +# crate's own tests instead. See the README in each rs//fuzz/. -# Fuzz a moq-net or moq-quic target by name: `just rs fuzz lite -- -max_total_time=300`. +# Fuzz a target of any rs//fuzz/ by name: `just rs fuzz lite -- -max_total_time=300`. # `just rs fuzz --minimize lite` shrinks its corpus instead. [positional-arguments] fuzz target *args: diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 2babfea57f5..20906aeafc4 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -12,6 +12,11 @@ rust-version.workspace = true keywords = ["quic", "http3", "webtransport", "media", "live"] categories = ["multimedia", "network-programming", "web-programming"] +[features] +# Exposes the media parsers' fuzz target bodies and seeds to the `fuzz/` harness through the +# hidden `fuzz` module. Off by default and hidden from the docs. +fuzz = [] + [dependencies] anyhow = { workspace = true } base64 = { workspace = true } @@ -56,6 +61,13 @@ workspace-hack = { path = "../workspace-hack" } [target.'cfg(all(target_os = "linux", target_env = "gnu"))'.dev-dependencies] libc = { workspace = true } +# Writes the generated fuzzer corpus to disk. Needs the `fuzz` feature, so a default +# `--all-targets` build skips it rather than failing to find the module. +[[example]] +name = "fuzz-seeds" +path = "examples/fuzz-seeds.rs" +required-features = ["fuzz"] + [[bench]] name = "catalog" harness = false diff --git a/rs/moq-mux/examples/fuzz-seeds.rs b/rs/moq-mux/examples/fuzz-seeds.rs new file mode 100644 index 00000000000..f7ce9332ce4 --- /dev/null +++ b/rs/moq-mux/examples/fuzz-seeds.rs @@ -0,0 +1,23 @@ +//! Write the generated fuzzer corpus to disk: `fuzz-seeds ` fills +//! `//` with one file per seed. +//! +//! `just rs fuzz` runs this before each session, so the corpus always matches the +//! fixtures and framing in `moq_mux::fuzz` rather than whatever was generated last. + +fn main() { + let root = std::env::args().nth(1).expect("usage: fuzz-seeds "); + let root = std::path::Path::new(&root); + + for (target, _) in moq_mux::fuzz::TARGETS { + let dir = root.join(target); + std::fs::create_dir_all(&dir).expect("could not create the seed directory"); + } + + let seeds = moq_mux::fuzz::seeds(); + for (index, seed) in seeds.iter().enumerate() { + let path = root.join(seed.target).join(format!("{index}.bin")); + std::fs::write(&path, &seed.data).expect("could not write a seed"); + } + + println!("wrote {} seeds to {}", seeds.len(), root.display()); +} diff --git a/rs/moq-mux/fuzz/.gitignore b/rs/moq-mux/fuzz/.gitignore new file mode 100644 index 00000000000..bfd3bed6c97 --- /dev/null +++ b/rs/moq-mux/fuzz/.gitignore @@ -0,0 +1,5 @@ +target +corpus +seeds +artifacts +coverage diff --git a/rs/moq-mux/fuzz/Cargo.toml b/rs/moq-mux/fuzz/Cargo.toml new file mode 100644 index 00000000000..3bc135f909b --- /dev/null +++ b/rs/moq-mux/fuzz/Cargo.toml @@ -0,0 +1,77 @@ +# The libFuzzer harness for moq-mux's media parsers. `just rs fuzz ` drives it, +# and nightly.yml runs every target for five minutes. +# +# A workspace member so it shares the root lockfile, which is the only thing that +# keeps its resolve from drifting when a sibling it depends on is bumped. It is +# excluded from every gate that means "all of it" instead: libFuzzer needs a nightly +# toolchain and sanitizer flags, so nothing in `check` or `test` may compile it. +# +# The target bodies live in `moq-mux`'s hidden `fuzz` module, not here: sharing them +# is what lets the crate's own tests replay the corpus on stable, allocation bound +# included. +[package] +name = "moq-mux-fuzz" +version = "0.0.0" +license = "MIT OR Apache-2.0" +publish = false +edition = "2024" +autobins = false + +[package.metadata] +cargo-fuzz = true + +[[bin]] +name = "fmp4" +path = "fuzz_targets/fmp4.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "mkv" +path = "fuzz_targets/mkv.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "ts" +path = "fuzz_targets/ts.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "flv" +path = "fuzz_targets/flv.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "cmaf" +path = "fuzz_targets/cmaf.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "annexb" +path = "fuzz_targets/annexb.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "catalog" +path = "fuzz_targets/catalog.rs" +test = false +doc = false +bench = false + +[dependencies] +libfuzzer-sys = "0.4" +moq-mux = { workspace = true, features = ["fuzz"] } + +[lints] +workspace = true diff --git a/rs/moq-mux/fuzz/README.md b/rs/moq-mux/fuzz/README.md new file mode 100644 index 00000000000..55853448c6d --- /dev/null +++ b/rs/moq-mux/fuzz/README.md @@ -0,0 +1,69 @@ +# Fuzzing moq-mux + +Coverage-guided fuzzing of the media parsers an untrusted publisher reaches: the fMP4, +Matroska, MPEG-TS, and FLV importers (fMP4, HLS, SRT, and RTMP ingest), the Annex-B and +AV1 OBU splitters, the CMAF fragment decoder, and the hang and MSF catalog consumers +(catalog subscription). + +```bash +cargo install --locked cargo-fuzz +just rs fuzz fmp4 +``` + +The targets are `fmp4`, `mkv`, `ts`, `flv`, `cmaf`, `annexb`, and `catalog`. Extra +arguments pass through to libFuzzer, so `just rs fuzz fmp4 -- -max_total_time=300` +bounds a run. + +The Nightly workflow runs every target for five minutes, resumes from the previous run's +minimized corpus, and uploads failure inputs as workflow artifacts. + +## How it fits together + +The target bodies are not in `fuzz_targets/`; they are in `moq-mux`'s hidden `fuzz` +module (`src/fuzz.rs`), and the files here are shims that also register its counting +allocator. `just check` replays the same bodies on the pinned stable toolchain, bound +included, so a failure found here becomes a regression test that CI runs without anyone +installing cargo-fuzz. + +`just rs fuzz` regenerates `seeds/` from `moq_mux::fuzz::seeds()` before each run: the +crate's media fixtures, wrapped in each target's framing. libFuzzer writes what it +discovers to `corpus//`, which is kept across runs and gitignored along with +`seeds/` and `artifacts/`. `just rs fuzz --minimize ` shrinks the corpus once it +has grown. + +Calling `cargo fuzz` directly skips `just rs fuzz`, which turns off the release +profile's fat LTO (it drops the sanitizer-coverage globals, so the link fails) and +symbol stripping. Prefix direct calls with +`CARGO_PROFILE_RELEASE_LTO=false CARGO_PROFILE_RELEASE_STRIP=none`. + +## Input framing + +- `fmp4`, `mkv`, `ts`, `flv`: a write size byte (0 for the whole input), then the + container bytes, written in pieces of that size as ingest reads a socket. +- `annexb`: a selector byte whose remainder modulo 3 picks H.264, H.265, or AV1 and whose + quotient is the write size, then the stream. +- `cmaf`: a big-endian `u16` init segment length, the init segment, then one moof+mdat + frame. +- `catalog`: a selector byte whose low bit picks the hang (0) or MSF (1) catalog track, + then that track's frames, separated by NUL. + +## What the targets assert + +Beyond "does not panic", `fuzz::run` counts the heap a target allocates on its thread +and fails an input whose total bytes or allocations exceed a fixed base plus a constant +per input byte. Every allocation counts, not just what stays live, so the bound catches +work that grows faster than the input, such as rebuffering or reparsing on each write, +as well as memory that does. The constants and what sets them are documented in +`src/fuzz.rs`. + +## When something fails + +libFuzzer writes the input to `artifacts//crash-` and prints the `cargo fuzz` +command to reproduce it (add the prefix above). Fix the bug, then commit the input: + +```bash +cp rs/moq-mux/fuzz/artifacts/fmp4/crash- rs/moq-mux/fuzz/regressions/fmp4/ +``` + +`fuzz::tests::regressions` replays everything under `regressions//` as part of +the normal test suite, so the case stays covered. diff --git a/rs/moq-mux/fuzz/fuzz_targets/annexb.rs b/rs/moq-mux/fuzz/fuzz_targets/annexb.rs new file mode 100644 index 00000000000..a2655bdca96 --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/annexb.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::annexb, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/catalog.rs b/rs/moq-mux/fuzz/fuzz_targets/catalog.rs new file mode 100644 index 00000000000..0d1048c2293 --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/catalog.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::catalog, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/cmaf.rs b/rs/moq-mux/fuzz/fuzz_targets/cmaf.rs new file mode 100644 index 00000000000..9b0a8f66ab9 --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/cmaf.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::cmaf, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/flv.rs b/rs/moq-mux/fuzz/fuzz_targets/flv.rs new file mode 100644 index 00000000000..e17709779d0 --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/flv.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::flv, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/fmp4.rs b/rs/moq-mux/fuzz/fuzz_targets/fmp4.rs new file mode 100644 index 00000000000..b68c9e4be0f --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/fmp4.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::fmp4, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/mkv.rs b/rs/moq-mux/fuzz/fuzz_targets/mkv.rs new file mode 100644 index 00000000000..c8b96be1d0a --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/mkv.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::mkv, data); +}); diff --git a/rs/moq-mux/fuzz/fuzz_targets/ts.rs b/rs/moq-mux/fuzz/fuzz_targets/ts.rs new file mode 100644 index 00000000000..1fda4c1b9d6 --- /dev/null +++ b/rs/moq-mux/fuzz/fuzz_targets/ts.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_mux::fuzz::Counting = moq_mux::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_mux::fuzz::run(moq_mux::fuzz::ts, data); +}); diff --git a/rs/moq-mux/fuzz/regressions/README.md b/rs/moq-mux/fuzz/regressions/README.md new file mode 100644 index 00000000000..eec5b302c2a --- /dev/null +++ b/rs/moq-mux/fuzz/regressions/README.md @@ -0,0 +1,8 @@ +# Regressions + +Failing inputs the fuzzer found, one file per input, in a directory named after its +target (`fmp4/`, `ts/`, `catalog/`, ...). `fuzz::tests::regressions` replays them on the +stable toolchain as part of `just check`, allocation bound included, so a bug found by +fuzzing stays fixed without anyone running the fuzzer again. + +See [../README.md](../README.md) for the workflow. diff --git a/rs/moq-mux/fuzz/regressions/mkv/short-block-header b/rs/moq-mux/fuzz/regressions/mkv/short-block-header new file mode 100644 index 00000000000..e2559ddbe10 --- /dev/null +++ b/rs/moq-mux/fuzz/regressions/mkv/short-block-header @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/rs/moq-mux/src/allocation.rs b/rs/moq-mux/src/allocation.rs new file mode 100644 index 00000000000..834ba1c418f --- /dev/null +++ b/rs/moq-mux/src/allocation.rs @@ -0,0 +1,73 @@ +//! Per-thread heap accounting for the crate's tests and the fuzz targets' allocation bound. +//! +//! [`Counting`] forwards every call to the system allocator unchanged and counts only on a thread +//! inside [`measure`], so concurrent tests are unaffected. A binary opts in by registering it as its +//! `#[global_allocator]`: this crate's unit tests do below, and each `fuzz/` target does too. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +/// The heap traffic of one [`measure`]d call. +#[derive(Clone, Copy, Debug, Default)] +pub struct Usage { + /// Bytes allocated, counting a realloc as a fresh allocation of its new size. + pub allocated: usize, + /// Bytes freed, counting a realloc as freeing its old size. + pub freed: usize, + /// Calls to allocate or reallocate. + pub allocations: usize, +} + +thread_local! { + /// `const` so the allocator can read it without allocating. + static USAGE: Cell> = const { Cell::new(None) }; +} + +fn record(update: impl FnOnce(&mut Usage)) { + // `try_with` because TLS is unavailable while a thread is being torn down. + let _ = USAGE.try_with(|usage| { + if let Some(mut current) = usage.get() { + update(&mut current); + usage.set(Some(current)); + } + }); +} + +/// Counts on the measuring thread; every call forwards to the system allocator unchanged. +pub struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + record(|usage| { + usage.allocated += layout.size(); + usage.allocations += 1; + }); + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + record(|usage| usage.freed += layout.size()); + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + record(|usage| { + usage.allocated += new_size; + usage.freed += layout.size(); + usage.allocations += 1; + }); + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[cfg(test)] +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// Run `f`, returning what it did to this thread's heap. Not reentrant. +pub fn measure(f: impl FnOnce() -> T) -> (T, Usage) { + USAGE.with(|usage| usage.set(Some(Usage::default()))); + let result = f(); + let usage = USAGE.with(|usage| usage.replace(None).unwrap()); + (result, usage) +} diff --git a/rs/moq-mux/src/catalog/mod.rs b/rs/moq-mux/src/catalog/mod.rs index 3dde971992e..aeb84b90cf2 100644 --- a/rs/moq-mux/src/catalog/mod.rs +++ b/rs/moq-mux/src/catalog/mod.rs @@ -43,6 +43,8 @@ pub use entry::Entry; pub(crate) use estimate::Reading; pub use estimate::{Estimate, Estimator}; pub use format::*; +#[cfg(any(test, feature = "fuzz"))] +pub(crate) use producer::to_msf; pub use producer::{Config, Guard, Producer}; pub use select::Select; pub use stream::Stream; diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index f5699d5c8f4..d2b96b98a93 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1397,7 +1397,7 @@ fn packaging_of(container: &hang::catalog::Container) -> moq_msf::Packaging { /// Convert a catalog to its MSF equivalent: the media sections become MSF tracks and the /// extension rides the MSF root under the same member names it uses in the hang catalog. -fn to_msf(catalog: &Catalog, base: &moq_net::Path) -> moq_msf::Catalog { +pub(crate) fn to_msf(catalog: &Catalog, base: &moq_net::Path) -> moq_msf::Catalog { let mut msf = to_msf_media(&catalog.media(), base); msf.ext = catalog.ext.clone(); msf diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 33b81f7fcfc..f6d5be2ff82 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -610,8 +610,12 @@ pub(crate) fn pack_constraint_flags(profile: &scuffle_h265::Profile) -> [u8; 6] } /// Real parameter sets, mostly from a single-frame x265 encode (1280x720, Main profile), -/// for tests that need an SPS rather than hand-built bytes. -#[cfg(test)] +/// for tests that need an SPS rather than hand-built bytes, and the fuzz seeds. +#[cfg(any(test, feature = "fuzz"))] +#[cfg_attr( + not(test), + allow(dead_code, reason = "the fuzz seeds use only the Main parameter sets") +)] pub(crate) mod fixtures { use bytes::Bytes; diff --git a/rs/moq-mux/src/fuzz.rs b/rs/moq-mux/src/fuzz.rs new file mode 100644 index 00000000000..e4e4cd38786 --- /dev/null +++ b/rs/moq-mux/src/fuzz.rs @@ -0,0 +1,459 @@ +//! Fuzz target bodies for the media parsers an untrusted publisher reaches, plus their seeds. +//! +//! Each target takes raw fuzzer bytes, drives one parser through its public entry point the way +//! ingest or a subscriber does, and returns whether the input was accepted. [`run`] wraps a target +//! with the invariant the fuzzer checks beyond "no panic": heap traffic bounded in proportion to +//! the input. The bodies live here rather than in `fuzz/fuzz_targets/` so the tests below replay +//! them on stable, which is what turns a crash the fuzzer found into a regression `just check` +//! runs. +//! +//! Compiled only under `cfg(test)` or the `fuzz` feature, so none of this is part of the published +//! API. See `fuzz/README.md` for the workflow. + +use std::task::Poll; + +use bytes::Bytes; + +pub use crate::allocation::{Counting, Usage, measure}; +use crate::import::{Container, ContainerFormat, ContainerInit, TrackStream, VideoFormat, VideoInit}; + +/// One fuzz target body: it returns whether the input was accepted, which is what [`seeds`] is +/// checked against. +pub type Target = fn(&[u8]) -> bool; + +/// Every target, keyed by the name of its `fuzz_targets/.rs` shim. +pub const TARGETS: &[(&str, Target)] = &[ + ("fmp4", fmp4), + ("mkv", mkv), + ("ts", ts), + ("flv", flv), + ("cmaf", cmaf), + ("annexb", annexb), + ("catalog", catalog), +]; + +/// Bytes a target may allocate per input byte, summed over the whole run. +/// +/// Counts every allocation, not what stays live, so it bounds copying as well as holding: a parser +/// that rebuffers or reparses its input on every chunk grows with the square of the input and +/// crosses any linear bound once the input is long enough. A keyframe of a dozen bytes opens a +/// group, whose state and HLS record cost a few KiB, which is what sets the slope. +const BYTES_PER_BYTE: usize = 1024; + +/// Allocations a target may make per input byte, for the same dozen-byte keyframe. +const ALLOCATIONS_PER_BYTE: usize = 8; + +/// What any input may allocate, however short: the broadcast, catalog, and importer a target sets +/// up before reading a byte, and the first catalog and HLS publishes, whose compressed groups each +/// open a DEFLATE encoder of a few hundred KiB. +const BASE_BYTES: usize = 4 << 20; + +/// Allocations any input may make, however short. +const BASE_ALLOCATIONS: usize = 4096; + +/// Run `target` on `data` and check its heap traffic stays within a bound linear in `data`. +/// +/// Measures only when the binary registers [`Counting`] as its global allocator; this crate's tests +/// and every `fuzz/` target do. +pub fn run(target: Target, data: &[u8]) -> bool { + let (accepted, usage) = measure(|| target(data)); + if !within(usage, data.len()) { + // Lazily initialized statics (tracing callsites, compiled regexes) allocate once per + // process, not per input, so only a second run's excess is the input's own. + let (_, usage) = measure(|| target(data)); + assert!( + within(usage, data.len()), + "{} input bytes allocated {} bytes in {} allocations", + data.len(), + usage.allocated, + usage.allocations, + ); + } + accepted +} + +fn within(usage: Usage, len: usize) -> bool { + usage.allocated <= BASE_BYTES + BYTES_PER_BYTE * len + && usage.allocations <= BASE_ALLOCATIONS + ALLOCATIONS_PER_BYTE * len +} + +/// Split off a selector byte, and turn it into a write size: 0 writes the input whole. +fn chunked(data: &[u8], selector: u8) -> std::slice::Chunks<'_, u8> { + data.chunks(match selector { + 0 => data.len().max(1), + n => n as usize, + }) +} + +fn broadcast() -> (moq_net::broadcast::Producer, crate::catalog::Producer) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()) + .expect("a fresh broadcast takes a catalog"); + (broadcast, catalog) +} + +/// Import `data` as `format`, written in pieces the leading byte sizes, as ingest reads a socket. +fn container(format: ContainerFormat, data: &[u8]) -> bool { + let Some((&selector, data)) = data.split_first() else { + return false; + }; + let (broadcast, catalog) = broadcast(); + let init = ContainerInit::new(format, Bytes::new()); + let Ok(mut import) = Container::<()>::new(broadcast, catalog.reserve(), &init) else { + return false; + }; + for chunk in chunked(data, selector) { + if import.decode(chunk).is_err() { + return false; + } + } + import.finish().is_ok() +} + +/// fMP4 import, as fMP4 and HLS ingest feed it. +pub fn fmp4(data: &[u8]) -> bool { + container(ContainerFormat::Fmp4, data) +} + +/// Matroska / WebM import. +pub fn mkv(data: &[u8]) -> bool { + container(ContainerFormat::Mkv, data) +} + +/// MPEG-TS import, as SRT and HLS ingest feed it. +pub fn ts(data: &[u8]) -> bool { + container(ContainerFormat::Ts, data) +} + +/// FLV import, as RTMP ingest feeds it. +pub fn flv(data: &[u8]) -> bool { + container(ContainerFormat::Flv, data) +} + +/// A subscriber decoding a CMAF track: a big-endian `u16` init length, the init segment a catalog +/// carries, then one moof+mdat frame. +pub fn cmaf(data: &[u8]) -> bool { + use crate::container::Container as _; + + let Some((length, rest)) = data.split_first_chunk::<2>() else { + return false; + }; + let Some((init, fragment)) = rest.split_at_checked(u16::from_be_bytes(*length) as usize) else { + return false; + }; + let Ok(wire) = crate::container::fmp4::Wire::from_init(init) else { + return false; + }; + let frame = moq_net::frame::Frame::new(None, Bytes::copy_from_slice(fragment)); + wire.decode(frame).is_ok() +} + +/// The codecs [`annexb`] splits, selected by the leading byte modulo their count. +const STREAMS: &[VideoFormat] = &[VideoFormat::Avc3, VideoFormat::Hev1, VideoFormat::Av01]; + +/// A raw byte stream of unknown frame boundaries: H.264 or H.265 Annex-B, or AV1 OBUs. +/// +/// The leading byte picks the codec, and what remains of it once divided by the codec count is +/// the write size. +pub fn annexb(data: &[u8]) -> bool { + let Some((&selector, data)) = data.split_first() else { + return false; + }; + let format = STREAMS[selector as usize % STREAMS.len()]; + let (broadcast, catalog) = broadcast(); + let request = broadcast + .reserve_track("video") + .expect("a fresh broadcast has no video track"); + let Ok(mut import) = TrackStream::video(request, catalog.reserve(), VideoInit::new(format, Bytes::new())) else { + return false; + }; + for chunk in chunked(data, selector / STREAMS.len() as u8) { + if import.decode(chunk).is_err() { + return false; + } + } + import.finish().is_ok() +} + +/// A subscriber reading a catalog track: the leading byte's low bit picks hang (0) or MSF (1), +/// and the rest is the frames of one group, split on NUL, which JSON text never contains. +/// +/// The consumer reads after each frame, as it does when updates trickle in, and every catalog it +/// yields has each audio and video rendition's container built, as a subscriber does before +/// reading the track. +pub fn catalog(data: &[u8]) -> bool { + let Some((&selector, data)) = data.split_first() else { + return false; + }; + let broadcast = moq_net::broadcast::Info::new().produce(); + let waiter = kio::Waiter::noop(); + + let name = if selector & 1 == 0 { + hang::Catalog::DEFAULT_NAME + } else { + moq_msf::DEFAULT_NAME + }; + // Uncompressed: inflating a compressed group is moq-net's, ahead of any catalog parser. + let info = hang::Catalog::default_track_info().with_compress(false); + let track = broadcast + .create_track(name, info) + .expect("a fresh broadcast has no catalog"); + let subscriber = track.subscribe(None); + let mut next: Box Poll> = if selector & 1 == 0 { + let mut consumer = crate::catalog::hang::Consumer::<()>::new(subscriber); + Box::new(move || consumer.poll_next(&waiter)) + } else { + let mut consumer = crate::catalog::msf::Consumer::<()>::new(subscriber); + Box::new(move || consumer.poll_next(&waiter)) + }; + + let mut group = track.append_group().expect("a fresh track takes a group"); + let mut accepted = false; + for frame in data.split(|&byte| byte == 0) { + if group + .write_frame(moq_net::Timestamp::ZERO, Bytes::copy_from_slice(frame)) + .is_err() + { + return false; + } + match next() { + Poll::Ready(Ok(Some(catalog))) => accepted = containers(&catalog), + Poll::Ready(Ok(None)) | Poll::Pending => {} + Poll::Ready(Err(_)) => return false, + } + } + accepted +} + +/// What a catalog consumer yields when polled. +type CatalogUpdate = crate::Result>>; + +/// Build every audio and video rendition's container, returning whether all of them parsed. +fn containers(catalog: &crate::catalog::hang::Catalog<()>) -> bool { + use crate::catalog::hang::Container; + + let video = catalog.video.renditions.values().map(Container::try_from); + let audio = catalog.audio.renditions.values().map(Container::try_from); + video.chain(audio).all(|container| container.is_ok()) +} + +/// One input to start a target from. +pub struct Seed { + /// The target it feeds, a key of [`TARGETS`]. + pub target: &'static str, + /// The input, selector bytes included. + pub data: Vec, +} + +/// The starting corpus: the crate's media fixtures, wrapped in each target's framing. +/// +/// Every seed is accepted, so the fuzzer starts from inputs that reach past the first header. +pub fn seeds() -> Vec { + let mut seeds = Vec::new(); + let mut add = |target: &'static str, selector: u8, data: &[u8]| { + let mut input = vec![selector]; + input.extend_from_slice(data); + seeds.push(Seed { target, data: input }); + }; + + // bbb.mp4 ends on a moof without its mdat, so keep its first four whole fragments. + let bbb = &include_bytes!("container/fmp4/test_data/bbb.mp4")[..2035]; + for fmp4 in [ + bbb, + include_bytes!("container/fmp4/test_data/av1.mp4"), + include_bytes!("container/fmp4/test_data/vp9.mp4"), + ] { + add("fmp4", 0, fmp4); + } + // Small writes too, so the corpus covers an atom split across reads. + add("fmp4", 100, bbb); + + add("mkv", 0, include_bytes!("container/mkv/test_data/opus.webm")); + add("mkv", 100, include_bytes!("container/mkv/test_data/opus.webm")); + + for ts in [ + &include_bytes!("container/ts/test_data/bbb.ts")[..], + include_bytes!("container/ts/test_data/aac_quad.ts"), + include_bytes!("container/ts/test_data/poc-avc.ts"), + include_bytes!("container/ts/test_data/poc-hevc.ts"), + ] { + add("ts", 0, ts); + } + add("ts", 100, include_bytes!("container/ts/test_data/bbb.ts")); + + add("flv", 0, include_bytes!("container/ts/test_data/he_aac.flv")); + add("flv", 100, include_bytes!("container/ts/test_data/he_aac_v2.flv")); + + for (stream, unit) in [h264_stream(), h265_stream(), av1_stream()].into_iter().enumerate() { + let selector = stream as u8; + let data = [unit.clone(), unit].concat(); + add("annexb", selector, &data); + // The write size is the selector divided by the codec count. + add("annexb", selector + 7 * STREAMS.len() as u8, &data); + } + + for fixture in cmaf_fixtures() { + let mut data = u16::try_from(fixture.init.len()) + .expect("a fixture init fits a u16 length") + .to_be_bytes() + .to_vec(); + data.extend_from_slice(&fixture.init); + data.extend_from_slice(&fixture.fragment); + seeds.push(Seed { target: "cmaf", data }); + } + + let mut add = |selector: u8, frames: &[&[u8]]| { + let mut data = vec![selector]; + data.extend_from_slice(&frames.join(&0)); + seeds.push(Seed { + target: "catalog", + data, + }); + }; + for (cmaf, catalog) in [(false, imported(false)), (true, imported(true))] { + let hang = catalog.to_json().expect("a published catalog serializes"); + add(0, &[hang.as_bytes()]); + let msf = crate::catalog::to_msf(&catalog, &moq_net::Path::empty()); + let msf = msf.to_json().expect("a published catalog serializes"); + add(1, &[msf.as_bytes()]); + if cmaf { + // A snapshot, then a merge-patch delta in the same group. + add(0, &[hang.as_bytes(), br#"{"user":{"name":"bunny"}}"#]); + } + } + add(0, &[include_bytes!("../../hang/fixtures/catalog-clock.json")]); + add(0, &[include_bytes!("../../hang/fixtures/catalog-stats-echo.json")]); + + seeds +} + +/// The catalog an import of bbb.mp4 publishes, with CMAF passthrough or without. +fn imported(passthrough: bool) -> crate::catalog::hang::Catalog<()> { + let (broadcast, catalog) = broadcast(); + let mut import = crate::container::fmp4::Import::new(broadcast, catalog.reserve()).with_passthrough(passthrough); + import + .decode(&include_bytes!("container/fmp4/test_data/bbb.mp4")[..2035]) + .expect("the fixture imports"); + catalog.snapshot() +} + +/// The moov of an MP4 fixture. +fn moov(mp4: &[u8]) -> mp4_atom::Moov { + use mp4_atom::DecodeMaybe; + + let mut cursor = std::io::Cursor::new(mp4); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).expect("the fixture parses") { + if let mp4_atom::Any::Moov(moov) = atom { + return moov; + } + } + panic!("the fixture has no moov"); +} + +/// The codec config of an MP4 fixture's first track. +fn codec(mp4: &[u8]) -> mp4_atom::Codec { + moov(mp4).trak.swap_remove(0).mdia.minf.stbl.stsd.codecs.swap_remove(0) +} + +/// Prefix each NAL unit with a start code. +fn start_codes(nals: &[&[u8]]) -> Vec { + nals.iter().flat_map(|nal| [&[0, 0, 0, 1][..], nal].concat()).collect() +} + +/// An H.264 access unit: bbb.mp4's SPS and PPS, then an IDR slice. +fn h264_stream() -> Vec { + let mp4_atom::Codec::Avc1(avc1) = codec(include_bytes!("container/fmp4/test_data/bbb.mp4")) else { + panic!("bbb.mp4 opens with its H.264 track"); + }; + let (sps, pps) = ( + &avc1.avcc.sequence_parameter_sets[0], + &avc1.avcc.picture_parameter_sets[0], + ); + start_codes(&[&[0x09, 0xf0], sps, pps, &[0x65, 0x88, 0x84, 0x21]]) +} + +/// An H.265 access unit: real parameter sets, then an IDR slice. +fn h265_stream() -> Vec { + use crate::codec::h265::fixtures::{PPS, SPS, VPS}; + start_codes(&[&[0x46, 0x01, 0x50], VPS, SPS, PPS, &[0x26, 0x01, 0xaf, 0x80]]) +} + +/// An AV1 temporal unit: a temporal delimiter, av1.mp4's sequence header, and a key frame. +fn av1_stream() -> Vec { + let mp4_atom::Codec::Av01(av01) = codec(include_bytes!("container/fmp4/test_data/av1.mp4")) else { + panic!("av1.mp4 carries AV1"); + }; + [&[0x12, 0x00][..], &av01.av1c.config_obus, &[0x32, 0x02, 0x10, 0x00]].concat() +} + +/// A CMAF init segment and a fragment to decode against it. +#[serde_with::serde_as] +#[derive(serde::Deserialize)] +struct CmafFixture { + #[serde_as(as = "serde_with::base64::Base64")] + init: Vec, + #[serde_as(as = "serde_with::base64::Base64")] + fragment: Vec, +} + +/// The fragments `js/hang` decodes too, each leaving sample values to a different default. +fn cmaf_fixtures() -> Vec { + #[derive(serde::Deserialize)] + struct File { + cases: Vec, + } + let file: File = serde_json::from_str(include_str!("container/fmp4/test_data/sample-defaults.json")) + .expect("the fixture parses"); + file.cases +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target(name: &str) -> Target { + TARGETS + .iter() + .find(|(target, _)| *target == name) + .unwrap_or_else(|| panic!("no such target: {name}")) + .1 + } + + /// Every seed is accepted within the bound, and every target has one. + /// + /// A corpus the parser refuses leaves the fuzzer mutating noise against an early `return`. + #[test] + fn seeds_are_accepted() { + let seeds = seeds(); + for (name, _) in TARGETS { + assert!(seeds.iter().any(|seed| seed.target == *name), "no seed for {name}"); + } + for (index, seed) in seeds.iter().enumerate() { + assert!( + run(target(seed.target), &seed.data), + "{} seed {index} refused", + seed.target + ); + } + } + + /// Replay the inputs committed under `fuzz/regressions/`, so a bug the fuzzer found stays + /// fixed on the stable toolchain, in the suite CI already runs. + #[test] + fn regressions() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("fuzz/regressions"); + + for (name, target) in TARGETS { + // Nothing has failed yet for this target; `fuzz/README.md` says what to drop in here + // when something does. + let Ok(entries) = std::fs::read_dir(root.join(name)) else { + continue; + }; + for entry in entries { + let path = entry.expect("could not read the regression directory").path(); + let data = std::fs::read(&path).expect("could not read a regression input"); + run(*target, &data); + } + } + } +} diff --git a/rs/moq-mux/src/import/mod.rs b/rs/moq-mux/src/import/mod.rs index 87416181592..3f11a23165c 100644 --- a/rs/moq-mux/src/import/mod.rs +++ b/rs/moq-mux/src/import/mod.rs @@ -34,14 +34,8 @@ pub use track::*; /// retires many sources would otherwise hold each one's largest input chunk or element. Each source /// here buffers a [`PAD`]-sized element (an atom, a tag, a block, an access unit) or chunk, then the /// test measures what dropping the finished importers frees while their tracks are still held. -/// -/// The counting allocator below serves this crate's whole unit-test binary, but it counts only on a -/// thread inside [`released`], so other tests are unaffected. #[cfg(test)] mod tests { - use std::alloc::{GlobalAlloc, Layout, System}; - use std::cell::Cell; - use super::*; use bytes::Bytes; @@ -51,45 +45,10 @@ mod tests { /// Small enough that an element spans many writes, so the importer accumulates it. const CHUNK: usize = 64 * 1024; - thread_local! { - /// Bytes this thread freed minus those it allocated while measuring. Per thread so concurrent - /// tests don't mix, and `const` so the allocator can read it without allocating. - static RELEASED: Cell> = const { Cell::new(None) }; - } - - fn count(bytes: isize) { - // `try_with` because TLS is unavailable while a thread is being torn down. - let _ = RELEASED.try_with(|released| released.set(released.get().map(|n| n + bytes))); - } - - /// Counts on the measuring thread; every call forwards to the system allocator unchanged. - struct Counting; - - unsafe impl GlobalAlloc for Counting { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - count(-(layout.size() as isize)); - unsafe { System.alloc(layout) } - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - count(layout.size() as isize); - unsafe { System.dealloc(ptr, layout) } - } - - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - count(layout.size() as isize - new_size as isize); - unsafe { System.realloc(ptr, layout, new_size) } - } - } - - #[global_allocator] - static ALLOCATOR: Counting = Counting; - /// The heap `f` gives back, net of what it allocates on the way. fn released(f: impl FnOnce()) -> isize { - RELEASED.with(|released| released.set(Some(0))); - f(); - RELEASED.with(|released| released.replace(None).unwrap()) + let ((), usage) = crate::allocation::measure(f); + usage.freed as isize - usage.allocated as isize } /// One source's importer, finished and kept the way a binding keeps it. diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 1ff3603d2f2..361581e8e8b 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -30,12 +30,19 @@ //! - [`Linger`] keeps a cleanly ended broadcast up for a fixed time, so a //! subscriber that arrives after the end still gets its tail. +#[cfg(any(test, feature = "fuzz"))] +mod allocation; pub mod catalog; mod clock; pub mod codec; pub mod container; mod error; pub mod follow; +// Not part of the public API: compiled only for the crate's own tests and for the `fuzz/` +// harness. +#[cfg(any(test, feature = "fuzz"))] +#[doc(hidden)] +pub mod fuzz; pub mod hls; pub mod import; mod jitter; diff --git a/rs/moq-rtmp/Cargo.toml b/rs/moq-rtmp/Cargo.toml index 3724f83a98b..08fb75b7d51 100644 --- a/rs/moq-rtmp/Cargo.toml +++ b/rs/moq-rtmp/Cargo.toml @@ -21,6 +21,9 @@ default = ["tls"] # `pub use rustls`. Plaintext RTMP needs none of this, so an embedder can drop # TLS with `default-features = false`. tls = ["dep:rustls", "dep:tokio-rustls"] +# Exposes the AMF0 fuzz target body and seeds to the `fuzz/` harness through the hidden `fuzz` +# module. Off by default and hidden from the docs. +fuzz = [] # Pure library: the RTMP <-> MoQ gateway. The CLI lives in `moq-cli` (the `rtmp` # subcommand); a relay can also embed `moq_rtmp::Server` against its own origin. @@ -64,6 +67,13 @@ tokio = { workspace = true, features = ["test-util"] } tracing-test = { workspace = true } workspace-hack = { path = "../workspace-hack" } +# Writes the generated fuzzer corpus to disk. Needs the `fuzz` feature, so a default +# `--all-targets` build skips it rather than failing to find the module. +[[example]] +name = "fuzz-seeds" +path = "examples/fuzz-seeds.rs" +required-features = ["fuzz"] + [[bench]] name = "chunk_interleaving" harness = false diff --git a/rs/moq-rtmp/examples/fuzz-seeds.rs b/rs/moq-rtmp/examples/fuzz-seeds.rs new file mode 100644 index 00000000000..f06093a315c --- /dev/null +++ b/rs/moq-rtmp/examples/fuzz-seeds.rs @@ -0,0 +1,23 @@ +//! Write the generated fuzzer corpus to disk: `fuzz-seeds ` fills +//! `//` with one file per seed. +//! +//! `just rs fuzz` runs this before each session, so the corpus always matches the +//! messages in `moq_rtmp::fuzz` rather than whatever was generated last. + +fn main() { + let root = std::env::args().nth(1).expect("usage: fuzz-seeds "); + let root = std::path::Path::new(&root); + + for (target, _) in moq_rtmp::fuzz::TARGETS { + let dir = root.join(target); + std::fs::create_dir_all(&dir).expect("could not create the seed directory"); + } + + let seeds = moq_rtmp::fuzz::seeds(); + for (index, seed) in seeds.iter().enumerate() { + let path = root.join(seed.target).join(format!("{index}.bin")); + std::fs::write(&path, &seed.data).expect("could not write a seed"); + } + + println!("wrote {} seeds to {}", seeds.len(), root.display()); +} diff --git a/rs/moq-rtmp/fuzz/.gitignore b/rs/moq-rtmp/fuzz/.gitignore new file mode 100644 index 00000000000..bfd3bed6c97 --- /dev/null +++ b/rs/moq-rtmp/fuzz/.gitignore @@ -0,0 +1,5 @@ +target +corpus +seeds +artifacts +coverage diff --git a/rs/moq-rtmp/fuzz/Cargo.toml b/rs/moq-rtmp/fuzz/Cargo.toml new file mode 100644 index 00000000000..857e85c0d7f --- /dev/null +++ b/rs/moq-rtmp/fuzz/Cargo.toml @@ -0,0 +1,35 @@ +# The libFuzzer harness for moq-rtmp's AMF0 receive views. `just rs fuzz amf0` drives it, +# and nightly.yml runs it for five minutes. +# +# A workspace member so it shares the root lockfile, which is the only thing that +# keeps its resolve from drifting when a sibling it depends on is bumped. It is +# excluded from every gate that means "all of it" instead: libFuzzer needs a nightly +# toolchain and sanitizer flags, so nothing in `check` or `test` may compile it. +# +# The target body lives in `moq-rtmp`'s hidden `fuzz` module, not here: the vendored +# `rml` module is private, and sharing the body is what lets the crate's own tests +# replay the corpus on stable. +[package] +name = "moq-rtmp-fuzz" +version = "0.0.0" +license = "MIT OR Apache-2.0" +publish = false +edition = "2024" +autobins = false + +[package.metadata] +cargo-fuzz = true + +[[bin]] +name = "amf0" +path = "fuzz_targets/amf0.rs" +test = false +doc = false +bench = false + +[dependencies] +libfuzzer-sys = "0.4" +moq-rtmp = { workspace = true, features = ["fuzz"] } + +[lints] +workspace = true diff --git a/rs/moq-rtmp/fuzz/README.md b/rs/moq-rtmp/fuzz/README.md new file mode 100644 index 00000000000..627f13e5a0b --- /dev/null +++ b/rs/moq-rtmp/fuzz/README.md @@ -0,0 +1,36 @@ +# Fuzzing moq-rtmp + +Coverage-guided fuzzing of the AMF0 decoder behind RTMP commands and script data, the +first bytes a publisher sends after the handshake. + +```bash +cargo install --locked cargo-fuzz +just rs fuzz amf0 +``` + +Extra arguments pass through to libFuzzer, so `just rs fuzz amf0 -- -max_total_time=300` +bounds a run. The Nightly workflow runs it for five minutes, resumes from the previous +run's minimized corpus, and uploads failure inputs as workflow artifacts. + +## How it fits together + +The target body is not in `fuzz_targets/`; it is in `moq-rtmp`'s hidden `fuzz` module +(`src/fuzz.rs`), because the vendored `rml` module is private, and so `just check` +replays it on the pinned stable toolchain. The shim here also registers the counting +allocator the body measures with. + +`just rs fuzz` regenerates `seeds/` from `moq_rtmp::fuzz::seeds()` before each run. The +first input byte picks the message type (AMF3 data, AMF3 command, AMF0 data, or AMF0 +command); the rest is its body. See [moq-mux's README](../../moq-mux/fuzz/README.md) for +the corpus layout and calling `cargo fuzz` directly. + +## What the target asserts + +Beyond "does not panic": reading a message through the session's borrowed views and +walking every value it holds allocates nothing. The views keep the encoded bytes rather +than a node per value, so what a peer sends cannot grow the heap. + +## When something fails + +Fix the bug, then commit the input under `regressions/amf0/`, which +`fuzz::tests::regressions` replays as part of the normal test suite. diff --git a/rs/moq-rtmp/fuzz/fuzz_targets/amf0.rs b/rs/moq-rtmp/fuzz/fuzz_targets/amf0.rs new file mode 100644 index 00000000000..ad1be1d443a --- /dev/null +++ b/rs/moq-rtmp/fuzz/fuzz_targets/amf0.rs @@ -0,0 +1,10 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +#[global_allocator] +static ALLOCATOR: moq_rtmp::fuzz::Counting = moq_rtmp::fuzz::Counting; + +fuzz_target!(|data: &[u8]| { + moq_rtmp::fuzz::amf0(data); +}); diff --git a/rs/moq-rtmp/fuzz/regressions/README.md b/rs/moq-rtmp/fuzz/regressions/README.md new file mode 100644 index 00000000000..85d4d01953d --- /dev/null +++ b/rs/moq-rtmp/fuzz/regressions/README.md @@ -0,0 +1,8 @@ +# Regressions + +Failing inputs the fuzzer found, one file per input, in a directory named after its +target (`amf0/`). `fuzz::tests::regressions` replays them on the +stable toolchain as part of `just check`, allocation check included, so a bug found by +fuzzing stays fixed without anyone running the fuzzer again. + +See [../README.md](../README.md) for the workflow. diff --git a/rs/moq-rtmp/src/fuzz.rs b/rs/moq-rtmp/src/fuzz.rs new file mode 100644 index 00000000000..cfb46f199a5 --- /dev/null +++ b/rs/moq-rtmp/src/fuzz.rs @@ -0,0 +1,187 @@ +//! The fuzz target body for the AMF0 receive views, plus the seeds it starts from. +//! +//! The target reads an RTMP AMF0 message through the borrowed views a session uses, walking every +//! value a peer supplied, and checks that it allocates nothing: the views keep the encoded bytes +//! instead of a node per value, so heap traffic stays independent of what the peer sends. The body +//! lives here rather than in `fuzz/fuzz_targets/` because the `rml` module is private, and so the +//! tests below replay it on stable. +//! +//! Compiled only under `cfg(test)` or the `fuzz` feature, so none of this is part of the published +//! API. See `fuzz/README.md` for the workflow. + +use crate::rml::amf0::{Amf0Value, Value, allocation::measure, serialize}; +use crate::rml::messages::MessagePayload; + +pub use crate::rml::amf0::allocation::Counting; + +/// One fuzz target body: it returns whether the input decoded. +pub type Target = fn(&[u8]) -> bool; + +/// Every target, keyed by the name of its `fuzz_targets/.rs` shim. +pub const TARGETS: &[(&str, Target)] = &[("amf0", amf0)]; + +/// The AMF0 message types, selected by the input's first byte: AMF3 data, AMF3 command (AMF0 +/// behind a zero byte), AMF0 data, and AMF0 command. +const TYPES: &[u8] = &[15, 17, 18, 20]; + +/// An AMF0 message read through the session's views, every value walked, with no allocation. +/// +/// The leading byte picks the message type; the rest is its body. Measures only when the binary +/// registers [`Counting`] as its global allocator; this crate's tests and the `fuzz/` target do. +pub fn amf0(data: &[u8]) -> bool { + let Some((&selector, body)) = data.split_first() else { + return false; + }; + let payload = MessagePayload { + type_id: TYPES[selector as usize % TYPES.len()], + data: bytes::Bytes::copy_from_slice(body), + ..MessagePayload::new() + }; + + let (decoded, peak) = measure(|| { + use crate::rml::messages::Amf0Message; + + match payload.amf0() { + Ok(Some(Amf0Message::Command { + name, + object, + arguments, + .. + })) => { + std::hint::black_box(name); + walk(object); + arguments.for_each(walk); + true + } + Ok(Some(Amf0Message::Data(values))) => { + values.for_each(walk); + true + } + Ok(None) | Err(_) => false, + } + }); + assert_eq!(peak, 0, "reading {} AMF0 bytes allocated", body.len()); + decoded +} + +/// Visit every value nested in `value`, and look a property up the way a session does. +fn walk(value: Value<'_>) { + match value { + Value::Object(properties) => { + std::hint::black_box(properties.get("width")); + for (name, value) in properties { + std::hint::black_box(name); + walk(value); + } + } + Value::StrictArray(values) => values.for_each(walk), + value => { + std::hint::black_box(value); + } + } +} + +/// One input to start a target from. +pub struct Seed { + /// The target it feeds, a key of [`TARGETS`]. + pub target: &'static str, + /// The input, selector byte included. + pub data: Vec, +} + +/// The starting corpus: the commands and metadata a publisher sends, one per message type. +pub fn seeds() -> Vec { + let object = |properties: &[(&str, Amf0Value)]| { + Amf0Value::Object( + properties + .iter() + .map(|(name, value)| (name.to_string(), value.clone())) + .collect(), + ) + }; + let string = |value: &str| Amf0Value::Utf8String(value.into()); + + let connect = vec![ + string("connect"), + Amf0Value::Number(1.0), + object(&[ + ("app", string("live")), + ("tcUrl", string("rtmp://localhost/live")), + ("fpad", Amf0Value::Boolean(false)), + ( + "fourCcList", + Amf0Value::StrictArray(vec![string("av01"), string("hvc1")]), + ), + ]), + ]; + let publish = vec![ + string("publish"), + Amf0Value::Number(5.0), + Amf0Value::Null, + string("stream"), + string("live"), + ]; + let metadata = vec![ + string("@setDataFrame"), + string("onMetaData"), + object(&[ + ("width", Amf0Value::Number(1280.0)), + ("height", Amf0Value::Number(720.0)), + ("videocodecid", Amf0Value::Number(7.0)), + ("framerate", Amf0Value::Number(30.0)), + ("encoder", string("obs")), + ("nested", object(&[("undefined", Amf0Value::Undefined)])), + ]), + ]; + + let mut seeds = Vec::new(); + for (selector, values) in [ + (1u8, &connect), + (3, &connect), + (3, &publish), + (0, &metadata), + (2, &metadata), + ] { + let mut data = vec![selector]; + // An AMF3 command wraps its AMF0 behind a zero byte. + if TYPES[selector as usize] == 17 { + data.push(0); + } + data.extend(serialize(values).expect("the seed serializes")); + seeds.push(Seed { target: "amf0", data }); + } + seeds +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every seed decodes, so the fuzzer starts past the first marker. + #[test] + fn seeds_are_accepted() { + for (index, seed) in seeds().iter().enumerate() { + assert!(amf0(&seed.data), "seed {index} refused"); + } + } + + /// Replay the inputs committed under `fuzz/regressions/`, so a bug the fuzzer found stays + /// fixed on the stable toolchain, in the suite CI already runs. + #[test] + fn regressions() { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("fuzz/regressions"); + + for (name, target) in TARGETS { + // Nothing has failed yet for this target; `fuzz/README.md` says what to drop in here + // when something does. + let Ok(entries) = std::fs::read_dir(root.join(name)) else { + continue; + }; + for entry in entries { + let path = entry.expect("could not read the regression directory").path(); + let data = std::fs::read(&path).expect("could not read a regression input"); + target(&data); + } + } + } +} diff --git a/rs/moq-rtmp/src/lib.rs b/rs/moq-rtmp/src/lib.rs index cbd0a42d812..6a558f11696 100644 --- a/rs/moq-rtmp/src/lib.rs +++ b/rs/moq-rtmp/src/lib.rs @@ -63,6 +63,11 @@ use std::time::Duration; mod dial; mod error; mod flv; +// Not part of the public API: compiled only for the crate's own tests and for the `fuzz/` +// harness. +#[cfg(any(test, feature = "fuzz"))] +#[doc(hidden)] +pub mod fuzz; mod metadata; // Vendored forks of rml_rtmp and rml_amf0; see the module docs. mod rml; diff --git a/rs/moq-rtmp/src/rml/amf0/allocation.rs b/rs/moq-rtmp/src/rml/amf0/allocation.rs index 589d9dc84a4..0468f32d5d6 100644 --- a/rs/moq-rtmp/src/rml/amf0/allocation.rs +++ b/rs/moq-rtmp/src/rml/amf0/allocation.rs @@ -1,4 +1,7 @@ -//! Allocation accounting for synchronous receive-path tests. +//! Allocation accounting for synchronous receive-path tests and the `amf0` fuzz target. +//! +//! A binary opts in by registering [`Counting`] as its `#[global_allocator]`: this crate's unit +//! tests do below, and the `fuzz/` target does too. use std::alloc::{GlobalAlloc, Layout, System}; use std::cell::Cell; @@ -7,7 +10,7 @@ thread_local! { static USAGE: Cell> = const { Cell::new(None) }; } -struct Counting; +pub struct Counting; // All memory operations still use System. Const TLS keeps accounting from // allocating recursively and isolates concurrent tests on other threads. @@ -35,9 +38,11 @@ unsafe impl GlobalAlloc for Counting { } } +#[cfg(test)] #[global_allocator] static ALLOCATOR: Counting = Counting; +/// Run `run`, returning the peak heap it held on this thread. Not reentrant. pub fn measure(run: impl FnOnce() -> T) -> (T, usize) { USAGE.with(|usage| usage.set(Some((0, 0)))); let result = run(); diff --git a/rs/moq-rtmp/src/rml/amf0/mod.rs b/rs/moq-rtmp/src/rml/amf0/mod.rs index 7b9f3cf7755..ebacda589b9 100644 --- a/rs/moq-rtmp/src/rml/amf0/mod.rs +++ b/rs/moq-rtmp/src/rml/amf0/mod.rs @@ -1,7 +1,7 @@ //! AMF0 codec vendored from rml_amf0 0.3.0 (MIT, Matthew Shapiro). //! See ../LICENSE. Container nesting is limited to 64 levels when decoding. -#[cfg(test)] +#[cfg(any(test, feature = "fuzz"))] pub(crate) mod allocation; mod deserialization; diff --git a/sh/coverage.test.py b/sh/coverage.test.py index e8a45561cfa..5e2bf0ff6fe 100644 --- a/sh/coverage.test.py +++ b/sh/coverage.test.py @@ -126,7 +126,7 @@ def test_rust_measures_native_workspace_without_fuzz_targets(self): self.assertEqual(calls[0][2:], ["llvm-cov", "clean", "--workspace"]) build = calls[1][2:] self.assertIn("--workspace", build) - for excluded in ("moq-net-fuzz", "moq-quic-fuzz"): + for excluded in ("moq-net-fuzz", "moq-quic-fuzz", "moq-mux-fuzz", "moq-rtmp-fuzz"): self.assertIn(excluded, build) self.assertIn("--tests", build) self.assertIn("--test-threads", build) diff --git a/sh/rs/coverage.sh b/sh/rs/coverage.sh index 53325225c3a..785fc804f7e 100644 --- a/sh/rs/coverage.sh +++ b/sh/rs/coverage.sh @@ -5,7 +5,7 @@ crate=${1:-workspace} report=() if [[ $crate == workspace ]]; then # Fuzz targets require nightly. - packages=(--workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz) + packages=(--workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz --exclude moq-mux-fuzz --exclude moq-rtmp-fuzz) elif [[ $crate =~ ^[a-zA-Z0-9_-]+$ ]]; then packages=(-p "$crate") report=(-p "$crate") diff --git a/sh/rs/fuzz.sh b/sh/rs/fuzz.sh index 027848ed2dc..e20de566ad7 100755 --- a/sh/rs/fuzz.sh +++ b/sh/rs/fuzz.sh @@ -2,9 +2,9 @@ # Coverage-guided fuzzing: sh/rs/fuzz.sh [--minimize] TARGET [ARGS...] # # TARGET names a file in some crate's fuzz/fuzz_targets/, such as moq-net's lite or -# moq-quic's connection; names are unique across crates. ARGS pass to +# moq-mux's fmp4; names are unique across crates. ARGS pass to # `cargo fuzz run`, so `just rs fuzz lite -- -max_total_time=300` bounds a run. -# See rs/moq-net/fuzz/README.md and rs/moq-quic/fuzz/README.md. +# See the README in each rs//fuzz/. # # Seeds are regenerated first so the corpus follows the code rather than a # stale run, and libFuzzer writes what it discovers to the FIRST corpus diff --git a/sh/rs/select.sh b/sh/rs/select.sh index 7747793c74e..c5ab247057b 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -136,7 +136,7 @@ fi # The `*-fuzz` crates need a nightly toolchain; only `just rs fuzz` compiles them. if [[ "$packages" == ALL ]]; then - flags=(--workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz) + flags=(--workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz --exclude moq-mux-fuzz --exclude moq-rtmp-fuzz) else # Space-separated `--package ` pairs; ids contain no spaces. # shellcheck disable=SC2206 diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index ba22669c1e4..30fa27aef70 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -9,6 +9,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] +FUZZ_EXCLUDES = " ".join(f"--exclude {crate}-fuzz" for crate in ("moq-net", "moq-quic", "moq-mux", "moq-rtmp")) DEVICE_FEATURES = "moq-video/capture moq-audio/capture moq-audio/playback moq-cli/play moq-cli/openh264" LINUX = os.uname().sysname == "Linux" @@ -55,7 +56,7 @@ def test_nextest_launcher_selects_workspace(self): suffix += " moq-relay/io-uring" if LINUX and action == "test" else "" suffix += " --timings" if github_actions and action == "test" else "" self.assertIn( - f"rs {action} --workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz{suffix}", + f"rs {action} --workspace {FUZZ_EXCLUDES}{suffix}", self.selected("sh/rs/nextest.sh", action=action, github_actions=github_actions), ) selected = self.selected("sh/rs/nextest.sh")