From a9dc743dfdf84e90e06bc82465c28c8d2eaa7cf1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:36 +0000 Subject: [PATCH 1/8] test(net): check resume against a model of one logical track (#6947) * chore(quest): claim quest/m1/resume-model-test.md Quest-Claim: 7bff411939571787 * test(net): check resume against a model of one logical track A seeded proptest drives an origin's fronts through route flips, route failures, give-ups, ranges, cache expiry, a newer epoch, and reader joins and leaves on the simulated clock, and checks every reader's groups, frames, drops, and end against a plain cursor on the logical track. Fixes what it found: - An ordered reader never ended once a finished track left nothing in its range, or the rest of the range was given up. - A drop poll with local bounds below the subscription's floor built a reversed span and panicked. - A resumed reader held a drop back for a replaced copy kept only to back a group still read, and was not woken once that read finished. * test(net): check JS resume against the logical-track model Mirror rs/moq-net's resume model as a fast-check property over the same ops, constants and checks, seeded by MOQ_RESUME_SEED and sized by MOQ_RESUME_CASES (default 64). It found three bugs, each fixed with a regression case: - A replacement copy's refusal ended a group the serving route still delivered; it is a verdict only once that route won't land it either. - A failed copy's cache eviction released a group recovering from another copy; only the copy a mirror reads from reclaims it. - A reported drop was final, so a group a later copy delivered was never handed out; the report is now once and not final. * ci: sweep the resume model nightly from a fresh seed * test(net): answer route fetches in the resume model A real route forwards a fetch for a group it lacks to the publisher. The model's routes refused every fetch, so a group continued across a failover read NotFound although the instance carried it. * test(net): let model routes hold a tail fill beside the live copy A frame-bounded fetch caches a partial copy of a group, so a later whole-group fetch must still be answered. A group a reader stitched from two copies is complete once any copy wrote its last frame. * fix(net): keep a recovered group when its old copy is let go Releasing a copy's subscription released every mirror it cached, including a group recovery had already moved to another copy, which then ended cleanly short of its frames. The JS model's routes now answer fetches, which exposed it. * quest: finish resume-model-test, plan delivering late groups below the resume point The maintainer kept the resume point policy on #6947; delivering the late lower groups is a nice-to-have follow-up in m2. * test(net): request the model's broadcast through the renamed origin API * test(net): resume readers only on a route Up or Down actually changes * test(net): restore Recover's refusal wait, and resume Rust readers only when the cheapest route changes The main merge took main's whole resume.ts and dropped the PR's Recover hunk. Co-authored-by: Claude Opus 5.5 --- .github/workflows/nightly.yml | 3 +- Cargo.lock | 1 + bun.lock | 1 + js/net/package.json | 1 + js/net/src/group.ts | 15 +- js/net/src/internal.ts | 12 +- js/net/src/resume.model.test.ts | 996 +++++++++++++++++++++++ js/net/src/resume.ts | 19 +- js/net/src/track.ts | 6 +- quest/m1/README.md | 1 - quest/m1/resume-model-test.md | 34 - quest/m1/time-addressing/model.md | 7 +- quest/m2/README.md | 1 + quest/m2/resume-late-groups.md | 34 + quest/m2/resume-redesign.md | 7 +- rs/moq-net/Cargo.toml | 1 + rs/moq-net/src/model/mod.rs | 3 + rs/moq-net/src/model/resume.rs | 20 +- rs/moq-net/src/model/resume_model.rs | 1088 ++++++++++++++++++++++++++ rs/moq-net/src/model/track.rs | 45 +- test/justfile | 4 + test/resume-model.sh | 12 + 22 files changed, 2245 insertions(+), 66 deletions(-) create mode 100644 js/net/src/resume.model.test.ts delete mode 100644 quest/m1/resume-model-test.md create mode 100644 quest/m2/resume-late-groups.md create mode 100644 rs/moq-net/src/model/resume_model.rs create mode 100755 test/resume-model.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 81c3d164218..6e4936f982c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -355,6 +355,7 @@ jobs: # viewer: diff-independent, since a break can arrive through the relay, # js/net, or the wire between them. # `rs quic-sim` sweeps moq-quic's simulated workloads from a fresh seed; PRs run a fixed 64. + # `test resume-model` sweeps resume against its model from a fresh seed; PRs run a fixed 64. # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. # `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time. @@ -364,7 +365,7 @@ jobs: # `rs multicast` runs the mDNS tests, which loop multicast back on the runner. # `rs render-sw` renders through wgpu on Mesa's software Vulkan driver. # `test qmux` runs js/qmux against the in-tree Rust qmux, per draft. - recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] + recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] include: - recipe: rs send-depth nightly: true diff --git a/Cargo.lock b/Cargo.lock index d268b93829b..54f44dab4d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5602,6 +5602,7 @@ dependencies = [ "moq-clock", "moq-pattern 0.1.2", "num_enum", + "proptest", "rand 0.10.3", "rangemap", "serde", diff --git a/bun.lock b/bun.lock index 51ac06ff5dc..a8e43248274 100644 --- a/bun.lock +++ b/bun.lock @@ -293,6 +293,7 @@ "@types/bun": "^1.4.3", "@types/node": "^26.6.4", "@typescript/lib-dom": "npm:@types/web@^0.0.358", + "fast-check": "^4.10.2", "rimraf": "^6.1.3", "typescript": "7.0.2", }, diff --git a/js/net/package.json b/js/net/package.json index 457c0ea0b0f..13b7b9a241a 100644 --- a/js/net/package.json +++ b/js/net/package.json @@ -38,6 +38,7 @@ "@types/bun": "^1.4.3", "@types/node": "^26.6.4", "@typescript/lib-dom": "npm:@types/web@^0.0.358", + "fast-check": "^4.10.2", "rimraf": "^6.1.3", "typescript": "7.0.2" } diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 3d5f8fc6be6..ae7965b68be 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -666,6 +666,8 @@ export class Consumer { readonly sequence: number; #state: GroupState; + // The copy it was mirrored from, which that copy's cache evicts. + readonly #mirrored: GroupState; #expiry?: Expiry; // Carries the group across its logical track's route changes; see `resume.ts`. #recover?: GroupRecover; @@ -685,6 +687,7 @@ export class Consumer { private constructor(state: GroupState) { this.#state = state; + this.#mirrored = state; this.sequence = state.sequence; } @@ -746,6 +749,7 @@ export class Consumer { static { makeConsumer = (state) => new Consumer(state); hooks.releaseGroupReader = (group, abort) => group.#release(abort); + hooks.readsMirrored = (group) => group.#state === group.#mirrored; hooks.groupTimestamp = (group) => group.#state.unverified || group.#state.offset !== 0 ? undefined : group.#state.timestamp; hooks.groupLatest = (group) => (group.#state.unverified ? undefined : group.#state.latest); @@ -777,9 +781,7 @@ export class Consumer { group.#closed = undefined; }; hooks.readGroupFrame = (group, compressed) => group.#readFrame(0, compressed); - hooks.evictGroup = (group) => { - group.#evict(); - }; + hooks.evictGroup = (group) => group.#evict(); } // Evaluate the drift budget for a read that has nothing buffered and is about to @@ -801,7 +803,9 @@ export class Consumer { return true; } - #evict() { + #evict(): boolean { + // A group read on from another copy is that copy's to reclaim. + if (this.#state !== this.#mirrored) return false; const frames = this.#bufferedFrames(); if (frames.length > 0) { // Unread content is going away either way. If the budget had already given up @@ -819,6 +823,9 @@ export class Consumer { this.#state.cacheBytes = 0; this.#state.flate = undefined; this.#state.frames.set([]); + // A failed copy holds nothing more to reclaim, and its recovery may still continue the group. + const closed = this.#state.closed.peek(); + return !(closed instanceof Error && this.#recovers(closed)); } // The next change a parked read waits for. diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 7136246c290..0b29b82f1d5 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -554,6 +554,8 @@ export const hooks: { * Assigned by `group.ts`. */ releaseGroupReader: (group: GroupConsumer, abort?: Error) => void; + /** Whether a group reader still reads the copy it was mirrored from, not one its recovery moved it to. */ + readsMirrored: (group: GroupConsumer) => boolean; groupTimestamp: (group: GroupConsumer) => Timestamp | undefined; groupLatest: (group: GroupConsumer) => Timestamp | undefined; /** Return when a group joined its track at this hop, on the installed clock. */ @@ -572,8 +574,11 @@ export const hooks: { recoverGroup: (group: GroupConsumer, recover: GroupRecover) => void; /** Read the next frame, as its stored DEFLATE slice when `compressed`, so a lite-07 publisher forwards it without inflating. */ readGroupFrame: (group: GroupConsumer, compressed: boolean) => Promise; - /** Make an evicted mirror terminal while its track timeline still contains it. */ - evictGroup: (group: GroupConsumer) => void; + /** + * Make an evicted mirror terminal while its track timeline still contains it. Whether the + * mirror is released too: not when its group reads on, or may yet, from another copy. + */ + evictGroup: (group: GroupConsumer) => boolean; /** Attach the origin advertisement of a created broadcast. */ attachAnnouncer: ( producer: BroadcastProducer, @@ -771,6 +776,9 @@ export const hooks: { releaseGroupReader: () => { throw new Error("group.ts not loaded"); }, + readsMirrored: () => { + throw new Error("group.ts not loaded"); + }, groupTimestamp: () => { throw new Error("group.ts not loaded"); }, diff --git a/js/net/src/resume.model.test.ts b/js/net/src/resume.model.test.ts new file mode 100644 index 00000000000..b7fb9dfd5ad --- /dev/null +++ b/js/net/src/resume.model.test.ts @@ -0,0 +1,996 @@ +/** + * Resume checked against a model of one logical track, mirroring the Rust `model::resume_model`. + * + * Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and reader + * joins and leaves drive an origin's fronts on the manual clock. Every reader's groups, frames, + * drops, and end are checked against what one plain cursor on the logical track would see: + * every group its range and budget allow, each once, in its order. A newer publisher epoch wins + * the path while older readers stay on theirs until it has no route left, then end with an error. + * + * `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick the cases; nightly sweeps a fresh seed. + */ + +import { expect, test } from "bun:test"; +import { Manual, Milli, Nano } from "@moq/clock"; +import fc from "fast-check"; +import { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; +import * as Epoch from "./epoch.ts"; +import { NotFound, StreamCode, StreamError, TooFarBehind } from "./error.ts"; +import type { Frame, Consumer as GroupConsumer, Producer as GroupProducer } from "./group.ts"; +import { HopSchema, Route } from "./hop.ts"; +import { hooks } from "./internal.ts"; +import { type Consumer as OriginConsumer, Producer as OriginProducer, type Requesting } from "./origin.ts"; +import * as Path from "./path.ts"; +import { routeFailed } from "./resume.ts"; +import { Timescale, Timestamp } from "./time.ts"; +import type * as track from "./track.ts"; +import { overrideBroadcastWire, wireOf } from "./wire.ts"; + +/** Groups in the logical track; group `g` has `1 + g % 3` frames. */ +const GROUPS = 12; +/** Routes per epoch, cheaper the higher the slot. */ +const SLOTS = 3; +/** The older publisher instance, then a newer one. */ +const EPOCHS = 2; +const READERS = 4; +const PATH = Path.from("room/alice"); +/** A budget nothing in a case outlives. */ +const HOUR = Milli(3_600_000); +/** A budget spanning about three groups. */ +const SHORT = Milli(30); + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function frames(group: number): number { + return 1 + (group % 3); +} + +/** Frame `frame` of `group` as `epoch` publishes it: every route of one epoch carries it. */ +function payload(epoch: number, group: number, frame: number): string { + return `e${epoch}:${group}.${frame}`; +} + +// UUIDv7s minted in order, so the second is the newer instance. +const EPOCH_IDS = [ + Epoch.parse("01900000-0000-7000-8000-000000000001"), + Epoch.parse("01900000-0001-7000-8000-000000000001"), +]; + +/** How a route fails: its session closes on the far side. */ +function routeFailure(): Error { + return new StreamError(StreamCode.SessionClosed); +} + +/** A verdict on content, not a route failing: another route may still deliver it. */ +function giveUp(): Error { + return new StreamError(StreamCode.DeliveryTimeout); +} + +type Budget = "hour" | "short"; +type Range = readonly [number, number] | undefined; + +interface Spec { + /** Sequence order through {@link track.Ordered} rather than arrival order. */ + ordered: boolean; + budget: Budget; + /** The subscription's one group range, or the default. */ + range: Range; + /** Polls drops from the start, as a publisher forwarding them does. */ + drops: boolean; +} + +type Op = + /** The route announces itself to the origin. */ + | { kind: "up"; epoch: number; slot: number } + /** The route's session fails, failing its copy. */ + | { kind: "down"; epoch: number; slot: number } + /** + * The route's copy gets up to `frames` more frames of `group`, or of the lowest group it + * lacks, finishing the group once complete. + */ + | { kind: "write"; epoch: number; slot: number; group?: number; frames: number } + /** The route gives up `group` with a verdict on its content. */ + | { kind: "giveUp"; epoch: number; slot: number; group: number } + /** The route's copy declares the track's end, once it holds the last group. */ + | { kind: "end"; epoch: number; slot: number } + | { kind: "advance"; ms: number } + /** Long enough for idle cached groups to expire. */ + | { kind: "expire" } + | { kind: "join"; reader: number; spec: Spec } + | { kind: "leave"; reader: number } + /** Replace the subscription's group range. */ + | { kind: "ranges"; reader: number; range: Range } + /** Replace the reader's local group bounds. */ + | { kind: "bounds"; reader: number; range: Range } + /** Poll the reader whether or not it was woken. */ + | { kind: "read"; reader: number }; + +interface Case { + ops: Op[]; + /** Every live route declares the end once it has caught up. */ + finish: boolean; +} + +const range: fc.Arbitrary = fc.option( + fc + .tuple(fc.nat({ max: GROUPS - 1 }), fc.integer({ min: 1, max: GROUPS })) + .map(([start, len]): Range => [start, Math.min(start + len, GROUPS)]), + { nil: undefined, freq: 2 }, +); +const epoch = fc.nat({ max: EPOCHS - 1 }); +const slot = fc.nat({ max: SLOTS - 1 }); +const reader = fc.nat({ max: READERS - 1 }); +const spec: fc.Arbitrary = fc.record({ + ordered: fc.boolean(), + budget: fc.oneof( + { arbitrary: fc.constant("hour"), weight: 3 }, + { arbitrary: fc.constant("short"), weight: 1 }, + ), + range, + drops: fc.boolean(), +}); +const op: fc.Arbitrary = fc.oneof( + { + weight: 30, + arbitrary: fc.record({ + kind: fc.constant("write" as const), + epoch, + slot, + group: fc.oneof( + { weight: 7, arbitrary: fc.constant(undefined) }, + { weight: 3, arbitrary: fc.nat({ max: GROUPS - 1 }) }, + ), + frames: fc.integer({ min: 1, max: 3 }), + }), + }, + { weight: 8, arbitrary: fc.record({ kind: fc.constant("up" as const), epoch, slot }) }, + { weight: 6, arbitrary: fc.record({ kind: fc.constant("down" as const), epoch, slot }) }, + { + weight: 4, + arbitrary: fc.record({ kind: fc.constant("giveUp" as const), epoch, slot, group: fc.nat({ max: GROUPS - 2 }) }), + }, + { weight: 2, arbitrary: fc.record({ kind: fc.constant("end" as const), epoch, slot }) }, + { + weight: 6, + arbitrary: fc.record({ kind: fc.constant("advance" as const), ms: fc.integer({ min: 1, max: 200 }) }), + }, + { weight: 1, arbitrary: fc.constant({ kind: "expire" as const }) }, + { weight: 10, arbitrary: fc.record({ kind: fc.constant("join" as const), reader, spec }) }, + { weight: 3, arbitrary: fc.record({ kind: fc.constant("leave" as const), reader }) }, + { weight: 3, arbitrary: fc.record({ kind: fc.constant("ranges" as const), reader, range }) }, + { weight: 2, arbitrary: fc.record({ kind: fc.constant("bounds" as const), reader, range }) }, + { weight: 8, arbitrary: fc.record({ kind: fc.constant("read" as const), reader }) }, +); +const cases: fc.Arbitrary = fc.record({ + ops: fc.array(op, { minLength: 1, maxLength: 63, size: "max" }), + finish: fc.boolean(), +}); + +/** One route's copy of the logical track, as its upstream publishes it. */ +interface Copy { + track: track.Producer; + /** Answers the front's requests for the route until the route goes. */ + serving: { close(): void }; + broadcast: BroadcastProducer; + /** Frames written per group, with the writer while the group is open. */ + groups: Map; + givenUp: Set; +} + +/** Answers a fetch of `sequence` that `local`, the route's own cache, may serve. */ +type Fill = (sequence: number, local: () => Promise) => Promise; + +/** A route's broadcast, which fetches upstream what its cache lacks, the way a relay forwards a fetch. */ +class Upstream extends BroadcastConsumer { + #fill: Fill; + + constructor(fill: Fill, state?: never) { + super(state); + const local = wireOf(this).fetchGroup; + if (!local) throw new Error("a produced broadcast always fetches"); + overrideBroadcastWire(this, { + fetchGroup: (name, sequence, options) => fill(sequence, () => local(name, sequence, options)), + }); + this.#fill = fill; + } + + /** Serve `broadcast`, fetching through `fill`. */ + static of(broadcast: BroadcastProducer, fill: Fill): Upstream { + const base = broadcast.consume(); + const upstream = new Upstream(fill, (base as Upstream).shareState()); + base.close(); + return upstream; + } + + override clone(): Upstream { + return new Upstream(this.#fill, this.shareState()); + } +} + +function complete(copy: Copy, group: number): boolean { + return copy.groups.get(group)?.written === frames(group); +} + +/** One publisher instance: its routes, and what any of them ever carried. */ +class Instance { + routes: (Copy | undefined)[] = Array.from({ length: SLOTS }, () => undefined); + /** Most frames of each group any route carried. */ + written = new Map(); + /** Groups some route finished. */ + complete = new Set(); + /** Groups some route gave up. */ + givenUp = new Set(); + /** Some route declared the track's end. */ + ended = false; + /** The newest group when idle cached groups last expired: any route may have let go of those. */ + expired?: number; + + live(): Copy[] { + return this.routes.filter((copy) => copy !== undefined); + } + + newest(): number | undefined { + return this.written.size === 0 ? undefined : Math.max(...this.written.keys()); + } +} + +/** A group a reader was handed, and how far it read it. */ +interface Reading { + group: GroupConsumer; + frames: number; + /** `null` once it ended cleanly, the error once it failed. */ + end?: Error | null; +} + +interface Reader { + instance: number; + spec: Spec; + request: Requesting; + sub: track.Subscriber; + ordered?: track.Ordered; + /** Set by a notification since the last poll. */ + flag: boolean; + unwatch?: () => void; + /** Left the case: whatever its groups do from here is not checked. */ + gone: boolean; + /** The subscription's group range. */ + ranges: Range; + /** + * The one group cursor reads honor. Unlike the Rust reader's two filters, a JS subscriber's + * subscription range only seeds it: an update resets it to that range, and local bounds + * replace it in either direction. + */ + cursor: Range; + /** Every group handed out, in order. */ + handed: number[]; + reading: Map; + /** Groups reported dropped. */ + dropped: Set; + /** Only groups past this one are owed: older ones were cached history at join, or may have expired. */ + floor?: number; + /** Its instance lost its last route while it read. */ + doomed: boolean; + /** + * The newest group handed out when its routes last changed. A route change resumes from + * there, so a group below it that lands only afterwards may be lost. + */ + resumed?: number; + /** `null` once the track ended cleanly, the error once it failed. */ + end?: Error | null; + /** + * The track ended cleanly with only groups past the reader's cap left: where Rust ends the + * reader, JS leaves the route but keeps it open with those claimable for the cap to rise. + */ + parked: boolean; + /** What every live route carried in full when the track ended cleanly. */ + carried: Set; +} + +/** + * Unlike Rust, which ends a reader capped below a finished track's end, JS keeps the groups past + * the cap claimable, so the reader stays open for the cap to rise. + */ +function capped(reader: Reader): boolean { + return (reader.cursor?.[1] ?? GROUPS) < GROUPS; +} + +function within(range: Range, group: number): boolean { + return range === undefined || (range[0] <= group && group < range[1]); +} + +function covers(reader: Reader, group: number): boolean { + return within(reader.cursor, group); +} + +/** Everything observed, to tell whether a poll found anything new. */ +function observed(reader: Reader): string { + let read = 0; + for (const reading of reader.reading.values()) read += reading.frames + (reading.end !== undefined ? 1 : 0); + return JSON.stringify([reader.handed.length, read, reader.end !== undefined, reader.dropped.size]); +} + +function groups(range: Range): track.Groups { + return range ? { start: { included: range[0] }, end: { excluded: range[1] } } : {}; +} + +function subscription(budget: Budget, range: Range): track.Subscription { + return { maxDelay: budget === "hour" ? HOUR : SHORT, groups: range ? groups(range) : undefined }; +} + +function max(values: Iterable): number | undefined { + let most: number | undefined; + for (const value of values) most = Math.max(most ?? value, value); + return most; +} + +function fail(reader: number, what: string): Error { + return new Error(`reader ${reader}: ${what}`); +} + +class Model { + clock: Manual; + origin = new OriginProducer(); + consumer: OriginConsumer; + instances = Array.from({ length: EPOCHS }, () => new Instance()); + readers: (Reader | undefined)[] = Array.from({ length: READERS }, () => undefined); + /** The first check a frame read failed, which ends the case once it settles. */ + failure?: Error; + /** Bumped by every frame read, so settling sees progress no reader flag shows. */ + progress = 0; + + constructor(clock: Manual) { + this.clock = clock; + this.consumer = this.origin.consume(); + } + + async apply(op: Op): Promise { + // Only adding or removing a route can move a reader to another one. Unlike Rust, which + // moves only when the cheapest route changes, a JS reader may keep reading from a route + // that is no longer the cheapest, so any change counts. + const rerouted = + (op.kind === "up" && !this.instances[op.epoch].routes[op.slot]) || + (op.kind === "down" && this.instances[op.epoch].routes[op.slot] !== undefined); + if (rerouted) { + for (const reader of this.readers) { + if (reader?.instance !== op.epoch) continue; + const newest = max(reader.handed); + if (newest !== undefined) reader.resumed = Math.max(reader.resumed ?? newest, newest); + } + } + switch (op.kind) { + case "up": + this.up(op.epoch, op.slot); + break; + case "down": { + const instance = this.instances[op.epoch]; + const copy = instance.routes[op.slot]; + if (!copy) return; + instance.routes[op.slot] = undefined; + copy.serving.close(); + for (const { writer } of copy.groups.values()) writer?.abort(routeFailure()); + copy.track.abort(routeFailure()); + copy.broadcast.close(); + if (instance.live().length === 0) { + for (const reader of this.readers) { + if (reader?.instance === op.epoch) reader.doomed = true; + } + } + break; + } + case "write": { + const copy = this.instances[op.epoch].routes[op.slot]; + if (!copy) return; + let group = op.group; + if (group === undefined) { + for (let candidate = 0; candidate < GROUPS; candidate++) { + if (!complete(copy, candidate) && !copy.givenUp.has(candidate)) { + group = candidate; + break; + } + } + } + if (group !== undefined) this.write(op.epoch, op.slot, group, op.frames); + break; + } + case "giveUp": { + const instance = this.instances[op.epoch]; + const copy = instance.routes[op.slot]; + if (!copy || complete(copy, op.group) || copy.givenUp.has(op.group)) return; + copy.givenUp.add(op.group); + const entry = copy.groups.get(op.group); + if (entry?.writer) { + entry.writer.abort(giveUp()); + entry.writer = undefined; + } else if (!entry) { + hooks.dropTrackGroups(copy.track, op.group, op.group + 1, giveUp()); + } + instance.givenUp.add(op.group); + break; + } + case "end": + this.end(op.epoch, op.slot); + break; + case "advance": + await this.clock.advance(Nano.fromMilli(Milli(op.ms))); + break; + case "expire": + await this.clock.advance(Nano.fromMilli(Milli(60_000))); + for (const instance of this.instances) instance.expired = instance.newest(); + for (const reader of this.readers) { + if (reader) reader.floor = this.instances[reader.instance].newest(); + } + break; + case "join": + await this.join(op.reader, op.spec); + break; + case "leave": + this.leave(op.reader); + break; + case "ranges": { + const reader = this.readers[op.reader]; + if (!reader) return; + reader.ranges = op.range; + reader.cursor = op.range; + reader.sub.update(subscription(reader.spec.budget, op.range)); + this.poll(op.reader); + break; + } + case "bounds": { + const reader = this.readers[op.reader]; + if (!reader) return; + reader.cursor = op.range; + (reader.ordered ?? reader.sub).withRange(groups(op.range)); + this.poll(op.reader); + break; + } + case "read": + this.poll(op.reader); + break; + } + } + + up(epoch: number, slot: number): void { + const instance = this.instances[epoch]; + if (instance.routes[slot]) return; + const route = Route.normalize({ + epoch: EPOCH_IDS[epoch], + hops: [HopSchema.parse(BigInt(10 * (epoch + 1) + slot))], + cost: BigInt(SLOTS - slot), + }); + const serving = wireOf(this.origin).receive(PATH, route); + const broadcast = new BroadcastProducer(); + const producer = broadcast.createTrack("video", { timescale: Timescale.MILLI }); + void (async () => { + for await (const request of serving.requested()) { + hooks.acceptRequest( + request, + Upstream.of(broadcast, (sequence, local) => this.fill(epoch, slot, sequence, local)), + ); + } + })(); + instance.routes[slot] = { track: producer, serving, broadcast, groups: new Map(), givenUp: new Set() }; + } + + write(epoch: number, slot: number, group: number, more: number): void { + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy || copy.givenUp.has(group) || complete(copy, group)) return; + let entry = copy.groups.get(group); + if (!entry) { + entry = { written: 0 }; + copy.groups.set(group, entry); + } + if (entry.written === 0 && !entry.writer) { + try { + entry.writer = copy.track.createGroup(group); + } catch { + return; + } + } + const writer = entry.writer; + if (!writer) return; + // The cache reclaimed it while idle: this route never carries the rest. + if (writer.isClosed) { + entry.writer = undefined; + return; + } + const until = Math.min(entry.written + more, frames(group)); + while (entry.written < until) { + const timestamp = Timestamp.fromMillis(group * 10 + entry.written); + writer.writeFrame({ payload: enc.encode(payload(epoch, group, entry.written)), timestamp }); + entry.written++; + } + if (entry.written === frames(group)) { + writer.finish(); + entry.writer = undefined; + instance.complete.add(group); + } + instance.written.set(group, Math.max(instance.written.get(group) ?? 0, entry.written)); + } + + /** + * Answer a fetch the route's cache missed from upstream, where the instance's publisher + * holds every frame any of its routes wrote. Later writes to the route continue the group. + */ + async fill( + epoch: number, + slot: number, + sequence: number, + local: () => Promise, + ): Promise { + this.progress++; + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy) throw routeFailure(); + if (copy.givenUp.has(sequence)) throw giveUp(); + if (copy.groups.has(sequence)) return local(); + const written = instance.written.get(sequence) ?? 0; + if (sequence >= GROUPS || written === 0) throw new NotFound(`group ${sequence}`); + const writer = copy.track.createGroup(sequence); + copy.groups.set(sequence, { written: 0, writer }); + const group = writer.consume(); + this.write(epoch, slot, sequence, written); + return group; + } + + end(epoch: number, slot: number): void { + const instance = this.instances[epoch]; + const copy = instance.routes[slot]; + if (!copy?.groups.has(GROUPS - 1)) return; + copy.track.finish(); + instance.ended = true; + } + + async join(index: number, spec: Spec): Promise { + if (this.readers[index]) return; + // The newest instance with a route wins the path. + let expected: number | undefined; + for (let epoch = EPOCHS - 1; epoch >= 0; epoch--) { + if (this.instances[epoch].live().length > 0) { + expected = epoch; + break; + } + } + if (expected === undefined) return; + const request = this.consumer.request(PATH); + // Answered by the routes' serving loops, without time passing. + for (let turn = 0; !request.active.peek(); turn++) { + const closed = request.closed.peek(); + if (closed !== undefined || turn === 64) { + request.close(); + throw fail(index, `request failed: ${closed ?? "never resolved"}`); + } + await this.clock.advance(Nano.zero); + } + const broadcast = request.active.peek(); + const instance = broadcast?.epoch === undefined ? -1 : EPOCH_IDS.indexOf(broadcast.epoch); + if (!broadcast || instance === -1) { + request.close(); + throw fail(index, "resolved no known instance"); + } + if (instance !== expected) { + request.close(); + throw fail(index, `joined instance ${instance}, not the newest ${expected}`); + } + const sub = broadcast.track("video").subscribe(subscription(spec.budget, spec.range)); + this.readers[index] = { + instance, + spec, + request, + sub, + ordered: spec.ordered ? sub.ordered() : undefined, + flag: false, + gone: false, + ranges: spec.range, + cursor: spec.range, + handed: [], + reading: new Map(), + dropped: new Set(), + floor: this.instances[instance].newest(), + doomed: false, + end: undefined, + parked: false, + carried: new Set(), + }; + this.poll(index); + } + + leave(index: number): void { + const reader = this.readers[index]; + if (!reader) return; + this.readers[index] = undefined; + reader.gone = true; + reader.unwatch?.(); + for (const reading of reader.reading.values()) reading.group.close(); + reader.sub.close(); + reader.request.close(); + } + + /** Read everything `index` has ready, checking each step against the model. */ + /** The track ended cleanly: a group landing below the end afterwards is not owed, since a reader stops at it. */ + #endClean(index: number, reader: Reader): void { + const instance = this.instances[reader.instance]; + if (!instance.ended) throw fail(index, "the track ended cleanly though no route declared an end"); + reader.carried = new Set(); + for (let group = 0; group < GROUPS; group++) { + if (instance.live().every((copy) => complete(copy, group))) reader.carried.add(group); + } + } + + poll(index: number): void { + const reader = this.readers[index]; + if (!reader) return; + const instance = this.instances[reader.instance]; + reader.flag = false; + reader.unwatch?.(); + + while (reader.end === undefined) { + const recv = reader.spec.ordered ? hooks.tryNextGroup(reader.sub) : hooks.tryRecvGroup(reader.sub, false); + if (recv.kind === "idle" || recv.kind === "boundary") { + if (!reader.parked && capped(reader) && reader.sub.closed.peek() === null) { + this.#endClean(index, reader); + reader.parked = true; + } + break; + } + if (recv.kind === "group") { + const group = recv.group; + const sequence = group.sequence; + if (reader.reading.has(sequence)) { + throw fail(index, `group ${sequence} handed out twice: ${reader.handed}`); + } + const last = reader.handed.at(-1); + if (reader.spec.ordered && last !== undefined && last >= sequence) { + throw fail(index, `group ${sequence} out of order after ${reader.handed}`); + } + if (!covers(reader, sequence)) { + throw fail(index, `group ${sequence} outside cursor ${reader.cursor}, ranges ${reader.ranges}`); + } + if (!instance.written.has(sequence)) { + throw fail(index, `group ${sequence} handed out before any route had it`); + } + reader.handed.push(sequence); + const reading: Reading = { group, frames: 0 }; + reader.reading.set(sequence, reading); + void this.#read(index, reader, reading); + } else if (recv.kind === "done") { + if (!reader.parked) this.#endClean(index, reader); + reader.end = null; + } else if (recv.kind === "error") { + if (!reader.doomed) throw fail(index, `the track failed while its instance had a route: ${recv.error}`); + reader.end = recv.error; + } else { + throw fail(index, `an arrival read without dispositions reported drop ${recv.start}..${recv.end}`); + } + } + + if (reader.spec.drops && !reader.spec.ordered) { + for (let drop = hooks.tryDrop(reader.sub); drop; drop = hooks.tryDrop(reader.sub)) { + for (let group = drop.start; group < drop.end; group++) { + if (reader.reading.has(group)) { + throw fail( + index, + `dropped ${drop.start}..${drop.end} (${drop.error}) after handing out ${group}`, + ); + } + } + for (let group = drop.start; group < Math.min(drop.end, GROUPS); group++) reader.dropped.add(group); + } + } + + reader.unwatch = hooks.groupChanged(reader.sub, () => { + reader.flag = true; + }); + } + + /** Read a group handed out to its end, checking each frame against the model. */ + async #read(index: number, reader: Reader, reading: Reading): Promise { + const sequence = reading.group.sequence; + for (;;) { + let frame: Frame | undefined; + try { + frame = await reading.group.readFrame(); + } catch (err) { + if (reader.gone) return; + reading.end = err instanceof Error ? err : new Error(String(err)); + this.progress++; + return; + } + if (reader.gone) return; + this.progress++; + if (!frame) { + if (reading.frames !== frames(sequence) || !this.instances[reader.instance].complete.has(sequence)) { + this.failure ??= fail(index, `group ${sequence} ended cleanly after ${reading.frames} frames`); + } + reading.end = null; + return; + } + const expected = payload(reader.instance, sequence, reading.frames); + const read = dec.decode(frame.payload); + if (read !== expected || reading.frames >= frames(sequence)) { + this.failure ??= fail( + index, + `group ${sequence} frame ${reading.frames} read ${read}, expected ${expected}`, + ); + return; + } + reading.frames++; + } + } + + /** Let the routes run without time passing, polling each reader woken. */ + async settle(): Promise { + let quiet = 0; + for (let round = 0; round < 10_000; round++) { + const progress = this.progress; + await this.clock.advance(Nano.zero); + if (this.failure) throw this.failure; + let woken = this.progress !== progress; + for (let index = 0; index < READERS; index++) { + if (this.readers[index]?.flag) { + woken = true; + this.poll(index); + } + } + quiet = woken ? 0 : quiet + 1; + if (quiet === 2) return; + } + throw new Error("readers never settled: something keeps waking them"); + } + + /** + * Every live route catches up on every group and, for `finish`, declares the end; then each + * reader must have read what it is owed. + */ + async conclude(finish: boolean): Promise { + const before = this.readers.map((reader) => reader && max(reader.handed)); + for (let epoch = 0; epoch < EPOCHS; epoch++) { + for (let slot = 0; slot < SLOTS; slot++) { + for (let group = 0; group < GROUPS; group++) this.write(epoch, slot, group, Number.POSITIVE_INFINITY); + if (finish) this.end(epoch, slot); + } + } + await this.settle(); + + const seen = this.readers.map((reader) => reader && observed(reader)); + for (let index = 0; index < READERS; index++) { + const reader = this.readers[index]; + if (!reader) continue; + this.poll(index); + // A frame read still waiting with a frame or its end ready was never woken. + for (const [sequence, reading] of reader.reading) { + if (reading.end !== undefined) continue; + const ready = reading.group.done ? "its end" : reading.group.tryReadFrame() ? "a frame" : undefined; + if (ready) throw fail(index, `group ${sequence} has ${ready} ready that never woke its read`); + } + if (observed(reader) !== seen[index]) { + throw fail( + index, + `a poll found progress that never woke it: ${seen[index]} -> ${observed(reader)}, handed ${reader.handed}, dropped ${[...reader.dropped]}, end ${reader.end}`, + ); + } + this.owed(index, reader, before[index], finish); + } + } + + owed(index: number, reader: Reader, before: number | undefined, finish: boolean): void { + const instance = this.instances[reader.instance]; + const ended = reader.end === null || reader.parked; + if (reader.doomed) { + if (reader.end instanceof Error || ((ended || capped(reader)) && instance.ended)) return; + throw fail(index, `outlived its instance's routes: ${reader.end}`); + } + // What every live route carries in full. + const carried = (group: number) => instance.live().every((copy) => complete(copy, group)); + let owed: number[] = []; + for (let group = 0; group < GROUPS; group++) { + if (reader.floor !== undefined && group <= reader.floor) continue; + if (reader.resumed !== undefined && group <= reader.resumed) continue; + if (covers(reader, group) && carried(group)) owed.push(group); + } + if (reader.spec.ordered && before !== undefined) owed = owed.filter((group) => group > before); + if (ended) owed = owed.filter((group) => reader.carried.has(group)); + if (reader.spec.budget === "short") owed = owed.filter((group) => group === GROUPS - 1); + const missing = owed.filter((group) => !reader.reading.has(group)); + let problems = ""; + if (missing.length > 0) problems += `missing groups ${missing}; `; + if (reader.spec.budget === "hour") { + for (const [sequence, reading] of reader.reading) { + if (reading.end === null || !carried(sequence) || instance.givenUp.has(sequence)) continue; + // Like a group under the floor, one older than an expiry may be gone from its own copy + // or from the route it would continue from. + const gone = reading.end instanceof TooFarBehind || reading.end instanceof NotFound; + if (gone && instance.expired !== undefined && sequence <= instance.expired) continue; + // Unlike Rust, an ended JS reader follows no route, so a route failure ends a group still read. + if (ended && reading.end instanceof Error && routeFailed(reading.end)) continue; + problems += `group ${sequence} stopped at frame ${reading.frames} (${reading.end}); `; + } + } + const live = finish && instance.live().length > 0; + const end = reader.end; + const ends = + (end === undefined && (!live || capped(reader))) || + (end === null && live) || + (end === null && !live && instance.ended); + if (!ends) problems += `track end ${end}; `; + if (problems === "") return; + throw fail( + index, + `${problems}handed out ${reader.handed}, dropped ${[...reader.dropped]}, floor ${reader.floor}, spec ${JSON.stringify(reader.spec)}`, + ); + } + + /** Let go of everything, so nothing outlives the case. */ + async close(): Promise { + for (let index = 0; index < READERS; index++) this.leave(index); + for (const instance of this.instances) { + for (const copy of instance.live()) { + copy.serving.close(); + copy.track.abort(routeFailure()); + copy.broadcast.close(); + } + } + this.origin.close(); + await this.clock.advance(Nano.zero); + } +} + +async function run(c: Case): Promise { + const clock = new Manual(); + const installed = clock.install(); + const model = new Model(clock); + try { + model.up(0, 0); + for (const op of c.ops) { + await model.apply(op); + await model.settle(); + } + await model.conclude(c.finish); + } finally { + await model.close(); + installed.close(); + } +} + +function setting(name: string, fallback: number, min: number): number { + const raw = process.env[name]; + if (raw !== undefined && !/^\d+$/.test(raw)) throw new Error(`${name} must be an integer`); + const value = Number(raw ?? fallback); + if (!Number.isInteger(value) || value < min || value > 0x7fffffff) { + throw new Error(`${name} must be an integer between ${min} and 2147483647`); + } + return value; +} + +test( + "readers match the logical track", + async () => { + const seed = setting("MOQ_RESUME_SEED", 0x6d6f71, 0); + const numRuns = setting("MOQ_RESUME_CASES", 64, 1); + await fc.assert(fc.asyncProperty(cases, run), { seed, numRuns }); + }, + { timeout: 3_600_000 }, +); + +/** An Up or Down that changes no route resumes nothing, so lower groups stay owed. */ +test("unchanged routes keep groups owed", async () => { + const clock = new Manual(); + const installed = clock.install(); + const model = new Model(clock); + try { + model.up(0, 0); + const ops: Op[] = [ + { kind: "join", reader: 0, spec: { ordered: false, budget: "hour", range: undefined, drops: false } }, + { kind: "write", epoch: 0, slot: 0, group: 8, frames: 1 }, + { kind: "up", epoch: 0, slot: 0 }, + { kind: "down", epoch: 0, slot: 1 }, + ]; + for (const op of ops) { + await model.apply(op); + await model.settle(); + } + const reader = model.readers[0]; + expect(reader?.handed).toContain(8); + expect(reader?.resumed).toBeUndefined(); + } finally { + await model.close(); + installed.close(); + } +}); + +/** Failures the sweep found, as it shrank them. */ +test("regressions", async () => { + const spec = (ordered: boolean, budget: Budget, range: Range, drops: boolean): Spec => ({ + ordered, + budget, + range, + drops, + }); + const regressions: Case[] = [ + // Found by the Rust run. An ordered reader whose range a finished track already passed ends. + { ops: [{ kind: "join", reader: 0, spec: spec(true, "hour", [0, 1], false) }], finish: true }, + // An ordered reader ends once the rest of its range was given up. + { + ops: [ + { kind: "up", epoch: 1, slot: 2 }, + { kind: "join", reader: 0, spec: spec(true, "hour", [0, 8], false) }, + { kind: "giveUp", epoch: 1, slot: 2, group: 7 }, + ], + finish: true, + }, + // Local bounds below the subscription's range leave no drops to report. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", [11, 12], true) }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "bounds", reader: 0, range: [0, 1] }, + ], + finish: false, + }, + // A drop held for a replaced copy is reported once that copy only backs a read. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "short", undefined, true) }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + { kind: "up", epoch: 0, slot: 2 }, + { kind: "write", epoch: 0, slot: 2, group: 6, frames: 1 }, + ], + finish: false, + }, + // A group whose route failed continues on the next route once it lands there. + { + ops: [ + { kind: "up", epoch: 0, slot: 1 }, + { kind: "join", reader: 1, spec: spec(false, "hour", undefined, false) }, + { kind: "write", epoch: 0, slot: 1, group: 1, frames: 1 }, + { kind: "write", epoch: 0, slot: 0, group: 2, frames: 1 }, + { kind: "down", epoch: 0, slot: 1 }, + ], + finish: false, + }, + // A group the replaced route failed is still continued once its cache let go of it. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "up", epoch: 0, slot: 2 }, + { kind: "write", epoch: 0, slot: 2, group: 4, frames: 1 }, + { kind: "down", epoch: 0, slot: 2 }, + { kind: "expire" }, + { kind: "write", epoch: 0, slot: 0, group: undefined, frames: 1 }, + ], + finish: false, + }, + // A group a route gave up and the next route carries is handed out after its drop. + { + ops: [ + { kind: "up", epoch: 1, slot: 0 }, + { kind: "up", epoch: 1, slot: 2 }, + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, true) }, + { kind: "giveUp", epoch: 1, slot: 2, group: 0 }, + { kind: "down", epoch: 1, slot: 2 }, + ], + finish: false, + }, + // A group read on from the serving copy outlives the replaced copy it was handed out from. + { + ops: [ + { kind: "join", reader: 0, spec: spec(false, "hour", undefined, false) }, + { kind: "up", epoch: 0, slot: 1 }, + { kind: "write", epoch: 0, slot: 0, group: 8, frames: 1 }, + { kind: "giveUp", epoch: 0, slot: 0, group: 8 }, + { kind: "write", epoch: 0, slot: 1, group: 0, frames: 1 }, + ], + finish: false, + }, + ]; + const failures: string[] = []; + for (const [i, c] of regressions.entries()) { + try { + await run(c); + } catch (err) { + failures.push(`case ${i}: ${err}`); + } + } + if (failures.length > 0) throw new Error(failures.join("\n")); +}); diff --git a/js/net/src/resume.ts b/js/net/src/resume.ts index 440e8a43b00..54a1adb89c7 100644 --- a/js/net/src/resume.ts +++ b/js/net/src/resume.ts @@ -1358,6 +1358,9 @@ class Recover implements GroupRecover { let fetch: Fetch | undefined; for (;;) { signal.throwIfAborted(); + // The serving subscription may still deliver a group its route refused to fetch. + let landing: PromiseLike | undefined; + let unwatch: Dispose | undefined; // Subscribe the reader to a new route from its newest group, which is this one or // later, so that route's subscription may deliver the rest of it. reader.sync(); @@ -1414,12 +1417,22 @@ class Recover implements GroupRecover { const outcome = asked?.outcome; if (outcome && "group" in outcome) return this.#replacement(outcome.group, serving); // That route failing too waits for the next one, and a refusal is a verdict only for - // a copy that failed. + // a copy that failed, once the serving subscription will not deliver the group either. if (outcome && failed && !routeFailed(outcome.error) && !(serving.sub.closed.peek() instanceof Error)) { - throw outcome.error; + const maxDelay = serving.sub.subscription.peek()?.maxDelay ?? Milli.zero; + const land = { start: this.sequence, end: this.sequence + 1, floor: true, grace: grace(maxDelay) }; + if (generation === current.generation || !hooks.mayLand(serving.sub, land)) throw outcome.error; + landing = new Promise((resolve) => { + unwatch = hooks.groupChanged(serving.sub, resolve); + }); } } - await until(Signal.race(...waits), signal); + try { + const changed = Signal.race(...waits); + await until(landing === undefined ? changed : race([changed, landing]), signal); + } finally { + unwatch?.(); + } } } diff --git a/js/net/src/track.ts b/js/net/src/track.ts index 278802c155f..824da133fcd 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -1039,7 +1039,8 @@ export class Producer { for (const entry of this.#cache) { const mirror = entry.mirrors.get(sink); if (mirror) { - hooks.releaseGroupReader(mirror, abort); + // A group its recovery moved to another copy no longer reads this one. + if (hooks.readsMirrored(mirror)) hooks.releaseGroupReader(mirror, abort); entry.mirrors.delete(sink); } } @@ -1102,8 +1103,7 @@ export class Producer { EVICTED.add(evicted); entry.group.abort(evicted); } - const mirrors = [...entry.mirrors.values()]; - for (const mirror of mirrors) hooks.evictGroup(mirror); + const mirrors = [...entry.mirrors.values()].filter((mirror) => hooks.evictGroup(mirror)); this.#unlink(entry); for (const mirror of mirrors) mirror.close(); } diff --git a/quest/m1/README.md b/quest/m1/README.md index 3f56b70aabd..cf6465ce776 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -122,7 +122,6 @@ removed). ## Required -- [Resume checked against a model](/quest/m1/resume-model-test.md) - a randomized test checks every resumed reader against a one-track reference model in Rust and JS, and passes before the cut - [Address groups by timestamp](/quest/m1/time-addressing/README.md) - group start timestamps replace sequence numbers in the moq-net API and lite-07 wire, while old subscribers, old publishers, and published JS clients keep working - [Release audit findings](/quest/m1/release-audit/README.md) - resolve newly identified resource, parser, playback, and interoperability risks before the cut - [Pre-cut audit findings](/quest/m1/cut-audit/README.md) - fix the crashes, hostile-input costs, and correctness bugs the 2026-10-24 audit of main found diff --git a/quest/m1/resume-model-test.md b/quest/m1/resume-model-test.md deleted file mode 100644 index 4b1adf05221..00000000000 --- a/quest/m1/resume-model-test.md +++ /dev/null @@ -1,34 +0,0 @@ -# [L] Resume checked against a model - -## Goal - -A randomized test drives resumed fronts through route flips, route failures, -drops, ranges, cache expiry, and reader joins and leaves, and checks every -reader's observed groups, frames, gaps, and end against a simple reference -model of one logical track. It runs in Rust and `@moq/net`, and it passes -before the cut. - -## Plan - -Decided 2026-10-25 (maintainer): resume must be flawless at the cut. Resume -has had about 18 fixes in its last 31 commits (rs/moq-net/src/model/resume.rs, -2026-10-18 to 10-25), and nothing checks it beyond hand-written route-change -cases, so an oracle comes before any rewrite. - -- The model is what one plain cursor on the logical track would see: a - reader gets every group its range and budget allow, each exactly once, in - its order, with drops reported per the drop rules. A reported drop is not - final: a group that arrives later is still served. -- Include a newer publisher epoch winning the path while an older-epoch - subscription is open: that subscription never sees the new instance's - groups, and it ends with an error once its own instance has no route. -- Drive it on a mocked clock with seeded randomness, and shrink a failure to - a minimal sequence that becomes a regression test. Prefer an existing - property-testing crate. -- Run a fixed seed set in the merge gate and a longer sweep nightly. -- Fix what it finds at the cause. If the failures cluster on the per-reader - route-copy design rather than single missed checks, bring back the - [producer-replacement redesign](/quest/m2/resume-redesign.md) to the - maintainer for m1, with this test as its oracle. - -Public API: none. Wire: none. diff --git a/quest/m1/time-addressing/model.md b/quest/m1/time-addressing/model.md index e8d0bd401d2..074d17841f5 100644 --- a/quest/m1/time-addressing/model.md +++ b/quest/m1/time-addressing/model.md @@ -106,7 +106,8 @@ Rust half of every consumer the API change breaks to compile; [consumers](/quest/m1/time-addressing/consumers.md) and [bindings](/quest/m1/time-addressing/bindings.md) own the behavior changes. Test the legacy mapping both ways on every wire, with the -[resume model](/quest/m1/resume-model-test.md) as the oracle for resume. +resume model (`rs/moq-net/src/model/resume_model.rs`, +`js/net/src/resume.model.test.ts`) as the oracle for resume. The PR's Impact section says how to migrate. Public API: breaking. `group::Info`, `create_group`, `append_group`, @@ -121,7 +122,3 @@ Wire: no format change; legacy and lite-07-wip group IDs become start ticks. - [Delete the old start bookkeeping](/quest/m1/subscribe-ranges/start-bookkeeping.md) - the lite cursor state lives in the model before the cut rekeys it - [Lite-05/06 timestamps pass through](/quest/m1/lite-timestamps-through.md) - lite-05/06 tracks stay timed, so their ingest keeps real starts - [Drop set](/quest/m1/drop-set.md) - its dropped and expired runs convert mechanically from sequences to starts - -## Related - -- [Resume checked against a model](/quest/m1/resume-model-test.md) - its reference model is the oracle for resume after the cut diff --git a/quest/m2/README.md b/quest/m2/README.md index f6d54a23242..7fc17cfabb8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -269,5 +269,6 @@ lite-07; bitrate enforcement is not part of this release. - [Clean ALSA playback onset](/quest/m2/audio-alsa-onset.md) - no onset restarts or clicks over PulseAudio - [Interop audit](/quest/m2/audit-interop.md) - our interop runner entries match what we ship, and every red interop cell with a moq.dev side is fixed, reported, or noted as down or unsupported - [Open-GOP MPEG-2 export](/quest/m2/mpeg2-open-gop-export.md) - a TS export resuming in an open GOP trims the B-pictures that reference the previous one +- [Resume delivers late groups below its resume point](/quest/m2/resume-late-groups.md) - a bonus: groups that arrive late after a route change are delivered instead of written off - [The HLS server follows each path once](/quest/m2/hls-follow-once.md) - `Following` reports synced absence, so moq-hls resolves with one follow - [moq-bench pins the instances it measures](/quest/m2/bench-epoch-pin.md) - pooled requests pin their announced epoch, so a takeover is not counted as the same broadcast diff --git a/quest/m2/resume-late-groups.md b/quest/m2/resume-late-groups.md new file mode 100644 index 00000000000..ec6f1fed5e9 --- /dev/null +++ b/quest/m2/resume-late-groups.md @@ -0,0 +1,34 @@ +# [M] Resume delivers late groups below its resume point + +## Goal + +After a route change, a resumed reader still gets a group below the newest +group it handed out when that group arrives late, reordered or in flight at +the failover, as one plain cursor on the logical track would. Today those +groups are written off. The resume model test drops its `resumed` +relaxation in Rust and `@moq/net` and still passes its gate seeds and a +nightly sweep. Nice to have, not required: the written-off policy is +correct as documented, and this is a bonus. + +## Plan + +Decided 2026-10-29 (maintainer, on +[moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)): keep the +current resume point, asking a new route from the newest group handed out, +and deliver the late lower groups as a follow-up. + +- Backfill only the gaps below the resume point, from the serving route + through the existing backfill copy, bounded by the reader's own budget. + Do not replay groups the reader already handed out. +- Retire the tests that encode the write-off, such as + `a_widened_floor_stays_above_the_replacement_resume_point`, in favor of + their delivering counterparts. +- If [resume as producer replacement](/quest/m2/resume-redesign.md) lands + first, readers become plain cursors and this falls out of it: verify + with the model and delete this quest. + +Public API: none. Wire: none. + +## Related + +- [Resume as producer replacement](/quest/m2/resume-redesign.md) - makes readers plain cursors, which delivers these groups for free diff --git a/quest/m2/resume-redesign.md b/quest/m2/resume-redesign.md index b69cd928195..542f4cf254e 100644 --- a/quest/m2/resume-redesign.md +++ b/quest/m2/resume-redesign.md @@ -15,8 +15,11 @@ From the 2026-10-24 pre-cut audit at 151a529670. Internal only: no public API or wire change, so it can ship after the cut. The audit recommended after the cut, not before: a rewrite now would discard the large resume test corpus right before release. Decided 2026-10-25 (maintainer): the -[model test](/quest/m1/resume-model-test.md) comes first and gates the cut; -this rewrite moves to m1 only if that test shows design-level bugs. +model test (`rs/moq-net/src/model/resume_model.rs`) came first and gated the +cut; this rewrite moves to m1 only if that test shows design-level bugs. +It found single missed checks +([moq-dev/moq#6947](https://github.com/moq-dev/moq/pull/6947)), so this +stays after the cut, with the model as its oracle. **Why.** Main replaced release's splice design with per-reader route copies in 0382d309f4 (2026-10-04). Since then about 30 of 58 commits to diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index 7d10eb510ed..1f775c10e32 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -57,6 +57,7 @@ criterion = { package = "bench-harness", path = "../bench-harness" } futures = { workspace = true } # The tests' executor, with simulated time. moq-clock = { path = "../moq-clock", features = ["sim"] } +proptest = { workspace = true } serde_json = { workspace = true } # The benches run on tokio. test-util (tokio::time::pause/advance) is test-only and # is NOT supported on wasm, so it must not leak into the normal dependency feature set. diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index b961c3a6e9a..00a715a1efb 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -28,6 +28,9 @@ mod flate; pub(crate) mod object; mod request_queue; mod resume; +// Resume checked against a model of one logical track. +#[cfg(all(test, feature = "async"))] +mod resume_model; mod subscription; mod time; mod timed; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f59d94ab348..0ceeaf3b682 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -419,6 +419,9 @@ struct Copy { /// Held by every group handed out from this copy, so a replaced copy stays subscribed /// while one is still read from it. lease: Arc<()>, + /// Whether it may still deliver a group the reader lacks, as [`Reader::let_go`] last + /// judged: a drop under its bound waits for it. + feeds: bool, } // Boxed: a subscriber is twice a pending answer, and a copy holds one for its whole life. @@ -1080,6 +1083,7 @@ impl Reader { datagrams_done: None, since: None, lease: Arc::new(()), + feeds: true, }); self.learn_end(None); } @@ -1176,15 +1180,18 @@ impl Reader { }; let mut waits = false; // The serving copy, or one between routes with nothing replacing it yet, stays. - self.copies - .retain_mut(|copy| match copy.replaced.then(|| copy.retains(&reader)) { + self.copies.retain_mut(|copy| { + let retains = copy.replaced.then(|| copy.retains(&reader)); + copy.feeds = matches!(retains, None | Some(Retains::Wait)); + match retains { None | Some(Retains::Keep) => true, Some(Retains::Wait) => { waits = true; true } Some(Retains::No) => false, - }); + } + }); waits } @@ -1640,12 +1647,14 @@ impl Reader { for run in covered { self.drops.remove(run); } + // Judged as the group path judges it, so a copy kept only for a group still read + // holds nothing back, and letting one go wakes this too. + self.let_go(delivered, waiter); let serving = self.generation; - let budget = self.mirrored.budget(); let (start, end) = self.groups; let mut feeding = RangeSet::new(); for copy in &self.copies { - if Some(copy.generation) == serving || copy.done.is_some() || copy.expired(budget, waiter) { + if Some(copy.generation) == serving || !copy.feeds { continue; } let from = copy.start(start); @@ -1752,6 +1761,7 @@ impl Reader { done: None, datagrams_done: None, lease: Arc::new(()), + feeds: true, }; // The last copy stays the newest route's own. self.copies.insert(self.copies.len() - 1, copy); diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs new file mode 100644 index 00000000000..203cabeb3af --- /dev/null +++ b/rs/moq-net/src/model/resume_model.rs @@ -0,0 +1,1088 @@ +//! Resume checked against a model of one logical track. +//! +//! Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and +//! reader joins and leaves drive an origin's fronts on the simulated clock. Every reader's +//! groups, frames, drops, and end are checked against what one plain cursor on the logical +//! track would see: every group its range and budget allow, each once, in its order. A +//! newer publisher epoch wins the path while older readers stay on theirs until it has no +//! route left, then end with an error. +//! +//! `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick the cases; nightly sweeps a fresh seed. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::task::Poll; +use std::time::Duration; + +use proptest::prelude::*; +use proptest::test_runner::{Config, RngSeed, TestCaseError, TestRunner}; + +use super::origin::{Producer, Route}; +use crate::{Epoch, Error, Hop, Hops, SessionError, StreamError, Timestamp, broadcast, group, origin, track}; + +/// Groups in the logical track; group `g` has `1 + g % 3` frames. +const GROUPS: u64 = 12; +/// Routes per epoch, cheaper the higher the slot. +const SLOTS: usize = 3; +/// The older publisher instance, then a newer one. +const EPOCHS: usize = 2; +const READERS: usize = 4; +const PATH: &str = "room/alice"; +/// A budget nothing in a case outlives. +const HOUR: Duration = Duration::from_secs(3600); +/// A budget spanning about three groups. +const SHORT: Duration = Duration::from_millis(30); + +fn frames(group: u64) -> usize { + 1 + (group % 3) as usize +} + +/// Frame `frame` of `group` as `epoch` publishes it: every route of one epoch carries it. +fn payload(epoch: usize, group: u64, frame: usize) -> Vec { + format!("e{epoch}:{group}.{frame}").into_bytes() +} + +fn epoch(index: usize) -> Epoch { + // UUIDv7s minted in order, so the second is the newer instance. + [ + "01900000-0000-7000-8000-000000000001", + "01900000-0001-7000-8000-000000000001", + ][index] + .parse() + .unwrap() +} + +/// How a route fails: its session closes on the far side. +fn route_failure() -> Error { + Error::Stream(StreamError::Session(SessionError::Internal)) +} + +/// A verdict on content, not a route failing: another route may still deliver it. +const GIVE_UP: StreamError = StreamError::DeliveryTimeout; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Budget { + Hour, + Short, +} + +impl Budget { + fn duration(self) -> Duration { + match self { + Self::Hour => HOUR, + Self::Short => SHORT, + } + } +} + +#[derive(Clone, Copy, Debug)] +struct Spec { + /// Sequence order through [`track::Ordered`] rather than arrival order. + ordered: bool, + budget: Budget, + /// The subscription's one group range, or the default. + range: Option<(u64, u64)>, + /// Polls drops from the start, as a publisher forwarding them does. + drops: bool, +} + +#[derive(Clone, Debug)] +enum Op { + /// The route announces itself to the origin. + Up { + epoch: usize, + slot: usize, + }, + /// The route's session fails, failing its copy. + Down { + epoch: usize, + slot: usize, + }, + /// The route's copy gets up to `frames` more frames of `group`, or of the lowest group + /// it lacks, finishing the group once complete. + Write { + epoch: usize, + slot: usize, + group: Option, + frames: usize, + }, + /// The route gives up `group` with a verdict on its content. + GiveUp { + epoch: usize, + slot: usize, + group: u64, + }, + /// The route's copy declares the track's end, once it holds the last group. + End { + epoch: usize, + slot: usize, + }, + Advance { + ms: u64, + }, + /// Long enough for idle cached groups to expire. + Expire, + Join { + reader: usize, + spec: Spec, + }, + Leave { + reader: usize, + }, + /// Replace the subscription's group range. + Ranges { + reader: usize, + range: Option<(u64, u64)>, + }, + /// Replace the reader's local group bounds. + Bounds { + reader: usize, + range: Option<(u64, u64)>, + }, + /// Poll the reader whether or not it was woken. + Read { + reader: usize, + }, +} + +#[derive(Clone, Debug)] +struct Case { + ops: Vec, + /// Every live route declares the end once it has caught up. + finish: bool, +} + +fn range() -> impl Strategy> { + proptest::option::of((0..GROUPS, 1..=GROUPS).prop_map(|(start, len)| (start, (start + len).min(GROUPS)))) +} + +fn op() -> impl Strategy { + let route = (0..EPOCHS, 0..SLOTS); + let spec = ( + any::(), + prop_oneof![3 => Just(Budget::Hour), 1 => Just(Budget::Short)], + range(), + any::(), + ) + .prop_map(|(ordered, budget, range, drops)| Spec { + ordered, + budget, + range, + drops, + }); + prop_oneof![ + 30 => (route.clone(), proptest::option::weighted(0.3, 0..GROUPS), 1..=3usize) + .prop_map(|((epoch, slot), group, frames)| Op::Write { epoch, slot, group, frames }), + 8 => route.clone().prop_map(|(epoch, slot)| Op::Up { epoch, slot }), + 6 => route.clone().prop_map(|(epoch, slot)| Op::Down { epoch, slot }), + 4 => (route.clone(), 0..GROUPS - 1).prop_map(|((epoch, slot), group)| Op::GiveUp { epoch, slot, group }), + 2 => route.prop_map(|(epoch, slot)| Op::End { epoch, slot }), + 6 => (1..=200u64).prop_map(|ms| Op::Advance { ms }), + 1 => Just(Op::Expire), + 10 => (0..READERS, spec).prop_map(|(reader, spec)| Op::Join { reader, spec }), + 3 => (0..READERS).prop_map(|reader| Op::Leave { reader }), + 3 => (0..READERS, range()).prop_map(|(reader, range)| Op::Ranges { reader, range }), + 2 => (0..READERS, range()).prop_map(|(reader, range)| Op::Bounds { reader, range }), + 8 => (0..READERS).prop_map(|reader| Op::Read { reader }), + ] +} + +fn case() -> impl Strategy { + (proptest::collection::vec(op(), 1..64), any::()).prop_map(|(ops, finish)| Case { ops, finish }) +} + +/// One route's copy of the logical track, as its upstream publishes it. +struct Copy { + track: track::Producer, + /// Answers the front's requests for the route until the route goes. + serving: moq_clock::sim::JoinHandle<()>, + _broadcast: broadcast::Producer, + /// Fetches of groups the copy lacks, which the route forwards to the publisher. + fills: track::Requests, + /// The cached copies of each group: the live feed's, and any a fetch for its tail filled. + groups: BTreeMap>, + given_up: BTreeSet, +} + +/// One cached copy of a group, from frame `from`. +struct Part { + from: usize, + /// One past the last frame written. + written: usize, + /// While the copy is open. + writer: Option, +} + +impl Copy { + /// Whether the route holds every frame of `group`. + fn complete(&self, group: u64) -> bool { + self.groups + .get(&group) + .is_some_and(|parts| parts.iter().any(|part| part.from == 0 && part.written == frames(group))) + } +} + +/// One publisher instance: its routes, and what any of them ever carried. +#[derive(Default)] +struct Instance { + routes: [Option; SLOTS], + /// Most frames of each group any route carried. + written: BTreeMap, + /// Groups some route finished. + complete: BTreeSet, + /// Groups some route gave up. + given_up: BTreeSet, + /// Some route declared the track's end. + ended: bool, +} + +impl Instance { + fn live(&self) -> impl Iterator { + self.routes.iter().flatten() + } + + /// The slot of the cheapest live route, which the origin serves the path from. + fn winner(&self) -> Option { + self.routes.iter().rposition(Option::is_some) + } + + fn newest(&self) -> Option { + self.written.last_key_value().map(|(group, _)| *group) + } +} + +#[derive(Default)] +struct Flag(AtomicBool); + +impl std::task::Wake for Flag { + fn wake(self: Arc) { + self.0.store(true, Ordering::Relaxed); + } +} + +enum Cursor { + Arrival(track::Subscriber), + Ordered(track::Ordered), +} + +/// A group a reader was handed, and how far it read it. +struct Reading { + consumer: Option, + frames: usize, + end: Option>, +} + +struct Reader { + instance: usize, + spec: Spec, + cursor: Option, + control: track::Control, + _broadcast: broadcast::Consumer, + flag: Arc, + waiter: kio::Waiter, + ranges: Option<(u64, u64)>, + bounds: Option<(u64, u64)>, + /// Every group handed out, in order. + handed: Vec, + reading: BTreeMap, + /// Groups reported dropped. + dropped: BTreeSet, + /// Only groups past this one are owed: older ones were cached history at join, or + /// may have expired. + floor: Option, + /// Its instance lost its last route while it read. + doomed: bool, + /// The newest group handed out when its routes last changed. A route change resumes + /// from there, so a group below it that lands only afterwards may be lost. + resumed: Option, + end: Option>, + /// What every live route carried in full, in range, when the track ended cleanly. + carried: BTreeSet, +} + +impl Reader { + fn covers(&self, group: u64) -> bool { + let within = |range: Option<(u64, u64)>| range.is_none_or(|(start, end)| (start..end).contains(&group)); + within(self.ranges) && within(self.bounds) + } + + /// Everything observed, to tell whether a poll found anything new. + fn observed(&self) -> (usize, usize, bool, usize) { + let frames = self + .reading + .values() + .map(|read| read.frames + usize::from(read.end.is_some())) + .sum(); + (self.handed.len(), frames, self.end.is_some(), self.dropped.len()) + } +} + +fn subscription(budget: Budget, range: Option<(u64, u64)>) -> track::Subscription { + let subscription = track::Subscription::default().with_max_delay(budget.duration()); + match range { + Some((start, end)) => subscription.with_range(start..end), + None => subscription, + } +} + +fn fail(reader: usize, what: impl std::fmt::Display) -> TestCaseError { + TestCaseError::fail(format!("reader {reader}: {what}")) +} + +struct Model { + origin: Producer, + consumer: origin::Consumer, + instances: [Instance; EPOCHS], + readers: [Option; READERS], +} + +impl Model { + fn new() -> Self { + let origin = origin::Config { + hop: Hop::new(1).unwrap(), + update_hold: Duration::ZERO, + ..Default::default() + }; + let (origin, driver) = Producer::new(origin); + moq_clock::sim::spawn(crate::time::run(driver)); + Self { + consumer: origin.consume(), + origin, + instances: Default::default(), + readers: Default::default(), + } + } + + async fn apply(&mut self, op: &Op) -> Result<(), TestCaseError> { + // A reader moves only when its instance's winning route, the cheapest live one, changes. + let rerouted = match *op { + Op::Up { epoch, slot } | Op::Down { epoch, slot } => { + let instance = &self.instances[epoch]; + let mut live: Vec = instance.routes.iter().map(Option::is_some).collect(); + live[slot] = matches!(op, Op::Up { .. }); + (live.iter().rposition(|live| *live) != instance.winner()).then_some(epoch) + } + _ => None, + }; + if let Some(epoch) = rerouted { + for reader in self + .readers + .iter_mut() + .flatten() + .filter(|reader| reader.instance == epoch) + { + reader.resumed = reader.resumed.max(reader.handed.iter().max().copied()); + } + } + match *op { + Op::Up { epoch, slot } => self.up(epoch, slot), + Op::Down { epoch, slot } => { + let instance = &mut self.instances[epoch]; + let Some(copy) = instance.routes[slot].take() else { + return Ok(()); + }; + copy.serving.abort(); + for part in copy.groups.into_values().flatten() { + if let Some(writer) = part.writer { + let _ = writer.abort(route_failure()); + } + } + let _ = copy.track.abort(route_failure()); + if instance.live().next().is_none() { + for reader in self.readers.iter_mut().flatten() { + if reader.instance == epoch { + reader.doomed = true; + } + } + } + } + Op::Write { + epoch, + slot, + group, + frames, + } => { + let Some(copy) = &self.instances[epoch].routes[slot] else { + return Ok(()); + }; + let group = group + .or_else(|| (0..GROUPS).find(|group| !copy.complete(*group) && !copy.given_up.contains(group))); + if let Some(group) = group { + self.write(epoch, slot, group, frames); + } + } + Op::GiveUp { epoch, slot, group } => { + let instance = &mut self.instances[epoch]; + let Some(copy) = &mut instance.routes[slot] else { + return Ok(()); + }; + if copy.complete(group) || !copy.given_up.insert(group) { + return Ok(()); + } + match copy.groups.get_mut(&group) { + Some(parts) => { + for writer in parts.iter_mut().filter_map(|part| part.writer.take()) { + let _ = writer.abort(Error::Stream(GIVE_UP)); + } + } + None => copy.track.drop_groups(group..group + 1, GIVE_UP), + } + instance.given_up.insert(group); + } + Op::End { epoch, slot } => self.end(epoch, slot), + Op::Advance { ms } => moq_clock::sim::advance(Duration::from_millis(ms)).await, + Op::Expire => { + moq_clock::sim::advance(Duration::from_secs(60)).await; + for reader in self.readers.iter_mut().flatten() { + reader.floor = self.instances[reader.instance].newest(); + } + } + Op::Join { reader, spec } => self.join(reader, spec).await?, + Op::Leave { reader } => self.readers[reader] = None, + Op::Ranges { reader: index, range } => { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + reader.ranges = range; + let _ = reader.control.update(subscription(reader.spec.budget, range)); + self.poll(index)?; + } + Op::Bounds { reader: index, range } => { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + reader.bounds = range; + let (start, end) = range.unwrap_or((0, u64::MAX)); + reader.cursor = reader.cursor.take().map(|cursor| match cursor { + Cursor::Arrival(sub) => Cursor::Arrival(sub.with_range(start..end)), + Cursor::Ordered(sub) => Cursor::Ordered(sub.with_range(start..end)), + }); + self.poll(index)?; + } + Op::Read { reader } => { + if self.readers[reader].is_some() { + self.poll(reader)?; + } + } + } + Ok(()) + } + + fn up(&mut self, epoch: usize, slot: usize) { + if self.instances[epoch].routes[slot].is_some() { + return; + } + let mut hops = Hops::new(); + hops.push(Hop::new(10 * (epoch as u64 + 1) + slot as u64).unwrap()) + .unwrap(); + let route = Route::default() + .with_epoch(self::epoch(epoch)) + .with_hops(hops) + .with_cost((SLOTS - slot) as u64); + let requests = self.origin.requests_route(PATH, route).unwrap(); + let broadcast = broadcast::Info::new().produce(); + let track = broadcast.create_track("video", track::Info::timed()).unwrap(); + let upstream = broadcast.consume(); + let serving = moq_clock::sim::spawn(async move { + while let Ok(request) = kio::wait(|waiter| requests.poll_requested_broadcast(waiter)).await { + request.accept(&upstream); + } + }); + self.instances[epoch].routes[slot] = Some(Copy { + fills: track.requests(), + track, + serving, + _broadcast: broadcast, + groups: BTreeMap::new(), + given_up: BTreeSet::new(), + }); + } + + /// The route's live feed brings up to `more` frames of `group` past what it holds. + fn write(&mut self, epoch: usize, slot: usize, group: u64, more: usize) { + let Some(copy) = &mut self.instances[epoch].routes[slot] else { + return; + }; + if copy.given_up.contains(&group) { + return; + } + let parts = copy.groups.entry(group).or_default(); + if !parts.iter().any(|part| part.from == 0) + && let Ok(writer) = copy.track.create_group(group::Info { sequence: group }) + { + parts.push(Part { + from: 0, + written: 0, + writer: Some(writer), + }); + } + let held = parts.iter().map(|part| part.written).max().unwrap_or(0); + self.catch_up(epoch, slot, group, held.saturating_add(more)); + } + + /// Write every open copy of `group` on the route up to frame `until`. + fn catch_up(&mut self, epoch: usize, slot: usize, group: u64, until: usize) { + let instance = &mut self.instances[epoch]; + let Some(parts) = instance.routes[slot] + .as_mut() + .and_then(|copy| copy.groups.get_mut(&group)) + else { + return; + }; + let until = until.min(frames(group)); + for part in parts.iter_mut() { + let Some(writer) = &mut part.writer else { continue }; + while part.written < until { + let at = Timestamp::from_millis(group * 10 + part.written as u64).unwrap(); + writer.write_frame(at, payload(epoch, group, part.written)).unwrap(); + part.written += 1; + } + if part.written == frames(group) { + part.writer.take().unwrap().finish().unwrap(); + instance.complete.insert(group); + } + let most = instance.written.entry(group).or_default(); + *most = (*most).max(part.written); + } + } + + /// Answer each route's fetches as its publisher would: with what the instance carries + /// so far, which later writes continue, or not found when no route has it yet. + fn fill(&mut self) -> bool { + let mut filled = false; + for epoch in 0..EPOCHS { + for slot in 0..SLOTS { + while let Some(Poll::Ready(Ok(fill))) = self.instances[epoch].routes[slot] + .as_ref() + .map(|copy| copy.fills.poll_requested_fill(&kio::Waiter::noop())) + { + filled = true; + let (start, end) = (fill.start().group, fill.end()); + for group in start..end.min(GROUPS) { + let instance = &mut self.instances[epoch]; + let copy = instance.routes[slot].as_mut().expect("polled above"); + let from = fill.frame_start(group) as usize; + let written = instance.written.get(&group).copied().unwrap_or(0); + let held = copy.groups.get(&group).into_iter().flatten(); + if copy.given_up.contains(&group) { + fill.refuse_groups(group..group + 1, Error::Stream(GIVE_UP)); + } else if written == 0 || written < from || held.into_iter().any(|part| part.from <= from) { + fill.refuse_groups(group..group + 1, Error::NotFound); + } else if let Ok(writer) = fill.accept(group, None) { + // The fill starts at the frame its readers want, which may not exist yet. + copy.groups.entry(group).or_default().push(Part { + from, + written: from, + writer: Some(writer), + }); + self.catch_up(epoch, slot, group, written); + } + } + if end > GROUPS { + fill.refuse_groups(GROUPS.max(start)..end, Error::NotFound); + } + } + } + } + filled + } + + fn end(&mut self, epoch: usize, slot: usize) { + let instance = &mut self.instances[epoch]; + let Some(copy) = &instance.routes[slot] else { + return; + }; + if copy.groups.contains_key(&(GROUPS - 1)) && copy.track.finish().is_ok() { + instance.ended = true; + } + } + + async fn join(&mut self, index: usize, spec: Spec) -> Result<(), TestCaseError> { + if self.readers[index].is_some() { + return Ok(()); + } + // The newest instance with a route wins the path. + let Some(expected) = (0..EPOCHS) + .rev() + .find(|epoch| self.instances[*epoch].live().next().is_some()) + else { + return Ok(()); + }; + let broadcast = self + .consumer + .request(PATH, None) + .await + .map_err(|err| fail(index, format!("request failed: {err}")))?; + let instance = (0..EPOCHS) + .find(|epoch| broadcast.info().epoch.as_ref() == Some(&self::epoch(*epoch))) + .ok_or_else(|| fail(index, "resolved no known instance"))?; + if instance != expected { + return Err(fail( + index, + format!("joined instance {instance}, not the newest {expected}"), + )); + } + let subscriber = broadcast + .track("video") + .map_err(|err| fail(index, format!("track: {err}")))? + .subscribe(subscription(spec.budget, spec.range)) + .await + .map_err(|err| fail(index, format!("subscribe failed: {err}")))?; + let flag = Arc::::default(); + let reader = Reader { + instance, + spec, + control: subscriber.control(), + cursor: Some(match spec.ordered { + true => Cursor::Ordered(subscriber.ordered()), + false => Cursor::Arrival(subscriber), + }), + _broadcast: broadcast, + waiter: kio::Waiter::new(flag.clone().into()), + flag, + ranges: spec.range, + bounds: None, + handed: Vec::new(), + reading: BTreeMap::new(), + dropped: BTreeSet::new(), + floor: self.instances[instance].newest(), + doomed: false, + resumed: None, + end: None, + carried: BTreeSet::new(), + }; + self.readers[index] = Some(reader); + self.poll(index) + } + + /// Read everything `index` has ready, checking each step against the model. + fn poll(&mut self, index: usize) -> Result<(), TestCaseError> { + let Some(reader) = &mut self.readers[index] else { + return Ok(()); + }; + let instance = &self.instances[reader.instance]; + reader.flag.0.store(false, Ordering::Relaxed); + let waiter = reader.waiter.clone(); + + while reader.end.is_none() { + let polled = match reader.cursor.as_mut().unwrap() { + Cursor::Arrival(sub) => sub.poll_recv_group(&waiter), + Cursor::Ordered(sub) => sub.poll_next_group(&waiter), + }; + match polled { + Poll::Pending => break, + Poll::Ready(Ok(Some(group))) => { + let sequence = group.sequence; + if reader.reading.contains_key(&sequence) { + return Err(fail( + index, + format!("group {sequence} handed out twice: {:?}", reader.handed), + )); + } + if reader.spec.ordered && reader.handed.last().is_some_and(|last| *last >= sequence) { + return Err(fail( + index, + format!("group {sequence} out of order after {:?}", reader.handed), + )); + } + if !reader.covers(sequence) { + return Err(fail( + index, + format!( + "group {sequence} outside ranges {:?} and bounds {:?}", + reader.ranges, reader.bounds + ), + )); + } + if !instance.written.contains_key(&sequence) { + return Err(fail( + index, + format!("group {sequence} handed out before any route had it"), + )); + } + reader.handed.push(sequence); + reader.reading.insert( + sequence, + Reading { + consumer: Some(group), + frames: 0, + end: None, + }, + ); + } + Poll::Ready(Ok(None)) => { + if !instance.ended { + return Err(fail(index, "the track ended cleanly though no route declared an end")); + } + // A group landing below the end afterwards is not owed: a reader stops at it. + reader.carried = (0..GROUPS) + .filter(|group| reader.covers(*group) && instance.live().all(|copy| copy.complete(*group))) + .collect(); + reader.end = Some(Ok(())); + } + Poll::Ready(Err(err)) => { + if !reader.doomed { + return Err(fail( + index, + format!("the track failed while its instance had a route: {err}"), + )); + } + reader.end = Some(Err(err)); + } + } + } + + if reader.spec.drops + && let Some(Cursor::Arrival(sub)) = &mut reader.cursor + { + while let Poll::Ready((range, err)) = sub.poll_drop(&waiter) { + for group in range.clone() { + if reader.reading.contains_key(&group) { + return Err(fail( + index, + format!("dropped {range:?} ({err}) after handing out {group}"), + )); + } + } + reader.dropped.extend(range.start..range.end.min(GROUPS)); + } + } + + for (sequence, read) in &mut reader.reading { + let Some(consumer) = &mut read.consumer else { + continue; + }; + loop { + match consumer.poll_read_frame(&waiter) { + Poll::Pending => break, + Poll::Ready(Ok(Some(frame))) => { + let expected = payload(reader.instance, *sequence, read.frames); + if frame.payload[..] != expected[..] || read.frames >= frames(*sequence) { + return Err(fail( + index, + format!( + "group {sequence} frame {} read {:?}, expected {:?}", + read.frames, + String::from_utf8_lossy(&frame.payload), + String::from_utf8_lossy(&expected) + ), + )); + } + read.frames += 1; + } + Poll::Ready(Ok(None)) => { + if read.frames != frames(*sequence) || !instance.complete.contains(sequence) { + return Err(fail( + index, + format!("group {sequence} ended cleanly after {} frames", read.frames), + )); + } + read.end = Some(Ok(())); + read.consumer = None; + break; + } + Poll::Ready(Err(err)) => { + read.end = Some(Err(err)); + read.consumer = None; + break; + } + } + } + } + Ok(()) + } + + /// Let the driver and the routes run without time passing, polling each reader woken. + async fn settle(&mut self) -> Result<(), TestCaseError> { + let mut quiet = 0; + for _ in 0..10_000 { + moq_clock::sim::yield_now().await; + let mut woken = self.fill(); + for index in 0..READERS { + if self.readers[index] + .as_ref() + .is_some_and(|reader| reader.flag.0.load(Ordering::Relaxed)) + { + woken = true; + self.poll(index)?; + } + } + quiet = if woken { 0 } else { quiet + 1 }; + if quiet == 16 { + return Ok(()); + } + } + Err(TestCaseError::fail( + "readers never settled: something keeps waking them", + )) + } + + /// Every live route catches up on every group and, for `finish`, declares the end; then + /// each reader must have read what it is owed. + async fn conclude(&mut self, finish: bool) -> Result<(), TestCaseError> { + let before: Vec> = self + .readers + .iter() + .map(|reader| reader.as_ref().and_then(|reader| reader.handed.iter().max().copied())) + .collect(); + for epoch in 0..EPOCHS { + for slot in 0..SLOTS { + for group in 0..GROUPS { + self.write(epoch, slot, group, usize::MAX); + } + if finish { + self.end(epoch, slot); + } + } + } + self.settle().await?; + + let observed: Vec<_> = self + .readers + .iter() + .map(|reader| reader.as_ref().map(Reader::observed)) + .collect(); + for index in 0..READERS { + if self.readers[index].is_some() { + self.poll(index)?; + } + } + for (index, reader) in self.readers.iter().enumerate() { + let Some(reader) = reader else { continue }; + if Some(reader.observed()) != observed[index] { + return Err(fail( + index, + format!( + "a poll found progress that never woke it: {:?} -> {:?}, handed {:?}, dropped {:?}, end {:?}", + observed[index], + reader.observed(), + reader.handed, + reader.dropped, + reader.end + ), + )); + } + self.owed(index, reader, before[index], finish)?; + } + Ok(()) + } + + fn owed(&self, index: usize, reader: &Reader, before: Option, finish: bool) -> Result<(), TestCaseError> { + let instance = &self.instances[reader.instance]; + if reader.doomed { + return match &reader.end { + Some(Err(_)) => Ok(()), + Some(Ok(())) if instance.ended => Ok(()), + end => Err(fail(index, format!("outlived its instance's routes: {end:?}"))), + }; + } + // What every live route carries in full. + let carried = |group: u64| instance.live().all(|copy| copy.complete(group)); + let mut owed: Vec = (0..GROUPS) + .filter(|group| reader.floor.is_none_or(|floor| *group > floor)) + .filter(|group| reader.resumed.is_none_or(|resumed| *group > resumed)) + .filter(|group| reader.covers(*group) && carried(*group)) + .collect(); + if reader.spec.ordered { + owed.retain(|group| before.is_none_or(|before| *group > before)); + } + if matches!(reader.end, Some(Ok(()))) { + owed.retain(|group| reader.carried.contains(group)); + } + if reader.spec.budget == Budget::Short { + owed.retain(|group| *group == GROUPS - 1); + } + let missing: Vec = owed + .iter() + .copied() + .filter(|group| !reader.reading.contains_key(group)) + .collect(); + let mut problems = String::new(); + if !missing.is_empty() { + let _ = write!(problems, "missing groups {missing:?}; "); + } + if reader.spec.budget == Budget::Hour { + for (sequence, read) in &reader.reading { + if carried(*sequence) && !instance.given_up.contains(sequence) && !matches!(read.end, Some(Ok(()))) { + let _ = write!( + problems, + "group {sequence} stopped at frame {} ({:?}); ", + read.frames, read.end + ); + } + } + } + match (&reader.end, finish && instance.live().next().is_some()) { + (None, false) | (Some(Ok(())), true) => {} + (Some(Ok(())), false) if instance.ended => {} + (end, _) => { + let _ = write!(problems, "track end {end:?}; "); + } + } + if problems.is_empty() { + return Ok(()); + } + Err(fail( + index, + format!( + "{problems}handed out {:?}, dropped {:?}, floor {:?}, spec {:?}", + reader.handed, reader.dropped, reader.floor, reader.spec + ), + )) + } +} + +fn run(case: &Case) -> Result<(), TestCaseError> { + moq_clock::sim::run(async { + let mut model = Model::new(); + model.up(0, 0); + for op in &case.ops { + model.apply(op).await?; + model.settle().await?; + } + model.conclude(case.finish).await + }) +} + +/// Runs the cases `MOQ_RESUME_SEED` and `MOQ_RESUME_CASES` pick. +#[test] +fn readers_match_the_logical_track() { + let setting = |name: &str, fallback: u64| std::env::var(name).map_or(fallback, |value| value.parse().expect(name)); + let seed = setting("MOQ_RESUME_SEED", 0x6d6f71); + let count = setting("MOQ_RESUME_CASES", 64) as u32; + eprintln!("MOQ_RESUME_SEED={seed} MOQ_RESUME_CASES={count}"); + let mut runner = TestRunner::new(Config { + cases: count, + rng_seed: RngSeed::Fixed(seed), + failure_persistence: None, + ..Default::default() + }); + runner.run(&case(), |case| run(&case)).unwrap(); +} + +/// An Up or Down that leaves the winning route serving resumes nothing, so lower groups stay owed. +#[test] +fn unchanged_routes_keep_groups_owed() { + moq_clock::sim::run(async { + let mut model = Model::new(); + model.up(0, 0); + let ops = [ + Op::Up { epoch: 0, slot: 2 }, + Op::Join { + reader: 0, + spec: Spec { + ordered: false, + budget: Budget::Hour, + range: None, + drops: false, + }, + }, + Op::Write { + epoch: 0, + slot: 2, + group: Some(8), + frames: 1, + }, + // Repeated, absent, and standby routes. + Op::Up { epoch: 0, slot: 2 }, + Op::Down { epoch: 0, slot: 1 }, + Op::Up { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 1 }, + Op::Down { epoch: 0, slot: 0 }, + ]; + for op in &ops { + model.apply(op).await.unwrap(); + model.settle().await.unwrap(); + } + let reader = model.readers[0].as_ref().unwrap(); + assert!(reader.handed.contains(&8), "handed out {:?}", reader.handed); + assert_eq!(reader.resumed, None); + }); +} + +/// Failures the sweep found, as it shrank them. +#[test] +fn regressions() { + let spec = |ordered, budget, range, drops| Spec { + ordered, + budget, + range, + drops, + }; + let cases = [ + // An ordered reader whose range a finished track already passed ends. + Case { + ops: vec![Op::Join { + reader: 0, + spec: spec(true, Budget::Hour, Some((0, 1)), false), + }], + finish: true, + }, + // An ordered reader ends once the rest of its range was given up. + Case { + ops: vec![ + Op::Up { epoch: 1, slot: 2 }, + Op::Join { + reader: 0, + spec: spec(true, Budget::Hour, Some((0, 8)), false), + }, + Op::GiveUp { + epoch: 1, + slot: 2, + group: 7, + }, + ], + finish: true, + }, + // Local bounds below the subscription's range leave no drops to report. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, Some((11, 12)), true), + }, + Op::Write { + epoch: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Bounds { + reader: 0, + range: Some((0, 1)), + }, + ], + finish: false, + }, + // A drop held for a replaced copy is reported once that copy only backs a read. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Short, None, true), + }, + Op::Write { + epoch: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Up { epoch: 0, slot: 2 }, + Op::Write { + epoch: 0, + slot: 2, + group: Some(6), + frames: 1, + }, + ], + finish: false, + }, + ]; + for case in cases { + if let Err(err) = run(&case) { + panic!("{case:?}: {err}"); + } + } +} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index a60081f0023..2543420fbfe 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1164,7 +1164,9 @@ impl TrackState { // If the exclusive end is already at or below where we'd resume, no // group can ever satisfy this call until the cap rises or the ranges widen. // Pending (not None) so the consumer is parked rather than told the stream - // is over. An empty range (`end == 0`) parks even at the first sequence. + // is over, unless the track declared its end: a finished track ends a reader + // with nothing left in range, as the arrival path does. An empty range + // (`end == 0`) parks even at the first sequence. let cap = match (end_sequence, window.end()) { (Some(cap), Some(end)) => Some(cap.min(end)), (cap, end) => cap.or(end), @@ -1172,7 +1174,7 @@ impl TrackState { if let Some(cap) = cap && cap <= next_sequence { - if closed { + if closed || self.final_sequence().is_some() { return Poll::Ready(end()); } return Poll::Pending; @@ -1203,10 +1205,11 @@ impl TrackState { // `final_sequence` is one past the last possible sequence. If our // floor is already at/past it, nothing else can land in range. // A closed track produces nothing more either, so a gap below its - // boundary is the end, not a wait. Cached in-range groups were - // returned above, so an aborted track drains what finished first. + // boundary is the end, not a wait, as is one in a complete track, which + // ends the arrival path too. Cached in-range groups were returned above, + // so an aborted track drains what finished first. // This is the cursor the ordered reader uses. - if closed || self.final_sequence().is_some_and(|fin| next_sequence >= fin) { + if closed || self.is_complete() || self.final_sequence().is_some_and(|fin| next_sequence >= fin) { return Poll::Ready(end()); } Poll::Pending @@ -6097,8 +6100,10 @@ impl Cursor { { self.probe(self.floor(), &drift, waiter); } + // A cap at or below the floor leaves nothing to report, not a reversed span. + let floor = self.floor(); let within = Within { - span: self.floor()..self.end_sequence.unwrap_or(u64::MAX), + span: floor..self.end_sequence.unwrap_or(u64::MAX).max(floor), window: &self.filter.window, revision: self.filter.revision, }; @@ -13178,6 +13183,34 @@ mod test { assert!(done.is_none(), "missing groups below the boundary are not an error"); } + /// A finished track ends a sequence reader with nothing left in its range, as it ends + /// an arrival reader: whether the range ends at the cursor or the rest was given up. + #[test] + fn ordered_ends_once_a_finished_track_leaves_nothing_in_range() { + for given_up in [false, true] { + let producer = track_producer("test", Info::timed()); + let end = if given_up { 4 } else { 3 }; + let mut ordered = producer.subscribe(replay().with_range(0..end)).ordered(); + if given_up { + producer.drop_groups(3..4, StreamError::DeliveryTimeout); + } + for sequence in [0, 1, 2, 4, 5] { + write_group(&producer, sequence); + } + producer.finish().unwrap(); + for sequence in 0..3 { + let Poll::Ready(Ok(Some(group))) = ordered.poll_next_group(&kio::Waiter::noop()) else { + panic!("group {sequence}, given_up={given_up}") + }; + assert_eq!(group.sequence, sequence); + } + assert!( + matches!(ordered.poll_next_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), + "given_up={given_up}" + ); + } + } + #[test] fn append_finish_cannot_be_rewritten() { let producer = track_producer("test", Info::timed()); diff --git a/test/justfile b/test/justfile index c138ec7c7cd..85adb4202c9 100644 --- a/test/justfile +++ b/test/justfile @@ -266,6 +266,10 @@ bare-fin: json-properties: bash ../sh/local.sh bash json-properties.sh +# Check resume against its model of one logical track from a fresh replayable seed. +resume-model: + bash ../sh/local.sh bash resume-model.sh + # The TypeScript qmux peer as client and server against the in-tree Rust qmux # interop examples, per draft. The manifest path names the in-tree crate, which # shares its package name with the 0.6 release its tests interoperate with. diff --git a/test/resume-model.sh b/test/resume-model.sh new file mode 100755 index 00000000000..40678bc9af4 --- /dev/null +++ b/test/resume-model.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Nightly explores fresh sequences; printed settings replay either implementation. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +MOQ_RESUME_SEED=${MOQ_RESUME_SEED:-$(python3 -c 'import secrets; print(secrets.randbits(31))')} +MOQ_RESUME_CASES=${MOQ_RESUME_CASES:-4096} +export MOQ_RESUME_SEED MOQ_RESUME_CASES +echo "Resume model: MOQ_RESUME_SEED=$MOQ_RESUME_SEED MOQ_RESUME_CASES=$MOQ_RESUME_CASES" +just rs test -p moq-net --lib -E 'test(=model::resume_model::readers_match_the_logical_track)' +bun install --frozen-lockfile +bun test js/net/src/resume.model.test.ts From 8551b0f6de5e86e199c449c77a6d402a2e46be34 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 2/8] feat(quic)!: namespace QUIC streams as send, recv, and stream (#6950) * chore(quest): claim quest/m1/quic-api/streams.md Quest-Claim: 3067217ff870abc5 * wip(quic): namespace moq-quic streams as send, recv, and stream send::{Stream, Offset, End, Options, Group, Groups, Error}, recv::{Stream, Lease, Chunks, Error}, and stream::{Id, Dir}. Writes take Buf, a lease moves its chunks out and holds no borrow, send ends stay queryable until released, and datagrams chain chunks without copying. * wip(qmux): use moq-quic send ends and Buf writes * wip(tokio): namespace the QUIC driver as send, recv, and stream send::{Stream, Finished, Group, Options, Offset, Error} with finish(self) typestate and consuming resets, recv::{Stream, Lease, Error} with a lease that owns its bytes and holds no lock, Buf writes and BufMut reads, and datagrams that chain a header. quic::web collapses onto one error per direction. moq-quic only reruns the stuck pass on restore when an out lease held it off. * wip(uring): namespace the QUIC driver as send, recv, and stream * wip(uring): port tests and moq-relay to the namespaced driver * wip(uring): format * wip(quic): name the namespaced stream API in the fork notes * wip(quic): fix a stale doc link * chore(quest): finish quic-api/streams * fix(quic): keep terminal stream state across the review's lifecycle gaps - moq-tokio and moq-uring cache a reset `received_reset` reports, since moq-quic frees the stream then, so later reads and leases fail with it instead of panicking. - moq-uring's web `poll_closed` observes a clean FIN again through an empty read, instead of a reset-only query that waits forever. - moq-uring's web `finish` keeps a finished stream's handle when called again. - qmux resets an unfinished stream on drop before releasing its ID, which refused the reset. * fix(quic): size a datagram before taking its payload Co-authored-by: Claude Opus 5.5 --- quest/m1/quic-api/README.md | 1 - quest/m1/quic-api/native-web.md | 2 +- quest/m1/quic-api/streams.md | 111 --- quest/m1/quic-api/waits.md | 6 +- quest/m2/perf/quic-lock-contention.md | 3 +- quest/m2/quic-ack-hook.md | 17 +- rs/moq-quic/README.md | 17 +- rs/moq-quic/benches/common/mod.rs | 49 +- rs/moq-quic/benches/receive.rs | 22 +- rs/moq-quic/benches/streams.rs | 26 +- rs/moq-quic/src/connection/datagrams.rs | 45 +- rs/moq-quic/src/connection/mod.rs | 31 +- rs/moq-quic/src/connection/mux/mod.rs | 40 +- rs/moq-quic/src/connection/mux/tests.rs | 74 +- rs/moq-quic/src/connection/stats.rs | 4 +- rs/moq-quic/src/connection/streams/lease.rs | 337 +++++--- rs/moq-quic/src/connection/streams/mod.rs | 547 +------------ rs/moq-quic/src/connection/streams/recv.rs | 184 ++++- .../src/connection/streams/saturated.rs | 34 +- rs/moq-quic/src/connection/streams/send.rs | 698 +++++++++++------ rs/moq-quic/src/connection/streams/state.rs | 497 +++++++----- rs/moq-quic/src/frame.rs | 93 ++- rs/moq-quic/src/fuzz.rs | 7 +- rs/moq-quic/src/fuzz/connection.rs | 40 +- rs/moq-quic/src/lib.rs | 97 +-- rs/moq-quic/src/stream.rs | 94 +++ rs/moq-quic/src/tests/acked.rs | 79 +- rs/moq-quic/src/tests/characterize/limits.rs | 14 +- rs/moq-quic/src/tests/characterize/paths.rs | 6 +- rs/moq-quic/src/tests/characterize/reset.rs | 43 +- .../src/tests/characterize/schedule.rs | 8 +- rs/moq-quic/src/tests/mod.rs | 360 +++++---- rs/moq-quic/src/tests/probe.rs | 14 +- rs/moq-quic/src/tests/reset_stream_at.rs | 81 +- rs/moq-quic/src/tests/sim.rs | 6 +- rs/moq-quic/src/tests/sim/workloads.rs | 22 +- rs/moq-quic/src/tests/util.rs | 8 +- rs/moq-relay/src/uring.rs | 2 - rs/moq-relay/tests/integration/peer_limits.rs | 4 +- rs/moq-relay/tests/integration/refusals.rs | 2 +- .../tests/integration/runtime_uring.rs | 4 +- rs/moq-tokio/src/quic/driver/connection.rs | 120 ++- rs/moq-tokio/src/quic/driver/mod.rs | 19 +- rs/moq-tokio/src/quic/driver/recv.rs | 526 +++++++++++++ rs/moq-tokio/src/quic/driver/recv_stream.rs | 732 ------------------ rs/moq-tokio/src/quic/driver/send.rs | 567 ++++++++++++++ rs/moq-tokio/src/quic/driver/send_group.rs | 88 --- rs/moq-tokio/src/quic/driver/send_stream.rs | 468 ----------- rs/moq-tokio/src/quic/driver/tests.rs | 189 +++-- rs/moq-tokio/src/quic/web/connect.rs | 16 +- rs/moq-tokio/src/quic/web/error.rs | 115 +-- rs/moq-tokio/src/quic/web/recv.rs | 126 +-- rs/moq-tokio/src/quic/web/send.rs | 202 +++-- rs/moq-tokio/src/quic/web/session.rs | 191 +++-- rs/moq-tokio/src/quic/web/settings.rs | 12 +- rs/moq-tokio/src/transport.rs | 15 +- rs/moq-tokio/src/transport/owned.rs | 12 +- .../tests/integration/receive_credit.rs | 8 +- .../tests/integration/reset_secret.rs | 4 +- rs/moq-tokio/tests/integration/stream_code.rs | 8 +- rs/moq-uring/benches/echo_quic.rs | 2 +- rs/moq-uring/src/quic/driver/connection.rs | 375 +++++---- rs/moq-uring/src/quic/driver/endpoint.rs | 8 +- rs/moq-uring/src/quic/driver/group.rs | 79 -- rs/moq-uring/src/quic/driver/mod.rs | 8 +- rs/moq-uring/src/quic/driver/recv.rs | 523 +++++++++++++ rs/moq-uring/src/quic/driver/send.rs | 466 +++++++++++ rs/moq-uring/src/quic/driver/stream.rs | 537 ------------- rs/moq-uring/src/quic/mod.rs | 6 +- rs/moq-uring/src/quic/web.rs | 426 +++++----- rs/moq-uring/src/transport.rs | 22 +- rs/moq-uring/tests/integration/credit.rs | 193 ++--- rs/moq-uring/tests/integration/echo.rs | 2 +- rs/moq-uring/tests/integration/endpoint.rs | 74 +- rs/moq-uring/tests/integration/identity.rs | 16 +- rs/moq-uring/tests/integration/mtu.rs | 25 +- rs/moq-uring/tests/integration/qlog.rs | 6 +- rs/moq-uring/tests/integration/session.rs | 22 +- rs/moq-uring/tests/integration/teardown.rs | 11 +- rs/moq-uring/tests/integration/web.rs | 159 ++-- rs/moq-uring/tests/integration/workers.rs | 14 +- rs/qmux/src/session.rs | 96 +-- rs/qmux/src/session/tests.rs | 13 + 83 files changed, 5194 insertions(+), 5036 deletions(-) delete mode 100644 quest/m1/quic-api/streams.md create mode 100644 rs/moq-quic/src/stream.rs create mode 100644 rs/moq-tokio/src/quic/driver/recv.rs delete mode 100644 rs/moq-tokio/src/quic/driver/recv_stream.rs create mode 100644 rs/moq-tokio/src/quic/driver/send.rs delete mode 100644 rs/moq-tokio/src/quic/driver/send_group.rs delete mode 100644 rs/moq-tokio/src/quic/driver/send_stream.rs delete mode 100644 rs/moq-uring/src/quic/driver/group.rs create mode 100644 rs/moq-uring/src/quic/driver/recv.rs create mode 100644 rs/moq-uring/src/quic/driver/send.rs delete mode 100644 rs/moq-uring/src/quic/driver/stream.rs diff --git a/quest/m1/quic-api/README.md b/quest/m1/quic-api/README.md index 1ea034586b4..a1b4b76533b 100644 --- a/quest/m1/quic-api/README.md +++ b/quest/m1/quic-api/README.md @@ -47,7 +47,6 @@ side by side and fix any name or signature that still differs, and check ## Required -- [Streams](/quest/m1/quic-api/streams.md) - `send::`/`recv::` namespaces, one error per direction, `finish(self) -> send::Finished`, consuming `reset`/`stop`, `Buf` writes, both drivers mirrored - [Crypto](/quest/m1/quic-api/crypto.md) - crypto traits sealed to rustls, concrete peer identity and handshake data - [Waits](/quest/m1/quic-api/waits.md) - moq-quic owns every wait as a `poll_*` on `&kio::Waiter`, deleting `StreamEvent` and the drivers' and qmux's waiter tables - [Poll forms](/quest/m1/quic-api/poll.md) - both drivers' waits poll as `&self, &kio::Waiter` forwards to moq-quic, the async forms wrap them, and `send::Finished` is a future diff --git a/quest/m1/quic-api/native-web.md b/quest/m1/quic-api/native-web.md index 4e7951832a2..be24ee2efef 100644 --- a/quest/m1/quic-api/native-web.md +++ b/quest/m1/quic-api/native-web.md @@ -36,7 +36,7 @@ Align the two stacks while each gets its impl: it has two `send_datagram`s with different framing: its own prefixes the WebTransport session id, the driver's does not. Drop the `Deref`. - The session id is chained onto each datagram with `send(impl Buf)` - ([streams](/quest/m1/quic-api/streams.md)) instead of allocated and copied + ([moq-dev/moq#6950](https://github.com/moq-dev/moq/pull/6950)) instead of allocated and copied per datagram (quinn-rs/quinn#1724). - `quic::web` is supported API in both crates (maintainer, 2026-10-23): a WebTransport-over-moq-quic library usable without MoQ, documented, no diff --git a/quest/m1/quic-api/streams.md b/quest/m1/quic-api/streams.md deleted file mode 100644 index 4f4e0f4ca57..00000000000 --- a/quest/m1/quic-api/streams.md +++ /dev/null @@ -1,111 +0,0 @@ -# [L] Streams that cannot be misused - -## Goal - -Streams and datagrams in moq-quic, `moq_tokio::quic::driver`, and -moq-uring's `quic` module share one namespaced shape: `send::{Stream, -Finished, Group, Options, Error}`, `recv::{Stream, Lease, Error}`, and -`stream::{Id, Dir, Event}`, with the same signatures in both drivers. One -error type per direction replaces quinn's zoo, a write after `finish` or any -use after `reset`/`stop` does not compile, writes and reads take `Buf` -and `BufMut`, and a receive lease holds no connection lock. - -## Plan - -Decided 2026-10-23 (maintainer): - -**Names.** No `SendXxx`, `RecvXxx`, or `XxxError` at a crate root. -`SendStream` is `send::Stream`, `SendGroup` is `send::Group`, `SendGroups` -is `send::Groups`, `SendOptions` is `send::Options`, `RecvStream` is -`recv::Stream`, the driver's `Lease` and moq-quic's `Leased` are -`recv::Lease`, and `StreamId` is `stream::Id`. `StreamEvent` keeps its name: -[waits](/quest/m1/quic-api/waits.md) deletes it. -`recv::Stream::set_priority` keeps its name, though it ranks eviction rather -than scheduling (maintainer: "priority" means "more important" both ways). -`send::Options` loses its generic group parameter; each layer defines its -own. - -**Errors.** `send::Error` covers streams and datagrams: `Stopped(code)`, -`TooLarge`, `Unsupported` (peer or local datagram support off), -`Connection(connection::Error)`. `recv::Error` covers -reads and leases: `Reset(code)`, `Evicted`, `FinishedEarly(n)`, `TooLong`, -`TooLarge` (a lease past the window), `Connection`. Neither has a 0-RTT variant. Both -`#[non_exhaustive]`, both convert to `io::Error`. moq-quic mirrors the names -and adds `Blocked`, plus `Closed` for a stale `stream::Id` (its handles are -borrowed views by id, so it cannot consume). This replaces `ReadError`, -`ReadExactError`, `ReadToEndError`, `ResetError`, `WriteError`, -`StoppedError`, `FinishError`, `ReadableError`, `LeaseError`, -`SendDatagramError` (two different types at two layers today), and -`ClosedStream`. `LeaseError::Blocked(ShouldTransmit)` stops carrying a -transmit signal inside an error. - -**Typestate.** In the drivers, `send::Stream::finish(self)` returns -`send::Finished`, which keeps what still applies to a finished stream: -`reset(self, ..)`, `reset_at(self, ..)`, `stopped()`, priority and group, -and acknowledgment. `reset` and `reset_at` take `self` on both types; -`recv::Stream::stop(self, code)`. Dropping an unfinished `send::Stream` -still resets it. The maintainer chose this over `finish(&mut self)` because -a reset after finish stays possible while a write after it cannot compile. -Setters take `&mut self` and return nothing in both drivers (today -moq-tokio's take `&self` and return `Result<(), ClosedStream>`). - -**Offsets.** `reset_at` and `acked` given an offset past the written data -return an error instead of panicking (maintainer). First explore an opaque -`send::Offset` minted only by `offset()`, so a past-the-end offset cannot be -built; if it is tagged to its stream, a foreign one is the only error left. - -**Buffers.** `Bytes` and `BytesMut` implement `Buf::copy_to_bytes` as a -zero-copy split, so one generic write covers owned and borrowed data. Send: -`write(&mut impl Buf) -> usize`, `write_all(impl Buf)`, `AsyncWrite`. Recv: -`read(&mut impl BufMut)`, `read_exact`, `read_chunk(max) -> Bytes`, -`read_to_end(limit)`, `lease`, `AsyncRead`. `write_chunk`, `write_chunks`, and -`write_all_chunks` go (`read_chunks` already went in -[moq-dev/moq#6809](https://github.com/moq-dev/moq/pull/6809)); moq-quic's `write` and -`write_chunks` become one `write(&mut impl Buf)`. Confirm on -`benches/streams.rs` that owned writes stay zero-copy. A buffer pool is out -of scope: [QUIC I/O boundary](/quest/m2/quic-io-boundary.md) owns it after a -profile. - -**Leases hold no lock** (maintainer, 2026-10-23). The driver's `Lease` wraps -the connection's `MutexGuard`, so while moq-net decodes a message from a -lease, the connection's packet processing, transmits, timers, and every other -stream wait on it. `recv::Lease` instead holds owned `Bytes` chunks, taken -under a short lock, and returns their credit under another short lock on -release or drop. A lease already copies its chunks out of the receive batch -as it fills, so owning them adds no copy. A lease stays `!Send` only if -something still needs it; otherwise it may cross an await. Test that the driver keeps sending and receiving on -other streams while a lease is held. - -**Datagrams.** `send_datagram(impl Buf, send::Options)`, so a caller -chains its own header (WebTransport's session id) instead of allocating and -copying each datagram. - -**Sans-IO state** (maintainer, 2026-10-23). moq-uring's driver and qmux each -keep a `sends: FxHashMap>` because `Finished` and -`Stopped` are fire-once events, and both hand-roll a `write_limit()` probe, -`copy_to_bytes`, and `debug_assert` around `write`. moq-quic keeps a -queryable send-stream end (finished and acknowledged, stopped with a code, -reset) until the id is released, and takes `write(&mut impl Buf)`; both -consumers' copies go. The per-stream waker tables move into moq-quic in -[waits](/quest/m1/quic-api/waits.md) (maintainer, 2026-10-25, replacing "stay -in the drivers"). - -**Consumers' error plumbing.** moq-tokio's `quic::web` error module rewraps -`WriteError`, `ReadError`, `ReadExactError`, `ReadToEndError`, and -`ClosedStream` one for one, with per-method `map_err`s, an `unreachable!` -for unordered reads, and a four-level match in `session.rs`; collapse it onto -`send::Error` and `recv::Error`. - -**Acked.** The driver's waiter method arrives with -[poll_acked](/quest/m2/quic-ack-hook.md). Decided 2026-10-27 (maintainer): -the drivers do not match `StreamEvent::Acked` explicitly here, since -[waits](/quest/m1/quic-api/waits.md) deletes `StreamEvent` right after, so it -would be throwaway work. - -**moq-uring.** Its driver gets the same names and signatures and passes -`send::Options` to the open, as moq-tokio's does, instead of applying them -through setters afterwards. Moved here from drop web-transport-trait. - -Update every consumer (`quic::web` in both crates, qmux, the backends, -moq-relay's tests). `poll_*` forms come in [poll](/quest/m1/quic-api/poll.md). The Impact bullet says how to migrate in moq-quic, moq-tokio, moq-uring, and -qmux if its public API changes. diff --git a/quest/m1/quic-api/waits.md b/quest/m1/quic-api/waits.md index 9557b7add4d..1c69af67965 100644 --- a/quest/m1/quic-api/waits.md +++ b/quest/m1/quic-api/waits.md @@ -38,7 +38,7 @@ kio?": which a write, open, reset, stop, close, or credit change wakes. Pacing and loss timers stay on `poll_timeout`. This replaces the kick each handle sends its driver after mutating the connection. -- **After [streams](/quest/m1/quic-api/streams.md)**, whose queryable +- **After [moq-dev/moq#6950](https://github.com/moq-dev/moq/pull/6950)**, whose queryable send-stream end is what `poll_finished` and `poll_acked` read, and whose names the new methods take. @@ -70,10 +70,6 @@ variants go. The drivers' and qmux's public APIs are unchanged here; the driver poll forms come in [poll](/quest/m1/quic-api/poll.md). Update moq-quic's README fork list and describe migration in the PR's Impact section. Wire: none. -## Required - -- [Streams](/quest/m1/quic-api/streams.md) - the send-stream end state and the `send::`/`recv::` names these waits use - ## Related - [Poll forms](/quest/m1/quic-api/poll.md) - the drivers' public poll forms forward to these diff --git a/quest/m2/perf/quic-lock-contention.md b/quest/m2/perf/quic-lock-contention.md index 7baba7a5d08..0b1a3aeaa3c 100644 --- a/quest/m2/perf/quic-lock-contention.md +++ b/quest/m2/perf/quic-lock-contention.md @@ -36,7 +36,7 @@ since the async-to-poll adapter and leases that hold the lock both go first. Levers, only where the profile shows a cost: - A per-stream staging queue: a write hands its owned `Bytes` (the `Buf` - API from [streams](/quest/m1/quic-api/streams.md)) to the stream, bounded + API from [moq-dev/moq#6950](https://github.com/moq-dev/moq/pull/6950)) to the stream, bounded by the stream's flow-control window so backpressure is unchanged, and the connection task moves it into moq-quic under its own lock. - Dispatch a whole GRO batch per connection in one message, coordinated @@ -57,7 +57,6 @@ staging queue and batch dispatch should need none. ## Required - [Native WebTransport](/quest/m1/quic-api/native-web.md) - removes the adapter's per-operation boxing, which would otherwise dominate the profile -- [Streams](/quest/m1/quic-api/streams.md) - owned writes and lock-free leases - [Waits](/quest/m1/quic-api/waits.md) - moves the waiter tables under the lock this profiles ## Related diff --git a/quest/m2/quic-ack-hook.md b/quest/m2/quic-ack-hook.md index 274cb268b92..6e4b81a6d64 100644 --- a/quest/m2/quic-ack-hook.md +++ b/quest/m2/quic-ack-hook.md @@ -50,22 +50,21 @@ impl send::Stream { ``` The same pair goes on `send::Finished`, the handle `finish(self)` returns -([quic-api streams](/quest/m1/quic-api/streams.md)); if that quest lands an -opaque `send::Offset`, the methods take it instead of a `u64`. The -`send::Error` says why the bytes will never be acknowledged: a reset, the -peer's STOP_SENDING, the connection's loss, or an offset past the data -written. Both drivers forward to `moq-quic`'s own `poll_acked` wait, which +([moq-dev/moq#6950](https://github.com/moq-dev/moq/pull/6950)), and the methods take its opaque `send::Offset`, which only +`offset()` mints, instead of a `u64`. The `send::Error` says why the bytes will +never be acknowledged: a reset, the peer's STOP_SENDING, or the connection's +loss. Both drivers forward to `moq-quic`'s own `poll_acked` wait, which [waits](/quest/m1/quic-api/waits.md) makes of today's `acked(offset)` and `StreamEvent::Acked`. Four traps, each settled in moq-quic where the wait lives: - A finished stream whose every byte was acknowledged must not look like a reset one. moq-quic answers from the queryable send-stream end - [streams](/quest/m1/quic-api/streams.md) keeps until the id is released: + it keeps until the id is released: finished, with no reset or STOP_SENDING, means delivered. -- An offset past the data written returns an error, never a panic, since a - panic under the connection lock poisons it for every handle (streams - makes moq-quic return it). +- An offset past the data written cannot be built, and a foreign one panics + before moq-tokio takes the connection lock, since a panic under the lock + poisons it for every handle. - `reset_at` below the acknowledged prefix, or STOP_SENDING after a reset (which truncates to 0), shrinks the acknowledged prefix. The waiters fail anyway; keep the prefix a `poll_acked` reports monotonic. diff --git a/rs/moq-quic/README.md b/rs/moq-quic/README.md index f4f728d57be..1863f8b0cbc 100644 --- a/rs/moq-quic/README.md +++ b/rs/moq-quic/README.md @@ -61,21 +61,22 @@ Changes on top of the upstream commit, besides the renames: - `transport::Config::handshake_timeout` (default 10s, as msquic's `HandshakeIdleTimeoutMs`) governs the idle timer until the handshake completes, and the negotiated idle timeout takes over once established. A 2s idle timeout otherwise expires 3x the initial PTO in, just before the second Initial probe, leaving a client two flights. A server also drops an Initial left waiting for `accept` past it, and a deadline too far out to represent never fires instead of panicking. Also [moq-dev/noq#34](https://github.com/moq-dev/noq/pull/34), whose third change (the handshake idle timeout picks noq's idle-based PTO interval cap) has nothing to apply to here, since quinn's PTO backoff has no such cap. quinn's `handshake_timeout` test in its async layer expects the handshake to end at the idle timeout, so the driver's copy sets both timeouts. - **Time on [moq-clock](../moq-clock)**: the crate's `Instant` is `moq_clock::Instant`, so one clock steps every timer, tokio's paused clock and the simulator included. qlog stamps events relative to `QlogConfig::start_time` on that clock rather than the streamer's own `std::time::Instant`. Address-validation tokens and their log read `moq_clock::wall()` instead of quinn's `TimeSource`, for portable 1970-2554 wall times; the simulator steps wall time with its clock, but paused tokio does not, so token expiry there follows the host's wall clock. -- **Send groups** (`streams/schedule.rs`, `Connection::send_groups`): streams outside every group go first, in quinn's order; groups are served strictly by priority, with equal priorities sharing bytes by deficit round robin. A stream's priority is an `i64` ordering it only within its group, and `Streams::open` takes a `SendOptions { group, priority }` to start it there, like W3C WebTransport's send options. This replaces quinn's single pending-stream heap, so a port touching `PendingStreamsQueue` has to be redone against the schedule. moq-tokio's `quic::driver` and moq-uring's driver wrap a group in a shared `SendGroup` handle that closes on drop. -- **Scheduled datagrams** (`Datagrams::send`): a datagram is queued by its `SendOptions`, in a send group (or ungrouped) at an `i64` priority, and scheduled with the group's streams, winning a tie with a stream of equal priority and spending the group's byte credit, instead of quinn's datagram-first FIFO in `populate_packet`. Sending never blocks: `datagram_send_buffer_size` is shared, and an overflow evicts the oldest datagram of the lowest-priority group. `SendDatagramError::Blocked` and `Event::DatagramsUnblocked` are gone, and a port touching the `// DATAGRAM` loop has to be redone against `StreamsState::write_frames`. +- **Send groups** (`streams/schedule.rs`, `Connection::send_groups`): streams outside every group go first, in quinn's order; groups are served strictly by priority, with equal priorities sharing bytes by deficit round robin. A stream's priority is an `i64` ordering it only within its group, and `Streams::open` takes a `send::Options { group, priority }` to start it there, like W3C WebTransport's send options. This replaces quinn's single pending-stream heap, so a port touching `PendingStreamsQueue` has to be redone against the schedule. moq-tokio's `quic::driver` and moq-uring's driver wrap a group in a shared `send::Group` handle that closes on drop. +- **Scheduled datagrams** (`Datagrams::send`): a datagram is queued by its `send::Options`, in a send group (or ungrouped) at an `i64` priority, and scheduled with the group's streams, winning a tie with a stream of equal priority and spending the group's byte credit, instead of quinn's datagram-first FIFO in `populate_packet`. Sending never blocks: `datagram_send_buffer_size` is shared, and an overflow evicts the oldest datagram of the lowest-priority group. `SendDatagramError::Blocked` and `Event::DatagramsUnblocked` are gone, and a port touching the `// DATAGRAM` loop has to be redone against `StreamsState::write_frames`. - **Sharded connection IDs** (`RandomConnectionIdGenerator::set_shard`): every CID names shard `index` of a `count`-endpoint group in its first byte, modulo `count`, and sets a marker in its fourth byte, so a `SO_REUSEPORT` steering filter delivers a packet to the endpoint that issued its CID. `validate` refuses a CID without this shard's prefix and marker, so a misrouted packet is dropped instead of answered with a stateless reset. Not offered upstream; it serves moq-sock's per-core sharding. -- **`RESET_STREAM_AT`** ([draft-ietf-quic-reliable-stream-reset](https://datatracker.ietf.org/doc/draft-ietf-quic-reliable-stream-reset/)): the `reset_stream_at` transport parameter is always advertised, `SendStream::reset_at` keeps the data written before an offset reliable, holding a plain `RESET_STREAM` until that data is acknowledged when the peer lacks the extension, and a received reset surfaces only once the reliable prefix has been read. Stopping a stream after it was reset no longer credits its unread data twice. -- **ACK progress** (`SendStream::offset`, `SendStream::acked`, `StreamEvent::Acked`): `acked` reports whether the peer acknowledged every byte before an offset, by comparing against the stream's contiguous acknowledged prefix. A `false` stores the lowest polled offset on the stream, and the ACK that advances the prefix past it, or a reset or `STOP_SENDING` that makes it fail, reports `StreamEvent::Acked` with the prefix, so a port touching `StreamsState::received_ack_of`, `received_stop_sending`, or the reset path has to keep that report. Not offered upstream: quinn has no API for when sent data was acknowledged. +- **`RESET_STREAM_AT`** ([draft-ietf-quic-reliable-stream-reset](https://datatracker.ietf.org/doc/draft-ietf-quic-reliable-stream-reset/)): the `reset_stream_at` transport parameter is always advertised, `send::Stream::reset_at` keeps the data written before an offset reliable, holding a plain `RESET_STREAM` until that data is acknowledged when the peer lacks the extension, and a received reset surfaces only once the reliable prefix has been read. Stopping a stream after it was reset no longer credits its unread data twice. +- **ACK progress** (`send::Stream::offset`, `send::Stream::acked`, `StreamEvent::Acked`): `acked` reports whether the peer acknowledged every byte before an offset, by comparing against the stream's contiguous acknowledged prefix. A `false` stores the lowest polled offset on the stream, and the ACK that advances the prefix past it, or a reset or `STOP_SENDING` that makes it fail, reports `StreamEvent::Acked` with the prefix, so a port touching `StreamsState::received_ack_of`, `received_stop_sending`, or the reset path has to keep that report. Not offered upstream: quinn has no API for when sent data was acknowledged. - **Fuzzing** runs through the hidden `fuzz` module behind a `fuzz` feature, with one-line shims in [`fuzz/`](fuzz/README.md): upstream's `packet`, `params`, `streamid`, and `streams` targets, plus `frame` and a connection op sequence. Upstream's `cfg(fuzzing)` `fuzzing` module and `arbitrary` feature are gone, so its fuzz patches port by hand. - **Draining endpoint event**: `EndpointEvent::is_draining` fires once per connection when it stops being active, before the drained event, so the driver's `Endpoint::wait_all_draining` can end a shutdown without sitting out every closing period. - **Late stream limits**: peer transport parameters that raise a stream limit after an open was refused emit `StreamEvent::Available`, as `MAX_STREAMS` does. A server accepted before the rest of a multi-packet ClientHello (a post-quantum key share is enough) otherwise leaves a 0.5-RTT `open_uni` parked until the idle timeout. Not offered upstream yet. -- **Receive leases** (`streams/lease.rs`, `RecvStream::lease`, `Connection::leased`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s, while stream credit advances as usual. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `LeaseError::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. +- **Receive leases** (`streams/lease.rs`, `recv::Stream::lease`), building on [#5675](https://github.com/moq-dev/moq/pull/5675)'s held credit: a lease reads a stream's ordered bytes out of the assembler but keeps their connection credit, counted in `StreamsState::held`, until the reader `advance`s or `take`s past them and `release`s, while stream credit advances as usual. A lease copies the bytes it reads out of their datagram buffers (`Leased::extend`), so a byte of credit never pins a receive batch, whether its reader waits, parks, or takes it. The lease lives in the stream's `Recv`, which a leased stream keeps, with its stream ID, until the application stops it, even past the FIN. When the peer has spent the connection window and no reader can make progress, the stream ranked lowest by `set_priority` (the newest on a tie) is evicted: its bytes and credit are freed, the stream is stopped without `STOP_SENDING`, and its next lease fails with `recv::Error::Evicted`; the application's `stop` then tells the peer and frees the stream once its final size is known. Leased streams that may hold credit are kept in `StreamsState::ranks`, in eviction order, so an eviction costs a logarithm rather than a scan of every stream. `MAX_DATA` goes out for any increase once the peer is within an eighth of the window (`max_data_due`), since what leases release may never add up to the usual threshold. Streams with data their reader has not looked at hold eviction off; they are kept in `StreamsState::unlooked`, so the stuck check stays constant time however many readers wait. `ConnectionStats::credit` reports the leased and available bytes, the streams evicted, and the peak credit spent. A `recv::Lease` owns the chunks it took, so a reader holds no connection borrow while it parses; `restore` or `release` hands it back, and an eviction pass waits while one is out. A port touching `Chunks` or `add_read_credits` has to keep `held` balanced. - **Owned datagrams**: a received datagram is copied out of its packet when queued, since `datagram_receive_buffer_size` charges only its payload and a slice keeps the whole receive batch alive. quinn has the same gap; not offered upstream yet. -- **Stream credit on stop**: `RecvStream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. -- **Drop resets** (`quic::driver::SendStream`): a send stream dropped before `finish` is reset with code 0, or with the peer's `STOP_SENDING` code, instead of quinn's implicit `finish`, so a cancelled writer never ends a truncated stream with a FIN. This matches moq-uring's driver and moq-net's transport contract. `quic::web` resets after the WebTransport header with the mapped code. quinn's `stream_stopped_2` finishes its stream before dropping it. Not offered upstream: it reverses a documented quinn behaviour. +- **Stream credit on stop**: `recv::Stream::stop` on a stream whose final size is already known (finished or reset) frees it and queues `MAX_STREAMS` at once, as reading the reset does, instead of leaving the credit until the next packet arrives. Not offered upstream yet; quinn `main` has the same gap. +- **Drop resets** (`quic::driver::send::Stream`): a send stream dropped before `finish` is reset with code 0, or with the peer's `STOP_SENDING` code, instead of quinn's implicit `finish`, so a cancelled writer never ends a truncated stream with a FIN. This matches moq-uring's driver and moq-net's transport contract. `quic::web` resets after the WebTransport header with the mapped code. quinn's `stream_stopped_2` finishes its stream before dropping it. Not offered upstream: it reverses a documented quinn behaviour. - **Reset secret** (`endpoint::Config::reset_secret`): derives the stateless reset key from an operator's secret with HKDF-SHA256, and keys a `HashedConnectionIdGenerator` from it too, since a restarted endpoint drops packets for connection IDs it can't validate before it would reset them. It also lowers `min_reset_interval` to 1ms, so a restart resets thousands of connections in seconds. Both drivers take one, so a restarted server resets its predecessor's connections. Not offered upstream: it only combines quinn's `reset_key` and `HashedConnectionIdGenerator::from_key`. - **Connection-blocked writers**: a deduplicated set removes a writer on finish, reset, `STOP_SENDING`, or free, so a peer withholding `MAX_DATA` cannot retain cancelled-stream history. Terminal writes report their existing error before connection credit is checked. This lives in the shared stream engine used by QUIC and QMux. Not offered upstream yet. -- **Stopped after a local reset**: moq-tokio's driver resolves `SendStream::stopped` with `None` once the stream is reset locally, as it already did once the reset was acknowledged and the stream freed. No stream event reports a local reset, so quinn leaves a pending `stopped` waiting, and its waker registered, until the connection closes. Not offered upstream yet; quinn 0.11 has the same gap. +- **Stopped after a local reset**: moq-tokio's driver resolves `send::Finished::stopped` with `None` once the stream is reset locally, as it already did once the reset was acknowledged and the stream freed. No stream event reports a local reset, so quinn leaves a pending `stopped` waiting, and its waker registered, until the connection closes. Not offered upstream yet; quinn 0.11 has the same gap. +- **Namespaced streams** (`send`, `recv`, `stream`): quinn's `SendStream`, `RecvStream`, and their error zoo become `send::Stream` and `recv::Stream` with one `send::Error` (streams and datagrams) and one `recv::Error` (reads and leases), adding `Blocked`, and `Closed` for an ID the stream no longer has state for. `send::Stream::write` takes `&mut impl Buf`, splitting a `Bytes` source off without a copy, in place of `write` and `write_chunks`, and `Datagrams::send` takes `impl Buf` the same way, so a chained header costs no copy. An outgoing stream's `send::End` (finished and acknowledged, stopped, or reset) stays queryable until `send::Stream::release`, so a driver needs no table of its own for the fire-once `Finished` and `Stopped` events. A `send::Offset` comes only from `offset()` and is tagged to its stream, so `reset_at` and `acked` cannot be handed one past the data written. A port touching a public stream method has to be redone against these names. - **Connection limits** (`Limits`, `Connection::set_limits`, and the drivers' `set_limits`): one validated call changes a live connection's stream limit and windows, refusing values a peer would reject and changing nothing then. moq-tokio and moq-uring re-export the type, so both runtimes check limits the same way. Not offered upstream: it only bundles quinn's existing setters. - **Indexed send buffer** (`SendBuffer::get`): each retained write keeps its stream offset, and a STREAM frame's data comes from one binary search, instead of quinn walking every write from the oldest unacknowledged one for each piece, which costs a triangular number of visits when many small writes are in flight. Not offered upstream yet. - **Bandwidth probing** (`Connection::set_probe_target`, `connection/bandwidth_probe.rs`): an application-limited sender below its target fills the headroom BBR3 opens, in Startup and in one ProbeBW\_UP per probe wait, with duplicates of STREAM data still in flight, paced to the target, so the controller's samples measure the path rather than the application. `Controller::probes` and `ControllerMetrics::probe_inflight` tell the sender when and how far it may fill; CUBIC never does. A duplicate is never retransmitted, a reset or stopped stream's data is never resent, and a loss retransmits only the ranges no copy delivered (`SendBuffer::retransmit`). An attempt that raises nothing within two round trips, or ends short of the target, sits out 3 s and then 4 s before retrying the same target; reaching the target or a new one retries at once. `PathStats::sent_bandwidth_probe_bytes` counts the redundant bytes. A cherry-pick touching `poll_transmit`'s space loop or `populate_packet` has to keep the probe after the ordinary frames. Not offered upstream. diff --git a/rs/moq-quic/benches/common/mod.rs b/rs/moq-quic/benches/common/mod.rs index 143c1ab10b7..87868c1329b 100644 --- a/rs/moq-quic/benches/common/mod.rs +++ b/rs/moq-quic/benches/common/mod.rs @@ -16,7 +16,7 @@ use std::{ time::Duration, }; -use bytes::{BufMut, Bytes, BytesMut}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; use cpu_time::ThreadTime; use criterion::{ Criterion, @@ -25,12 +25,13 @@ use criterion::{ use moq_clock::Instant; use moq_congestion::{Ack, Ce, Controller, ControllerFactory, ControllerMetrics, Init, Loss, Packet, Sent}; use moq_quic::{ - ConnectionId, ConnectionIdGenerator, Event, LeaseError, RandomConnectionIdGenerator, ReadError, StreamEvent, - StreamId, VarInt, client, + ConnectionId, ConnectionIdGenerator, Event, RandomConnectionIdGenerator, StreamEvent, VarInt, client, connection::Connection, endpoint::{self, ConnectionHandle, DatagramEvent, Dial, Endpoint, Received as Inbound}, fuzz::Plaintext, - server, transport, + recv, server, + stream::Id as StreamId, + transport, }; /// The most datagrams one `poll_transmit` may batch, as a GSO-capable socket would take. @@ -249,8 +250,8 @@ fn drain(conn: &mut Connection, id: StreamId, sink: Sink, received: &mut Receive received.finished += 1; break; } - Err(ReadError::Blocked) => break, - Err(ReadError::Reset(_)) => { + Err(recv::Error::Blocked) => break, + Err(recv::Error::Reset(_)) => { received.reset += 1; break; } @@ -270,29 +271,27 @@ fn drain(conn: &mut Connection, id: StreamId, sink: Sink, received: &mut Receive /// discards its lease. fn lease(conn: &mut Connection, id: StreamId, want: usize, received: &mut Received) { loop { - let ended = match conn.recv_stream(id).lease(want) { - Ok(_) => None, - Err(LeaseError::Blocked(_) | LeaseError::ClosedStream) => return, - Err(LeaseError::Reset(_)) => Some(&mut received.reset), - Err(LeaseError::Evicted) => Some(&mut received.evicted), - Err(err) => panic!("lease failed: {err}"), - }; - if let Some(count) = ended { - *count += 1; - let _ = conn.recv_stream(id).stop(VarInt::from_u32(0)); - return; - } - let fin = conn.leased(id).expect("leased above").is_fin(); let mut stream = conn.recv_stream(id); - // Taken a chunk at a time, as moq-net hands a frame's payload on. - loop { - let chunk = stream.take(usize::MAX); - if chunk.is_empty() { - break; + let mut lease = match stream.lease(want).0 { + Ok(lease) => lease, + Err(recv::Error::Blocked | recv::Error::Closed) => return, + Err(err) => { + match err { + recv::Error::Reset(_) => received.reset += 1, + recv::Error::Evicted => received.evicted += 1, + err => panic!("lease failed: {err}"), + } + let _ = stream.stop(VarInt::from_u32(0)); + return; } + }; + let fin = lease.is_fin(); + // Taken a chunk at a time, as moq-net hands a frame's payload on. + while lease.has_remaining() { + let chunk = lease.copy_to_bytes(lease.chunk().len()); received.bytes += chunk.len() as u64; } - let _ = stream.release(); + let _ = stream.release(lease); if fin { received.finished += 1; let _ = stream.stop(VarInt::from_u32(0)); diff --git a/rs/moq-quic/benches/receive.rs b/rs/moq-quic/benches/receive.rs index c43269a764f..8785e651bca 100644 --- a/rs/moq-quic/benches/receive.rs +++ b/rs/moq-quic/benches/receive.rs @@ -3,7 +3,7 @@ use std::time::Duration; use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_quic::{Dir, ReadError, VarInt}; +use moq_quic::{VarInt, recv, stream::Dir}; #[allow( dead_code, @@ -44,7 +44,7 @@ fn bench_loss(c: &mut Criterion) { for _ in 0..streams { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); let mut stream = conn.send_stream(id); - assert_eq!(stream.write(&data).unwrap(), data.len()); + assert_eq!(stream.write(&mut data.clone()).unwrap(), data.len()); stream.finish().unwrap(); } pair.drive_until(|pair| pair.server.received.finished >= target); @@ -79,7 +79,7 @@ fn bench_reset(c: &mut Criterion) { let ids: Vec<_> = (0..streams) .map(|_| { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), bytes); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), bytes); id }) .collect(); @@ -127,7 +127,7 @@ fn bench_lease(c: &mut Criterion) { for _ in 0..streams { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); let mut stream = conn.send_stream(id); - assert_eq!(stream.write(&data).unwrap(), data.len()); + assert_eq!(stream.write(&mut data.clone()).unwrap(), data.len()); stream.finish().unwrap(); } pair.drive_until(|pair| pair.server.received.finished >= target); @@ -175,7 +175,7 @@ fn bench_evict(c: &mut Criterion) { let ids: Vec<_> = (0..streams) .map(|_| { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), PACKET); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), PACKET); id }) .collect(); @@ -218,7 +218,7 @@ fn bench_evict_churn(c: &mut Criterion) { let conn = pair.client(); for _ in 0..streams { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), PACKET); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), PACKET); } pair.drive_until(|pair| pair.server.received.evicted >= 1); pair.settle(); @@ -232,7 +232,7 @@ fn bench_evict_churn(c: &mut Criterion) { let target = pair.server.received.evicted + 1; let conn = pair.client(); let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), PACKET); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), PACKET); pair.drive_until(|pair| pair.server.received.evicted >= target); busy += pair.server.busy; // The evicted stream's stop and reset settle untimed. @@ -275,7 +275,7 @@ fn bench_reset_debt(c: &mut Criterion) { let conn = pair.client(); for _ in 0..holders { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), PACKET); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), PACKET); } pair.settle(); pair.server.sink = Sink::Ignore; @@ -293,7 +293,7 @@ fn bench_reset_debt(c: &mut Criterion) { let ids: Vec<_> = (0..resets) .map(|_| { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), PACKET); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), PACKET); id }) .collect(); @@ -312,7 +312,7 @@ fn bench_reset_debt(c: &mut Criterion) { while let Some(id) = conn.streams().accept(Dir::Uni) { let mut stream = conn.recv_stream(id); let mut chunks = stream.read().unwrap(); - assert!(matches!(chunks.next(usize::MAX), Err(ReadError::Reset(_)))); + assert!(matches!(chunks.next(usize::MAX), Err(recv::Error::Reset(_)))); let _ = chunks.finalize(); } let mut limits = moq_quic::Limits::default(); @@ -351,7 +351,7 @@ fn bench_stop(c: &mut Criterion) { let conn = pair.client(); for _ in 0..streams { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(&data).unwrap(), bytes); + assert_eq!(conn.send_stream(id).write(&mut data.clone()).unwrap(), bytes); } pair.drive_until(|pair| { pair.server.received.stopped >= target && pair.client().streams().send_streams() == 0 diff --git a/rs/moq-quic/benches/streams.rs b/rs/moq-quic/benches/streams.rs index 49d0ce72682..d5fef4f2f68 100644 --- a/rs/moq-quic/benches/streams.rs +++ b/rs/moq-quic/benches/streams.rs @@ -6,8 +6,9 @@ //! connection setup and dropping the remaining state outside the CPU measurement. The //! reprioritize case moves streams that stay queued, never sending them. +use bytes::Bytes; use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; -use moq_quic::Dir; +use moq_quic::stream::Dir; #[allow( dead_code, @@ -82,7 +83,7 @@ fn send(pair: &mut Pair, streams: usize, priorities: usize, writes: usize, size: // `as _` follows the priority type, which a scheduler rewrite widens. stream.set_priority((index % priorities) as _).unwrap(); for _ in 0..writes { - assert_eq!(stream.write(&data).unwrap(), size, "write was cut short"); + assert_eq!(stream.write(&mut data.clone()).unwrap(), size, "write was cut short"); } stream.finish().unwrap(); } @@ -122,7 +123,7 @@ fn bench_reprioritize(c: &mut Criterion) { let ids: Vec<_> = (0..queued) .map(|_| { let id = conn.streams().open(Dir::Uni, Default::default()).expect("stream limit"); - assert_eq!(conn.send_stream(id).write(b"x").unwrap(), 1); + assert_eq!(conn.send_stream(id).write(&mut &b"x"[..]).unwrap(), 1); id }) .collect(); @@ -154,12 +155,15 @@ fn send_windowed(pair: &mut Pair, total: usize, size: usize) { .expect("stream limit"); let mut written = 0; while written < total { - if pair.client().send_stream(id).write_limit() == 0 { - pair.drive_until(|pair| pair.client().send_stream(id).write_limit() > 0); - } // A write the window cuts short resumes mid-chunk, so every chunk boundary stays put. let start = written % size; - written += pair.client().send_stream(id).write(&data[start..]).unwrap(); + match pair.client().send_stream(id).write(&mut data.slice(start..)) { + Ok(n) => written += n, + Err(moq_quic::send::Error::Blocked) => { + pair.drive_until(|pair| pair.client().send_stream(id).write(&mut Bytes::new()).is_ok()) + } + Err(err) => panic!("{err}"), + } } pair.client().send_stream(id).finish().unwrap(); pair.drive_until(|pair| pair.server.received.finished >= target); @@ -232,12 +236,12 @@ fn bench_connection_blocked(c: &mut Criterion) { }); // A window can't be zero, so one stream spends its only byte. let id = pair.client().streams().open(Dir::Uni, Default::default()).unwrap(); - assert_eq!(pair.client().send_stream(id).write(b"x"), Ok(1)); + assert_eq!(pair.client().send_stream(id).write(&mut &b"x"[..]), Ok(1)); for _ in 0..live { let id = pair.client().streams().open(Dir::Uni, Default::default()).unwrap(); assert_eq!( - pair.client().send_stream(id).write(b"x"), - Err(moq_quic::WriteError::Blocked) + pair.client().send_stream(id).write(&mut &b"x"[..]), + Err(moq_quic::send::Error::Blocked) ); } pair @@ -246,7 +250,7 @@ fn bench_connection_blocked(c: &mut Criterion) { for _ in 0..churn { let id = pair.client().streams().open(Dir::Uni, Default::default()).unwrap(); let mut stream = pair.client().send_stream(id); - assert_eq!(stream.write(b"x"), Err(moq_quic::WriteError::Blocked)); + assert_eq!(stream.write(&mut &b"x"[..]), Err(moq_quic::send::Error::Blocked)); stream.reset(0u32.into()).unwrap(); } // The peer never grants credit. Packet exchange and dropping the live diff --git a/rs/moq-quic/src/connection/datagrams.rs b/rs/moq-quic/src/connection/datagrams.rs index 2538f65239f..b0d1d24bafb 100644 --- a/rs/moq-quic/src/connection/datagrams.rs +++ b/rs/moq-quic/src/connection/datagrams.rs @@ -1,13 +1,12 @@ use std::collections::VecDeque; -use bytes::Bytes; -use thiserror::Error; +use bytes::{Buf, Bytes}; use tracing::{debug, trace}; -use super::{Connection, SendOptions}; +use super::{Connection, send}; use crate::{ TransportError, - frame::{Datagram, FrameStruct}, + frame::{Datagram, FrameStruct, OutgoingDatagram}, }; /// API to control datagram traffic @@ -28,20 +27,26 @@ impl Datagrams<'_> { /// the oldest datagram of the lowest-priority group is dropped, and this datagram is dropped /// instead once it is the one to go. /// - /// Returns `Err` iff `data` cannot currently be sent. + /// Each chunk of `data` is taken with [`Buf::copy_to_bytes`], so a header chained in front + /// of a [`Bytes`] payload costs no copy of the payload. + /// + /// Fails with [`send::Error::Unsupported`] when datagrams are disabled locally or by the peer, + /// or [`send::Error::TooLarge`] when `data` exceeds [`max_size`](Self::max_size). /// /// # Panics /// - when `options.group` was opened on another connection - pub fn send(&mut self, data: Bytes, options: SendOptions<'_>) -> Result<(), SendDatagramError> { + pub fn send(&mut self, data: impl Buf, options: send::Options<'_>) -> Result<(), send::Error> { if self.conn.config.datagram.receive_buffer.is_none() { - return Err(SendDatagramError::Disabled); + return Err(send::Error::Unsupported); } - let max = self.max_size().ok_or(SendDatagramError::UnsupportedByPeer)?; + let max = self.max_size().ok_or(send::Error::Unsupported)?; let budget = self.conn.config.datagram.send_buffer; - let datagram = Datagram { data }; - if datagram.data.len() > max || datagram.size(true) > budget { - return Err(SendDatagramError::TooLarge); + // Checked before taking the payload, which copies whatever is not already `Bytes`. + let len = data.remaining(); + if len > max || OutgoingDatagram::frame_size(len) > budget { + return Err(send::Error::TooLarge); } + let datagram = OutgoingDatagram::new(data); if !self.conn.streams.queue_datagram(datagram, options, budget) { trace!("dropping outgoing datagram"); } @@ -206,21 +211,3 @@ mod tests { assert_eq!(state.recv().unwrap(), &[7][..]); } } - -/// Errors that can arise when sending a datagram -#[derive(Debug, Error, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[non_exhaustive] -pub enum SendDatagramError { - /// The peer does not support receiving datagrams - #[error("datagrams not supported by peer")] - UnsupportedByPeer, - /// Datagram support is disabled locally - #[error("datagram support disabled")] - Disabled, - /// The datagram is larger than the connection can currently accommodate - /// - /// Indicates that the path MTU minus overhead or the limit advertised by the peer has been - /// exceeded. - #[error("datagram too large")] - TooLarge, -} diff --git a/rs/moq-quic/src/connection/mod.rs b/rs/moq-quic/src/connection/mod.rs index 9973889acfd..53f8616e277 100644 --- a/rs/moq-quic/src/connection/mod.rs +++ b/rs/moq-quic/src/connection/mod.rs @@ -43,7 +43,6 @@ mod ack_frequency; use ack_frequency::AckFrequencyState; mod assembler; -pub use assembler::Chunk; mod bandwidth_probe; use bandwidth_probe::BandwidthProbe; @@ -53,7 +52,7 @@ use cid_state::CidState; mod datagrams; use datagrams::DatagramState; -pub use datagrams::{Datagrams, SendDatagramError}; +pub use datagrams::Datagrams; mod mtud; #[cfg(feature = "mux")] @@ -101,10 +100,7 @@ pub use stats::{CreditStats, FrameStats, PathStats, Stats, UdpStats}; mod streams; pub(crate) use streams::StreamsState; -pub use streams::{ - Chunks, ClosedStream, FinishError, LeaseError, Leased, ReadError, ReadableError, RecvStream, SendGroup, SendGroups, - SendOptions, SendStream, ShouldTransmit, StreamEvent, Streams, WriteError, Written, -}; +pub use streams::{ShouldTransmit, StreamEvent, Streams, recv, send}; mod timer; use crate::congestion::{self, Controller}; @@ -123,8 +119,8 @@ use timer::{Timer, TimerTable}; /// - B. Handlers for incoming events from the network or system, named `handle_*`. /// - C. State machine mutators, for incoming commands from the application. For convenience we /// refer to this as "performing I/O" below, however as per the design of this library none of the -/// functions actually perform system-level I/O. For example, [`read`](RecvStream::read) and -/// [`write`](SendStream::write), but also things like [`reset`](SendStream::reset). +/// functions actually perform system-level I/O. For example, [`read`](recv::Stream::read) and +/// [`write`](send::Stream::write), but also things like [`reset`](send::Stream::reset). /// - D. Polling functions for outgoing events or actions for the caller to /// take, named `poll_*`. /// @@ -458,17 +454,17 @@ impl Connection { /// Provide control over the send groups that schedule outgoing streams #[must_use] - pub fn send_groups(&mut self) -> SendGroups<'_> { - SendGroups { + pub fn send_groups(&mut self) -> send::Groups<'_> { + send::Groups { state: &mut self.streams, } } /// Provide control over streams #[must_use] - pub fn recv_stream(&mut self, id: StreamId) -> RecvStream<'_> { + pub fn recv_stream(&mut self, id: StreamId) -> recv::Stream<'_> { assert!(id.dir() == Dir::Bi || id.initiator() != self.side.side()); - RecvStream { + recv::Stream { id, state: &mut self.streams, pending: &mut self.spaces[SpaceId::Data].pending, @@ -477,9 +473,9 @@ impl Connection { /// Provide control over streams #[must_use] - pub fn send_stream(&mut self, id: StreamId) -> SendStream<'_> { + pub fn send_stream(&mut self, id: StreamId) -> send::Stream<'_> { assert!(id.dir() == Dir::Bi || id.initiator() == self.side.side()); - SendStream { + send::Stream { id, state: &mut self.streams, pending: &mut self.spaces[SpaceId::Data].pending, @@ -1339,7 +1335,7 @@ impl Connection { /// /// This does not ensure delivery of outstanding data. It is the application's responsibility to /// call this only when all important communications have been completed, e.g. by calling - /// [`SendStream::finish`] on outstanding streams and waiting for the corresponding + /// [`send::Stream::finish`] on outstanding streams and waiting for the corresponding /// [`StreamEvent::Finished`] event. /// /// If [`Streams::send_streams`] returns 0, all outstanding stream data has been @@ -1406,11 +1402,6 @@ impl Connection { stats } - /// The bytes leased from receive stream `id` with [`RecvStream::lease()`], if it has a lease - pub fn leased(&self, id: StreamId) -> Option<&Leased> { - self.streams.leased(id) - } - /// Assert that the receive lease books balance, for the fuzz target. #[cfg(any(test, feature = "fuzz"))] pub(crate) fn check_leases(&self) { diff --git a/rs/moq-quic/src/connection/mux/mod.rs b/rs/moq-quic/src/connection/mux/mod.rs index 865f9cfc2e5..71b21364514 100644 --- a/rs/moq-quic/src/connection/mux/mod.rs +++ b/rs/moq-quic/src/connection/mux/mod.rs @@ -14,7 +14,7 @@ use std::{collections::VecDeque, num::NonZeroU64}; -use bytes::Bytes; +use bytes::{Buf, Bytes}; use tracing::{debug, trace}; use super::{ @@ -22,12 +22,11 @@ use super::{ datagrams::DatagramState, spaces::Retransmits, stats::FrameStats, - streams::{Leased, RecvStream, SendGroups, SendOptions, SendStream, StreamEvent, Streams, StreamsState}, + streams::{StreamEvent, Streams, StreamsState, recv, send}, }; use crate::{ - ApplicationClose, Dir, Duration, Instant, Limits, MAX_STREAM_COUNT, SendDatagramError, Side, TransportError, - VarInt, - frame::{self, Close, Datagram}, + ApplicationClose, Dir, Duration, Instant, Limits, MAX_STREAM_COUNT, Side, TransportError, VarInt, + frame::{self, Close, Datagram, OutgoingDatagram}, transport_parameters::TransportParameters, }; @@ -373,15 +372,15 @@ impl Connection { } /// Schedule outgoing streams in send groups - pub fn send_groups(&mut self) -> SendGroups<'_> { - SendGroups { + pub fn send_groups(&mut self) -> send::Groups<'_> { + send::Groups { state: &mut self.streams, } } /// The send half of stream `id` - pub fn send_stream(&mut self, id: crate::StreamId) -> SendStream<'_> { - SendStream { + pub fn send_stream(&mut self, id: crate::StreamId) -> send::Stream<'_> { + send::Stream { id, state: &mut self.streams, pending: &mut self.pending, @@ -390,8 +389,8 @@ impl Connection { } /// The receive half of stream `id` - pub fn recv_stream(&mut self, id: crate::StreamId) -> RecvStream<'_> { - RecvStream { + pub fn recv_stream(&mut self, id: crate::StreamId) -> recv::Stream<'_> { + recv::Stream { id, state: &mut self.streams, pending: &mut self.pending, @@ -403,11 +402,6 @@ impl Connection { Datagrams { conn: self } } - /// The bytes leased from receive stream `id` with [`RecvStream::lease()`], if it has a lease - pub fn leased(&self, id: crate::StreamId) -> Option<&Leased> { - self.streams.leased(id) - } - /// The frame space of a record: the peer's `max_record_size`, under the STREAM ceiling fn budget(&self) -> usize { let peer = self @@ -826,16 +820,18 @@ impl Datagrams<'_> { /// /// # Panics /// - when `options.group` was opened on another connection - pub fn send(&mut self, data: Bytes, options: SendOptions<'_>) -> Result<(), SendDatagramError> { + pub fn send(&mut self, data: impl Buf, options: send::Options<'_>) -> Result<(), send::Error> { if self.conn.config.max_datagram_frame_size.is_none() { - return Err(SendDatagramError::Disabled); + return Err(send::Error::Unsupported); } - let max = self.max_size().ok_or(SendDatagramError::UnsupportedByPeer)?; + let max = self.max_size().ok_or(send::Error::Unsupported)?; let budget = self.conn.config.datagram_send_buffer_size; - let datagram = Datagram { data }; - if datagram.data.len() > max || datagram.size(true) > budget { - return Err(SendDatagramError::TooLarge); + // Checked before taking the payload, which copies whatever is not already `Bytes`. + let len = data.remaining(); + if len > max || OutgoingDatagram::frame_size(len) > budget { + return Err(send::Error::TooLarge); } + let datagram = OutgoingDatagram::new(data); if !self.conn.streams.queue_datagram(datagram, options, budget) { trace!("dropping outgoing datagram"); } diff --git a/rs/moq-quic/src/connection/mux/tests.rs b/rs/moq-quic/src/connection/mux/tests.rs index ec5af48e680..f81b702f7f4 100644 --- a/rs/moq-quic/src/connection/mux/tests.rs +++ b/rs/moq-quic/src/connection/mux/tests.rs @@ -1,7 +1,7 @@ use bytes::{BufMut, Bytes}; use super::*; -use crate::{ReadError, StreamId, TransportErrorCode, coding::BufMutExt, frame::FrameType}; +use crate::{StreamId, TransportErrorCode, coding::BufMutExt, frame::FrameType, recv, send}; /// Two connections joined by an in-memory transport, on a clock the test advances struct Pair { @@ -75,7 +75,7 @@ fn read_all(conn: &mut Connection, id: StreamId) -> (Vec, bool) { fin = true; break; } - Err(ReadError::Blocked) => break, + Err(recv::Error::Blocked) => break, Err(err) => panic!("read failed: {err}"), } } @@ -105,11 +105,11 @@ fn handshake() { fn streams_carry_data_and_fin() { for &version in Version::ALL { let mut pair = Pair::new(config(version)); - let id = pair.client.streams().open(Dir::Bi, SendOptions::default()).unwrap(); + let id = pair.client.streams().open(Dir::Bi, send::Options::default()).unwrap(); let payload = vec![7u8; 100_000]; let mut sent = 0; while sent < payload.len() { - sent += pair.client.send_stream(id).write(&payload[sent..]).unwrap(); + sent += pair.client.send_stream(id).write(&mut &payload[sent..]).unwrap(); pair.drive(); } pair.client.send_stream(id).finish().unwrap(); @@ -147,9 +147,9 @@ fn credit_returns_for_unread_and_stopped_streams() { // Far more than the connection window, a stream at a time for round in 0..64 { - let id = pair.client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); + let id = pair.client.streams().open(Dir::Uni, send::Options::default()).unwrap(); assert_eq!( - pair.client.send_stream(id).write(&chunk).unwrap(), + pair.client.send_stream(id).write(&mut &chunk[..]).unwrap(), chunk.len(), "round {round}" ); @@ -160,7 +160,7 @@ fn credit_returns_for_unread_and_stopped_streams() { // Stopped before the rest arrives, which then arrives anyway 0 => { pair.server.recv_stream(accepted).stop(VarInt::from_u32(1)).unwrap(); - let _ = pair.client.send_stream(id).write(&chunk); + let _ = pair.client.send_stream(id).write(&mut &chunk[..]); } // Read to the end _ => { @@ -339,8 +339,8 @@ fn records_stay_under_the_stream_ceiling() { let mut cfg = config(Version::QMux02); cfg.max_record_size = VarInt::from_u32(1 << 20); let mut pair = Pair::new(cfg); - let id = pair.client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); - pair.client.send_stream(id).write(&[0; 100_000]).unwrap(); + let id = pair.client.streams().open(Dir::Uni, send::Options::default()).unwrap(); + pair.client.send_stream(id).write(&mut &[0; 100_000][..]).unwrap(); while let Some(record) = pair.client.poll_transmit(pair.now) { assert!(record.len() <= MAX_RECORD_BUDGET); pair.server.handle_record(record, pair.now); @@ -355,8 +355,8 @@ fn draft00_sends_one_length_prefixed_frame_per_message() { while let Some(frame) = server.poll_transmit(now) { client.handle_record(frame, now); } - let id = client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); - client.send_stream(id).write(&[3; 40_000]).unwrap(); + let id = client.streams().open(Dir::Uni, send::Options::default()).unwrap(); + client.send_stream(id).write(&mut &[3; 40_000][..]).unwrap(); client.send_stream(id).finish().unwrap(); let mut messages = 0; while let Some(message) = client.poll_transmit(now) { @@ -426,10 +426,12 @@ fn stream_offsets_follow_the_transport_order() { #[test] fn reset_stream_at_delivers_the_prefix() { let mut pair = Pair::new(config(Version::QMux02)); - let id = pair.client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); - pair.client.send_stream(id).write(b"header").unwrap(); - pair.client.send_stream(id).write(b"payload").unwrap(); - pair.client.send_stream(id).reset_at(6, VarInt::from_u32(9)).unwrap(); + let id = pair.client.streams().open(Dir::Uni, send::Options::default()).unwrap(); + pair.client.send_stream(id).write(&mut &b"header"[..]).unwrap(); + pair.client.send_stream(id).write(&mut &b"payload"[..]).unwrap(); + let mut stream = pair.client.send_stream(id); + let at = stream.offset().rewind(6); + stream.reset_at(at, VarInt::from_u32(9)).unwrap(); pair.drive(); let accepted = pair.server.streams().accept(Dir::Uni).unwrap(); @@ -445,7 +447,7 @@ fn reset_stream_at_delivers_the_prefix() { }; let _ = chunks.finalize(); assert!(data.starts_with(b"header"), "{data:?} {err:?}"); - assert_eq!(err, ReadError::Reset(VarInt::from_u32(9))); + assert_eq!(err, recv::Error::Reset(VarInt::from_u32(9))); } #[test] @@ -472,7 +474,7 @@ fn datagrams() { assert!(max > 16_000); pair.client .datagrams() - .send(Bytes::from_static(b"hello"), SendOptions::default()) + .send(Bytes::from_static(b"hello"), send::Options::default()) .unwrap(); pair.drive(); assert!( @@ -487,8 +489,8 @@ fn datagrams() { assert_eq!( pair.client .datagrams() - .send(Bytes::from_static(b"hello"), SendOptions::default()), - Err(SendDatagramError::Disabled) + .send(Bytes::from_static(b"hello"), send::Options::default()), + Err(send::Error::Unsupported) ); } @@ -516,7 +518,7 @@ fn datagram_limits_count_the_encoded_frame() { assert_eq!(pair.client.datagrams().max_size(), Some(1197)); pair.client .datagrams() - .send(vec![7; 1197].into(), SendOptions::default()) + .send(Bytes::from(vec![7; 1197]), send::Options::default()) .unwrap(); pair.drive(); assert_eq!(pair.server.datagrams().recv().unwrap().len(), 1197); @@ -529,7 +531,7 @@ fn datagram_limits_count_the_encoded_frame() { assert!( pair.client .datagrams() - .send(Bytes::new(), SendOptions::default()) + .send(Bytes::new(), send::Options::default()) .is_err() ); } @@ -679,13 +681,13 @@ fn golden_records() { record.iter().map(|b| format!("{b:02x}")).collect::() }; - let id = pair.client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); - pair.client.send_stream(id).write(b"hi").unwrap(); + let id = pair.client.streams().open(Dir::Uni, send::Options::default()).unwrap(); + pair.client.send_stream(id).write(&mut &b"hi"[..]).unwrap(); pair.client.send_stream(id).finish().unwrap(); let stream = record(&mut pair.client, pair.now); - let id = pair.client.streams().open(Dir::Uni, SendOptions::default()).unwrap(); - pair.client.send_stream(id).write(b"x").unwrap(); + let id = pair.client.streams().open(Dir::Uni, send::Options::default()).unwrap(); + pair.client.send_stream(id).write(&mut &b"x"[..]).unwrap(); pair.client.send_stream(id).reset(VarInt::from_u32(5)).unwrap(); let reset = record(&mut pair.client, pair.now); @@ -723,9 +725,9 @@ fn stream_order(pair: &mut Pair) -> Vec<(u64, usize)> { } /// Open a client uni stream with `options` and queue `len` bytes and a FIN on it -fn queue(pair: &mut Pair, options: SendOptions<'_>, len: usize) -> u64 { +fn queue(pair: &mut Pair, options: send::Options<'_>, len: usize) -> u64 { let id = pair.client.streams().open(Dir::Uni, options).unwrap(); - assert_eq!(pair.client.send_stream(id).write(&vec![0; len]).unwrap(), len); + assert_eq!(pair.client.send_stream(id).write(&mut &vec![0; len][..]).unwrap(), len); pair.client.send_stream(id).finish().unwrap(); u64::from(id) } @@ -736,7 +738,7 @@ fn records_follow_strict_priority() { let mut pair = Pair::new(config(Version::QMux02)); let low = queue( &mut pair, - SendOptions { + send::Options { group: None, priority: 0, }, @@ -744,7 +746,7 @@ fn records_follow_strict_priority() { ); let high = queue( &mut pair, - SendOptions { + send::Options { group: None, priority: 1, }, @@ -757,8 +759,8 @@ fn records_follow_strict_priority() { #[test] fn records_share_equal_priority() { let mut pair = Pair::new(config(Version::QMux02)); - let a = queue(&mut pair, SendOptions::default(), 30_000); - let b = queue(&mut pair, SendOptions::default(), 30_000); + let a = queue(&mut pair, send::Options::default(), 30_000); + let b = queue(&mut pair, send::Options::default(), 30_000); let order = stream_order(&mut pair); assert!(order.len() >= 4, "{order:?}"); for (id, _) in &order { @@ -778,7 +780,7 @@ fn records_order_send_groups() { let high = pair.client.send_groups().open(1); let in_low = queue( &mut pair, - SendOptions { + send::Options { group: Some(&low), priority: 0, }, @@ -786,13 +788,13 @@ fn records_order_send_groups() { ); let in_high = queue( &mut pair, - SendOptions { + send::Options { group: Some(&high), priority: 0, }, 20_000, ); - let ungrouped = queue(&mut pair, SendOptions::default(), 20_000); + let ungrouped = queue(&mut pair, send::Options::default(), 20_000); assert_eq!( stream_order(&mut pair), [(ungrouped, 20_000), (in_high, 20_000), (in_low, 20_000)] @@ -801,7 +803,7 @@ fn records_order_send_groups() { // Raising a group's priority reorders what it has left to send. let first = queue( &mut pair, - SendOptions { + send::Options { group: Some(&high), priority: 0, }, @@ -809,7 +811,7 @@ fn records_order_send_groups() { ); let second = queue( &mut pair, - SendOptions { + send::Options { group: Some(&low), priority: 0, }, diff --git a/rs/moq-quic/src/connection/stats.rs b/rs/moq-quic/src/connection/stats.rs index 010618e81b6..180488f75a5 100644 --- a/rs/moq-quic/src/connection/stats.rs +++ b/rs/moq-quic/src/connection/stats.rs @@ -196,14 +196,14 @@ pub struct Stats { #[derive(Debug, Default, Copy, Clone, PartialEq, Eq)] #[non_exhaustive] pub struct CreditStats { - /// Bytes leased with [`RecvStream::lease()`](crate::RecvStream::lease) and not consumed yet, + /// Bytes leased with [`recv::Stream::lease()`](crate::recv::Stream::lease) and not consumed yet, /// which keep their credit pub leased: u64, /// Bytes the peer may still send before it needs more credit, including credit not yet /// advertised pub available: u64, /// Streams evicted so far because incomplete leases filled the window; see - /// [`RecvStream::lease()`](crate::RecvStream::lease) + /// [`recv::Stream::lease()`](crate::recv::Stream::lease) pub evicted: u64, /// The most credit the peer has had spent at once, leased or not yet read: how close the /// connection came to filling its receive window diff --git a/rs/moq-quic/src/connection/streams/lease.rs b/rs/moq-quic/src/connection/streams/lease.rs index fff33b4cb93..94d686b7274 100644 --- a/rs/moq-quic/src/connection/streams/lease.rs +++ b/rs/moq-quic/src/connection/streams/lease.rs @@ -4,10 +4,10 @@ use std::cmp::Reverse; use std::collections::VecDeque; -use bytes::{Buf, Bytes, BytesMut}; -use thiserror::Error; +use bytes::{Buf, BufMut, Bytes, BytesMut}; -use super::{RecvStream, ShouldTransmit, StreamEvent, StreamId}; +use super::recv::{Error, Stream}; +use super::{ShouldTransmit, StreamEvent, StreamId}; use super::{StreamsState, state::get_or_insert_recv}; use crate::VarInt; use crate::connection::assembler::Assembler; @@ -15,10 +15,10 @@ use crate::connection::assembler::Assembler; /// The bytes of a receive stream read in order but not yet released. /// /// They keep their connection flow control credit until released, so the connection receive -/// window bounds what every lease on the connection holds together. Borrowed with -/// [`RecvStream::lease()`] or [`Connection::leased()`](crate::connection::Connection::leased). +/// window bounds what every lease on the connection holds together. Handed out as a [`Lease`] +/// by [`Stream::lease()`]. #[derive(Debug, Default)] -pub struct Leased { +pub(super) struct Leased { chunks: VecDeque, /// Bytes in `chunks`. len: u64, @@ -36,6 +36,9 @@ pub struct Leased { filled: bool, /// Whether the stream is in [`StreamsState::ranks`]. ranked: bool, + /// The chunks are out in a [`Lease`], whose reader may be about to free the window, which + /// holds eviction off until it comes back. + out: bool, /// Discarded to free a stuck window, with the reader and the peer yet to hear of it from a /// stop. pub(super) evicted: bool, @@ -50,7 +53,7 @@ pub(super) type Rank = (i64, Reverse); /// bytes all sit past a gap, since its lease took every byte in order, and retransmitting the /// gap spends no credit. Nothing is evicted meanwhile. fn unlooked(unread: u64, lease: &Leased) -> bool { - (unread > 0 && (!lease.filled || lease.want.is_some())) || lease.ready + (unread > 0 && (!lease.filled || lease.want.is_some())) || lease.ready || lease.out } #[derive(Debug, Clone, Copy)] @@ -60,29 +63,26 @@ enum End { } impl Leased { - /// The number of leased bytes. - pub fn len(&self) -> usize { - self.len as usize - } - - /// Whether nothing is leased. - pub fn is_empty(&self) -> bool { - self.len == 0 - } - /// Whether the peer finished the stream right after the leased bytes. - pub fn is_fin(&self) -> bool { + pub(super) fn is_fin(&self) -> bool { matches!(self.end, Some(End::Fin)) } - /// The leased bytes in order, one contiguous chunk at a time. - pub fn chunks(&self) -> impl Iterator { + /// The number of leased bytes not out in a [`Lease`]. + #[cfg(test)] + pub(super) fn len(&self) -> usize { + self.len as usize + } + + /// The leased bytes not out in a [`Lease`], one contiguous chunk at a time. + #[cfg(test)] + pub(super) fn chunks(&self) -> impl Iterator { self.chunks.iter().map(|chunk| &chunk[..]) } - /// Hand out up to `max` leading bytes without copying, stopping at a chunk boundary. - /// - /// Their credit stays held until the next [`Self::release()`]. + /// Hand out up to `max` leading bytes without copying, stopping at a chunk boundary, as a + /// [`Lease`] read of them does. + #[cfg(test)] fn take(&mut self, max: usize) -> Bytes { let Some(front) = self.chunks.front_mut() else { return Bytes::new(); @@ -95,23 +95,7 @@ impl Leased { chunk } - /// Drop the first `n` leased bytes, keeping their credit until the next [`Self::release()`]. - fn advance(&mut self, n: usize) { - let mut left = n; - assert!(n as u64 <= self.len, "advanced {n} past {} leased bytes", self.len); - self.len -= n as u64; - while left > 0 { - let front = self.chunks.front_mut().expect("counted above"); - if front.len() > left { - front.advance(left); - break; - } - left -= front.len(); - self.chunks.pop_front(); - } - } - - /// Return the credit of every byte taken or advanced past. + /// Return the credit of every byte read out of a [`Lease`]. fn release(&mut self) -> u64 { let freed = self.held - self.len; self.held = self.len; @@ -182,8 +166,8 @@ impl Leased { self.len - len } - /// Discard the leased bytes for an eviction, returning how many. Taken bytes keep their - /// credit until the stop. + /// Discard the leased bytes for an eviction, returning how many. Bytes out in a [`Lease`] + /// keep their credit until the stop. fn evict(&mut self) -> u64 { let leased = std::mem::take(&mut self.len); self.held -= leased; @@ -204,47 +188,84 @@ impl Leased { } } -/// Errors from [`RecvStream::lease()`] -#[derive(Debug, Error, Clone, Copy, Eq, PartialEq)] -pub enum LeaseError { - /// Fewer bytes than wanted have arrived; the stream is readable again once more do. - /// - /// Carries whether frames were queued meanwhile, for stream credit or an eviction. - #[error("blocked")] - Blocked(ShouldTransmit), - /// The peer abandoned transmitting data on this stream, after the leased bytes. - #[error("reset by peer: code {0}")] - Reset(VarInt), - /// The connection window was stuck and this stream was discarded to free it. - /// - /// The peer has not been told: stop the stream with [`RecvStream::stop()`] and a code of the - /// caller's choosing. Until then the peer may keep sending, which costs bandwidth but no - /// window, since it is credited on arrival, and the bytes the reader took keep their credit, - /// which holds further eviction off. - #[error("evicted")] - Evicted, - /// More bytes were wanted than the connection receive window can ever hold. - #[error("want exceeds the receive window")] - TooLarge, - /// The stream was stopped, or was never opened. - #[error("closed stream")] - ClosedStream, +/// Bytes of a receive stream leased with [`Stream::lease()`], read as a [`Buf`] +/// +/// Reading moves past bytes for good. Hand the lease back with [`Stream::release()`] to return +/// the connection credit of the bytes read, or with [`Stream::restore()`] to keep it, as for a +/// message read only in part. Either way the bytes not read are leased again next time. +#[derive(Debug, Default)] +#[must_use = "hand a lease back with `restore` or `release`"] +pub struct Lease { + chunks: VecDeque, + /// Bytes in `chunks`. + len: usize, + fin: bool, +} + +impl Lease { + /// Whether the peer finished the stream right after the leased bytes. + pub fn is_fin(&self) -> bool { + self.fin + } + + /// The bytes not yet read, in order, one contiguous chunk at a time. + pub fn chunks(&self) -> impl Iterator { + self.chunks.iter().map(|chunk| &chunk[..]) + } } -impl RecvStream<'_> { +impl Buf for Lease { + fn remaining(&self) -> usize { + self.len + } + + fn chunk(&self) -> &[u8] { + self.chunks.front().map_or(&[], |chunk| &chunk[..]) + } + + fn advance(&mut self, n: usize) { + assert!(n <= self.len, "advanced {n} past {} leased bytes", self.len); + self.len -= n; + let mut left = n; + while left > 0 { + let front = self.chunks.front_mut().expect("counted above"); + if front.len() > left { + front.advance(left); + break; + } + left -= front.len(); + self.chunks.pop_front(); + } + } + + /// Without copying when the bytes lie within one chunk. + fn copy_to_bytes(&mut self, n: usize) -> Bytes { + let Some(front) = self.chunks.front_mut().filter(|front| front.len() >= n) else { + let mut bytes = BytesMut::with_capacity(n); + bytes.put(self.take(n)); + return bytes.freeze(); + }; + let chunk = match front.len() > n { + true => front.split_to(n), + false => self.chunks.pop_front().expect("front exists"), + }; + self.len -= n; + chunk + } +} + +impl Stream<'_> { /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the - /// stream ended, then read them with [`Connection::leased()`](crate::connection::Connection::leased). + /// stream ended. /// - /// The bytes stay buffered, with their connection credit, until [`Self::advance()`] or - /// [`Self::take()`] moves past them, and keep that credit until [`Self::release()`]; leasing - /// again without moving returns them again. If [`ShouldTransmit::should_transmit()`] - /// returns `true`, here or in [`LeaseError::Blocked`], transmit soon; the other errors queue - /// nothing. + /// The bytes keep their connection credit until the [`Lease`] is handed back with + /// [`Self::release()`], and those not read are leased again next time. Hand every lease + /// back before leasing again. If the returned [`ShouldTransmit::should_transmit()`] is + /// `true`, transmit soon, whatever the result. /// - /// [`LeaseError::Blocked`] records the `want`, so - /// the next [`StreamEvent::Readable`] for this stream means it may be met, and so that a - /// connection window every reader is waiting on can be recovered by evicting the stream ranked - /// lowest by [`Self::set_priority()`]. + /// [`Error::Blocked`] records the `want`, so the next [`StreamEvent::Readable`] for this + /// stream means it may be met, and so that a connection window every reader is waiting on + /// can be recovered by evicting the stream ranked lowest by [`Self::set_priority()`]. /// /// Nothing is evicted while the peer has connection credit left, so while the leased and /// unread bytes of every stream together stay below the connection receive window. A receiver @@ -252,31 +273,47 @@ impl RecvStream<'_> { /// [`CreditStats::evicted`](crate::CreditStats::evicted) counts any that happen. /// /// Eviction also waits for every stream with data its reader has not leased yet, since that - /// reader may be about to free the window, and for every reader waiting behind a gap, which - /// a retransmission fills without new credit. A reader that gives up on a stream must stop it - /// rather than leave it unpolled. + /// reader may be about to free the window, for every reader waiting behind a gap, which a + /// retransmission fills without new credit, and for every lease not handed back yet. A + /// reader that gives up on a stream must stop it rather than leave it unpolled. /// /// A leased stream keeps its state, and its stream ID, until it is stopped, even once it /// ended, so stop it when done. /// /// Do not mix with [`Self::read()`] on one stream: a read skips the leased bytes. - pub fn lease(&mut self, want: usize) -> Result { - // Taken bytes do not count: a unit that outgrows the window with them is one the stuck - // window evicts by rank, so it gives way like any other rather than failing alone. + /// + /// # Panics + /// - when the last lease of the stream was not handed back + #[must_use = "a lease may need a transmit"] + pub fn lease(&mut self, want: usize) -> (Result, ShouldTransmit) { + // Bytes out in a lease do not count: a unit that outgrows the window with them is one + // the stuck window evicts by rank, so it gives way like any other rather than failing + // alone. let want = want as u64; if want > self.state.receive_window { - return Err(LeaseError::TooLarge); + return (Err(Error::TooLarge), ShouldTransmit(false)); } - let transmit = self.fill()?; + let transmit = match self.fill() { + Ok(transmit) => transmit, + Err(err) => return (Err(err), ShouldTransmit(false)), + }; let lease = self.state.lease_mut(self.id).expect("filled above"); - if lease.len >= want { - return Ok(transmit); + if lease.len >= want || lease.is_fin() { + lease.out = true; + let out = Lease { + chunks: std::mem::take(&mut lease.chunks), + len: std::mem::take(&mut lease.len) as usize, + fin: lease.is_fin(), + }; + // Its reader may be about to free the window. + self.state.unlooked.insert(self.id); + return (Ok(out), transmit); } match lease.end { - Some(End::Fin) => Ok(transmit), + Some(End::Fin) => unreachable!("handed out above"), Some(End::Reset(code)) => { debug_assert!(!transmit.should_transmit(), "a reset stream takes no credit"); - Err(LeaseError::Reset(code)) + (Err(Error::Reset(code)), transmit) } None => { lease.want = Some(want); @@ -284,43 +321,38 @@ impl RecvStream<'_> { self.state.unlooked.insert(self.id); // An eviction always has a reader to wake, even with no `MAX_DATA` due yet. let evicted = self.unstick(); - Err(LeaseError::Blocked(ShouldTransmit( - transmit.should_transmit() || evicted, - ))) + ( + Err(Error::Blocked), + ShouldTransmit(transmit.should_transmit() || evicted), + ) } } } - /// Hand out up to `max` leading leased bytes without copying, stopping at a chunk boundary. + /// Hand a lease back, keeping the connection credit of the bytes read from it /// - /// They leave the lease but keep their connection credit until the next [`Self::release()`], - /// which is how a message reaches its consumer before the whole of it has arrived. Empty when - /// nothing is leased. - pub fn take(&mut self, max: usize) -> Bytes { - match self.state.lease_mut(self.id) { - Some(lease) => lease.take(max), - None => Bytes::new(), - } + /// They wait for a later [`Self::release()`], as the start of a message read only in part + /// does. If [`ShouldTransmit::should_transmit()`] returns `true`, transmit soon: the window + /// may have been stuck on this lease. + pub fn restore(&mut self, lease: Lease) -> ShouldTransmit { + self.put_back(lease); + // Only a stuck pass that waited reruns: a reader handing back what it looked at is not + // stuck itself, and one that is waits for more with its next lease. + ShouldTransmit(self.state.stuck_waiting && self.unstick()) } - /// Drop the first `n` leased bytes, which keep their connection credit until the next - /// [`Self::release()`]. - /// - /// # Panics - /// If `n` exceeds the leased bytes. - pub fn advance(&mut self, n: usize) { - match self.state.lease_mut(self.id) { - Some(lease) => lease.advance(n), - None => assert_eq!(n, 0, "advanced {n} past an empty lease"), - } - } - - /// Return the connection credit of every byte taken or advanced past, once the units they - /// belong to are complete. + /// Hand a lease back, returning the connection credit of every byte read from it or from an + /// earlier one, once the units they belong to are complete /// /// If [`ShouldTransmit::should_transmit()`] returns `true`, transmit soon so the peer learns /// of the larger window. - pub fn release(&mut self) -> ShouldTransmit { + pub fn release(&mut self, lease: Lease) -> ShouldTransmit { + self.put_back(lease); + self.release_credit() + } + + /// Return the credit of every byte read out of a lease. + fn release_credit(&mut self) -> ShouldTransmit { let Some(lease) = self.state.lease_mut(self.id) else { return ShouldTransmit(false); }; @@ -332,6 +364,56 @@ impl RecvStream<'_> { ShouldTransmit(transmit.should_transmit() || self.unstick()) } + /// [`Self::lease()`] and hand it straight back, for tests that read through the lease's + /// bookkeeping rather than a [`Lease`]. + #[cfg(test)] + pub(crate) fn peek(&mut self, want: usize) -> (Result<(), Error>, ShouldTransmit) { + let (lease, transmit) = self.lease(want); + (lease.map(|lease| self.put_back(lease)), transmit) + } + + /// Read up to `max` leased bytes out, keeping their credit, as reading a [`Lease`] does. + #[cfg(test)] + pub(crate) fn take(&mut self, max: usize) -> Bytes { + self.state + .lease_mut(self.id) + .map(|lease| lease.take(max)) + .unwrap_or_default() + } + + /// Drop `n` leased bytes, keeping their credit, as advancing a [`Lease`] does. + #[cfg(test)] + pub(crate) fn advance(&mut self, n: usize) { + let mut left = n; + while left > 0 { + let taken = self.take(left).len(); + assert!(taken > 0, "advanced {n} past the leased bytes"); + left -= taken; + } + } + + /// Return the credit of every byte read out, as [`Self::release()`] does. + #[cfg(test)] + pub(crate) fn release_taken(&mut self) -> ShouldTransmit { + self.release_credit() + } + + /// Put the bytes `lease` did not read back in front of the stream's lease. + /// + /// Dropped instead once the stream was stopped, which discarded its lease and its credit. + fn put_back(&mut self, lease: Lease) { + let Some(state) = self.state.lease_mut(self.id) else { + return; + }; + if !std::mem::take(&mut state.out) { + return; + } + // An out lease holds eviction off, and nothing fills a lease while it is out. + debug_assert!(!state.evicted && state.chunks.is_empty()); + state.chunks = lease.chunks; + state.len = lease.len as u64; + } + /// Rank this stream for eviction: when the connection window is stuck, the stream with the /// lowest priority goes first, and the newest of those that tie. pub fn set_priority(&mut self, priority: i64) { @@ -347,16 +429,17 @@ impl RecvStream<'_> { } /// Move what has arrived in order from the stream's receive buffer into its lease. - fn fill(&mut self) -> Result { + fn fill(&mut self) -> Result { let Some(slot) = self.state.recv.get_mut(&self.id) else { - return Err(LeaseError::ClosedStream); + return Err(Error::Closed); }; let recv = get_or_insert_recv(slot, self.state.initial_stream_receive_window); + assert!(!recv.lease.out, "leased again before the last lease was handed back"); if recv.lease.evicted { - return Err(LeaseError::Evicted); + return Err(Error::Evicted); } if recv.stopped { - return Err(LeaseError::ClosedStream); + return Err(Error::Closed); } self.state.unlooked.remove(&self.id); self.state.held += recv.lease.fill(&mut recv.assembler); @@ -401,7 +484,8 @@ impl RecvStream<'_> { impl StreamsState { /// The lease of a receive stream, if a reader leased it and has not stopped it since. - pub(crate) fn leased(&self, id: StreamId) -> Option<&Leased> { + #[cfg(test)] + pub(super) fn leased(&self, id: StreamId) -> Option<&Leased> { let recv = self.recv.get(&id)?.as_ref()?.as_open_recv()?; let lease = &recv.lease; (lease.filled && (!recv.stopped || lease.evicted)).then_some(lease) @@ -457,6 +541,7 @@ impl StreamsState { /// Returns `None` if nothing was evicted, or whether to transmit the `MAX_DATA` the evictions /// freed; each evicted stream's reader is woken by a [`StreamEvent::Readable`]. pub(super) fn evict_if_stuck(&mut self) -> Option { + self.stuck_waiting = false; if self.local_max_data > self.data_recvd { return None; } @@ -507,12 +592,14 @@ impl StreamsState { { self.scans += 1; } - let waiting = self + let recv = self .recv .get(&id) .and_then(|slot| slot.as_ref()?.as_open_recv()) - .is_some_and(|recv| !recv.stopped && unlooked(recv.unread_credit(), &recv.lease)); - if waiting { + .filter(|recv| !recv.stopped && unlooked(recv.unread_credit(), &recv.lease)); + if let Some(recv) = recv { + // Nothing else reruns the pass once the lease comes back. + self.stuck_waiting = recv.lease.out; found = true; break; } diff --git a/rs/moq-quic/src/connection/streams/mod.rs b/rs/moq-quic/src/connection/streams/mod.rs index c119faa939d..a1e9a26ced8 100644 --- a/rs/moq-quic/src/connection/streams/mod.rs +++ b/rs/moq-quic/src/connection/streams/mod.rs @@ -1,31 +1,19 @@ -use std::{collections::hash_map, io}; - -use bytes::Bytes; -use thiserror::Error; use tracing::trace; use super::spaces::{Retransmits, ThinRetransmits}; -use crate::{ - Dir, StreamId, VarInt, - connection::{ConnectionTag, streams::state::get_or_insert_recv}, - frame, -}; +use crate::{Dir, StreamId, VarInt, frame}; mod lease; -pub use lease::{LeaseError, Leased}; -mod recv; +pub mod recv; use recv::Recv; #[cfg(test)] mod saturated; mod schedule; -pub use recv::{Chunks, ReadError, ReadableError}; use schedule::{Next, Schedule}; -mod send; -pub(crate) use send::{ByteSlice, BytesArray}; -use send::{BytesSource, Send, SendState}; -pub use send::{FinishError, WriteError, Written}; +pub mod send; +use send::{Send, SendState}; mod state; pub(crate) use state::StreamsState; @@ -49,7 +37,7 @@ impl<'a> Streams<'a> { /// /// # Panics /// - when `options.group` was opened on another connection - pub fn open(&mut self, dir: Dir, options: SendOptions<'_>) -> Option { + pub fn open(&mut self, dir: Dir, options: send::Options<'_>) -> Option { let group = options.group.map(|g| g.index(self.state.tag)); if self.conn_state.is_closed() { return None; @@ -110,457 +98,6 @@ impl<'a> Streams<'a> { } } -/// Access to streams -pub struct RecvStream<'a> { - pub(super) id: StreamId, - pub(super) state: &'a mut StreamsState, - pub(super) pending: &'a mut Retransmits, -} - -impl RecvStream<'_> { - /// Read the stream's next contiguous bytes without copying. - pub fn read(&mut self) -> Result, ReadableError> { - Chunks::new(self.id, self.state, self.pending) - } - - /// Stop accepting data on the given receive stream - /// - /// Discards unread and leased data, returning its credit, and notifies the peer to stop - /// transmitting. Once stopped, further attempts to operate on a stream will yield - /// `ClosedStream` errors. - pub fn stop(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { - let hash_map::Entry::Occupied(mut entry) = self.state.recv.entry(self.id) else { - return Err(ClosedStream { _private: () }); - }; - let stream = get_or_insert_recv(entry.get_mut(), self.state.initial_stream_receive_window); - - // An eviction already discarded the buffer, leaving the peer and the bytes the reader - // took to this stop. - let (read_credits, stop_sending) = match stream.lease.evicted { - true => (0, stream.final_offset_unknown()), - false => { - let (credits, stop_sending) = stream.stop()?; - (credits, stop_sending.should_transmit()) - } - }; - let mut lease = std::mem::take(&mut stream.lease); - if stop_sending { - self.pending.stop_sending.push(frame::StopSending { - id: self.id, - error_code, - }); - } - - // We need to keep stopped streams around until they're finished or reset so we can update - // connection-level flow control to account for discarded data. Otherwise, we can discard - // state immediately. - if !stream.final_offset_unknown() { - let recv = entry.remove().expect("must have recv when stopping"); - // The lease was taken out above, so this returns no credit. - let _ = self.state.stream_recv_freed(self.id, recv); - self.state.queue_max_stream_id(self.pending); - } - - let leased = self.state.discard_lease(self.id, &mut lease); - let transmit = self.state.add_read_credits(read_credits); - self.pending.max_data |= leased.should_transmit() || transmit.should_transmit(); - // The credit returned may only have paid a shrunk window's debt. - self.unstick(); - - Ok(()) - } - - /// Check whether this stream has been reset by the peer, returning the reset error code if so - /// - /// After returning `Ok(Some(_))` once, stream state will be discarded and all future calls will - /// return `Err(ClosedStream)`. - pub fn received_reset(&mut self) -> Result, ClosedStream> { - let hash_map::Entry::Occupied(entry) = self.state.recv.entry(self.id) else { - return Err(ClosedStream { _private: () }); - }; - let Some(s) = entry.get().as_ref().and_then(|s| s.as_open_recv()) else { - return Ok(None); - }; - if s.stopped { - return Err(ClosedStream { _private: () }); - } - let Some(code) = s.reset_code() else { - return Ok(None); - }; - - // Clean up state after application observes the reset, since there's no reason for the - // application to attempt to read or stop the stream once it knows it's reset. A lease still - // holding credit returns it here, since nothing is left to release or stop it. - let (_, recv) = entry.remove_entry(); - let transmit = self - .state - .stream_recv_freed(self.id, recv.expect("must have recv on reset")); - self.pending.max_data |= transmit.should_transmit(); - self.state.queue_max_stream_id(self.pending); - // The credit returned may only have paid a shrunk window's debt. - self.unstick(); - - Ok(Some(code)) - } -} - -/// Access to streams -pub struct SendStream<'a> { - pub(super) id: StreamId, - pub(super) state: &'a mut StreamsState, - pub(super) pending: &'a mut Retransmits, - pub(super) conn_state: &'a super::State, -} - -impl<'a> SendStream<'a> { - /// Drive a bare `StreamsState` without a connection, for the `streams` fuzz target - #[cfg(any(test, feature = "fuzz"))] - pub(crate) fn new( - id: StreamId, - state: &'a mut StreamsState, - pending: &'a mut Retransmits, - conn_state: &'a super::State, - ) -> Self { - Self { - id, - state, - pending, - conn_state, - } - } - - /// Send data on the given stream - /// - /// Returns the number of bytes successfully written. - pub fn write(&mut self, data: &[u8]) -> Result { - Ok(self.write_source(&mut ByteSlice::from_slice(data))?.bytes) - } - - /// Send data on the given stream - /// - /// Returns the number of bytes and chunks successfully written. - /// Note that this method might also write a partial chunk. In this case - /// [`Written::chunks`] will not count this chunk as fully written. However - /// the chunk will be advanced and contain only non-written data after the call. - pub fn write_chunks(&mut self, data: &mut [Bytes]) -> Result { - self.write_source(&mut BytesArray::from_chunks(data)) - } - - /// How many bytes a write would accept right now - /// - /// Zero when a write would block or fail. Lets a caller size a chunk before taking it from a - /// buffer it cannot put bytes back into. - pub fn write_limit(&self) -> usize { - if self.conn_state.is_closed() { - return 0; - } - let budget = match self.state.send.get(&self.id) { - None => return 0, - Some(None) => self.state.max_send_data(self.id).into_inner(), - Some(Some(stream)) if !stream.is_writable() || stream.stop_reason.is_some() => { - return 0; - } - Some(Some(stream)) => stream.max_data - stream.offset(), - }; - usize::try_from(budget.min(self.state.write_limit())).unwrap_or(usize::MAX) - } - - fn write_source(&mut self, source: &mut B) -> Result { - if self.conn_state.is_closed() { - trace!(%self.id, "write blocked; connection draining"); - return Err(WriteError::Blocked); - } - - let limit = self.state.write_limit(); - - let max_send_data = self.state.max_send_data(self.id); - - let stream = self - .state - .send - .get_mut(&self.id) - .ok_or(WriteError::ClosedStream)? - .get_or_insert_with(|| Send::new(max_send_data)); - - let was_pending = stream.is_pending(); - let written = match stream.write(source, limit) { - Ok(written) => written, - Err(WriteError::Blocked) => { - if limit == 0 { - trace!( - stream = %self.id, max_data = self.state.max_data, data_sent = self.state.data_sent, - "write blocked by connection-level flow control or send window" - ); - self.state.connection_blocked.insert(self.id); - // Only report the peer's limit, not our own send window. - if self.state.data_sent == self.state.max_data - && self.state.data_blocked_limit != Some(self.state.max_data) - { - self.state.data_blocked_limit = Some(self.state.max_data); - self.pending.data_blocked = true; - } - } else if stream.data_blocked_limit != Some(stream.max_data) { - stream.data_blocked_limit = Some(stream.max_data); - self.pending.stream_data_blocked.insert(self.id); - } - return Err(WriteError::Blocked); - } - Err(e) => return Err(e), - }; - self.state.data_sent += written.bytes as u64; - self.state.buffered_data += written.bytes as u64; - trace!(stream = %self.id, "wrote {} bytes", written.bytes); - if !was_pending { - self.state.pending.push(self.id, &mut stream.slot, stream.priority); - } - Ok(written) - } - - /// Check if this stream was stopped, get the reason if it was - pub fn stopped(&self) -> Result, ClosedStream> { - match self.state.send.get(&self.id).as_ref() { - Some(Some(s)) => Ok(s.stop_reason), - Some(None) => Ok(None), - None => Err(ClosedStream { _private: () }), - } - } - - /// Finish a send stream, signalling that no more data will be sent. - /// - /// If this fails, no [`StreamEvent::Finished`] will be generated. - /// - /// [`StreamEvent::Finished`]: crate::StreamEvent::Finished - pub fn finish(&mut self) -> Result<(), FinishError> { - let max_send_data = self.state.max_send_data(self.id); - let stream = self - .state - .send - .get_mut(&self.id) - .ok_or(FinishError::ClosedStream)? - .get_or_insert_with(|| Send::new(max_send_data)); - - let was_pending = stream.is_pending(); - stream.finish()?; - self.state.connection_blocked.swap_remove(&self.id); - if !was_pending { - self.state.pending.push(self.id, &mut stream.slot, stream.priority); - } - - Ok(()) - } - - /// Abandon transmitting data on a stream - /// - /// # Panics - /// - when applied to a receive stream - pub fn reset(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { - self.reset_at(0, error_code) - } - - /// How many bytes were written to the stream so far, the offset the next write starts at - /// - /// After a reset, the bytes the reset still delivers. Zero before the first write, or once - /// the stream is gone. - /// - /// # Panics - /// - when applied to a receive stream - pub fn offset(&self) -> u64 { - match self.state.send.get(&self.id) { - Some(Some(stream)) => stream.offset(), - _ => 0, - } - } - - /// Whether the peer acknowledged every byte before `offset` - /// - /// A `false` notes to report a [`StreamEvent::Acked`] once the answer changes, so poll again - /// then. The stream holds only its lowest waiting offset, so every waiter on it polls again - /// on that event. Like a [`StreamEvent::Finished`] stream, an acknowledgment only means the - /// peer's QUIC stack holds the data, not that its application read it. - /// - /// Fails with [`ClosedStream`] once the peer will never acknowledge it, or no longer reads - /// it: a reset dropped the data before `offset`, the peer sent STOP_SENDING, or the - /// connection closed. It also fails once the stream is freed, after its - /// [`StreamEvent::Finished`], which means every byte was acknowledged. - /// - /// # Panics - /// - when applied to a receive stream - /// - when `offset` is past the data written to a stream that was not reset - pub fn acked(&mut self, offset: u64) -> Result { - if self.conn_state.is_closed() { - return Err(ClosedStream { _private: () }); - } - match self.state.send.get_mut(&self.id) { - Some(Some(stream)) => stream.acked(offset), - Some(None) => { - assert_eq!(offset, 0, "acked past the data written"); - Ok(true) - } - None => Err(ClosedStream { _private: () }), - } - } - - /// Abandon the stream, but still deliver its first `offset` bytes - /// - /// Nothing past `offset` is sent or retransmitted. A peer that accepts RESET_STREAM_AT gets - /// it at once, and reads the data before the error. Otherwise the data keeps being - /// retransmitted, and a plain RESET_STREAM follows once the peer has acknowledged all of it, a - /// round trip later. An acknowledgment only means the peer's QUIC stack holds the data: - /// RFC 9000 lets it discard data its application has not read yet on RESET_STREAM, so this - /// only reliably delivers data the peer parses on arrival, like a WebTransport stream header. - /// A peer that asked us to stop sending gets the reset at once, without the data. - /// - /// Fails with [`ClosedStream`] if the stream was already reset or is gone. - /// - /// # Panics - /// - when applied to a receive stream - /// - when `offset` is past the data written - pub fn reset_at(&mut self, offset: u64, error_code: VarInt) -> Result<(), ClosedStream> { - let max_send_data = self.state.max_send_data(self.id); - let at = self.state.peer_reset_stream_at; - let stream = self - .state - .send - .get_mut(&self.id) - .ok_or(ClosedStream { _private: () })? - .get_or_insert_with(|| Send::new(max_send_data)); - - if stream.is_reset() { - // Redundant reset call - return Err(ClosedStream { _private: () }); - } - assert!(offset <= stream.offset(), "reset past the data written"); - // The peer that sent STOP_SENDING will not read the prefix - let reliable_size = match stream.stop_reason { - Some(_) => 0, - None => offset, - }; - - // Restore the portion of the send window consumed by the data that we aren't about to - // send. We leave flow control alone because the peer's responsible for issuing additional - // credit based on the final offset communicated in the RESET_STREAM frame we send. - let buffered = stream.pending.buffered(); - let send = stream.reset(error_code, reliable_size, at); - self.state.connection_blocked.swap_remove(&self.id); - self.state.buffered_data -= buffered - stream.pending.buffered(); - if let Some(offset) = stream.take_waiting() { - self.state.events.push_back(StreamEvent::Acked { id: self.id, offset }); - } - // Only data below the Reliable Size is left to send - if !stream.is_pending() { - self.state.pending.remove(&mut stream.slot); - } - if let Some(reliable_size) = send { - self.pending.reset_stream.push((self.id, reliable_size)); - } - - // Don't reopen an already-closed stream we haven't forgotten yet - Ok(()) - } - - /// Set the priority within the stream's send group, or session-wide when ungrouped - /// - /// Higher values go first; equal values share bytes round robin. A queued stream moves at - /// once, behind the streams already queued at its new priority. With - /// [`transport::Config::send_fairness`](crate::transport::Config::send_fairness) off, a stream - /// its group is partway through keeps going first until it runs out of data. - /// - /// # Panics - /// - when applied to a receive stream - pub fn set_priority(&mut self, priority: i64) -> Result<(), ClosedStream> { - let max_send_data = self.state.max_send_data(self.id); - let stream = self - .state - .send - .get_mut(&self.id) - .ok_or(ClosedStream { _private: () })? - .get_or_insert_with(|| Send::new(max_send_data)); - - stream.priority = priority; - self.state.pending.set_stream_priority(&mut stream.slot, priority); - Ok(()) - } - - /// Get the priority of a stream - /// - /// # Panics - /// - when applied to a receive stream - pub fn priority(&self) -> Result { - let stream = self.state.send.get(&self.id).ok_or(ClosedStream { _private: () })?; - - Ok(stream.as_ref().map(|s| s.priority).unwrap_or_default()) - } - - /// Move the stream into a send group, or with `None` back to the ungrouped streams - /// - /// Ungrouped streams are sent before any send group. A queued stream keeps its place. - /// - /// # Panics - /// - when applied to a receive stream - /// - when `group` was opened on another connection - pub fn set_group(&mut self, group: Option<&SendGroup>) -> Result<(), ClosedStream> { - let max_send_data = self.state.max_send_data(self.id); - let stream = self - .state - .send - .get_mut(&self.id) - .ok_or(ClosedStream { _private: () })? - .get_or_insert_with(|| Send::new(max_send_data)); - - let group = group.map(|g| g.index(self.state.tag)); - self.state - .pending - .join(self.id, &mut stream.slot, group, stream.priority); - Ok(()) - } -} - -/// Scheduling of outgoing streams between send groups -/// -/// A higher priority group preempts every lower one, and groups of equal priority get an -/// equal share of bytes no matter how many streams each has queued. Streams outside any group -/// go before all of it. A group's datagrams, from [`Datagrams::send`](crate::Datagrams::send), -/// share its bytes in the same order. -pub struct SendGroups<'a> { - pub(super) state: &'a mut StreamsState, -} - -impl SendGroups<'_> { - /// Open a send group at `priority` - pub fn open(&mut self, priority: i64) -> SendGroup { - SendGroup { - index: self.state.pending.open_group(priority), - tag: self.state.tag, - } - } - - /// Release a send group; its streams keep their place until they finish - /// - /// # Panics - /// - when `group` was opened on another connection - pub fn close(&mut self, group: SendGroup) { - let index = group.index(self.state.tag); - self.state.pending.close_group(index); - } - - /// Change a group's priority, moving any data it has queued along with it - /// - /// # Panics - /// - when `group` was opened on another connection - pub fn set_priority(&mut self, group: &SendGroup, priority: i64) { - let index = group.index(self.state.tag); - self.state.pending.set_group_priority(index, priority); - } - - /// The group's priority - /// - /// # Panics - /// - when `group` was opened on another connection - pub fn priority(&self, group: &SendGroup) -> i64 { - self.state.pending.group_priority(group.index(self.state.tag)) - } -} - impl StreamsState { /// Queue a datagram scheduled by `options`, within `budget` bytes of queued datagrams. /// @@ -570,12 +107,12 @@ impl StreamsState { /// - when `options.group` was opened on another connection pub(crate) fn queue_datagram( &mut self, - datagram: frame::Datagram, - options: SendOptions<'_>, + datagram: frame::OutgoingDatagram, + options: send::Options<'_>, budget: usize, ) -> bool { let group = options.group.map(|g| g.index(self.tag)); - let frame = datagram.size(true); + let frame = datagram.size(); self.pending .push_datagram(group, options.priority, datagram, frame, budget) } @@ -583,11 +120,11 @@ impl StreamsState { /// Drop queued datagrams with a payload over `max_payload` bytes, returning whether any was. pub(crate) fn drop_oversized_datagrams(&mut self, max_payload: usize) -> bool { self.pending.retain_datagrams(|datagram| { - let keep = datagram.data.len() <= max_payload; + let keep = datagram.len() <= max_payload; if !keep { trace!( "dropping {} byte datagram violating {} byte limit", - datagram.data.len(), + datagram.len(), max_payload ); } @@ -601,53 +138,6 @@ impl StreamsState { } } -/// How an outgoing stream or datagram is scheduled, like W3C WebTransport's send options -/// -/// The default is ungrouped at priority 0. Generic so the drivers can carry their own group -/// handle in the same shape. -#[derive(Debug)] -pub struct SendOptions<'a, G = SendGroup> { - /// The send group to join, or `None` for the ungrouped streams that go before every group - pub group: Option<&'a G>, - /// The priority within the group: higher goes first, and a datagram goes ahead of a - /// stream of equal priority - pub priority: i64, -} - -impl Default for SendOptions<'_, G> { - fn default() -> Self { - Self { - group: None, - priority: 0, - } - } -} - -impl Clone for SendOptions<'_, G> { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for SendOptions<'_, G> {} - -/// A send group, from [`SendGroups::open`] -/// -/// Release it with [`SendGroups::close`] on the connection that opened it. -#[derive(Debug, PartialEq, Eq)] -#[must_use = "a send group lives until it is closed"] -pub struct SendGroup { - index: usize, - tag: ConnectionTag, -} - -impl SendGroup { - fn index(&self, tag: ConnectionTag) -> usize { - assert_eq!(self.tag, tag, "send group used on another connection"); - self.index - } -} - /// Application events about streams #[derive(Debug, PartialEq, Eq)] pub enum StreamEvent { @@ -675,12 +165,12 @@ pub enum StreamEvent { /// Which stream has been finished id: StreamId, }, - /// The answer changed for the lowest offset a [`SendStream::acked`] call is waiting for + /// The answer changed for the lowest offset a [`send::Stream::acked`] call is waiting for /// /// The stream's contiguous acknowledged prefix reached it, or a reset or STOP_SENDING may have /// failed any waiter. Every waiter on the stream polls again: one at or below `offset` is /// acknowledged, and one past it learns its error or registers again. Reported before a [`StreamEvent::Finished`] from the same ACK. - /// Not reported when the connection closes, which fails [`SendStream::acked`] instead. + /// Not reported when the connection closes, which fails [`send::Stream::acked`] instead. Acked { /// Which stream was acknowledged id: StreamId, @@ -716,19 +206,6 @@ impl ShouldTransmit { } } -/// Error indicating that a stream has not been opened or has already been finished or reset -#[derive(Debug, Default, Error, Clone, PartialEq, Eq)] -#[error("closed stream")] -pub struct ClosedStream { - _private: (), -} - -impl From for io::Error { - fn from(x: ClosedStream) -> Self { - Self::new(io::ErrorKind::NotConnected, x) - } -} - #[derive(Debug, Copy, Clone, Eq, PartialEq)] enum StreamHalf { Send, diff --git a/rs/moq-quic/src/connection/streams/recv.rs b/rs/moq-quic/src/connection/streams/recv.rs index b1010a41e82..d7c122abf5f 100644 --- a/rs/moq-quic/src/connection/streams/recv.rs +++ b/rs/moq-quic/src/connection/streams/recv.rs @@ -1,15 +1,117 @@ -use std::collections::hash_map::Entry; -use std::mem; +//! Incoming streams: their handles, leases, and errors. + +use std::collections::hash_map::{self, Entry}; +use std::{io, mem}; use thiserror::Error; use tracing::debug; +use super::lease::Leased; use super::state::get_or_insert_recv; -use super::{ClosedStream, Leased, Retransmits, ShouldTransmit, StreamId, StreamsState}; -use crate::connection::assembler::{Assembler, Chunk}; +use super::{Retransmits, ShouldTransmit, StreamId, StreamsState}; +use crate::connection::assembler::Assembler; +pub use crate::connection::assembler::Chunk; use crate::connection::streams::state::StreamRecv; use crate::{TransportError, VarInt, frame}; +pub use super::lease::Lease; + +/// An incoming stream, from [`Connection::recv_stream`](crate::connection::Connection::recv_stream) +/// +/// A borrowed view by ID: the stream lives on in the connection after this is dropped, and +/// calls on an ID the stream no longer has state for fail with [`Error::Closed`]. +pub struct Stream<'a> { + pub(crate) id: StreamId, + pub(crate) state: &'a mut StreamsState, + pub(crate) pending: &'a mut Retransmits, +} + +impl Stream<'_> { + /// Read the stream's next contiguous bytes without copying. + pub fn read(&mut self) -> Result, Error> { + Chunks::new(self.id, self.state, self.pending) + } + + /// Stop accepting data on the given receive stream + /// + /// Discards unread and leased data, returning its credit, and notifies the peer to stop + /// transmitting. Once stopped, further calls on the stream fail with [`Error::Closed`]. + pub fn stop(&mut self, error_code: VarInt) -> Result<(), Error> { + let hash_map::Entry::Occupied(mut entry) = self.state.recv.entry(self.id) else { + return Err(Error::Closed); + }; + let stream = get_or_insert_recv(entry.get_mut(), self.state.initial_stream_receive_window); + + // An eviction already discarded the buffer, leaving the peer and the bytes the reader + // took to this stop. + let (read_credits, stop_sending) = match stream.lease.evicted { + true => (0, stream.final_offset_unknown()), + false => { + let (credits, stop_sending) = stream.stop()?; + (credits, stop_sending.should_transmit()) + } + }; + let mut lease = std::mem::take(&mut stream.lease); + if stop_sending { + self.pending.stop_sending.push(frame::StopSending { + id: self.id, + error_code, + }); + } + + // We need to keep stopped streams around until they're finished or reset so we can update + // connection-level flow control to account for discarded data. Otherwise, we can discard + // state immediately. + if !stream.final_offset_unknown() { + let recv = entry.remove().expect("must have recv when stopping"); + // The lease was taken out above, so this returns no credit. + let _ = self.state.stream_recv_freed(self.id, recv); + self.state.queue_max_stream_id(self.pending); + } + + let leased = self.state.discard_lease(self.id, &mut lease); + let transmit = self.state.add_read_credits(read_credits); + self.pending.max_data |= leased.should_transmit() || transmit.should_transmit(); + // The credit returned may only have paid a shrunk window's debt. + self.unstick(); + + Ok(()) + } + + /// Check whether this stream has been reset by the peer, returning the reset error code if so + /// + /// After returning `Ok(Some(_))` once, stream state will be discarded and all future calls will + /// fail with [`Error::Closed`]. + pub fn received_reset(&mut self) -> Result, Error> { + let hash_map::Entry::Occupied(entry) = self.state.recv.entry(self.id) else { + return Err(Error::Closed); + }; + let Some(s) = entry.get().as_ref().and_then(|s| s.as_open_recv()) else { + return Ok(None); + }; + if s.stopped { + return Err(Error::Closed); + } + let Some(code) = s.reset_code() else { + return Ok(None); + }; + + // Clean up state after application observes the reset, since there's no reason for the + // application to attempt to read or stop the stream once it knows it's reset. A lease still + // holding credit returns it here, since nothing is left to release or stop it. + let (_, recv) = entry.remove_entry(); + let transmit = self + .state + .stream_recv_freed(self.id, recv.expect("must have recv on reset")); + self.pending.max_data |= transmit.should_transmit(); + self.state.queue_max_stream_id(self.pending); + // The credit returned may only have paid a shrunk window's debt. + self.unstick(); + + Ok(Some(code)) + } +} + #[derive(Debug, Default)] pub(super) struct Recv { // NB: when adding or removing fields, remember to update `reinit`. @@ -111,9 +213,9 @@ impl Recv { Ok((new_bytes, frame.fin && self.stopped)) } - pub(super) fn stop(&mut self) -> Result<(u64, ShouldTransmit), ClosedStream> { + pub(super) fn stop(&mut self) -> Result<(u64, ShouldTransmit), Error> { if self.stopped { - return Err(ClosedStream { _private: () }); + return Err(Error::Closed); } let read_credits = self.unread_credit(); @@ -294,7 +396,7 @@ impl Recv { } } -/// Chunks returned from [`RecvStream::read()`][crate::RecvStream::read]. +/// Chunks returned from [`Stream::read()`]. /// /// ### Note: Finalization Needed /// Bytes read from the stream are not released from the congestion window until @@ -318,15 +420,15 @@ impl<'a> Chunks<'a> { id: StreamId, streams: &'a mut StreamsState, pending: &'a mut Retransmits, - ) -> Result { + ) -> Result { let mut entry = match streams.recv.entry(id) { Entry::Occupied(entry) => entry, - Entry::Vacant(_) => return Err(ReadableError::ClosedStream), + Entry::Vacant(_) => return Err(Error::Closed), }; let recv = get_or_insert_recv(entry.get_mut(), streams.initial_stream_receive_window); if recv.stopped { - return Err(ReadableError::ClosedStream); + return Err(Error::Closed); } let recv = entry.remove().unwrap().into_inner(); // initialized above Ok(Self { @@ -338,14 +440,15 @@ impl<'a> Chunks<'a> { }) } - /// Next + /// The next contiguous chunk of at most `max_length` bytes, or `None` once the stream finished /// - /// Should call finalize() when done calling this. - pub fn next(&mut self, max_length: usize) -> Result, ReadError> { + /// Fails with [`Error::Blocked`] until more arrives, or [`Error::Reset`]. Call + /// [`Self::finalize()`] when done. + pub fn next(&mut self, max_length: usize) -> Result, Error> { let rs = match &mut self.state { ChunksState::Readable(rs) => rs, ChunksState::Reset(error_code) => { - return Err(ReadError::Reset(*error_code)); + return Err(Error::Reset(*error_code)); } ChunksState::Finished => { return Ok(None); @@ -361,7 +464,7 @@ impl<'a> Chunks<'a> { match rs.state { RecvState::ResetRecvd { reliable_size, .. } if rs.assembler.unread_before(reliable_size) > 0 => { // The data below the Reliable Size is still in flight - Err(ReadError::Blocked) + Err(Error::Blocked) } RecvState::ResetRecvd { error_code, .. } => { let state = mem::replace(&mut self.state, ChunksState::Reset(error_code)); @@ -371,7 +474,7 @@ impl<'a> Chunks<'a> { _ => unreachable!("state must be ChunkState::Readable"), }; self.pending.max_data |= self.streams.stream_recv_freed(self.id, recv).should_transmit(); - Err(ReadError::Reset(error_code)) + Err(Error::Reset(error_code)) } RecvState::Recv { size } => { if size == Some(rs.end) && rs.assembler.bytes_read() == rs.end { @@ -388,7 +491,7 @@ impl<'a> Chunks<'a> { // retrying a read harmlessly re-traces our steps back to returning // `Err(Blocked)` again. The buffers can't refill and the stream's own state // can't change so long as this `Chunks` exists. - Err(ReadError::Blocked) + Err(Error::Blocked) } } } @@ -451,28 +554,43 @@ enum ChunksState { Finalized, } -/// Errors triggered when reading from a recv stream -#[derive(Debug, Error, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum ReadError { - /// No more data is currently available on this stream. - /// - /// If more data on this stream is received from the peer, an `Event::StreamReadable` will be - /// generated for this stream, indicating that retrying the read might succeed. +/// Why a read or lease failed +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// Nothing more has arrived yet; a [`StreamEvent::Readable`](super::StreamEvent::Readable) + /// says when to retry #[error("blocked")] Blocked, - /// The peer abandoned transmitting data on this stream. - /// - /// Carries an application-defined error code. + /// The peer abandoned the stream, with its code #[error("reset by peer: code {0}")] Reset(VarInt), + /// The connection window was stuck and this stream was discarded to free it + /// + /// The peer has not been told: stop the stream with [`Stream::stop()`] and a code of the + /// caller's choosing. Until then the peer may keep sending, which costs bandwidth but no + /// window, since it is credited on arrival, and the bytes the reader took keep their credit, + /// which holds further eviction off. + #[error("evicted")] + Evicted, + /// A lease wanted more bytes than the connection receive window can ever hold + #[error("want exceeds the receive window")] + TooLarge, + /// The stream was stopped, or never opened, or already read to its end + #[error("closed stream")] + Closed, } -/// Errors triggered when opening a recv stream for reading -#[derive(Debug, Error, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum ReadableError { - /// The stream has not been opened or was already stopped, finished, or reset - #[error("closed stream")] - ClosedStream, +impl From for io::Error { + fn from(err: Error) -> Self { + let kind = match err { + Error::Blocked => io::ErrorKind::WouldBlock, + Error::Reset(_) | Error::Evicted => io::ErrorKind::ConnectionReset, + Error::TooLarge => io::ErrorKind::InvalidInput, + Error::Closed => io::ErrorKind::NotConnected, + }; + Self::new(kind, err) + } } #[derive(Debug, Copy, Clone, Eq, PartialEq)] diff --git a/rs/moq-quic/src/connection/streams/saturated.rs b/rs/moq-quic/src/connection/streams/saturated.rs index 19da2dc1b55..f72fad558f9 100644 --- a/rs/moq-quic/src/connection/streams/saturated.rs +++ b/rs/moq-quic/src/connection/streams/saturated.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use bytes::{BufMut, Bytes, BytesMut}; use super::state::StreamsState; -use super::{SendGroup, SendGroups, SendOptions, SendStream, Streams}; +use super::{Streams, send}; use crate::{ Dir, Side, StreamId, VarInt, connection::{State as ConnState, spaces::Retransmits}, @@ -83,16 +83,16 @@ impl Sender { } } - fn groups(&mut self) -> SendGroups<'_> { - SendGroups { state: &mut self.state } + fn groups(&mut self) -> send::Groups<'_> { + send::Groups { state: &mut self.state } } - fn group(&mut self, priority: i64) -> SendGroup { + fn group(&mut self, priority: i64) -> send::Group { self.groups().open(priority) } - fn send(&mut self, id: StreamId) -> SendStream<'_> { - SendStream { + fn send(&mut self, id: StreamId) -> send::Stream<'_> { + send::Stream { id, state: &mut self.state, pending: &mut self.pending, @@ -101,32 +101,32 @@ impl Sender { } /// Open a stream in `group` at `priority` and queue `bytes` on it. - fn open(&mut self, group: Option<&SendGroup>, priority: i64, bytes: usize) -> StreamId { + fn open(&mut self, group: Option<&send::Group>, priority: i64, bytes: usize) -> StreamId { let id = Streams { state: &mut self.state, conn_state: &self.conn, } - .open(Dir::Uni, SendOptions { group, priority }) + .open(Dir::Uni, send::Options { group, priority }) .unwrap(); self.write(id, bytes); id } fn write(&mut self, id: StreamId, bytes: usize) { - assert_eq!(self.send(id).write(&vec![0; bytes]), Ok(bytes)); + assert_eq!(self.send(id).write(&mut &vec![0; bytes][..]), Ok(bytes)); } /// Queue a `bytes` payload datagram in `group` at `priority`, returning its tag. - fn datagram(&mut self, group: Option<&SendGroup>, priority: i64, bytes: usize) -> u64 { + fn datagram(&mut self, group: Option<&send::Group>, priority: i64, bytes: usize) -> u64 { let tag = self.tag; self.tag += 1; let mut data = BytesMut::with_capacity(bytes); data.put_u64(tag); data.resize(bytes, 0); - let datagram = frame::Datagram { data: data.freeze() }; + let datagram = frame::OutgoingDatagram::new(data.freeze()); assert!( self.state - .queue_datagram(datagram, SendOptions { group, priority }, usize::MAX) + .queue_datagram(datagram, send::Options { group, priority }, usize::MAX) ); tag } @@ -159,7 +159,7 @@ impl Sender { match frame.unwrap() { Frame::Stream(stream) => self.order.push(Sent::Stream(stream.id)), Frame::Datagram(datagram) => { - let wire = datagram.size(true) as u64; + let wire = frame::OutgoingDatagram::new(datagram.data.clone()).size() as u64; let tag = u64::from_be_bytes(datagram.data[..8].try_into().unwrap()); *self.datagram_wire.entry(tag).or_default() += wire; datagram_bytes += wire; @@ -441,7 +441,9 @@ fn reliable_reset_sends_its_prefix_then_frees_its_group() { sender.send(finished).finish().unwrap(); sender.groups().close(group); - sender.send(reset).reset_at(3_000, 0u32.into()).unwrap(); + let mut stream = sender.send(reset); + let at = stream.offset().rewind(3_000); + stream.reset_at(at, 0u32.into()).unwrap(); assert!(sender.state.can_send_stream_data()); sender.drain(100); assert_eq!(sender.bytes(&[reset]), 3_000, "only the reliable prefix"); @@ -470,7 +472,9 @@ fn stop_sending_drops_a_reliable_prefix_from_the_schedule() { let mut sender = Sender::new(); let group = sender.group(0); let reset = sender.open(Some(&group), 0, 10_000); - sender.send(reset).reset_at(3_000, 0u32.into()).unwrap(); + let mut stream = sender.send(reset); + let at = stream.offset().rewind(3_000); + stream.reset_at(at, 0u32.into()).unwrap(); assert!(sender.state.can_send_stream_data()); assert_eq!(sender.state.received_stop_sending(reset, 0u32.into()), Ok(Some(0))); diff --git a/rs/moq-quic/src/connection/streams/send.rs b/rs/moq-quic/src/connection/streams/send.rs index a3d009ce626..39fb6080750 100644 --- a/rs/moq-quic/src/connection/streams/send.rs +++ b/rs/moq-quic/src/connection/streams/send.rs @@ -1,8 +1,14 @@ -use bytes::Bytes; +//! Outgoing streams and datagrams: their handles, scheduling, and errors. + +use std::io; + +use bytes::Buf; use thiserror::Error; +use tracing::trace; -use super::{ClosedStream, schedule::Slot}; -use crate::{StreamId, VarInt, connection::send_buffer::SendBuffer, frame}; +use super::{StreamEvent, StreamsState, schedule::Slot}; +use crate::connection::{ConnectionTag, State as ConnState, send_buffer::SendBuffer, spaces::Retransmits}; +use crate::{StreamId, VarInt, frame}; #[derive(Debug)] pub(super) struct Send { @@ -19,8 +25,10 @@ pub(super) struct Send { pub(super) data_blocked_limit: Option, /// The reason the peer wants us to stop, if `STOP_SENDING` was received pub(super) stop_reason: Option, - /// The lowest offset a [`SendStream::acked`](super::SendStream::acked) call is waiting for + /// The lowest offset a [`Stream::acked`] call is waiting for pub(super) notify: Option, + /// The application gave up the stream's ID with [`Stream::release`], so its end is not kept + pub(super) released: bool, } impl Send { @@ -35,6 +43,7 @@ impl Send { data_blocked_limit: None, stop_reason: None, notify: None, + released: false, }) } @@ -43,46 +52,58 @@ impl Send { matches!(self.state, SendState::ResetSent { .. }) } - pub(super) fn finish(&mut self) -> Result<(), FinishError> { + pub(super) fn finish(&mut self) -> Result<(), Error> { if let Some(error_code) = self.stop_reason { - Err(FinishError::Stopped(error_code)) + Err(Error::Stopped(error_code)) } else if self.state == SendState::Ready { self.state = SendState::DataSent { finish_acked: false }; self.fin_pending = true; Ok(()) } else { - Err(FinishError::ClosedStream) + Err(Error::Closed) } } - pub(super) fn write(&mut self, source: &mut S, limit: u64) -> Result { + /// Queue as much of `data` as `limit` and the stream's flow control allow. + /// + /// Each chunk is taken with [`Buf::copy_to_bytes`], which a [`Bytes`](bytes::Bytes) source + /// splits off without copying. + pub(super) fn write(&mut self, data: &mut impl Buf, limit: u64) -> Result { if !self.is_writable() { - return Err(WriteError::ClosedStream); + return Err(Error::Closed); } if let Some(error_code) = self.stop_reason { - return Err(WriteError::Stopped(error_code)); + return Err(Error::Stopped(error_code)); } let budget = self.max_data - self.pending.offset(); if budget == 0 || limit == 0 { - return Err(WriteError::Blocked); + return Err(Error::Blocked); } - let mut limit = limit.min(budget) as usize; + let mut limit = usize::try_from(limit.min(budget)).unwrap_or(usize::MAX); - let mut result = Written::default(); - loop { - let (chunk, chunks_consumed) = source.pop_chunk(limit); - result.chunks += chunks_consumed; - result.bytes += chunk.len(); - - if chunk.is_empty() { + let mut written = 0; + while limit > 0 { + let take = data.chunk().len().min(limit); + if take == 0 { break; } - - limit -= chunk.len(); - self.pending.write(chunk); + self.pending.write(data.copy_to_bytes(take)); + limit -= take; + written += take; } + Ok(written) + } - Ok(result) + /// How the stream ended, if it has: a peer's stop wins over the end it interrupted. + pub(super) fn end(&self) -> Option { + if let Some(code) = self.stop_reason { + return Some(End::Stopped(code)); + } + match self.state { + SendState::ResetSent { error_code, .. } => Some(End::Reset(error_code)), + SendState::DataSent { finish_acked: true } if self.pending.is_fully_acked() => Some(End::Finished), + _ => None, + } } /// Update stream state due to a reset sent by the local application @@ -229,15 +250,10 @@ impl Send { } /// Whether the peer acknowledged everything before `offset`, noting to report it if not - pub(super) fn acked(&mut self, offset: u64) -> Result { - // A reset cuts the buffer short, so only an unreset stream knows how much was written - assert!( - self.is_reset() || offset <= self.pending.offset(), - "acked past the data written" - ); + pub(super) fn acked(&mut self, offset: u64) -> Result { // An acknowledgment no longer means the peer reads the data, and a reset dropped the rest if self.stop_reason.is_some() || offset > self.pending.offset() { - return Err(ClosedStream::default()); + return Err(Error::Closed); } if offset <= self.pending.acked() { return Ok(true); @@ -288,267 +304,447 @@ impl Send { } } -/// A [`BytesSource`] implementation for `&'a mut [Bytes]` +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(super) enum SendState { + /// Sending new data + Ready, + /// Stream was finished; now sending retransmits only + DataSent { finish_acked: bool }, + /// Reset by the application, while data below `reliable_size` is still delivered + /// + /// The peer gets RESET_STREAM_AT, or, without it, a RESET_STREAM `held` until it has + /// acknowledged that data. + ResetSent { + final_size: u64, + reliable_size: u64, + error_code: VarInt, + held: bool, + /// The smallest Reliable Size an acknowledged reset frame carried + acked: Option, + }, +} + +/// An outgoing stream, from [`Connection::send_stream`](crate::connection::Connection::send_stream) /// -/// The type allows to dequeue [`Bytes`] chunks from an array of chunks, up to -/// a configured limit. -pub(crate) struct BytesArray<'a> { - /// The wrapped slice of `Bytes` - chunks: &'a mut [Bytes], - /// The amount of chunks consumed from this source - consumed: usize, +/// A borrowed view by ID: the stream lives on in the connection after this is dropped, and +/// calls on an ID the stream no longer has state for fail with [`Error::Closed`]. +pub struct Stream<'a> { + pub(crate) id: StreamId, + pub(crate) state: &'a mut StreamsState, + pub(crate) pending: &'a mut Retransmits, + pub(crate) conn_state: &'a ConnState, } -impl<'a> BytesArray<'a> { - pub(crate) fn from_chunks(chunks: &'a mut [Bytes]) -> Self { - Self { chunks, consumed: 0 } +impl<'a> Stream<'a> { + /// Drive a bare `StreamsState` without a connection, for the `streams` fuzz target + #[cfg(any(test, feature = "fuzz"))] + pub(crate) fn new( + id: StreamId, + state: &'a mut StreamsState, + pending: &'a mut Retransmits, + conn_state: &'a ConnState, + ) -> Self { + Self { + id, + state, + pending, + conn_state, + } } -} -impl BytesSource for BytesArray<'_> { - fn pop_chunk(&mut self, limit: usize) -> (Bytes, usize) { - // The loop exists to skip empty chunks while still marking them as - // consumed - let mut chunks_consumed = 0; - - while self.consumed < self.chunks.len() { - let chunk = &mut self.chunks[self.consumed]; - - if chunk.len() <= limit { - let chunk = std::mem::take(chunk); - self.consumed += 1; - chunks_consumed += 1; - if chunk.is_empty() { - continue; + /// Queue as much of `data` as the stream and connection accept now, returning how many bytes + /// + /// Each chunk is taken with [`Buf::copy_to_bytes`], which a [`Bytes`](bytes::Bytes) source + /// splits off without copying. Fails with [`Error::Blocked`] when nothing fits; a + /// [`StreamEvent::Writable`] says when to retry. + pub fn write(&mut self, data: &mut impl Buf) -> Result { + if self.conn_state.is_closed() { + trace!(%self.id, "write blocked; connection draining"); + return Err(Error::Blocked); + } + + let limit = self.state.write_limit(); + let max_data = self.state.max_send_data(self.id); + let stream = StreamsState::send_mut(&mut self.state.send, self.id, max_data)?; + let was_pending = stream.is_pending(); + let written = match stream.write(data, limit) { + Ok(written) => written, + Err(Error::Blocked) => { + if limit == 0 { + trace!( + stream = %self.id, max_data = self.state.max_data, data_sent = self.state.data_sent, + "write blocked by connection-level flow control or send window" + ); + self.state.connection_blocked.insert(self.id); + // Only report the peer's limit, not our own send window. + if self.state.data_sent == self.state.max_data + && self.state.data_blocked_limit != Some(self.state.max_data) + { + self.state.data_blocked_limit = Some(self.state.max_data); + self.pending.data_blocked = true; + } + } else if stream.data_blocked_limit != Some(stream.max_data) { + stream.data_blocked_limit = Some(stream.max_data); + self.pending.stream_data_blocked.insert(self.id); } - return (chunk, chunks_consumed); - } else if limit > 0 { - let chunk = chunk.split_to(limit); - return (chunk, chunks_consumed); - } else { - break; + return Err(Error::Blocked); } + Err(e) => return Err(e), + }; + self.state.data_sent += written as u64; + self.state.buffered_data += written as u64; + trace!(stream = %self.id, "wrote {} bytes", written); + if !was_pending { + self.state.pending.push(self.id, &mut stream.slot, stream.priority); } - - (Bytes::new(), chunks_consumed) + Ok(written) } -} - -/// A [`BytesSource`] implementation for `&[u8]` -/// -/// The type allows to dequeue a single [`Bytes`] chunk, which will be lazily -/// created from a reference. This allows to defer the allocation until it is -/// known how much data needs to be copied. -pub(crate) struct ByteSlice<'a> { - /// The wrapped byte slice - data: &'a [u8], -} -impl<'a> ByteSlice<'a> { - pub(crate) fn from_slice(data: &'a [u8]) -> Self { - Self { data } + /// How the stream ended, or `None` while it is still going + /// + /// The peer's stop, a reset, or a finish every byte of which the peer acknowledged. Kept + /// until [`Self::release`], so it can be asked after the stream's state is gone. + pub fn end(&self) -> Result, Error> { + match self.state.send.get(&self.id) { + Some(Some(stream)) if stream.released => Err(Error::Closed), + Some(Some(stream)) => Ok(stream.end()), + Some(None) => Ok(None), + None => self.state.ends.get(&self.id).map(|&end| Some(end)).ok_or(Error::Closed), + } } -} -impl BytesSource for ByteSlice<'_> { - fn pop_chunk(&mut self, limit: usize) -> (Bytes, usize) { - let limit = limit.min(self.data.len()); - if limit == 0 { - return (Bytes::new(), 0); + /// Give up the stream's ID: its [`end`](Self::end) is no longer kept + /// + /// The stream itself carries on, so finish or reset it first. Every later call on the ID + /// fails with [`Error::Closed`]. Without it, the end of every finished or reset stream is + /// kept for the connection's life. + pub fn release(self) { + if self.state.ends.remove(&self.id).is_some() { + return; } + let max_data = self.state.max_send_data(self.id); + if let Ok(stream) = StreamsState::send_mut(&mut self.state.send, self.id, max_data) { + stream.released = true; + } + } - let chunk = Bytes::from(self.data[..limit].to_owned()); - self.data = &self.data[chunk.len()..]; + /// Finish the stream, signalling that no more data will be sent + /// + /// If this fails, the stream never reaches [`End::Finished`]. + pub fn finish(&mut self) -> Result<(), Error> { + let max_data = self.state.max_send_data(self.id); + let stream = StreamsState::send_mut(&mut self.state.send, self.id, max_data)?; + let was_pending = stream.is_pending(); + stream.finish()?; + self.state.connection_blocked.swap_remove(&self.id); + if !was_pending { + self.state.pending.push(self.id, &mut stream.slot, stream.priority); + } - let chunks_consumed = usize::from(self.data.is_empty()); - (chunk, chunks_consumed) + Ok(()) } -} -/// A source of one or more buffers which can be converted into `Bytes` buffers on demand -/// -/// The purpose of this data type is to defer conversion as long as possible, -/// so that no heap allocation is required in case no data is writable. -pub(super) trait BytesSource { - /// Returns the next chunk from the source of owned chunks. - /// - /// This method will consume parts of the source. - /// Calling it will yield `Bytes` elements up to the configured `limit`. + /// Abandon transmitting data on the stream /// - /// The method returns a tuple: - /// - The first item is the yielded `Bytes` element. The element will be - /// empty if the limit is zero or no more data is available. - /// - The second item returns how many complete chunks inside the source had - /// had been consumed. This can be less than 1, if a chunk inside the - /// source had been truncated in order to adhere to the limit. It can also - /// be more than 1, if zero-length chunks had been skipped. - fn pop_chunk(&mut self, limit: usize) -> (Bytes, usize); -} + /// Fails with [`Error::Closed`] if the stream was already reset or is gone. + pub fn reset(&mut self, error_code: VarInt) -> Result<(), Error> { + self.reset_to(0, error_code) + } -/// Indicates how many bytes and chunks had been transferred in a write operation -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -pub struct Written { - /// The amount of bytes which had been written - pub bytes: usize, - /// The amount of full chunks which had been written + /// The offset the next write starts at, which is how many bytes were written so far /// - /// If a chunk was only partially written, it will not be counted by this field. - pub chunks: usize, -} + /// After a reset, the bytes the reset still delivers. Zero before the first write, or once + /// the stream is gone. + pub fn offset(&self) -> Offset { + let value = match self.state.send.get(&self.id) { + Some(Some(stream)) => stream.offset(), + _ => 0, + }; + Offset { + id: self.id, + tag: self.state.tag, + value, + } + } -/// Errors triggered while writing to a send stream -#[derive(Debug, Error, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum WriteError { - /// The peer is not able to accept additional data, or the connection is congested. + /// Whether the peer acknowledged every byte before `offset` /// - /// If the peer issues additional flow control credit, a [`StreamEvent::Writable`] event will - /// be generated, indicating that retrying the write might succeed. + /// A `false` notes to report a [`StreamEvent::Acked`] once the answer changes, so poll again + /// then. The stream holds only its lowest waiting offset, so every waiter on it polls again + /// on that event. Like [`End::Finished`], an acknowledgment only means the peer's QUIC stack + /// holds the data, not that its application read it. /// - /// [`StreamEvent::Writable`]: crate::StreamEvent::Writable - #[error("unable to accept further writes")] - Blocked, - /// The peer is no longer accepting data on this stream, and it has been implicitly reset. The - /// stream cannot be finished or further written to. + /// Fails with [`Error::Closed`] once the peer will never acknowledge it, or no longer reads + /// it: a reset dropped the data before `offset`, the peer sent STOP_SENDING, or the + /// connection closed. /// - /// Carries an application-defined error code. + /// # Panics + /// - when `offset` came from another stream + pub fn acked(&mut self, offset: Offset) -> Result { + let offset = self.own(offset); + if self.conn_state.is_closed() { + return Err(Error::Closed); + } + match self.state.send.get_mut(&self.id) { + Some(Some(stream)) if !stream.released => stream.acked(offset), + // Nothing was written, so an offset of it is 0. + Some(None) => Ok(true), + Some(Some(_)) => Err(Error::Closed), + None => match self.state.ends.get(&self.id) { + Some(End::Finished) => Ok(true), + _ => Err(Error::Closed), + }, + } + } + + /// Abandon the stream, but still deliver the bytes before `offset` /// - /// [`StreamEvent::Finished`]: crate::StreamEvent::Finished - #[error("stopped by peer: code {0}")] - Stopped(VarInt), - /// The stream has not been opened or has already been finished or reset - #[error("closed stream")] - ClosedStream, -} + /// Nothing past `offset` is sent or retransmitted. A peer that accepts RESET_STREAM_AT gets + /// it at once, and reads the data before the error. Otherwise the data keeps being + /// retransmitted, and a plain RESET_STREAM follows once the peer has acknowledged all of it, a + /// round trip later. An acknowledgment only means the peer's QUIC stack holds the data: + /// RFC 9000 lets it discard data its application has not read yet on RESET_STREAM, so this + /// only reliably delivers data the peer parses on arrival, like a WebTransport stream header. + /// A peer that asked us to stop sending gets the reset at once, without the data. + /// + /// Fails with [`Error::Closed`] if the stream was already reset or is gone. + /// + /// # Panics + /// - when `offset` came from another stream + pub fn reset_at(&mut self, offset: Offset, error_code: VarInt) -> Result<(), Error> { + let offset = self.own(offset); + self.reset_to(offset, error_code) + } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub(super) enum SendState { - /// Sending new data - Ready, - /// Stream was finished; now sending retransmits only - DataSent { finish_acked: bool }, - /// Reset by the application, while data below `reliable_size` is still delivered + /// The value of an offset minted for this stream. + fn own(&self, offset: Offset) -> u64 { + assert!( + offset.id == self.id && offset.tag == self.state.tag, + "offset used on another stream" + ); + offset.value + } + + fn reset_to(&mut self, offset: u64, error_code: VarInt) -> Result<(), Error> { + let at = self.state.peer_reset_stream_at; + let max_data = self.state.max_send_data(self.id); + let stream = StreamsState::send_mut(&mut self.state.send, self.id, max_data)?; + if stream.is_reset() { + // Redundant reset call + return Err(Error::Closed); + } + // An offset is minted from what was written, which only grows until a reset. + debug_assert!(offset <= stream.offset(), "reset past the data written"); + // The peer that sent STOP_SENDING will not read the prefix + let reliable_size = match stream.stop_reason { + Some(_) => 0, + None => offset, + }; + + // Restore the portion of the send window consumed by the data that we aren't about to + // send. We leave flow control alone because the peer's responsible for issuing additional + // credit based on the final offset communicated in the RESET_STREAM frame we send. + let buffered = stream.pending.buffered(); + let send = stream.reset(error_code, reliable_size, at); + self.state.connection_blocked.swap_remove(&self.id); + self.state.buffered_data -= buffered - stream.pending.buffered(); + if let Some(offset) = stream.take_waiting() { + self.state.events.push_back(StreamEvent::Acked { id: self.id, offset }); + } + // Only data below the Reliable Size is left to send + if !stream.is_pending() { + self.state.pending.remove(&mut stream.slot); + } + if let Some(reliable_size) = send { + self.pending.reset_stream.push((self.id, reliable_size)); + } + + Ok(()) + } + + /// Set the priority within the stream's send group, or session-wide when ungrouped /// - /// The peer gets RESET_STREAM_AT, or, without it, a RESET_STREAM `held` until it has - /// acknowledged that data. - ResetSent { - final_size: u64, - reliable_size: u64, - error_code: VarInt, - held: bool, - /// The smallest Reliable Size an acknowledged reset frame carried - acked: Option, - }, -} + /// Higher values go first; equal values share bytes round robin. A queued stream moves at + /// once, behind the streams already queued at its new priority. With + /// [`transport::Config::send_fairness`](crate::transport::Config::send_fairness) off, a stream + /// its group is partway through keeps going first until it runs out of data. + pub fn set_priority(&mut self, priority: i64) -> Result<(), Error> { + let max_data = self.state.max_send_data(self.id); + let stream = StreamsState::send_mut(&mut self.state.send, self.id, max_data)?; + stream.priority = priority; + self.state.pending.set_stream_priority(&mut stream.slot, priority); + Ok(()) + } -/// Reasons why attempting to finish a stream might fail -#[derive(Debug, Error, Clone, PartialEq, Eq)] -pub enum FinishError { - /// The peer is no longer accepting data on this stream. No - /// [`StreamEvent::Finished`] event will be emitted for this stream. + /// The priority of the stream + pub fn priority(&self) -> Result { + match self.state.send.get(&self.id) { + Some(Some(stream)) if stream.released => Err(Error::Closed), + Some(stream) => Ok(stream.as_ref().map(|s| s.priority).unwrap_or_default()), + None => Err(Error::Closed), + } + } + + /// Move the stream into a send group, or with `None` back to the ungrouped streams /// - /// Carries an application-defined error code. + /// Ungrouped streams are sent before any send group. A queued stream keeps its place. /// - /// [`StreamEvent::Finished`]: crate::StreamEvent::Finished - #[error("stopped by peer: code {0}")] + /// # Panics + /// - when `group` was opened on another connection + pub fn set_group(&mut self, group: Option<&Group>) -> Result<(), Error> { + let group = group.map(|g| g.index(self.state.tag)); + let max_data = self.state.max_send_data(self.id); + let stream = StreamsState::send_mut(&mut self.state.send, self.id, max_data)?; + self.state + .pending + .join(self.id, &mut stream.slot, group, stream.priority); + Ok(()) + } +} + +/// A position in an outgoing stream, from [`Stream::offset`] +/// +/// Only the stream it came from takes it, so it never points past the data written. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Offset { + id: StreamId, + tag: ConnectionTag, + value: u64, +} + +impl Offset { + /// The same stream's offset `value` bytes in, at or before this one, for tests and the fuzz + /// target. + #[cfg(any(test, feature = "fuzz"))] + pub(crate) fn rewind(self, value: u64) -> Self { + assert!(value <= self.value, "rewound past {}", self.value); + Self { value, ..self } + } +} + +impl From for u64 { + fn from(offset: Offset) -> Self { + offset.value + } +} + +/// How an outgoing stream ended, from [`Stream::end`] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum End { + /// Finished, and the peer acknowledged every byte + Finished, + /// The peer asked us to stop sending, with its code Stopped(VarInt), - /// The stream has not been opened or was already finished or reset - #[error("closed stream")] - ClosedStream, + /// Reset by us, with our code + Reset(VarInt), } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn bytes_array() { - let full = b"Hello World 123456789 ABCDEFGHJIJKLMNOPQRSTUVWXYZ".to_owned(); - for limit in 0..full.len() { - let mut chunks = [ - Bytes::from_static(b""), - Bytes::from_static(b"Hello "), - Bytes::from_static(b"Wo"), - Bytes::from_static(b""), - Bytes::from_static(b"r"), - Bytes::from_static(b"ld"), - Bytes::from_static(b""), - Bytes::from_static(b" 12345678"), - Bytes::from_static(b"9 ABCDE"), - Bytes::from_static(b"F"), - Bytes::from_static(b"GHJIJKLMNOPQRSTUVWXYZ"), - ]; - let num_chunks = chunks.len(); - let last_chunk_len = chunks[chunks.len() - 1].len(); - - let mut array = BytesArray::from_chunks(&mut chunks); - - let mut buf = Vec::new(); - let mut chunks_popped = 0; - let mut chunks_consumed = 0; - let mut remaining = limit; - loop { - let (chunk, consumed) = array.pop_chunk(remaining); - chunks_consumed += consumed; - - if !chunk.is_empty() { - buf.extend_from_slice(&chunk); - remaining -= chunk.len(); - chunks_popped += 1; - } else { - break; - } - } +/// Scheduling of outgoing streams between send groups +/// +/// A higher priority group preempts every lower one, and groups of equal priority get an +/// equal share of bytes no matter how many streams each has queued. Streams outside any group +/// go before all of it. A group's datagrams, from [`Datagrams::send`](crate::Datagrams::send), +/// share its bytes in the same order. +pub struct Groups<'a> { + pub(crate) state: &'a mut StreamsState, +} - assert_eq!(&buf[..], &full[..limit]); - - if limit == full.len() { - // Full consumption of the last chunk - assert_eq!(chunks_consumed, num_chunks); - // Since there are empty chunks, we consume more than there are popped - assert_eq!(chunks_consumed, chunks_popped + 3); - } else if limit > full.len() - last_chunk_len { - // Partial consumption of the last chunk - assert_eq!(chunks_consumed, num_chunks - 1); - assert_eq!(chunks_consumed, chunks_popped + 2); - } +impl Groups<'_> { + /// Open a send group at `priority` + pub fn open(&mut self, priority: i64) -> Group { + Group { + index: self.state.pending.open_group(priority), + tag: self.state.tag, } } - #[test] - fn byte_slice() { - let full = b"Hello World 123456789 ABCDEFGHJIJKLMNOPQRSTUVWXYZ".to_owned(); - for limit in 0..full.len() { - let mut array = ByteSlice::from_slice(&full[..]); + /// Release a send group; its streams keep their place until they finish + /// + /// # Panics + /// - when `group` was opened on another connection + pub fn close(&mut self, group: Group) { + let index = group.index(self.state.tag); + self.state.pending.close_group(index); + } - let mut buf = Vec::new(); - let mut chunks_popped = 0; - let mut chunks_consumed = 0; - let mut remaining = limit; - loop { - let (chunk, consumed) = array.pop_chunk(remaining); - chunks_consumed += consumed; + /// Change a group's priority, moving any data it has queued along with it + /// + /// # Panics + /// - when `group` was opened on another connection + pub fn set_priority(&mut self, group: &Group, priority: i64) { + let index = group.index(self.state.tag); + self.state.pending.set_group_priority(index, priority); + } - if !chunk.is_empty() { - buf.extend_from_slice(&chunk); - remaining -= chunk.len(); - chunks_popped += 1; - } else { - break; - } - } + /// The group's priority + /// + /// # Panics + /// - when `group` was opened on another connection + pub fn priority(&self, group: &Group) -> i64 { + self.state.pending.group_priority(group.index(self.state.tag)) + } +} - assert_eq!(&buf[..], &full[..limit]); - if limit != 0 { - assert_eq!(chunks_popped, 1); - } else { - assert_eq!(chunks_popped, 0); - } +/// How an outgoing stream or datagram is scheduled, like W3C WebTransport's send options +/// +/// The default is ungrouped at priority 0. +#[derive(Debug, Default, Clone, Copy)] +pub struct Options<'a> { + /// The send group to join, or `None` for the ungrouped streams that go before every group + pub group: Option<&'a Group>, + /// The priority within the group: higher goes first, and a datagram goes ahead of a + /// stream of equal priority + pub priority: i64, +} - if limit == full.len() { - assert_eq!(chunks_consumed, 1); - } else { - assert_eq!(chunks_consumed, 0); - } - } +/// A send group, from [`Groups::open`] +/// +/// Release it with [`Groups::close`] on the connection that opened it. +#[derive(Debug, PartialEq, Eq)] +#[must_use = "a send group lives until it is closed"] +pub struct Group { + index: usize, + tag: ConnectionTag, +} + +impl Group { + pub(crate) fn index(&self, tag: ConnectionTag) -> usize { + assert_eq!(self.tag, tag, "send group used on another connection"); + self.index + } +} + +/// Why a stream write or datagram failed +#[derive(Debug, Error, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// Flow control or the send buffer has no room; a [`StreamEvent::Writable`] says when to + /// retry + #[error("blocked")] + Blocked, + /// The peer asked us to stop sending, with its code + #[error("stopped by peer: code {0}")] + Stopped(VarInt), + /// The datagram is larger than the path or the peer's limit allows + #[error("datagram too large")] + TooLarge, + /// Datagrams are disabled locally or unsupported by the peer + #[error("datagrams unsupported")] + Unsupported, + /// The stream was finished, reset, or released, or was never opened + #[error("closed stream")] + Closed, +} + +impl From for io::Error { + fn from(err: Error) -> Self { + let kind = match err { + Error::Blocked => io::ErrorKind::WouldBlock, + Error::Stopped(_) => io::ErrorKind::ConnectionReset, + Error::TooLarge => io::ErrorKind::InvalidInput, + Error::Unsupported => io::ErrorKind::Unsupported, + Error::Closed => io::ErrorKind::NotConnected, + }; + Self::new(kind, err) } } diff --git a/rs/moq-quic/src/connection/streams/state.rs b/rs/moq-quic/src/connection/streams/state.rs index 2ab7a00ee4e..ce66d36c3c6 100644 --- a/rs/moq-quic/src/connection/streams/state.rs +++ b/rs/moq-quic/src/connection/streams/state.rs @@ -13,6 +13,7 @@ use tracing::{debug, trace}; use super::{ Next, Recv, Retransmits, Schedule, Send, SendState, ShouldTransmit, StreamEvent, StreamHalf, ThinRetransmits, lease::Rank, + send::{End, Error}, }; use crate::{ Dir, MAX_STREAM_COUNT, Side, StreamId, TransportError, VarInt, @@ -87,6 +88,8 @@ pub(crate) struct StreamsState { pub(super) side: Side, // Set of streams that are currently open, or could be immediately opened by the peer pub(super) send: FxHashMap>>, + /// How each send stream whose state is gone ended, until the application releases its ID + pub(super) ends: FxHashMap, pub(super) recv: FxHashMap>, pub(super) free_recv: Vec, pub(super) next: [u64; 2], @@ -116,7 +119,7 @@ pub(crate) struct StreamsState { /// permitted to open but which have not yet been opened. pub(super) send_streams: usize, /// Streams with outgoing data queued and outgoing datagrams, in the order to send them - pub(super) pending: Schedule, + pub(super) pending: Schedule, pub(super) events: VecDeque, /// Streams blocked on connection-level flow control or stream window space @@ -149,6 +152,8 @@ pub(crate) struct StreamsState { /// Receive streams whose reader may progress without new credit, which holds eviction off: /// every such stream, and possibly some that no longer are, trimmed by the stuck pass pub(super) unlooked: FxHashSet, + /// A stuck pass waited on a lease out with its reader, so handing it back reruns the pass + pub(super) stuck_waiting: bool, /// Credit still held by evicted readers that have not stopped their streams, which holds /// further eviction off pub(super) evicted_held: u64, @@ -201,6 +206,7 @@ impl StreamsState { Self { side, send: FxHashMap::default(), + ends: FxHashMap::default(), recv: FxHashMap::default(), free_recv: Vec::new(), next: [0, 0], @@ -226,6 +232,7 @@ impl StreamsState { evicted: 0, peak_spent: 0, unlooked: FxHashSet::default(), + stuck_waiting: false, evicted_held: 0, #[cfg(test)] scans: 0, @@ -471,8 +478,8 @@ impl StreamsState { // A reliable reset lingers until the data below its Reliable Size is acknowledged too, and // one since shrunk by STOP_SENDING until its replacement is if e.get_mut().as_mut().is_some_and(|s| s.reset_acked(reliable_size)) { - if let (_, Some(mut stream)) = e.remove_entry() { - self.pending.leave(&mut stream.slot); + if let (_, Some(stream)) = e.remove_entry() { + self.send_freed(id, stream); } self.stream_freed(id, StreamHalf::Send); } @@ -733,8 +740,8 @@ impl StreamsState { continue; } let datagram = self.pending.sent_datagram(group); - trace!(len = datagram.data.len(), "DATAGRAM"); - datagram.encode(true, buf); + trace!(len = datagram.len(), "DATAGRAM"); + datagram.encode(buf); written.datagrams += 1; continue; } @@ -866,8 +873,8 @@ impl StreamsState { // A reset is not reported as finished, like one without a Reliable Size let reset = stream.is_reset(); - if let (_, Some(mut stream)) = entry.remove_entry() { - self.pending.leave(&mut stream.slot); + if let (_, Some(stream)) = entry.remove_entry() { + self.send_freed(id, stream); } self.stream_freed(id, StreamHalf::Send); if !reset { @@ -1162,6 +1169,34 @@ impl StreamsState { } } + /// Drop a send stream's state, keeping how it ended until the application releases its ID + fn send_freed(&mut self, id: StreamId, mut stream: Box) { + self.pending.leave(&mut stream.slot); + if !stream.released + && let Some(end) = stream.end() + { + self.ends.insert(id, end); + } + } + + /// The state of send stream `id`, allocated on first use + /// + /// A free function over the map, so the caller can still reach the other fields. + pub(super) fn send_mut( + send: &mut FxHashMap>>, + id: StreamId, + max_data: VarInt, + ) -> Result<&mut Send, Error> { + let stream = send + .get_mut(&id) + .ok_or(Error::Closed)? + .get_or_insert_with(|| Send::new(max_data)); + match stream.released { + true => Err(Error::Closed), + false => Ok(stream), + } + } + /// Update counters for removal of a stream pub(super) fn stream_freed(&mut self, id: StreamId, half: StreamHalf) { if id.initiator() != self.side { @@ -1225,10 +1260,12 @@ pub(super) fn get_or_insert_recv(opt: &mut Option, initial_max_data: mod tests { use super::*; use crate::{ - LeaseError, ReadableError, RecvStream, SendStream, TransportErrorCode, WriteError, - connection::State as ConnState, connection::Streams, + TransportErrorCode, + connection::State as ConnState, + connection::Streams, + connection::streams::{recv, send}, }; - use bytes::Bytes; + use bytes::{Buf, Bytes}; fn make(side: Side) -> StreamsState { StreamsState::new( @@ -1258,21 +1295,21 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - assert_eq!(stream.write(b"blocked"), Err(WriteError::Blocked)); - assert_eq!(stream.write(b"blocked"), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &b"blocked"[..]), Err(send::Error::Blocked)); + assert_eq!(stream.write(&mut &b"blocked"[..]), Err(send::Error::Blocked)); assert_eq!(stream.state.connection_blocked.len(), 1); stream.reset(0u32.into()).unwrap(); assert!( stream.state.connection_blocked.is_empty(), "reset writer no longer waits for credit" ); - assert_eq!(stream.write(b"blocked"), Err(WriteError::ClosedStream)); + assert_eq!(stream.write(&mut &b"blocked"[..]), Err(send::Error::Closed)); server.reset_acked(id, 0); assert!(server.send.is_empty()); assert!(server.connection_blocked.is_empty()); @@ -1295,15 +1332,15 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - assert_eq!(stream.write(b"blocked"), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &b"blocked"[..]), Err(send::Error::Blocked)); stream.finish().unwrap(); - assert_eq!(stream.write(b"closed"), Err(WriteError::ClosedStream)); + assert_eq!(stream.write(&mut &b"closed"[..]), Err(send::Error::Closed)); server.received_ack_of(frame::StreamMeta { id, offsets: 0..0, @@ -1333,13 +1370,13 @@ mod tests { .open(Dir::Uni, Default::default()) .unwrap(); assert_eq!( - SendStream { + send::Stream { id: holder, state: &mut server, pending: &mut pending, conn_state: &state } - .write(b"x"), + .write(&mut &b"x"[..]), Ok(1) ); let id = Streams { @@ -1348,21 +1385,21 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - assert_eq!(stream.write(b"blocked"), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &b"blocked"[..]), Err(send::Error::Blocked)); let code = 7u32.into(); stream.state.received_stop_sending(id, code).unwrap(); assert_eq!(stream.state.poll(), Some(StreamEvent::Stopped { id, error_code: code })); - assert_eq!(stream.write(b"stopped"), Err(WriteError::Stopped(code))); - assert_eq!(stream.write(&[]), Err(WriteError::Stopped(code))); - let mut chunks = [Bytes::from_static(b"stopped")]; - assert_eq!(stream.write_chunks(&mut chunks), Err(WriteError::Stopped(code))); - assert_eq!(chunks[0], b"stopped"[..]); + assert_eq!(stream.write(&mut &b"stopped"[..]), Err(send::Error::Stopped(code))); + assert_eq!(stream.write(&mut &[][..]), Err(send::Error::Stopped(code))); + let mut chunk = Bytes::from_static(b"stopped"); + assert_eq!(stream.write(&mut chunk), Err(send::Error::Stopped(code))); + assert_eq!(chunk, b"stopped"[..]); assert!(stream.state.connection_blocked.is_empty()); assert_eq!(stream.state.write_limit(), 0); assert_eq!(stream.state.send[&holder].as_ref().unwrap().pending.buffered(), 1); @@ -1386,19 +1423,19 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - assert_eq!(stream.write(b"x"), Err(WriteError::Blocked)); - assert_eq!(stream.write(b"x"), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &b"x"[..]), Err(send::Error::Blocked)); + assert_eq!(stream.write(&mut &b"x"[..]), Err(send::Error::Blocked)); id }) .collect(); server.received_stop_sending(ids[0], 9u32.into()).unwrap(); - SendStream { + send::Stream { id: ids[1], state: &mut server, pending: &mut pending, @@ -1406,7 +1443,7 @@ mod tests { } .reset(0u32.into()) .unwrap(); - SendStream { + send::Stream { id: ids[2], state: &mut server, pending: &mut pending, @@ -1443,13 +1480,13 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - SendStream { + send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, } - .write(&[42; 24]) + .write(&mut &[42; 24][..]) .unwrap(); let send = server.send.get_mut(&id).unwrap().as_mut().unwrap(); assert_eq!(send.pending.poll_transmit(16).0, 0..16); @@ -1478,13 +1515,13 @@ mod tests { fin: false, }); assert_eq!(server.write_limit(), 24); - SendStream { + send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, } - .write(&[7; 24]) + .write(&mut &[7; 24][..]) .unwrap(); assert_eq!(server.write_limit(), 0); } @@ -1507,13 +1544,13 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - SendStream { + send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, } - .write(&[42; 24]) + .write(&mut &[42; 24][..]) .unwrap(); let mut buf = Vec::with_capacity(1000); let sent = server.write_frames(&mut buf, 0..1000, true).streams; @@ -1544,13 +1581,13 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - SendStream { + send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, } - .write(&[42; 24]) + .write(&mut &[42; 24][..]) .unwrap(); let send = server.send.get_mut(&id).unwrap().as_mut().unwrap(); send.pending.poll_transmit(16); @@ -1560,7 +1597,7 @@ mod tests { offsets: 16..23, fin: false, }); - SendStream { + send::Stream { id, state: &mut server, pending: &mut pending, @@ -1625,7 +1662,7 @@ mod tests { assert_eq!(client.local_max_data - initial_max, 0); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -1666,7 +1703,7 @@ mod tests { ); let mut pending = Retransmits::default(); - RecvStream { + recv::Stream { id, state: &mut client, pending: &mut pending, @@ -1699,7 +1736,7 @@ mod tests { assert_eq!(client.local_max_data - initial_max, 0); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -1725,13 +1762,13 @@ mod tests { assert_eq!(client.local_max_data - initial_max, 4096); // Ensure reading after a reset doesn't issue redundant credit - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, }; let mut chunks = recv.read().unwrap(); - assert_eq!(chunks.next(1024).unwrap_err(), crate::ReadError::Reset(0u32.into())); + assert_eq!(chunks.next(1024).unwrap_err(), crate::recv::Error::Reset(0u32.into())); let _ = chunks.finalize(); assert_eq!(client.data_recvd, 4096); assert_eq!(client.local_max_data - initial_max, 4096); @@ -1825,7 +1862,7 @@ mod tests { assert_eq!(client.local_max_data, initial_max); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -1836,7 +1873,7 @@ mod tests { assert!(!recv.pending.max_data); assert!(recv.stop(0u32.into()).is_err()); - assert_eq!(recv.read().err(), Some(ReadableError::ClosedStream)); + assert_eq!(recv.read().err(), Some(recv::Error::Closed)); assert_eq!(client.local_max_data - initial_max, 32); assert_eq!( @@ -1880,7 +1917,7 @@ mod tests { ); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -1929,7 +1966,7 @@ mod tests { .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, @@ -1941,10 +1978,10 @@ mod tests { assert!(stream.state.events.contains(&StreamEvent::Stopped { id, error_code })); stream.state.events.clear(); - assert_eq!(stream.write(&[]), Err(WriteError::Stopped(error_code))); + assert_eq!(stream.write(&mut &[][..]), Err(send::Error::Stopped(error_code))); stream.reset(0u32.into()).unwrap(); - assert_eq!(stream.write(&[]), Err(WriteError::ClosedStream)); + assert_eq!(stream.write(&mut &[][..]), Err(send::Error::Closed)); // A duplicate frame is a no-op stream.state.received_stop_sending(id, error_code).unwrap(); @@ -1988,31 +2025,31 @@ mod tests { let id_mid = streams.open(Dir::Bi, Default::default()).unwrap(); let id_low = streams.open(Dir::Bi, Default::default()).unwrap(); - let mut mid = SendStream { + let mut mid = send::Stream { id: id_mid, state: &mut server, pending: &mut pending, conn_state: &state, }; - mid.write(b"mid").unwrap(); + mid.write(&mut &b"mid"[..]).unwrap(); - let mut low = SendStream { + let mut low = send::Stream { id: id_low, state: &mut server, pending: &mut pending, conn_state: &state, }; low.set_priority(-1).unwrap(); - low.write(b"low").unwrap(); + low.write(&mut &b"low"[..]).unwrap(); - let mut high = SendStream { + let mut high = send::Stream { id: id_high, state: &mut server, pending: &mut pending, conn_state: &state, }; high.set_priority(1).unwrap(); - high.write(b"high").unwrap(); + high.write(&mut &b"high"[..]).unwrap(); let mut buf = Vec::with_capacity(40); let meta = server.write_frames(&mut buf, 0..40, true).streams; @@ -2043,14 +2080,14 @@ mod tests { let id_low = streams.open(Dir::Bi, Default::default()).unwrap(); let id_high = streams.open(Dir::Bi, Default::default()).unwrap(); - let mut low = SendStream { + let mut low = send::Stream { id: id_low, state: &mut server, pending: &mut pending, conn_state: &state, }; low.set_priority(-1).unwrap(); - assert_eq!(low.write(b"low").unwrap(), 3); + assert_eq!(low.write(&mut &b"low"[..]).unwrap(), 3); let mut buf = Vec::with_capacity(40); let mut sent = server.write_frames(&mut buf, 0..40, fair).streams; @@ -2065,14 +2102,14 @@ mod tests { server.retransmit(lost); assert_eq!(server.pending.len(), 1); - let mut high = SendStream { + let mut high = send::Stream { id: id_high, state: &mut server, pending: &mut pending, conn_state: &state, }; high.set_priority(1).unwrap(); - assert_eq!(high.write(b"high").unwrap(), 4); + assert_eq!(high.write(&mut &b"high"[..]).unwrap(), 4); buf.clear(); let sent = server.write_frames(&mut buf, 0..40, fair).streams; @@ -2105,27 +2142,27 @@ mod tests { let id_high = streams.open(Dir::Bi, Default::default()).unwrap(); let id_mid = streams.open(Dir::Bi, Default::default()).unwrap(); - let mut mid = SendStream { + let mut mid = send::Stream { id: id_mid, state: &mut server, pending: &mut pending, conn_state: &state, }; - assert_eq!(mid.write(b"mid").unwrap(), 3); + assert_eq!(mid.write(&mut &b"mid"[..]).unwrap(), 3); assert_eq!(server.pending.len(), 1); - let mut high = SendStream { + let mut high = send::Stream { id: id_high, state: &mut server, pending: &mut pending, conn_state: &state, }; high.set_priority(1).unwrap(); - assert_eq!(high.write(&[0; 200]).unwrap(), 200); + assert_eq!(high.write(&mut &[0; 200][..]).unwrap(), 200); assert_eq!(server.pending.len(), 2); // Lowering the queued high priority stream moves it behind mid at once. - let mut high = SendStream { + let mut high = send::Stream { id: id_high, state: &mut server, pending: &mut pending, @@ -2167,14 +2204,14 @@ mod tests { let (busy, starved) = (&ids[..2], ids[2]); for (&id, priority) in ids.iter().zip([5, 5, 0]) { - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; stream.set_priority(priority).unwrap(); - stream.write(&[0; 200]).unwrap(); + stream.write(&mut &[0; 200][..]).unwrap(); } let mut buf = Vec::with_capacity(1000); @@ -2184,7 +2221,7 @@ mod tests { assert!(busy.contains(&meta[0].id), "{meta:?}"); } - SendStream { + send::Stream { id: starved, state: &mut server, pending: &mut pending, @@ -2219,29 +2256,29 @@ mod tests { let id_b = streams.open(Dir::Bi, Default::default()).unwrap(); let id_c = streams.open(Dir::Bi, Default::default()).unwrap(); - let mut stream_a = SendStream { + let mut stream_a = send::Stream { id: id_a, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream_a.write(&[b'a'; 100]).unwrap(); + stream_a.write(&mut &[b'a'; 100][..]).unwrap(); - let mut stream_b = SendStream { + let mut stream_b = send::Stream { id: id_b, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream_b.write(&[b'b'; 100]).unwrap(); + stream_b.write(&mut &[b'b'; 100][..]).unwrap(); - let mut stream_c = SendStream { + let mut stream_c = send::Stream { id: id_c, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream_c.write(&[b'c'; 100]).unwrap(); + stream_c.write(&mut &[b'c'; 100][..]).unwrap(); let mut metas = vec![]; let mut buf = Vec::with_capacity(1024); @@ -2293,21 +2330,21 @@ mod tests { let id_b = streams.open(Dir::Bi, Default::default()).unwrap(); let id_c = streams.open(Dir::Bi, Default::default()).unwrap(); - let mut stream_a = SendStream { + let mut stream_a = send::Stream { id: id_a, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream_a.write(&[b'a'; 100]).unwrap(); + stream_a.write(&mut &[b'a'; 100][..]).unwrap(); - let mut stream_b = SendStream { + let mut stream_b = send::Stream { id: id_b, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream_b.write(&[b'b'; 100]).unwrap(); + stream_b.write(&mut &[b'b'; 100][..]).unwrap(); let mut metas = vec![]; let mut buf = Vec::with_capacity(1024); @@ -2319,14 +2356,14 @@ mod tests { metas.extend(meta); // Queue stream_c which has higher priority - let mut stream_c = SendStream { + let mut stream_c = send::Stream { id: id_c, state: &mut server, pending: &mut pending, conn_state: &state, }; stream_c.set_priority(1).unwrap(); - stream_c.write(&[b'b'; 100]).unwrap(); + stream_c.write(&mut &[b'b'; 100][..]).unwrap(); // loop until all the streams are written loop { @@ -2367,7 +2404,7 @@ mod tests { ) .unwrap(); let mut pending = Retransmits::default(); - let mut stream = RecvStream { + let mut stream = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -2393,13 +2430,13 @@ mod tests { }; let id = streams.open(Dir::Uni, Default::default()).unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello").unwrap(); + stream.write(&mut &b"hello"[..]).unwrap(); stream.reset(0u32.into()).unwrap(); assert_eq!(pending.reset_stream, &[(id, 0)]); @@ -2441,7 +2478,7 @@ mod tests { // Free stream 127 let mut pending = Retransmits::default(); - let mut stream = RecvStream { + let mut stream = recv::Stream { id: StreamId::new(Side::Server, Dir::Uni, 127), state: &mut client, pending: &mut pending, @@ -2536,7 +2573,7 @@ mod tests { // Free stream 127 let mut pending = Retransmits::default(); - let mut stream = RecvStream { + let mut stream = recv::Stream { id: StreamId::new(Side::Server, Dir::Uni, 127), state: &mut client, pending: &mut pending, @@ -2571,7 +2608,7 @@ mod tests { Ok(ShouldTransmit(false)) ); let mut pending = Retransmits::default(); - let mut stream = RecvStream { + let mut stream = recv::Stream { id: StreamId::new(Side::Server, Dir::Uni, 126), state: &mut client, pending: &mut pending, @@ -2656,7 +2693,7 @@ mod tests { ); let id = StreamId::new(Side::Server, Dir::Uni, 3); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -2685,7 +2722,7 @@ mod tests { ); // Drain the stream so it's fully freed let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -2734,7 +2771,7 @@ mod tests { Ok(ShouldTransmit(false)) ); let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, @@ -2838,7 +2875,7 @@ mod tests { .open(Dir::Uni, Default::default()) .expect("should be able to open a stream"); - let mut stream = SendStream { + let mut stream = send::Stream { id: stream_id, state: &mut server, pending: &mut retransmits, @@ -2849,18 +2886,18 @@ mod tests { let initial_send_len = initial_send_window as usize; let data = vec![0xFFu8; initial_send_len]; - assert_eq!(stream.write(&data), Ok(initial_send_len)); + assert_eq!(stream.write(&mut &data[..]), Ok(initial_send_len)); // Try to write the same data again, observe that it's blocked - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); // Check that we get a `Writable` event after increasing the send window stream.state.set_send_window(larger_send_window); assert_eq!(stream.state.poll(), Some(StreamEvent::Writable { id: stream_id })); // Check that the stream accepts the exact same amount of data again - assert_eq!(stream.write(&data), Ok(initial_send_len)); - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Ok(initial_send_len)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); assert_eq!(stream.state.poll(), None); @@ -2874,9 +2911,9 @@ mod tests { assert_eq!(stream.state.poll(), Some(StreamEvent::Writable { id: stream_id })); // Check that our full send window is available again - assert_eq!(stream.write(&data), Ok(initial_send_len)); - assert_eq!(stream.write(&data), Ok(initial_send_len)); - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Ok(initial_send_len)); + assert_eq!(stream.write(&mut &data[..]), Ok(initial_send_len)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); } #[test] @@ -2907,7 +2944,7 @@ mod tests { .open(Dir::Uni, Default::default()) .expect("should be able to open a stream"); - let mut stream = SendStream { + let mut stream = send::Stream { id: stream_id, state: &mut server, pending: &mut retransmits, @@ -2919,8 +2956,8 @@ mod tests { let data = vec![0xFFu8; initial_send_len]; // Assert that the full send window is accepted - assert_eq!(stream.write(&data), Ok(initial_send_len)); - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Ok(initial_send_len)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); assert_eq!(stream.state.write_limit(), 0); assert_eq!(stream.state.poll(), None); @@ -2931,7 +2968,7 @@ mod tests { assert_eq!(stream.state.poll(), None); // Assert that data is still not accepted - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); // Ack some data, assert that writes are still not accepted due to outstanding sends stream.state.received_ack_of(frame::StreamMeta { @@ -2940,7 +2977,7 @@ mod tests { fin: false, }); - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); // Ack the rest of the data stream.state.received_ack_of(frame::StreamMeta { @@ -2954,8 +2991,8 @@ mod tests { assert_eq!(stream.state.write_limit(), smaller_send_window); // Assert that only `smaller_send_window` bytes are accepted - assert_eq!(stream.write(&data), Ok(smaller_send_window as usize)); - assert_eq!(stream.write(&data), Err(WriteError::Blocked)); + assert_eq!(stream.write(&mut &data[..]), Ok(smaller_send_window as usize)); + assert_eq!(stream.write(&mut &data[..]), Err(send::Error::Blocked)); } #[test] @@ -2997,9 +3034,9 @@ mod tests { } /// Read everything readable, returning the byte count and how the read stopped - fn drain(state: &mut StreamsState, id: StreamId) -> (u64, crate::ReadError) { + fn drain(state: &mut StreamsState, id: StreamId) -> (u64, recv::Error) { let mut pending = Retransmits::default(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state, pending: &mut pending, @@ -3032,7 +3069,7 @@ mod tests { // Data past the Reliable Size is dropped without counting twice receive(&mut client, id, 4..20); assert_eq!(client.data_recvd, 100); - assert_eq!(drain(&mut client, id), (10, crate::ReadError::Reset(7u32.into()))); + assert_eq!(drain(&mut client, id), (10, crate::recv::Error::Reset(7u32.into()))); assert_eq!(client.local_max_data - initial_max, 100); } @@ -3043,9 +3080,9 @@ mod tests { let initial_max = client.local_max_data; let _ = client.received_reset(reset_at(id, 7, 20, Some(10))).unwrap(); receive(&mut client, id, 4..20); - assert_eq!(drain(&mut client, id), (0, crate::ReadError::Blocked)); + assert_eq!(drain(&mut client, id), (0, crate::recv::Error::Blocked)); receive(&mut client, id, 0..4); - assert_eq!(drain(&mut client, id), (10, crate::ReadError::Reset(7u32.into()))); + assert_eq!(drain(&mut client, id), (10, crate::recv::Error::Reset(7u32.into()))); assert_eq!(client.local_max_data - initial_max, 20); } @@ -3064,7 +3101,7 @@ mod tests { ); assert_eq!(client.local_max_data - initial_max, 16); receive(&mut client, id, 0..20); - assert_eq!(drain(&mut client, id), (4, crate::ReadError::Reset(7u32.into()))); + assert_eq!(drain(&mut client, id), (4, crate::recv::Error::Reset(7u32.into()))); assert_eq!(client.local_max_data - initial_max, 20); } @@ -3077,7 +3114,7 @@ mod tests { receive(&mut client, id, 0..2); let _ = client.received_reset(reset_at(id, 7, 20, Some(10))).unwrap(); let _ = client.received_reset(reset_at(id, 7, 20, None)).unwrap(); - assert_eq!(drain(&mut client, id), (0, crate::ReadError::Reset(7u32.into()))); + assert_eq!(drain(&mut client, id), (0, crate::recv::Error::Reset(7u32.into()))); assert_eq!(client.local_max_data - initial_max, 20); } @@ -3130,7 +3167,7 @@ mod tests { receive(&mut client, id, 0..10); let _ = client.received_reset(reset_at(id, 7, 20, reliable_size)).unwrap(); let mut pending = Retransmits::default(); - RecvStream { + recv::Stream { id, state: &mut client, pending: &mut pending, @@ -3169,16 +3206,16 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); let mut buf = Vec::new(); let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; - stream.reset_at(4, 7u32.into()).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); assert!(pending.reset_stream.is_empty()); assert_eq!(server.buffered_data, 4); @@ -3213,14 +3250,14 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); - stream.reset_at(4, 7u32.into()).unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); assert_eq!(pending.reset_stream, &[(id, 4)]); assert_eq!(server.received_stop_sending(id, 9u32.into()), Ok(Some(0))); @@ -3242,16 +3279,16 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); let mut buf = Vec::new(); let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; - stream.reset_at(4, 7u32.into()).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); server.reset_acked(id, 4); assert_eq!(server.received_stop_sending(id, 9u32.into()), Ok(Some(0))); @@ -3283,14 +3320,14 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); - stream.reset_at(4, 7u32.into()).unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); assert_eq!(pending.reset_stream.is_empty(), !reset_stream_at); assert_eq!(server.received_stop_sending(id, 9u32.into()), Ok(Some(0))); @@ -3316,17 +3353,17 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); let mut buf = Vec::new(); let sent = stream.state.write_frames(&mut buf, 0..100, true).streams; assert_eq!(sent[0].offsets, 0..11); - stream.reset_at(4, 7u32.into()).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); assert_eq!(server.buffered_data, 4); // Only the lost prefix is retransmitted, without the FIN of a stream that was never finished @@ -3357,15 +3394,15 @@ mod tests { } .open(Dir::Uni, Default::default()) .unwrap(); - let mut stream = SendStream { + let mut stream = send::Stream { id, state: &mut server, pending: &mut pending, conn_state: &state, }; - stream.write(b"hello world").unwrap(); + stream.write(&mut &b"hello world"[..]).unwrap(); stream.state.received_stop_sending(id, 9u32.into()).unwrap(); - stream.reset_at(4, 7u32.into()).unwrap(); + stream.reset_at(stream.offset().rewind(4), 7u32.into()).unwrap(); let send = server.send[&id].as_ref().unwrap(); let frame = send.reset_frame(id, true).unwrap(); @@ -3400,8 +3437,8 @@ mod tests { .unwrap(); } - fn recv<'a>(client: &'a mut StreamsState, pending: &'a mut Retransmits, id: StreamId) -> RecvStream<'a> { - RecvStream { + fn recv<'a>(client: &'a mut StreamsState, pending: &'a mut Retransmits, id: StreamId) -> recv::Stream<'a> { + recv::Stream { id, state: client, pending, @@ -3411,15 +3448,24 @@ mod tests { /// Lease whatever has arrived on `id`, returning how many bytes are leased. fn lease(client: &mut StreamsState, id: StreamId) -> usize { let mut pending = Retransmits::default(); - let _ = recv(client, &mut pending, id).lease(0).unwrap(); + recv(client, &mut pending, id).peek(0).0.unwrap(); client.check_leases(); client.leased(id).unwrap().len() } /// Wait on `id` for more than it has leased, as a reader decoding a message would. - fn wait(client: &mut StreamsState, id: StreamId, want: usize) -> Result<(), LeaseError> { + fn wait(client: &mut StreamsState, id: StreamId, want: usize) -> Result<(), recv::Error> { + wait_transmit(client, id, want).0 + } + + /// [`wait`], and whether it queued a frame. + fn wait_transmit( + client: &mut StreamsState, + id: StreamId, + want: usize, + ) -> (Result<(), recv::Error>, ShouldTransmit) { let mut pending = Retransmits::default(); - let result = recv(client, &mut pending, id).lease(want).map(|_| ()); + let result = recv(client, &mut pending, id).peek(want); client.check_leases(); result } @@ -3428,7 +3474,7 @@ mod tests { let mut pending = Retransmits::default(); let mut recv = recv(client, &mut pending, id); recv.advance(n); - let _ = recv.release(); + let _ = recv.release_taken(); client.check_leases(); } @@ -3509,7 +3555,7 @@ mod tests { if reset_first { reset(&mut client, id, 128); - assert_eq!(wait(&mut client, id, 33), Err(LeaseError::Reset(0u32.into()))); + assert_eq!(wait(&mut client, id, 33), Err(recv::Error::Reset(0u32.into()))); assert_eq!(client.local_max_data, initial + 128 - 32, "all but the lease"); } let _ = stop(&mut client, id, 0); @@ -3544,13 +3590,13 @@ mod tests { let id = StreamId::new(Side::Server, Dir::Uni, 0); data(&mut client, id, 0, 32, false); let window = client.receive_window as usize; - assert!(matches!(wait(&mut client, id, window), Err(LeaseError::Blocked(_)))); - assert_eq!(wait(&mut client, id, window + 1), Err(LeaseError::TooLarge)); + assert!(matches!(wait(&mut client, id, window), Err(recv::Error::Blocked))); + assert_eq!(wait(&mut client, id, window + 1), Err(recv::Error::TooLarge)); // Taken bytes do not: a frame past the window waits, and gives way to eviction. let mut pending = Retransmits::default(); let _ = recv(&mut client, &mut pending, id).take(10); - assert!(matches!(wait(&mut client, id, window - 9), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, id, window - 9), Err(recv::Error::Blocked))); } /// A connection with a 2 * `window` receive window, and its streams 0 and 1. @@ -3602,7 +3648,7 @@ mod tests { client.set_receive_window(VarInt::from_u64(3 * WINDOW / 2).unwrap()); let want = WINDOW as usize + 1; - assert!(matches!(wait(&mut client, ids[0], want), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, ids[0], want), Err(recv::Error::Blocked))); assert_eq!(client.credit_stats().evicted, 2); assert_eq!(readable(&mut client), [ids[2], ids[1]], "newest first"); assert_eq!(client.local_max_data, max_data + WINDOW / 2); @@ -3648,12 +3694,10 @@ mod tests { let mut pending = Retransmits::default(); let mut recv = recv(&mut client, &mut pending, id); match reader { - Reader::Wait => assert!(matches!(recv.lease(buffers.len() + 1), Err(LeaseError::Blocked(_)))), - Reader::Park => { - let _ = recv.lease(1).unwrap(); - } + Reader::Wait => assert!(matches!(recv.peek(buffers.len() + 1).0, Err(recv::Error::Blocked))), + Reader::Park => recv.peek(1).0.unwrap(), Reader::Take => { - let _ = recv.lease(1).unwrap(); + recv.peek(1).0.unwrap(); taken.push(recv.take(usize::MAX)); } } @@ -3693,7 +3737,7 @@ mod tests { data(&mut client, id, 0, 1, false); } for &id in &ids { - assert!(matches!(wait(&mut client, id, 2), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, id, 2), Err(recv::Error::Blocked))); } assert_eq!(client.credit_stats().evicted, 1); assert!( @@ -3729,7 +3773,7 @@ mod tests { for id in [ids[1], ids[2]] { assert!(matches!( wait(&mut client, id, WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); } assert_eq!(client.credit_stats().evicted, 0); @@ -3755,13 +3799,13 @@ mod tests { assert_eq!(lease(&mut client, ids[0]), 100); assert!(matches!( wait(&mut client, ids[0], WINDOW as usize), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); data(&mut client, ids[1], 0, WINDOW as usize, false); assert_eq!(lease(&mut client, ids[1]), WINDOW as usize); assert!(matches!( wait(&mut client, ids[1], WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); assert_eq!(client.credit_stats().evicted, 0); assert_eq!(client.local_max_data, client.data_recvd, "the peer is blocked"); @@ -3798,7 +3842,7 @@ mod tests { for id in [ids[1], ids[2]] { assert!(matches!( wait(&mut client, id, WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); } readable(&mut client); @@ -3836,7 +3880,7 @@ mod tests { assert_eq!(lease(&mut client, id), WINDOW as usize); assert!(matches!( wait(&mut client, id, WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); } client.set_receive_window(VarInt::from_u64(2 * WINDOW).unwrap()); @@ -3879,7 +3923,7 @@ mod tests { assert_eq!(lease(&mut client, ids[0]), WINDOW as usize); assert!(matches!( wait(&mut client, ids[0], WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); data(&mut client, ids[1], 0, WINDOW as usize, false); assert_eq!(lease(&mut client, ids[1]), WINDOW as usize); @@ -3887,12 +3931,12 @@ mod tests { reset(&mut client, ids[0], WINDOW as u32); assert!(matches!( wait(&mut client, ids[1], WINDOW as usize + 1), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); assert_eq!(client.credit_stats().evicted, 0); assert_eq!( wait(&mut client, ids[0], WINDOW as usize + 1), - Err(LeaseError::Reset(0u32.into())) + Err(recv::Error::Reset(0u32.into())) ); } @@ -3935,13 +3979,13 @@ mod tests { // Debt as large as one stream's bytes, so the first stop only pays it. client.set_receive_window(VarInt::from_u64(WINDOW).unwrap()); - assert!(matches!(wait(&mut client, ids[0], 1), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, ids[0], 1), Err(recv::Error::Blocked))); assert_eq!(readable(&mut client), [ids[1]]); assert_eq!(client.credit_stats().evicted, 1); assert_eq!(client.credit_stats().leased, 2 * WINDOW, "taken credit stays held"); - assert!(matches!(wait(&mut client, ids[0], 1), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, ids[0], 1), Err(recv::Error::Blocked))); assert_eq!(client.credit_stats().evicted, 1, "waits for the stop"); - assert_eq!(wait(&mut client, ids[1], 1), Err(LeaseError::Evicted)); + assert_eq!(wait(&mut client, ids[1], 1), Err(recv::Error::Evicted)); // The stop pays the debt alone, so eviction resumes with the other stream. let _ = stop(&mut client, ids[1], 0); @@ -3968,8 +4012,8 @@ mod tests { assert_eq!(readable(&mut client), [high, low]); assert_eq!(lease(&mut client, high), WINDOW as usize); assert_eq!(lease(&mut client, low), WINDOW as usize - 1); - assert!(matches!(wait(&mut client, high, 2000), Err(LeaseError::Blocked(_)))); - assert!(matches!(wait(&mut client, low, 2000), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, high, 2000), Err(recv::Error::Blocked))); + assert!(matches!(wait(&mut client, low, 2000), Err(recv::Error::Blocked))); assert!( client.leased(low).is_some_and(|lease| !lease.evicted), "a byte of credit left" @@ -3980,10 +4024,10 @@ mod tests { // The arrival woke the reader of `low`, which may yet consume. assert_eq!(readable(&mut client), [low]); assert!(!client.leased(low).unwrap().evicted); - let blocked = wait(&mut client, low, 2000); - assert_eq!(blocked, Err(LeaseError::Blocked(ShouldTransmit(true)))); + let blocked = wait_transmit(&mut client, low, 2000); + assert_eq!(blocked, (Err(recv::Error::Blocked), ShouldTransmit(true))); assert_eq!(readable(&mut client), [low], "the evicted reader wakes"); - assert_eq!(wait(&mut client, low, 1), Err(LeaseError::Evicted)); + assert_eq!(wait(&mut client, low, 1), Err(recv::Error::Evicted)); assert_eq!(client.credit_stats().leased, WINDOW); assert_eq!(client.credit_stats().evicted, 1); assert_eq!(client.local_max_data, 3 * WINDOW, "the evicted credit returns"); @@ -4011,10 +4055,10 @@ mod tests { assert_eq!(taken.len(), WINDOW as usize); assert_eq!( - wait(&mut client, frame, 1), - Err(LeaseError::Blocked(ShouldTransmit(true))) + wait_transmit(&mut client, frame, 1), + (Err(recv::Error::Blocked), ShouldTransmit(true)) ); - assert_eq!(wait(&mut client, frame, 1), Err(LeaseError::Evicted)); + assert_eq!(wait(&mut client, frame, 1), Err(recv::Error::Evicted)); assert_eq!(client.local_max_data, WINDOW, "the taken bytes are still held"); drop(taken); let _ = stop(&mut client, frame, 0); @@ -4031,11 +4075,66 @@ mod tests { assert_eq!(lease(&mut client, old), WINDOW as usize); data(&mut client, new, 0, WINDOW as usize, false); let _ = readable(&mut client); - assert!(matches!(wait(&mut client, new, 2000), Err(LeaseError::Blocked(_)))); - assert_eq!(wait(&mut client, new, 1), Err(LeaseError::Evicted)); + assert!(matches!(wait(&mut client, new, 2000), Err(recv::Error::Blocked))); + assert_eq!(wait(&mut client, new, 1), Err(recv::Error::Evicted)); assert_eq!(lease(&mut client, old), WINDOW as usize); } + /// A lease moves its bytes out; restoring it keeps their credit, and releasing it after a + /// partial read returns only the read bytes' credit. + #[test] + fn lease_moves_out_and_back() { + const WINDOW: u64 = 1000; + let (mut client, [id, _]) = stuck(WINDOW); + data(&mut client, id, 0, 600, false); + let mut pending = Retransmits::default(); + let mut stream = recv(&mut client, &mut pending, id); + + let lease = stream.lease(0).0.unwrap(); + assert_eq!(lease.remaining(), 600); + assert!(!lease.is_fin()); + assert_eq!(stream.state.leased(id).unwrap().len(), 0, "moved out, not copied"); + let _ = stream.restore(lease); + assert_eq!(stream.state.credit_stats().leased, 600); + + let mut lease = stream.lease(0).0.unwrap(); + assert_eq!(lease.remaining(), 600, "restored bytes are leased again"); + lease.advance(250); + let _ = stream.release(lease); + assert_eq!(stream.state.credit_stats().leased, 350); + assert_eq!(stream.lease(0).0.unwrap().remaining(), 350); + } + + /// Leasing again before handing the last lease back would lose its bytes. + #[test] + #[should_panic(expected = "leased again before the last lease was handed back")] + fn lease_twice_panics() { + let (mut client, [id, _]) = stuck(1000); + data(&mut client, id, 0, 100, false); + let mut pending = Retransmits::default(); + let mut stream = recv(&mut client, &mut pending, id); + let _lease = stream.lease(0).0.unwrap(); + let _ = stream.lease(0); + } + + /// A lease out with its reader may be about to free the window, so nothing is evicted until + /// it comes back. + #[test] + fn lease_out_holds_eviction_off() { + const WINDOW: u64 = 1000; + let (mut client, [old, new]) = stuck(WINDOW); + data(&mut client, old, 0, WINDOW as usize, false); + let mut pending = Retransmits::default(); + let held = recv(&mut client, &mut pending, old).lease(0).0.unwrap(); + data(&mut client, new, 0, WINDOW as usize, false); + let _ = readable(&mut client); + assert!(matches!(wait(&mut client, new, 2000), Err(recv::Error::Blocked))); + assert_eq!(client.credit_stats().evicted, 0); + + let _ = recv(&mut client, &mut pending, old).restore(held); + assert_eq!(wait(&mut client, new, 1), Err(recv::Error::Evicted)); + } + /// A stream nobody has leased yet may have a reader about to run, so it holds eviction off /// until its reader looks. #[test] @@ -4047,13 +4146,14 @@ mod tests { data(&mut client, fresh, 0, WINDOW as usize, false); let mut pending = Retransmits::default(); recv(&mut client, &mut pending, fresh).set_priority(-1); - assert!(matches!(wait(&mut client, waiting, 2000), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, waiting, 2000), Err(recv::Error::Blocked))); assert_eq!(client.credit_stats().evicted, 0); assert_eq!(lease(&mut client, fresh), WINDOW as usize); - let blocked = wait(&mut client, fresh, 2000); - assert_eq!(blocked, Err(LeaseError::Blocked(ShouldTransmit(true)))); - assert_eq!(wait(&mut client, fresh, 1), Err(LeaseError::Evicted)); + let (blocked, transmit) = wait_transmit(&mut client, fresh, 2000); + assert_eq!(blocked, Err(recv::Error::Blocked)); + assert!(transmit.should_transmit()); + assert_eq!(wait(&mut client, fresh, 1), Err(recv::Error::Evicted)); assert_eq!(lease(&mut client, waiting), WINDOW as usize); } @@ -4082,8 +4182,8 @@ mod tests { let mut pending = Retransmits::default(); assert_eq!( - recv(&mut client, &mut pending, waiting).lease(2 * WINDOW), - Err(LeaseError::Blocked(ShouldTransmit(true))) + recv(&mut client, &mut pending, waiting).peek(2 * WINDOW), + (Err(recv::Error::Blocked), ShouldTransmit(true)) ); assert!(client.leased(parked).unwrap().evicted); assert!(!pending.max_stream_id[Dir::Uni as usize], "its reader has yet to hear"); @@ -4103,8 +4203,8 @@ mod tests { data(&mut client, b, 0, WINDOW as usize, false); assert_eq!(lease(&mut client, a), WINDOW as usize); release(&mut client, a, 1); - assert!(matches!(wait(&mut client, b, 2000), Err(LeaseError::Blocked(_)))); - assert!(matches!(wait(&mut client, a, 2000), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, b, 2000), Err(recv::Error::Blocked))); + assert!(matches!(wait(&mut client, a, 2000), Err(recv::Error::Blocked))); assert_eq!(client.credit_stats().evicted, 0); } @@ -4138,7 +4238,7 @@ mod tests { assert_eq!(lease(&mut client, id), 10); let mut pending = Retransmits::default(); assert!(recv(&mut client, &mut pending, id).stop(0u32.into()).is_ok()); - assert_eq!(wait(&mut client, id, 0), Err(LeaseError::ClosedStream)); + assert_eq!(wait(&mut client, id, 0), Err(recv::Error::Closed)); assert!(recv(&mut client, &mut pending, id).stop(0u32.into()).is_err()); assert!(client.leased(id).is_none()); } @@ -4162,10 +4262,10 @@ mod tests { // One byte taken, so the reader waits for a unit the window can hold. let mut pending = Retransmits::default(); let _ = recv(&mut client, &mut pending, waiting).take(1); - assert!(matches!(wait(&mut client, waiting, room), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, waiting, room), Err(recv::Error::Blocked))); assert_eq!(client.credit_stats().evicted, 1); - assert_eq!(wait(&mut client, waiting, 1), Err(LeaseError::Evicted)); - assert_eq!(wait(&mut client, reset_id, 1), Err(LeaseError::Reset(0u32.into()))); + assert_eq!(wait(&mut client, waiting, 1), Err(recv::Error::Evicted)); + assert_eq!(wait(&mut client, reset_id, 1), Err(recv::Error::Reset(0u32.into()))); } /// A reset below what a lease already read and partly took leaves those bytes leased, and @@ -4182,7 +4282,7 @@ mod tests { let _ = client.received_reset(reset_at(id, 7, 200, Some(50))).unwrap(); assert_eq!(client.local_max_data, initial + 100, "the abandoned tail"); - assert_eq!(wait(&mut client, id, 41), Err(LeaseError::Reset(7u32.into()))); + assert_eq!(wait(&mut client, id, 41), Err(recv::Error::Reset(7u32.into()))); assert_eq!(client.leased(id).unwrap().len(), 40, "read before the reset"); release(&mut client, id, 0); @@ -4232,7 +4332,7 @@ mod tests { for index in 0..STREAMS + CYCLES { let id = StreamId::new(Side::Server, Dir::Uni, index); data(&mut client, id, 0, 1, false); - assert!(matches!(wait(&mut client, id, 2), Err(LeaseError::Blocked(_)))); + assert!(matches!(wait(&mut client, id, 2), Err(recv::Error::Blocked))); if index == STREAMS - 1 { client.scans = 0; } @@ -4261,11 +4361,14 @@ mod tests { data(&mut client, id, 0, 64, false); let mut pending = Retransmits::default(); assert_eq!( - recv(&mut client, &mut pending, id).lease(1001), - Err(LeaseError::TooLarge) + recv(&mut client, &mut pending, id).peek(1001), + (Err(recv::Error::TooLarge), ShouldTransmit(false)) ); assert!(pending.max_stream_data.is_empty()); - assert_eq!(recv(&mut client, &mut pending, id).lease(0), Ok(ShouldTransmit(true))); + assert_eq!( + recv(&mut client, &mut pending, id).peek(0), + (Ok(()), ShouldTransmit(true)) + ); assert!(pending.max_stream_data.contains(&id)); let id = StreamId::new(Side::Server, Dir::Uni, 1); @@ -4273,8 +4376,8 @@ mod tests { reset(&mut client, id, 10); let mut pending = Retransmits::default(); assert_eq!( - recv(&mut client, &mut pending, id).lease(11), - Err(LeaseError::Reset(0u32.into())) + recv(&mut client, &mut pending, id).peek(11), + (Err(recv::Error::Reset(0u32.into())), ShouldTransmit(false)) ); assert!(!pending.max_stream_id[Dir::Uni as usize]); let pending = stop(&mut client, id, 0); @@ -4326,9 +4429,9 @@ mod tests { let _ = recv(&mut client, &mut pending, id).take(1); assert!(matches!( wait(&mut client, id, WINDOW as usize), - Err(LeaseError::Blocked(_)) + Err(recv::Error::Blocked) )); - assert_eq!(wait(&mut client, id, 1), Err(LeaseError::Evicted)); + assert_eq!(wait(&mut client, id, 1), Err(recv::Error::Evicted)); let mut frames = Retransmits::default(); if fin_first { @@ -4357,9 +4460,9 @@ mod tests { fn send_group_from_another_connection() { let mut a = make(Side::Client); let mut b = make(Side::Client); - let group = crate::SendGroups { state: &mut a }.open(0); - let _other = crate::SendGroups { state: &mut b }.open(0); - crate::SendGroups { state: &mut b }.set_priority(&group, 1); + let group = send::Groups { state: &mut a }.open(0); + let _other = send::Groups { state: &mut b }.open(0); + send::Groups { state: &mut b }.set_priority(&group, 1); } /// Ingest and read rounds across many streams, with a plain read or a lease released whole. @@ -4395,15 +4498,15 @@ mod tests { data: chunk.clone(), }; let _ = client.received(frame, payload).unwrap(); - let mut recv = RecvStream { + let mut recv = recv::Stream { id, state: &mut client, pending: &mut pending, }; if leased { - let _ = recv.lease(1).unwrap(); - std::hint::black_box(recv.take(usize::MAX)); - let _ = recv.release(); + let mut lease = recv.lease(1).0.unwrap(); + std::hint::black_box(lease.copy_to_bytes(lease.remaining())); + let _ = recv.release(lease); } else { let mut chunks = recv.read().unwrap(); std::hint::black_box(chunks.next(usize::MAX).unwrap()); diff --git a/rs/moq-quic/src/frame.rs b/rs/moq-quic/src/frame.rs index 2cdb8fe656a..3f9d4a536af 100644 --- a/rs/moq-quic/src/frame.rs +++ b/rs/moq-quic/src/frame.rs @@ -928,23 +928,6 @@ impl FrameStruct for Datagram { } impl Datagram { - pub(crate) fn encode(&self, length: bool, out: &mut Vec) { - out.write(FrameType(*DATAGRAM_TYS.start() | u64::from(length))); // 1 byte - if length { - // Safe to unwrap because we check length sanity before queueing datagrams - out.write(VarInt::from_u64(self.data.len() as u64).unwrap()); // <= 8 bytes - } - out.extend_from_slice(&self.data); - } - - pub(crate) fn size(&self, length: bool) -> usize { - 1 + if length { - VarInt::from_u64(self.data.len() as u64).unwrap().size() - } else { - 0 - } + self.data.len() - } - /// The largest payload whose length-prefixed frame fits in `budget` bytes, if any frame does /// /// Exact, rather than reserving [`SIZE_BOUND`](FrameStruct::SIZE_BOUND)'s 8-byte length: a @@ -959,6 +942,53 @@ impl Datagram { } } +/// An outgoing datagram's payload, kept as the chunks it was handed in +/// +/// A header chained in front of a payload costs no copy of the payload. +#[derive(Debug)] +pub(crate) struct OutgoingDatagram { + chunks: tinyvec::TinyVec<[Bytes; 2]>, + len: usize, +} + +impl OutgoingDatagram { + /// Take all of `data`, a chunk at a time. + pub(crate) fn new(mut data: impl Buf) -> Self { + let len = data.remaining(); + let mut chunks = tinyvec::TinyVec::new(); + while data.has_remaining() { + let chunk = data.chunk().len(); + chunks.push(data.copy_to_bytes(chunk)); + } + Self { chunks, len } + } + + /// The payload's length. + pub(crate) fn len(&self) -> usize { + self.len + } + + /// The length-prefixed frame's size. + pub(crate) fn size(&self) -> usize { + Self::frame_size(self.len) + } + + /// The size of the length-prefixed frame of a `len`-byte payload. + pub(crate) fn frame_size(len: usize) -> usize { + 1 + VarInt::from_u64(len as u64).map_or(8, |len| len.size()) + len + } + + /// Write the length-prefixed frame. + pub(crate) fn encode(&self, out: &mut Vec) { + out.write(FrameType(*DATAGRAM_TYS.start() | 1)); // 1 byte + // Safe to unwrap because we check length sanity before queueing datagrams + out.write(VarInt::from_u64(self.len as u64).unwrap()); // <= 8 bytes + for chunk in &self.chunks { + out.extend_from_slice(chunk); + } + } +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) struct AckFrequency { pub(crate) sequence: VarInt, @@ -1167,6 +1197,28 @@ mod test { VarInt::MAX.into_inner(), ]; + #[test] + fn outgoing_datagram_chains_without_copying() { + let payload = Bytes::from(vec![7; 100]); + let datagram = OutgoingDatagram::new((&b"hdr"[..]).chain(payload.clone())); + assert_eq!(datagram.len(), 103); + assert_eq!( + datagram.chunks[1].as_ptr(), + payload.as_ptr(), + "the payload is not copied" + ); + + let mut out = Vec::new(); + datagram.encode(&mut out); + assert_eq!(out.len(), datagram.size()); + let frame = Iter::new(Bytes::from(out)).unwrap().next().unwrap().unwrap(); + let Frame::Datagram(frame) = frame else { + panic!("{frame:?}") + }; + assert_eq!(&frame.data[..3], b"hdr"); + assert_eq!(frame.data[3..], payload[..]); + } + #[test] fn datagram_max_payload_is_exact() { // Not even an empty frame fits @@ -1174,12 +1226,7 @@ mod test { assert_eq!(Datagram::max_payload(1), None); for budget in [2u64, 3, 64, 65, 66, 1200, 16385, 16386, 1 << 20] { let payload = Datagram::max_payload(budget).unwrap(); - let frame = |n: u64| { - Datagram { - data: Bytes::from(vec![0; n as usize]), - } - .size(true) as u64 - }; + let frame = |n: u64| OutgoingDatagram::new(Bytes::from(vec![0; n as usize])).size() as u64; assert!(frame(payload) <= budget, "{budget}: {payload} overflows"); assert!(frame(payload + 1) > budget, "{budget}: {payload} isn't the largest"); } diff --git a/rs/moq-quic/src/fuzz.rs b/rs/moq-quic/src/fuzz.rs index 1a86e252faf..08e35446261 100644 --- a/rs/moq-quic/src/fuzz.rs +++ b/rs/moq-quic/src/fuzz.rs @@ -23,7 +23,7 @@ use bytes::{Bytes, BytesMut}; use rand::{Rng, SeedableRng, rngs::StdRng}; use crate::{ - ConnectionId, Dir, MAX_CID_SIZE, RESET_TOKEN_SIZE, SendStream, Side, StreamId, Streams, VarInt, + ConnectionId, Dir, MAX_CID_SIZE, RESET_TOKEN_SIZE, Side, StreamId, Streams, VarInt, coding::BufMutExt, connection::{Retransmits, State, StreamsState}, frame::{self, Frame}, @@ -185,7 +185,7 @@ pub fn streams(data: &[u8]) -> bool { Streams::new(&mut state, &conn_state).accept(dir); } StreamOp::Finish { id } => { - let _ = SendStream::new(StreamId(id), &mut state, &mut pending, &conn_state).finish(); + let _ = crate::send::Stream::new(StreamId(id), &mut state, &mut pending, &conn_state).finish(); } StreamOp::ReceivedStopSending { id, error_code } => { let _ = state.received_stop_sending(StreamId(id), varint(error_code)); @@ -204,7 +204,8 @@ pub fn streams(data: &[u8]) -> bool { }); } StreamOp::Reset { id } => { - let _ = SendStream::new(StreamId(id), &mut state, &mut pending, &conn_state).reset(0u32.into()); + let _ = + crate::send::Stream::new(StreamId(id), &mut state, &mut pending, &conn_state).reset(0u32.into()); } } } diff --git a/rs/moq-quic/src/fuzz/connection.rs b/rs/moq-quic/src/fuzz/connection.rs index 519ffa7be39..d8b935f17bb 100644 --- a/rs/moq-quic/src/fuzz/connection.rs +++ b/rs/moq-quic/src/fuzz/connection.rs @@ -14,16 +14,16 @@ use std::{ }; use arbitrary::{Arbitrary, Unstructured}; -use bytes::{Bytes, BytesMut}; +use bytes::{Buf, Bytes, BytesMut}; use super::plaintext::{Clear, Plaintext, TAG_LEN}; use crate::{ - Dir, Duration, Event, Instant, LeaseError, Limits, ReadError, Side, StreamId, client, + Dir, Duration, Event, Instant, Limits, Side, StreamId, client, connection::Connection, endpoint::{self, ConnectionHandle, DatagramEvent, Dial, Endpoint, Received}, frame::{self, Frame}, packet::{FixedLengthConnectionIdParser, PartialDecode}, - server, transport, + recv, server, transport, }; /// The connection ID length both endpoints issue, which a captured packet needs to parse. @@ -578,14 +578,16 @@ impl Pair { } => { if let Some((id, conn)) = send_stream(self.peer(who), stream) { let mut stream = conn.send_stream(id); - let offset = u64::from(offset) % (stream.offset() + 1); + let end = stream.offset(); + let offset = end.rewind(u64::from(offset) % (u64::from(end) + 1)); let _ = stream.reset_at(offset, code.into()); } } Op::Acked { who, stream, offset } => { if let Some((id, conn)) = send_stream(self.peer(who), stream) { let mut stream = conn.send_stream(id); - let offset = u64::from(offset) % (stream.offset() + 1); + let end = stream.offset(); + let offset = end.rewind(u64::from(offset) % (u64::from(end) + 1)); let _ = stream.acked(offset); } } @@ -671,7 +673,7 @@ impl Pair { .datagrams() .send( data, - crate::SendOptions { + crate::send::Options { group: None, priority: priority.into(), }, @@ -752,7 +754,7 @@ impl Pair { let data: Vec = (sent.len..sent.len + len) .map(|offset| stream_byte(side, id, offset)) .collect(); - if let Ok(written) = conn.send_stream(id).write(&data) { + if let Ok(written) = conn.send_stream(id).write(&mut &data[..]) { sent.len += written as u64; } } @@ -847,7 +849,8 @@ fn read(peer: &mut Peer, sent: &HashMap<(Side, StreamId), Sent>, id: StreamId, m } break; } - Err(ReadError::Blocked | ReadError::Reset(_)) => break, + Err(recv::Error::Blocked | recv::Error::Reset(_)) => break, + Err(err) => unreachable!("{err}"), } } @@ -868,9 +871,9 @@ fn lease(peer: &mut Peer, sent: &HashMap<(Side, StreamId), Sent>, id: StreamId, let Some((_, conn)) = &mut peer.conn else { return; }; - match conn.recv_stream(id).lease(want) { - Ok(_) => {} - Err(LeaseError::Evicted) => { + let mut lease = match conn.recv_stream(id).lease(want).0 { + Ok(lease) => lease, + Err(recv::Error::Evicted) => { conn.recv_stream(id) .stop(0u32.into()) .expect("an evicted stream stops once"); @@ -878,23 +881,22 @@ fn lease(peer: &mut Peer, sent: &HashMap<(Side, StreamId), Sent>, id: StreamId, } // Waiting, reset, or stopped: nothing to check yet. Err(_) => return, - } - let Some(lease) = conn.leased(id) else { - return; }; let offset = peer.read.entry(id).or_default(); - let n = max.min(lease.len()); + let n = max.min(lease.remaining()); if !forged { let written = sent.get(&(writer, id)).map_or(0, |sent| sent.len); - assert!(*offset + lease.len() as u64 <= written, "leased past what was written"); + assert!( + *offset + lease.remaining() as u64 <= written, + "leased past what was written" + ); for (at, &byte) in (*offset..).zip(lease.chunks().flatten().take(n)) { assert_eq!(byte, stream_byte(writer, id, at), "corrupt stream data"); } } *offset += n as u64; - let mut recv = conn.recv_stream(id); - recv.advance(n); - let _ = recv.release(); + lease.advance(n); + let _ = conn.recv_stream(id).release(lease); } /// The byte a writer puts at `offset` of a stream, unique enough to catch misplaced data. diff --git a/rs/moq-quic/src/lib.rs b/rs/moq-quic/src/lib.rs index 7e5ec942f0a..7814a5f58d2 100644 --- a/rs/moq-quic/src/lib.rs +++ b/rs/moq-quic/src/lib.rs @@ -15,7 +15,6 @@ #![cfg_attr(test, allow(dead_code, reason = "test builds compile helpers only some tests call"))] use std::{ - fmt, net::{IpAddr, SocketAddr}, ops, }; @@ -42,13 +41,14 @@ use bloom_token_log::BloomTokenLog; pub mod connection; pub use crate::connection::{ - Chunk, Chunks, ClosedStream, CreditStats, Datagrams, Event, FinishError, FrameStats, LeaseError, Leased, PathStats, - ReadError, ReadableError, RecvStream, SendDatagramError, SendGroup, SendGroups, SendOptions, SendStream, - ShouldTransmit, StreamEvent, Streams, UdpStats, WriteError, Written, + CreditStats, Datagrams, Event, FrameStats, PathStats, ShouldTransmit, StreamEvent, Streams, UdpStats, recv, send, }; + +pub mod stream; #[cfg(feature = "mux")] #[doc(hidden)] pub use connection::mux; +pub(crate) use stream::{Dir, Id as StreamId}; #[cfg(any(feature = "ring", feature = "aws-lc-rs"))] pub use rustls; @@ -152,95 +152,6 @@ impl ops::Not for Side { } } -/// Whether a stream communicates data in both directions or only from the initiator -#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub enum Dir { - /// Data flows in both directions - Bi = 0, - /// Data flows only from the stream's initiator - Uni = 1, -} - -impl Dir { - fn iter() -> impl Iterator { - [Self::Bi, Self::Uni].iter().cloned() - } -} - -impl fmt::Display for Dir { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use Dir::*; - f.pad(match *self { - Bi => "bidirectional", - Uni => "unidirectional", - }) - } -} - -/// Identifier for a stream within a particular connection -#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub struct StreamId(u64); - -impl fmt::Display for StreamId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let initiator = match self.initiator() { - Side::Client => "client", - Side::Server => "server", - }; - let dir = match self.dir() { - Dir::Uni => "uni", - Dir::Bi => "bi", - }; - write!(f, "{} {}directional stream {}", initiator, dir, self.index()) - } -} - -impl StreamId { - /// Create a new StreamId - pub fn new(initiator: Side, dir: Dir, index: u64) -> Self { - Self((index << 2) | ((dir as u64) << 1) | initiator as u64) - } - /// Which side of a connection initiated the stream - pub fn initiator(self) -> Side { - if self.0 & 0x1 == 0 { Side::Client } else { Side::Server } - } - /// Which directions data flows in - pub fn dir(self) -> Dir { - if self.0 & 0x2 == 0 { Dir::Bi } else { Dir::Uni } - } - /// Distinguishes streams of the same initiator and directionality - pub fn index(self) -> u64 { - self.0 >> 2 - } -} - -impl From for VarInt { - fn from(x: StreamId) -> Self { - unsafe { Self::from_u64_unchecked(x.0) } - } -} - -impl From for StreamId { - fn from(v: VarInt) -> Self { - Self(v.0) - } -} - -impl From for u64 { - fn from(x: StreamId) -> Self { - x.0 - } -} - -impl coding::Codec for StreamId { - fn decode(buf: &mut B) -> coding::Result { - VarInt::decode(buf).map(|x| Self(x.into_inner())) - } - fn encode(&self, buf: &mut B) { - VarInt::from_u64(self.0).unwrap().encode(buf); - } -} - /// An outgoing packet #[derive(Debug)] #[must_use] diff --git a/rs/moq-quic/src/stream.rs b/rs/moq-quic/src/stream.rs new file mode 100644 index 00000000000..ea10754f4b7 --- /dev/null +++ b/rs/moq-quic/src/stream.rs @@ -0,0 +1,94 @@ +//! Stream identity, shared by both directions. + +use std::fmt; + +use crate::{Side, VarInt, coding}; + +/// Whether a stream communicates data in both directions or only from the initiator +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub enum Dir { + /// Data flows in both directions + Bi = 0, + /// Data flows only from the stream's initiator + Uni = 1, +} + +impl Dir { + pub(crate) fn iter() -> impl Iterator { + [Self::Bi, Self::Uni].iter().cloned() + } +} + +impl fmt::Display for Dir { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Dir::*; + f.pad(match *self { + Bi => "bidirectional", + Uni => "unidirectional", + }) + } +} + +/// Identifier for a stream within a particular connection +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct Id(pub(crate) u64); + +impl fmt::Display for Id { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let initiator = match self.initiator() { + Side::Client => "client", + Side::Server => "server", + }; + let dir = match self.dir() { + Dir::Uni => "uni", + Dir::Bi => "bi", + }; + write!(f, "{} {}directional stream {}", initiator, dir, self.index()) + } +} + +impl Id { + /// Create a new stream ID + pub fn new(initiator: Side, dir: Dir, index: u64) -> Self { + Self((index << 2) | ((dir as u64) << 1) | initiator as u64) + } + /// Which side of a connection initiated the stream + pub fn initiator(self) -> Side { + if self.0 & 0x1 == 0 { Side::Client } else { Side::Server } + } + /// Which directions data flows in + pub fn dir(self) -> Dir { + if self.0 & 0x2 == 0 { Dir::Bi } else { Dir::Uni } + } + /// Distinguishes streams of the same initiator and directionality + pub fn index(self) -> u64 { + self.0 >> 2 + } +} + +impl From for VarInt { + fn from(x: Id) -> Self { + unsafe { Self::from_u64_unchecked(x.0) } + } +} + +impl From for Id { + fn from(v: VarInt) -> Self { + Self(v.0) + } +} + +impl From for u64 { + fn from(x: Id) -> Self { + x.0 + } +} + +impl coding::Codec for Id { + fn decode(buf: &mut B) -> coding::Result { + VarInt::decode(buf).map(|x| Self(x.into_inner())) + } + fn encode(&self, buf: &mut B) { + VarInt::from_u64(self.0).unwrap().encode(buf); + } +} diff --git a/rs/moq-quic/src/tests/acked.rs b/rs/moq-quic/src/tests/acked.rs index 555c14db065..7be6b220f92 100644 --- a/rs/moq-quic/src/tests/acked.rs +++ b/rs/moq-quic/src/tests/acked.rs @@ -1,4 +1,4 @@ -//! Tests for [`SendStream::acked`], which reports whether the peer acknowledged a send stream's prefix +//! Tests for [`send::Stream::acked`], which reports whether the peer acknowledged a send stream's prefix use super::*; @@ -21,13 +21,13 @@ fn open(pair: &mut Pair, client_ch: ConnectionHandle, server_ch: ConnectionHandl .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"header").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"header"[..]).unwrap(); pair.drive(); assert_eq!(pair.server_streams(server_ch).accept(Dir::Uni), Some(s)); s } -fn acked(pair: &mut Pair, ch: ConnectionHandle, s: StreamId, offset: u64) -> Result { +fn acked(pair: &mut Pair, ch: ConnectionHandle, s: StreamId, offset: send::Offset) -> Result { pair.client_send(ch, s).acked(offset) } @@ -62,12 +62,12 @@ fn waits_for_contiguous_prefix() { pair.drive(); reported(&mut pair, client_ch, s); - pair.client_send(client_ch, s).write(b"lost").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"lost"[..]).unwrap(); let lost = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); // Acknowledged, but behind the lost packet: the middle of the stream, not its prefix - pair.client_send(client_ch, s).write(b"after").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"after"[..]).unwrap(); let after = pair.client_send(client_ch, s).offset(); let acks = pair.client_conn_mut(client_ch).stats().frame_rx.acks; while pair.client_conn_mut(client_ch).stats().frame_rx.acks == acks { @@ -95,11 +95,11 @@ fn later_waiter_registers_again() { reported(&mut pair, client_ch, s); // The first packet is held, and the second lost - pair.client_send(client_ch, s).write(b"first").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"first"[..]).unwrap(); let first = pair.client_send(client_ch, s).offset(); pair.client.drive(pair.time, pair.server.addr); pair.client.delay_outbound(); - pair.client_send(client_ch, s).write(b"second").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"second"[..]).unwrap(); let second = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); assert_eq!(acked(&mut pair, client_ch, s, second), Ok(false)); @@ -110,12 +110,12 @@ fn later_waiter_registers_again() { while acked(&mut pair, client_ch, s, first) != Ok(true) { assert!(pair.step(), "never acknowledged"); } - assert_eq!(reported(&mut pair, client_ch, s), Some(first)); + assert_eq!(reported(&mut pair, client_ch, s), Some(u64::from(first))); assert_eq!(acked(&mut pair, client_ch, s, second), Ok(false)); // The retransmission reports the second pair.drive(); - assert_eq!(reported(&mut pair, client_ch, s), Some(second)); + assert_eq!(reported(&mut pair, client_ch, s), Some(u64::from(second))); } /// Without a poll waiting on it, an acknowledgment is not reported. @@ -126,7 +126,7 @@ fn unpolled_is_not_reported() { let (client_ch, server_ch) = pair.connect(); let s = open(&mut pair, client_ch, server_ch); - pair.client_send(client_ch, s).write(b"payload").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"payload"[..]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); pair.drive(); assert_eq!(reported(&mut pair, client_ch, s), None); @@ -147,7 +147,7 @@ fn already_acked() { let empty = pair.client_send(client_ch, s).offset(); assert_eq!(acked(&mut pair, client_ch, s, empty), Ok(true)); - pair.client_send(client_ch, s).write(b"payload").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"payload"[..]).unwrap(); pair.drive(); let all = pair.client_send(client_ch, s).offset(); assert_eq!(acked(&mut pair, client_ch, s, all), Ok(true)); @@ -164,7 +164,7 @@ fn finished_stream_reports_before_freed() { pair.drive(); reported(&mut pair, client_ch, s); - pair.client_send(client_ch, s).write(b"payload").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"payload"[..]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); assert_eq!(acked(&mut pair, client_ch, s, offset), Ok(false)); pair.client_send(client_ch, s).finish().unwrap(); @@ -183,7 +183,12 @@ fn finished_stream_reports_before_freed() { StreamEvent::Finished { id: s } ] ); - assert_eq!(acked(&mut pair, client_ch, s, offset), Err(ClosedStream::default())); + // The freed stream's end is kept until released, so the offset is still answered. + assert_eq!(acked(&mut pair, client_ch, s, offset), Ok(true)); + assert_eq!(pair.client_send(client_ch, s).end(), Ok(Some(send::End::Finished))); + pair.client_send(client_ch, s).release(); + assert_eq!(acked(&mut pair, client_ch, s, offset), Err(send::Error::Closed)); + assert_eq!(pair.client_send(client_ch, s).end(), Err(send::Error::Closed)); } /// A reset fails the offsets past its Reliable Size at once, while those before it are @@ -195,16 +200,16 @@ fn reset_fails_past_reliable_size() { let (client_ch, server_ch) = pair.connect(); let s = open(&mut pair, client_ch, server_ch); - pair.client_send(client_ch, s).write(b"kept").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"kept"[..]).unwrap(); let kept = pair.client_send(client_ch, s).offset(); let reset = pair.client_send(client_ch, s).offset(); - pair.client_send(client_ch, s).write(b"dropped").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"dropped"[..]).unwrap(); let dropped = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); reported(&mut pair, client_ch, s); pair.client_send(client_ch, s).reset_at(reset, VarInt(42)).unwrap(); - assert_eq!(acked(&mut pair, client_ch, s, dropped), Err(ClosedStream::default())); + assert_eq!(acked(&mut pair, client_ch, s, dropped), Err(send::Error::Closed)); assert_eq!(acked(&mut pair, client_ch, s, kept), Ok(false)); assert_eq!(reported(&mut pair, client_ch, s), None); @@ -224,9 +229,9 @@ fn reset_reports_a_dropped_waiter() { let s = open(&mut pair, client_ch, server_ch); // A lower waiter the reset keeps doesn't hide the one it drops - pair.client_send(client_ch, s).write(b"kept").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"kept"[..]).unwrap(); let kept = pair.client_send(client_ch, s).offset(); - pair.client_send(client_ch, s).write(b"dropped").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"dropped"[..]).unwrap(); let dropped = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); reported(&mut pair, client_ch, s); @@ -236,7 +241,7 @@ fn reset_reports_a_dropped_waiter() { pair.client_send(client_ch, s).reset_at(kept, VarInt(42)).unwrap(); assert_eq!(pair.client_send(client_ch, s).offset(), kept); assert!(reported(&mut pair, client_ch, s).is_some()); - assert_eq!(acked(&mut pair, client_ch, s, dropped), Err(ClosedStream::default())); + assert_eq!(acked(&mut pair, client_ch, s, dropped), Err(send::Error::Closed)); } /// A peer that stopped reading will not read what it acknowledges, before or after the @@ -248,18 +253,21 @@ fn stop_sending_fails() { let (client_ch, server_ch) = pair.connect(); let s = open(&mut pair, client_ch, server_ch); - pair.client_send(client_ch, s).write(b"unread").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"unread"[..]).unwrap(); let before = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); assert_eq!(acked(&mut pair, client_ch, s, before), Ok(false)); pair.server_recv(server_ch, s).stop(VarInt(7)).unwrap(); pair.drive(); - assert_eq!(pair.client_send(client_ch, s).stopped(), Ok(Some(VarInt(7)))); + assert_eq!( + pair.client_send(client_ch, s).end(), + Ok(Some(send::End::Stopped(VarInt(7)))) + ); let after = pair.client_send(client_ch, s).offset(); assert_eq!(reported(&mut pair, client_ch, s), Some(b"header".len() as u64)); for offset in [before, after] { - assert_eq!(acked(&mut pair, client_ch, s, offset), Err(ClosedStream::default())); + assert_eq!(acked(&mut pair, client_ch, s, offset), Err(send::Error::Closed)); } } @@ -271,35 +279,24 @@ fn close_fails() { let (client_ch, server_ch) = pair.connect(); let s = open(&mut pair, client_ch, server_ch); - pair.client_send(client_ch, s).write(b"unacked").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"unacked"[..]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); lose_client_packets(&mut pair); let now = pair.time; pair.client_conn_mut(client_ch).close(now, VarInt(0), String::new()); - assert_eq!(acked(&mut pair, client_ch, s, offset), Err(ClosedStream::default())); -} - -/// Asking about data never written is a caller bug. -#[test] -#[should_panic(expected = "acked past the data written")] -fn acked_past_written_panics() { - let _guard = subscribe(); - let mut pair = pair(true); - let (client_ch, server_ch) = pair.connect(); - let s = open(&mut pair, client_ch, server_ch); - let past = pair.client_send(client_ch, s).offset() + 1; - let _ = acked(&mut pair, client_ch, s, past); + assert_eq!(acked(&mut pair, client_ch, s, offset), Err(send::Error::Closed)); } -/// Resetting past the data written is a caller bug. +/// An offset from another stream is a caller bug, the one misuse its type leaves possible. #[test] -#[should_panic(expected = "reset past the data written")] -fn reset_past_written_panics() { +#[should_panic(expected = "offset used on another stream")] +fn foreign_offset_panics() { let _guard = subscribe(); let mut pair = pair(true); let (client_ch, server_ch) = pair.connect(); let s = open(&mut pair, client_ch, server_ch); - let past = pair.client_send(client_ch, s).offset() + 1; - let _ = pair.client_send(client_ch, s).reset_at(past, VarInt(42)); + let other = open(&mut pair, client_ch, server_ch); + let foreign = pair.client_send(client_ch, other).offset(); + let _ = pair.client_send(client_ch, s).reset_at(foreign, VarInt(42)); } diff --git a/rs/moq-quic/src/tests/characterize/limits.rs b/rs/moq-quic/src/tests/characterize/limits.rs index 6166e35e2fc..85fae809ee3 100644 --- a/rs/moq-quic/src/tests/characterize/limits.rs +++ b/rs/moq-quic/src/tests/characterize/limits.rs @@ -12,7 +12,7 @@ fn openable(pair: &mut Pair, ch: ConnectionHandle) -> Vec { /// Finishes each client stream and has the server read it to the end fn finish_all(pair: &mut Pair, client_ch: ConnectionHandle, server_ch: ConnectionHandle, streams: &[StreamId]) { for &s in streams { - pair.client_send(client_ch, s).write(b"x").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"x"[..]).unwrap(); pair.client_send(client_ch, s).finish().unwrap(); } pair.drive(); @@ -59,7 +59,7 @@ fn max_concurrent_streams_lowered() { /// How many bytes a fresh client stream can write, up to `limit` fn writable(pair: &mut Pair, ch: ConnectionHandle, limit: usize) -> usize { let s = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - pair.client_send(ch, s).write(&vec![0; limit]).unwrap_or(0) + pair.client_send(ch, s).write(&mut &vec![0; limit][..]).unwrap_or(0) } /// Reads everything the server has buffered on every client uni stream, returning the total @@ -178,11 +178,17 @@ fn set_limits_changes_the_stream_window() { .open(Dir::Uni, Default::default()) .unwrap(); let data = vec![0; 8 * WINDOW as usize]; - assert_eq!(pair.client_send(client_ch, s).write(&data), Ok(WINDOW as usize)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &data[..]), + Ok(WINDOW as usize) + ); pair.drive(); assert_eq!(read_buffered(&mut pair, server_ch, &mut streams), WINDOW as usize); pair.drive(); - assert_eq!(pair.client_send(client_ch, s).write(&data), Ok(4 * WINDOW as usize)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &data[..]), + Ok(4 * WINDOW as usize) + ); limits.stream_receive_window = NonZeroU64::new(100); pair.server_conn_mut(server_ch).set_limits(limits).unwrap(); diff --git a/rs/moq-quic/src/tests/characterize/paths.rs b/rs/moq-quic/src/tests/characterize/paths.rs index d27ae329ba0..4a76081a84c 100644 --- a/rs/moq-quic/src/tests/characterize/paths.rs +++ b/rs/moq-quic/src/tests/characterize/paths.rs @@ -146,7 +146,7 @@ fn duplicate_packet_is_discarded() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive_client(); assert_eq!(pair.server.inbound.len(), 1); let duplicate = pair.server.inbound[0].clone(); @@ -159,7 +159,7 @@ fn duplicate_packet_is_discarded() { let mut recv = pair.server_recv(server_ch, s); let mut chunks = recv.read().unwrap(); assert_eq!(chunks.next(usize::MAX).unwrap().unwrap().bytes, MSG); - assert_eq!(chunks.next(usize::MAX), Err(ReadError::Blocked)); + assert_eq!(chunks.next(usize::MAX), Err(recv::Error::Blocked)); let _ = chunks.finalize(); } @@ -316,7 +316,7 @@ fn path_validation_failure_reverts() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive(); assert_eq!(pair.server_streams(server_ch).accept(Dir::Uni), Some(s)); } diff --git a/rs/moq-quic/src/tests/characterize/reset.rs b/rs/moq-quic/src/tests/characterize/reset.rs index 808c221a2c2..15eb165a4f6 100644 --- a/rs/moq-quic/src/tests/characterize/reset.rs +++ b/rs/moq-quic/src/tests/characterize/reset.rs @@ -5,7 +5,7 @@ use super::*; const CODE: VarInt = VarInt(7); /// Every chunk a server stream yields, and how it ended -fn read_all(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, ReadError) { +fn read_all(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, recv::Error) { let mut recv = pair.server_recv(ch, s); let mut chunks = recv.read().unwrap(); let mut data = Vec::new(); @@ -26,7 +26,7 @@ fn open(pair: &mut Pair, client_ch: ConnectionHandle, server_ch: ConnectionHandl .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(data).unwrap(); + pair.client_send(client_ch, s).write(&mut &data[..]).unwrap(); pair.drive(); assert_eq!(pair.server_streams(server_ch).accept(Dir::Uni), Some(s)); s @@ -45,10 +45,7 @@ fn received_reset_reports_the_code_once() { // Observing the reset frees the stream, so every later access finds it closed assert_eq!(pair.server_recv(server_ch, s).received_reset(), Ok(Some(CODE))); assert!(pair.server_recv(server_ch, s).received_reset().is_err()); - assert_eq!( - pair.server_recv(server_ch, s).read().err(), - Some(ReadableError::ClosedStream) - ); + assert_eq!(pair.server_recv(server_ch, s).read().err(), Some(recv::Error::Closed)); assert!(pair.server_recv(server_ch, s).stop(CODE).is_err()); } @@ -102,14 +99,11 @@ fn chunks_end_in_reset() { let mut recv = pair.server_recv(server_ch, s); let mut chunks = recv.read().unwrap(); - assert_eq!(chunks.next(usize::MAX), Err(ReadError::Reset(CODE))); + assert_eq!(chunks.next(usize::MAX), Err(recv::Error::Reset(CODE))); // The end repeats until the chunks are finalized - assert_eq!(chunks.next(usize::MAX), Err(ReadError::Reset(CODE))); + assert_eq!(chunks.next(usize::MAX), Err(recv::Error::Reset(CODE))); let _ = chunks.finalize(); - assert_eq!( - pair.server_recv(server_ch, s).read().err(), - Some(ReadableError::ClosedStream) - ); + assert_eq!(pair.server_recv(server_ch, s).read().err(), Some(recv::Error::Closed)); } #[test] @@ -126,10 +120,7 @@ fn chunks_end_in_finished() { assert_eq!(chunks.next(usize::MAX), Ok(None)); assert_eq!(chunks.next(usize::MAX), Ok(None)); let _ = chunks.finalize(); - assert_eq!( - pair.server_recv(server_ch, s).read().err(), - Some(ReadableError::ClosedStream) - ); + assert_eq!(pair.server_recv(server_ch, s).read().err(), Some(recv::Error::Closed)); } /// The reader sees exactly the code the writer reset with, across the varint range @@ -141,14 +132,17 @@ fn reset_codes() { let s = open(&mut pair, client_ch, server_ch, b"x"); pair.client_send(client_ch, s).reset(code).unwrap(); pair.drive(); - assert_eq!(read_all(&mut pair, server_ch, s), (Vec::new(), ReadError::Reset(code))); + assert_eq!( + read_all(&mut pair, server_ch, s), + (Vec::new(), recv::Error::Reset(code)) + ); let s = open(&mut pair, client_ch, server_ch, b"x"); pair.server_recv(server_ch, s).stop(code).unwrap(); pair.drive(); assert_eq!( - pair.client_send(client_ch, s).write(b"y"), - Err(WriteError::Stopped(code)) + pair.client_send(client_ch, s).write(&mut &b"y"[..]), + Err(send::Error::Stopped(code)) ); } } @@ -163,7 +157,7 @@ fn reset_returns_stream_credit_once_observed() { pair.drive(); assert_eq!(pair.client_streams(client_ch).open(Dir::Uni, Default::default()), None); - assert_eq!(read_all(&mut pair, server_ch, s).1, ReadError::Reset(CODE)); + assert_eq!(read_all(&mut pair, server_ch, s).1, recv::Error::Reset(CODE)); pair.drive(); assert!( pair.client_streams(client_ch) @@ -185,14 +179,19 @@ fn blocked() -> (Pair, ConnectionHandle, ConnectionHandle, StreamId) { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, other).write(b"x"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, other).write(&mut &b"x"[..]), + Err(send::Error::Blocked) + ); (pair, client_ch, server_ch, s) } /// How many bytes a fresh client stream can write fn writable(pair: &mut Pair, ch: ConnectionHandle) -> usize { let s = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - pair.client_send(ch, s).write(&[0; 2 * WINDOW as usize]).unwrap_or(0) + pair.client_send(ch, s) + .write(&mut &[0; 2 * WINDOW as usize][..]) + .unwrap_or(0) } #[test] diff --git a/rs/moq-quic/src/tests/characterize/schedule.rs b/rs/moq-quic/src/tests/characterize/schedule.rs index a98f89c5a33..cee09fce624 100644 --- a/rs/moq-quic/src/tests/characterize/schedule.rs +++ b/rs/moq-quic/src/tests/characterize/schedule.rs @@ -41,7 +41,7 @@ fn assert_golden(pair: &mut Pair, golden: &str) { fn queue(pair: &mut Pair, ch: ConnectionHandle, priority: i64) -> StreamId { let s = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); pair.client_send(ch, s).set_priority(priority).unwrap(); - assert_eq!(pair.client_send(ch, s).write(&[0; LEN]).unwrap(), LEN); + assert_eq!(pair.client_send(ch, s).write(&mut &[0; LEN][..]).unwrap(), LEN); pair.client_send(ch, s).finish().unwrap(); s } @@ -152,12 +152,12 @@ fn datagrams_interleaved() { queue(&mut pair, ch, 0); for _ in 0..3 { pair.client_datagrams(ch) - .send(vec![0; 500].into(), Default::default()) + .send(Bytes::from(vec![0; 500]), Default::default()) .unwrap(); } queue(&mut pair, ch, 0); pair.client_datagrams(ch) - .send(vec![0; 1000].into(), Default::default()) + .send(Bytes::from(vec![0; 1000]), Default::default()) .unwrap(); assert_golden( &mut pair, @@ -180,7 +180,7 @@ fn retransmit_within_stream() { let _guard = subscribe(); let (mut pair, ch) = idle(true); let s = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - assert_eq!(pair.client_send(ch, s).write(&[0; 16_000]).unwrap(), 16_000); + assert_eq!(pair.client_send(ch, s).write(&mut &[0; 16_000][..]).unwrap(), 16_000); // Lose the second packet of the first flight let now = pair.time; diff --git a/rs/moq-quic/src/tests/mod.rs b/rs/moq-quic/src/tests/mod.rs index 9a1d08cd3d1..00b23c2c85b 100644 --- a/rs/moq-quic/src/tests/mod.rs +++ b/rs/moq-quic/src/tests/mod.rs @@ -13,7 +13,7 @@ use std::{ }; use assert_matches::assert_matches; -use bytes::{Bytes, BytesMut}; +use bytes::{Buf, Bytes, BytesMut}; use hex_literal::hex; use rand::Rng; #[cfg(all(feature = "aws-lc-rs", not(feature = "ring")))] @@ -925,7 +925,7 @@ fn finish_stream_simple() { .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); assert_eq!(pair.client_streams(client_ch).send_streams(), 1); pair.client_send(client_ch, s).finish().unwrap(); pair.drive(); @@ -972,7 +972,7 @@ fn reset_stream() { .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive(); info!("resetting stream"); @@ -987,7 +987,7 @@ fn reset_stream() { assert_matches!(pair.server_streams(server_ch).accept(Dir::Uni), Some(stream) if stream == s); let mut recv = pair.server_recv(server_ch, s); let mut chunks = recv.read().unwrap(); - assert_matches!(chunks.next(usize::MAX), Err(ReadError::Reset(ERROR))); + assert_matches!(chunks.next(usize::MAX), Err(recv::Error::Reset(ERROR))); let _ = chunks.finalize(); assert_matches!(pair.client_conn_mut(client_ch).poll(), None); } @@ -1003,7 +1003,7 @@ fn stop_stream() { .open(Dir::Uni, Default::default()) .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive(); info!("stopping stream"); @@ -1018,12 +1018,12 @@ fn stop_stream() { assert_matches!(pair.server_streams(server_ch).accept(Dir::Uni), Some(stream) if stream == s); assert_matches!( - pair.client_send(client_ch, s).write(b"foo"), - Err(WriteError::Stopped(ERROR)) + pair.client_send(client_ch, s).write(&mut &b"foo"[..]), + Err(send::Error::Stopped(ERROR)) ); assert_matches!( pair.client_send(client_ch, s).finish(), - Err(FinishError::Stopped(ERROR)) + Err(send::Error::Stopped(ERROR)) ); } @@ -1110,14 +1110,14 @@ fn congestion() { .unwrap(); // Send data without receiving ACKs until the congestion state falls below target while pair.client_conn_mut(client_ch).congestion_window() > TARGET { - let n = pair.client_send(client_ch, s).write(&[42; 1024]).unwrap(); + let n = pair.client_send(client_ch, s).write(&mut &[42; 1024][..]).unwrap(); assert_eq!(n, 1024); pair.drive_client(); } // Ensure that the congestion state recovers after receiving the ACKs pair.drive(); assert!(pair.client_conn_mut(client_ch).congestion_window() >= TARGET); - pair.client_send(client_ch, s).write(&[42; 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 1024][..]).unwrap(); } #[test] @@ -1148,7 +1148,7 @@ fn full_initial_window() { .unwrap(); let data = vec![42; 2 * window as usize]; assert_eq!( - pair.client_send(client_ch, s).write(&data), + pair.client_send(client_ch, s).write(&mut &data[..]), Ok(data.len()), "the test must be limited by congestion control, not by flow control" ); @@ -1488,7 +1488,7 @@ fn stream_id_limit() { ); // Generate some activity to allow the server to see the stream const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.client_send(client_ch, s).finish().unwrap(); pair.drive(); assert_matches!( @@ -1592,7 +1592,7 @@ fn streams_blocked() { assert_eq!(pair.client_streams(client_ch).open(Dir::Uni, Default::default()), None); // Send data so the STREAMS_BLOCKED piggybacks on an outgoing packet - pair.client_send(client_ch, s).write(b"hi").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hi"[..]).unwrap(); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.streams_blocked_uni, 1); @@ -1610,7 +1610,7 @@ fn streams_blocked_not_sent_under_limit() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .expect("open stream"); - pair.client_send(client_ch, s).write(b"hi").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hi"[..]).unwrap(); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.streams_blocked_uni, 0); @@ -1637,14 +1637,23 @@ fn data_blocked() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.data_blocked, 1); assert_eq!(pair.server_conn_mut(server_ch).stats().frame_rx.data_blocked, 1); // Being refused again at the same limit does not repeat the frame - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.data_blocked, 1); @@ -1655,8 +1664,14 @@ fn data_blocked() { assert_matches!(chunks.next(usize::MAX), Ok(Some(chunk)) if chunk.bytes.len() == 10); let _ = chunks.finalize(); pair.drive(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); let stats = pair.client_conn_mut(client_ch).stats(); assert_eq!(stats.frame_tx.data_blocked, 2); @@ -1684,14 +1699,23 @@ fn stream_data_blocked() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.stream_data_blocked, 1); assert_eq!(pair.server_conn_mut(server_ch).stats().frame_rx.stream_data_blocked, 1); // Being refused again at the same limit does not repeat the frame - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.stream_data_blocked, 1); @@ -1702,8 +1726,14 @@ fn stream_data_blocked() { assert_matches!(chunks.next(usize::MAX), Ok(Some(chunk)) if chunk.bytes.len() == 10); let _ = chunks.finalize(); pair.drive(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive(); let stats = pair.client_conn_mut(client_ch).stats(); assert_eq!(stats.frame_tx.stream_data_blocked, 2); @@ -1721,7 +1751,7 @@ fn data_blocked_not_sent_under_limit() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hi").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hi"[..]).unwrap(); pair.drive(); let stats = pair.client_conn_mut(client_ch).stats(); @@ -1741,8 +1771,14 @@ fn data_blocked_not_sent_for_local_send_window() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789").unwrap(), 5); - assert_matches!(pair.client_send(client_ch, s).write(b"x"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789"[..]).unwrap(), + 5 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"x"[..]), + Err(send::Error::Blocked) + ); pair.drive(); assert_eq!(pair.client_conn_mut(client_ch).stats().frame_tx.data_blocked, 0); @@ -1768,8 +1804,14 @@ fn data_blocked_dropped_when_limit_raised() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); // The peer raises the limit before the queued frame gets transmitted pair.server_conn_mut(server_ch).set_receive_window(100u32.into()); @@ -1799,8 +1841,14 @@ fn stream_data_blocked_not_sent_after_reset() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); // Resetting the stream before the queued frame gets transmitted discards it pair.client_send(client_ch, s).reset(0u32.into()).unwrap(); @@ -1831,8 +1879,14 @@ fn data_blocked_retransmit() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write(b"0123456789ab").unwrap(), 10); - assert_matches!(pair.client_send(client_ch, s).write(b"c"), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &b"0123456789ab"[..]).unwrap(), + 10 + ); + assert_matches!( + pair.client_send(client_ch, s).write(&mut &b"c"[..]), + Err(send::Error::Blocked) + ); pair.drive_client(); pair.server.inbound.clear(); // Lose the packet carrying the frame pair.drive(); @@ -1858,7 +1912,7 @@ fn key_update_simple() { .expect("couldn't open first stream"); const MSG1: &[u8] = b"hello1"; - pair.client_send(client_ch, s).write(MSG1).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG1[..]).unwrap(); pair.drive(); assert_matches!( @@ -1883,7 +1937,7 @@ fn key_update_simple() { pair.client_conn_mut(client_ch).force_key_update(); const MSG2: &[u8] = b"hello2"; - pair.client_send(client_ch, s).write(MSG2).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG2[..]).unwrap(); pair.drive(); assert_matches!(pair.server_conn_mut(server_ch).poll(), Some(Event::Stream(StreamEvent::Readable { id })) if id == s); @@ -1915,7 +1969,7 @@ fn key_update_reordered() { .expect("couldn't open first stream"); const MSG1: &[u8] = b"1"; - pair.client_send(client_ch, s).write(MSG1).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG1[..]).unwrap(); pair.client.drive(pair.time, pair.server.addr); assert!(!pair.client.outbound.is_empty()); pair.client.delay_outbound(); @@ -1924,7 +1978,7 @@ fn key_update_reordered() { info!("updated keys"); const MSG2: &[u8] = b"two"; - pair.client_send(client_ch, s).write(MSG2).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG2[..]).unwrap(); pair.client.drive(pair.time, pair.server.addr); pair.client.finish_delay(); pair.drive(); @@ -2196,7 +2250,7 @@ fn unreachable_is_loss_once_established() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hello").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hello"[..]).unwrap(); pair.drive(); assert_matches!( pair.server_conn_mut(server_ch).poll(), @@ -2240,7 +2294,9 @@ fn connection_close_while_congestion_blocked() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 1024 * 1024]).unwrap(); + pair.client_send(client_ch, s) + .write(&mut &[42; 1024 * 1024][..]) + .unwrap(); pair.drive_client(); // Close while the window is full and stream data is still pending @@ -2343,22 +2399,19 @@ fn test_flow_control(config: transport::Config, window_size: usize) { .open(Dir::Uni, Default::default()) .unwrap(); info!("writing"); - assert_eq!(pair.client_send(client_ch, s).write_limit(), window_size); - assert_eq!(pair.client_send(client_ch, s).write(&msg), Ok(window_size)); - assert_eq!(pair.client_send(client_ch, s).write_limit(), 0); + assert_eq!(pair.client_send(client_ch, s).write(&mut &msg[..]), Ok(window_size)); assert_eq!( - pair.client_send(client_ch, s).write(&msg[window_size..]), - Err(WriteError::Blocked) + pair.client_send(client_ch, s).write(&mut &msg[window_size..]), + Err(send::Error::Blocked) ); pair.drive(); info!("resetting"); pair.client_send(client_ch, s).reset(VarInt(42)).unwrap(); - assert_eq!(pair.client_send(client_ch, s).write_limit(), 0); pair.drive(); let mut recv = pair.server_recv(server_ch, s); let mut chunks = recv.read().unwrap(); - assert_eq!(chunks.next(usize::MAX).err(), Some(ReadError::Reset(VarInt(42)))); + assert_eq!(chunks.next(usize::MAX).err(), Some(recv::Error::Reset(VarInt(42)))); let _ = chunks.finalize(); // Happy path @@ -2367,12 +2420,10 @@ fn test_flow_control(config: transport::Config, window_size: usize) { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, s).write_limit(), window_size); - assert_eq!(pair.client_send(client_ch, s).write(&msg), Ok(window_size)); - assert_eq!(pair.client_send(client_ch, s).write_limit(), 0); + assert_eq!(pair.client_send(client_ch, s).write(&mut &msg[..]), Ok(window_size)); assert_eq!( - pair.client_send(client_ch, s).write(&msg[window_size..]), - Err(WriteError::Blocked) + pair.client_send(client_ch, s).write(&mut &msg[window_size..]), + Err(send::Error::Blocked) ); pair.drive(); @@ -2387,7 +2438,7 @@ fn test_flow_control(config: transport::Config, window_size: usize) { Ok(None) => { panic!("end of stream"); } - Err(ReadError::Blocked) => { + Err(recv::Error::Blocked) => { break; } Err(e) => { @@ -2414,20 +2465,18 @@ fn test_flow_control(config: transport::Config, window_size: usize) { drive_until(&mut pair, |pair| grants(pair) > before); let lost = grants(&mut pair); assert_eq!( - pair.client_send(client_ch, s).write(&msg), - Err(WriteError::Blocked), + pair.client_send(client_ch, s).write(&mut &msg[..]), + Err(send::Error::Blocked), "the dropped grant did not reach the sender" ); pair.to_client.loss = 0.0; pair.drive(); assert!(grants(&mut pair) > lost, "flow-control credit was retransmitted"); info!("writing"); - assert_eq!(pair.client_send(client_ch, s).write_limit(), window_size); - assert_eq!(pair.client_send(client_ch, s).write(&msg), Ok(window_size)); - assert_eq!(pair.client_send(client_ch, s).write_limit(), 0); + assert_eq!(pair.client_send(client_ch, s).write(&mut &msg[..]), Ok(window_size)); assert_eq!( - pair.client_send(client_ch, s).write(&msg[window_size..]), - Err(WriteError::Blocked) + pair.client_send(client_ch, s).write(&mut &msg[window_size..]), + Err(send::Error::Blocked) ); pair.drive(); @@ -2442,7 +2491,7 @@ fn test_flow_control(config: transport::Config, window_size: usize) { Ok(None) => { panic!("end of stream"); } - Err(ReadError::Blocked) => { + Err(recv::Error::Blocked) => { break; } Err(e) => { @@ -2514,12 +2563,12 @@ fn stop_opens_bidi() { let mut recv = pair.server_recv(server_ch, s); let mut chunks = recv.read().unwrap(); - assert_matches!(chunks.next(usize::MAX), Err(ReadError::Blocked)); + assert_matches!(chunks.next(usize::MAX), Err(recv::Error::Blocked)); let _ = chunks.finalize(); assert_matches!( - pair.server_send(server_ch, s).write(b"foo"), - Err(WriteError::Stopped(ERROR)) + pair.server_send(server_ch, s).write(&mut &b"foo"[..]), + Err(send::Error::Stopped(ERROR)) ); assert_matches!( pair.server_conn_mut(server_ch).poll(), @@ -2544,7 +2593,7 @@ fn implicit_open() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s2).write(b"hello").unwrap(); + pair.client_send(client_ch, s2).write(&mut &b"hello"[..]).unwrap(); pair.drive(); assert_matches!( pair.server_conn_mut(server_ch).poll(), @@ -2716,7 +2765,7 @@ fn finish_stream_flow_control_reordered() { .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive_client(); // Send stream data pair.server.drive(pair.time, pair.client.addr); // Receive @@ -2774,7 +2823,7 @@ fn handshake_1rtt_handling() { .open(Dir::Uni, Default::default()) .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.client_send(client_ch, s).finish().unwrap(); pair.client.drive(pair.time, pair.server.addr); let early = pair.client.outbound.len(); @@ -2817,7 +2866,7 @@ fn stop_before_finish() { .open(Dir::Uni, Default::default()) .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive(); info!("stopping stream"); @@ -2827,7 +2876,7 @@ fn stop_before_finish() { assert_matches!( pair.client_send(client_ch, s).finish(), - Err(FinishError::Stopped(ERROR)) + Err(send::Error::Stopped(ERROR)) ); } @@ -2842,7 +2891,7 @@ fn stop_during_finish() { .open(Dir::Uni, Default::default()) .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive(); assert_matches!(pair.server_streams(server_ch).accept(Dir::Uni), Some(stream) if stream == s); @@ -2873,7 +2922,7 @@ fn congested_tail_loss() { .unwrap(); // Send data without receiving ACKs until the congestion state falls below target while pair.client_conn_mut(client_ch).congestion_window() > TARGET { - let n = pair.client_send(client_ch, s).write(&[42; 1024]).unwrap(); + let n = pair.client_send(client_ch, s).write(&mut &[42; 1024][..]).unwrap(); assert_eq!(n, 1024); pair.drive_client(); } @@ -2883,7 +2932,7 @@ fn congested_tail_loss() { info!("recovering"); pair.drive(); assert!(pair.client_conn_mut(client_ch).congestion_window() > TARGET); - pair.client_send(client_ch, s).write(&[42; 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 1024][..]).unwrap(); } // Send a tail-loss probe when GSO segment_size is less than INITIAL_MTU @@ -2926,7 +2975,7 @@ fn tail_loss_small_segment_size() { info!("Sending datagram batch"); for _ in 0..DGRAM_NUM { pair.client_datagrams(client_ch) - .send(vec![0; DGRAM_LEN].into(), Default::default()) + .send(Bytes::from(vec![0; DGRAM_LEN]), Default::default()) .unwrap(); } @@ -2977,7 +3026,7 @@ fn tail_loss_probe_keeps_ping_when_datagram_does_not_fit() { let datagram_len = pair.client_datagrams(client_ch).max_size().unwrap(); assert!(datagram_len > INITIAL_MTU as usize); pair.client_datagrams(client_ch) - .send(vec![0; datagram_len].into(), Default::default()) + .send(Bytes::from(vec![0; datagram_len]), Default::default()) .unwrap(); pair.drive(); @@ -3023,13 +3072,13 @@ fn tail_loss_probe_ended_short_ends_the_gso_batch() { // Overhead: flags + CID + PN + tag + frame type + frame length = 1 + 8 + 1 + 16 + 1 + 2 = 29 const DATAGRAM_LEN: usize = INITIAL_MTU as usize - 29; pair.client_datagrams(client_ch) - .send(vec![0; DATAGRAM_LEN].into(), Default::default()) + .send(Bytes::from(vec![0; DATAGRAM_LEN]), Default::default()) .unwrap(); let s = pair .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[0; 4096]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[0; 4096][..]).unwrap(); pair.client_conn_mut(client_ch).ping(); pair.drive(); @@ -3071,7 +3120,7 @@ fn tail_loss_respect_max_datagrams() { info!("Sending datagram batch"); for _ in 0..DGRAM_NUM { pair.client_datagrams(client_ch) - .send(vec![0; DGRAM_LEN].into(), Default::default()) + .send(Bytes::from(vec![0; DGRAM_LEN]), Default::default()) .unwrap(); } @@ -3092,7 +3141,7 @@ fn datagram_send_recv() { const DATA: &[u8] = b"whee"; pair.client_datagrams(client_ch) - .send(DATA.into(), Default::default()) + .send(Bytes::from(DATA), Default::default()) .unwrap(); pair.drive(); assert_matches!(pair.server_conn_mut(server_ch).poll(), Some(Event::DatagramReceived)); @@ -3114,19 +3163,19 @@ fn first_flight_datagrams(stream: i64, datagram: i64) -> usize { .streams() .open( Dir::Uni, - SendOptions { + send::Options { group: Some(&streams), priority: 0, }, ) .unwrap(); - conn.send_stream(s).write(&[0; 256 * 1024]).unwrap(); + conn.send_stream(s).write(&mut &[0; 256 * 1024][..]).unwrap(); for _ in 0..DATAGRAMS { - let options = SendOptions { + let options = send::Options { group: Some(&datagrams), priority: 0, }; - conn.datagrams().send(vec![1; 100].into(), options).unwrap(); + conn.datagrams().send(Bytes::from(vec![1; 100]), options).unwrap(); } // One congestion-limited flight. @@ -3187,13 +3236,13 @@ fn datagram_recv_buffer_overflow() { const DATA2: &[u8] = &[0xBC; (PAYLOAD_WINDOW / 3) + 1]; const DATA3: &[u8] = &[0xCD; (PAYLOAD_WINDOW / 3) + 1]; pair.client_datagrams(client_ch) - .send(DATA1.into(), Default::default()) + .send(Bytes::from(DATA1), Default::default()) .unwrap(); pair.client_datagrams(client_ch) - .send(DATA2.into(), Default::default()) + .send(Bytes::from(DATA2), Default::default()) .unwrap(); pair.client_datagrams(client_ch) - .send(DATA3.into(), Default::default()) + .send(Bytes::from(DATA3), Default::default()) .unwrap(); pair.drive(); assert_matches!(pair.server_conn_mut(server_ch).poll(), Some(Event::DatagramReceived)); @@ -3202,7 +3251,7 @@ fn datagram_recv_buffer_overflow() { assert_matches!(pair.server_datagrams(server_ch).recv(), None); pair.client_datagrams(client_ch) - .send(DATA1.into(), Default::default()) + .send(Bytes::from(DATA1), Default::default()) .unwrap(); pair.drive(); assert_eq!(pair.server_datagrams(server_ch).recv().unwrap(), DATA1); @@ -3232,7 +3281,7 @@ fn datagram_of_max_size_keeps_connection() { assert_eq!(max_size, Datagram::max_payload(WINDOW as u64).unwrap() as usize); pair.client_datagrams(client_ch) - .send(vec![0xAB; max_size].into(), Default::default()) + .send(Bytes::from(vec![0xAB; max_size]), Default::default()) .unwrap(); pair.drive(); assert_matches!(pair.server_conn_mut(server_ch).poll(), None); @@ -3240,7 +3289,7 @@ fn datagram_of_max_size_keeps_connection() { const DATA: &[u8] = &[0xCD; 100]; pair.client_datagrams(client_ch) - .send(DATA.into(), Default::default()) + .send(Bytes::from(DATA), Default::default()) .unwrap(); pair.drive(); assert_matches!(pair.server_conn_mut(server_ch).poll(), Some(Event::DatagramReceived)); @@ -3287,12 +3336,12 @@ fn datagram_larger_than_send_buffer_is_too_large() { assert_matches!( pair.client_datagrams(client_ch) .send(Bytes::from_static(&[0; 2]), Default::default()), - Err(SendDatagramError::TooLarge) + Err(send::Error::TooLarge) ); assert_matches!( pair.client_datagrams(client_ch) .send(Bytes::from_static(&[0; 2]), Default::default()), - Err(SendDatagramError::TooLarge) + Err(send::Error::TooLarge) ); } @@ -3315,7 +3364,7 @@ fn datagram_unsupported() { assert_matches!(pair.client_datagrams(client_ch).max_size(), None); match pair.client_datagrams(client_ch).send(Bytes::new(), Default::default()) { - Err(SendDatagramError::UnsupportedByPeer) => {} + Err(send::Error::Unsupported) => {} Err(e) => panic!("unexpected error: {e}"), Ok(_) => panic!("unexpected success"), } @@ -3342,7 +3391,7 @@ fn datagram_zero_limit_unsupported() { assert_matches!(pair.client_datagrams(client_ch).max_size(), None); assert_matches!( pair.client_datagrams(client_ch).send(Bytes::new(), Default::default()), - Err(SendDatagramError::UnsupportedByPeer) + Err(send::Error::Unsupported) ); } @@ -3378,7 +3427,7 @@ fn finish_acked() { .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); info!("client sends data to server"); pair.drive_client(); // send data to server info!("server acknowledges data"); @@ -3403,7 +3452,7 @@ fn finish_acked() { chunks.next(usize::MAX), Ok(Some(chunk)) if chunk.bytes == MSG ); - assert_matches!(chunks.next(usize::MAX), Err(ReadError::Blocked)); + assert_matches!(chunks.next(usize::MAX), Err(recv::Error::Blocked)); let _ = chunks.finalize(); // Finish before receiving data ack @@ -3440,7 +3489,7 @@ fn finish_retransmit() { .unwrap(); const MSG: &[u8] = b"hello"; - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.drive_client(); // send data to server pair.server.inbound.clear(); // Lose it @@ -3502,7 +3551,7 @@ fn repeated_request_response() { .open(Dir::Bi, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(REQUEST).unwrap(); + pair.client_send(client_ch, s).write(&mut &REQUEST[..]).unwrap(); pair.client_send(client_ch, s).finish().unwrap(); pair.drive(); @@ -3517,7 +3566,7 @@ fn repeated_request_response() { assert_matches!(chunks.next(usize::MAX), Ok(None)); let _ = chunks.finalize(); - pair.server_send(server_ch, s).write(RESPONSE).unwrap(); + pair.server_send(server_ch, s).write(&mut &RESPONSE[..]).unwrap(); pair.server_send(server_ch, s).finish().unwrap(); pair.drive(); @@ -3660,7 +3709,7 @@ fn a_peer_heard_after_an_outage_brings_the_probe_forward() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hello").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hello"[..]).unwrap(); let probes = probe_through_outage(&mut pair, (client_ch, server_ch), Duration::from_secs(5), false, None); assert_backs_off(&probes); @@ -3696,7 +3745,7 @@ fn a_peer_heard_through_a_one_way_outage_leaves_the_backoff() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hello").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hello"[..]).unwrap(); let probes = probe_through_outage( &mut pair, @@ -3722,7 +3771,7 @@ fn a_peer_heard_as_the_data_left_still_brings_the_probe_forward() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hello").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hello"[..]).unwrap(); // The PING arrives now and the data leaves in the same drive, into an outage that lasts until // the client's first probe. @@ -3760,7 +3809,7 @@ fn a_peer_heard_on_a_new_path_brings_no_probe_forward() { .server_streams(server_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.server_send(server_ch, s).write(b"hello").unwrap(); + pair.server_send(server_ch, s).write(&mut &b"hello"[..]).unwrap(); // The server's data and two probes go unheard. pair.to_server.loss = 1.0; @@ -3823,7 +3872,7 @@ fn an_unprotected_packet_does_not_bring_the_probe_forward() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(b"hello").unwrap(); + pair.client_send(client_ch, s).write(&mut &b"hello"[..]).unwrap(); let probes = probe_through_outage(&mut pair, (client_ch, server_ch), Duration::from_secs(5), false, None); assert_backs_off(&probes); @@ -4283,7 +4332,7 @@ fn blackhole_after_mtu_change_repairs_itself() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&payload).unwrap(); + pair.client_send(client_ch, s).write(&mut &payload[..]).unwrap(); let out_of_bounds = pair.drive_bounded(); if out_of_bounds { @@ -4335,7 +4384,7 @@ fn packet_splitting_with_default_mtu() { .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&payload).unwrap(); + pair.client_send(client_ch, s).write(&mut &payload[..]).unwrap(); pair.client.drive(pair.time, pair.server.addr); assert_eq!(pair.client.outbound.len(), 2); @@ -4359,7 +4408,7 @@ fn packet_splitting_not_necessary_after_higher_mtu_discovered() { .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&payload).unwrap(); + pair.client_send(client_ch, s).write(&mut &payload[..]).unwrap(); pair.client.drive(pair.time, pair.server.addr); assert_eq!(pair.client.outbound.len(), 1); @@ -4762,7 +4811,7 @@ fn ack_frequency_update_max_delay() { assert!(pair.server_conn_mut(server_ch).stats().frame_rx.ack_frequency >= 2); } -fn stream_chunks(mut recv: RecvStream<'_>) -> Vec { +fn stream_chunks(mut recv: recv::Stream<'_>) -> Vec { let mut buf = Vec::new(); let mut chunks = recv.read().unwrap(); @@ -4909,7 +4958,7 @@ fn stream_gso() { // datagram batches info!("sending"); for _ in 0..20 { - pair.client_send(client_ch, s).write(&[0; 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[0; 1024][..]).unwrap(); } pair.client_send(client_ch, s).finish().unwrap(); pair.drive(); @@ -4966,7 +5015,7 @@ fn gso_truncation() { const SIZES: [usize; 3] = [1024, 768, 768]; for len in SIZES { pair.client_datagrams(client_ch) - .send(vec![0; len].into(), Default::default()) + .send(Bytes::from(vec![0; len]), Default::default()) .unwrap(); } pair.drive(); @@ -5071,7 +5120,7 @@ fn min_mtu_limits_application_loss_probes() { let client = pair.client_conn_mut(client_ch); client.handle_timeout(now); let stream = client.streams().open(Dir::Uni, Default::default()).unwrap(); - client.send_stream(stream).write(&[42; 3000]).unwrap(); + client.send_stream(stream).write(&mut &[42; 3000][..]).unwrap(); let mut buf = Vec::new(); let transmit = client.poll_transmit(now, 1, &mut buf).unwrap(); assert_eq!(transmit.size, usize::from(MIN_MTU)); @@ -5168,10 +5217,10 @@ fn pad_to_mtu() { const LEN_1: usize = 800; const LEN_2: usize = 600; pair.client_datagrams(client_ch) - .send(vec![0; LEN_1].into(), Default::default()) + .send(Bytes::from(vec![0; LEN_1]), Default::default()) .unwrap(); pair.client_datagrams(client_ch) - .send(vec![0; LEN_2].into(), Default::default()) + .send(Bytes::from(vec![0; LEN_2]), Default::default()) .unwrap(); pair.client.drive(pair.time, pair.server.addr); @@ -5246,7 +5295,7 @@ fn voluntary_ack_with_large_datagrams() { for _ in 0..COUNT { let max_size = pair.client_datagrams(client_ch).max_size().unwrap(); pair.client_datagrams(client_ch) - .send(vec![0; max_size].into(), Default::default()) + .send(Bytes::from(vec![0; max_size]), Default::default()) .unwrap(); pair.drive(); } @@ -5268,7 +5317,7 @@ fn ack_bundled_with_datagrams() { // Send packet from client and then send from server. the packet from server should include ACKs pair.client_datagrams(client_ch) - .send(vec![0; 1].into(), Default::default()) + .send(Bytes::from(vec![0; 1]), Default::default()) .unwrap(); pair.drive_client(); pair.drive_server(); @@ -5277,7 +5326,7 @@ fn ack_bundled_with_datagrams() { let server_tx_packets_before_datagram = pair.server_conn_mut(server_ch).stats().udp_tx.datagrams; pair.server_datagrams(server_ch) - .send(vec![0; 1].into(), Default::default()) + .send(Bytes::from(vec![0; 1]), Default::default()) .unwrap(); pair.drive_server(); @@ -5387,7 +5436,7 @@ fn oversized_datagrams_are_dropped() { let data = vec![0; max_size]; while pair.client_datagrams(client_ch).send_buffer_space() >= max_size { pair.client_datagrams(client_ch) - .send(data.clone().into(), Default::default()) + .send(Bytes::from(data.clone()), Default::default()) .unwrap(); } // Set the MTU to a smaller value so the queued datagrams cannot be sent. @@ -5400,11 +5449,11 @@ fn oversized_datagrams_are_dropped() { while sent.is_ok() && pair.client_datagrams(client_ch).send_buffer_space() >= max_size { sent = pair .client_datagrams(client_ch) - .send(data.clone().into(), Default::default()); + .send(Bytes::from(data.clone()), Default::default()); } match sent { Ok(()) => {} - Err(SendDatagramError::TooLarge) => break, + Err(send::Error::TooLarge) => break, Err(err) => panic!("unexpected error: {err}"), } } @@ -5838,7 +5887,7 @@ fn cwnd_limited_is_not_reported_when_only_pacing_blocks() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 64 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 64 * 1024][..]).unwrap(); pair.drive(); assert_eq!( @@ -5918,7 +5967,7 @@ fn send_quantum_bounds_the_gso_batch() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 64 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 64 * 1024][..]).unwrap(); let now = pair.time; let mut buf = Vec::new(); @@ -6126,7 +6175,7 @@ fn congestion_callbacks_resolve_each_packet_once() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 16 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 16 * 1024][..]).unwrap(); pair.drive(); let logs = factory.logs.lock().unwrap().clone(); @@ -6183,7 +6232,7 @@ fn old_ce_marks_report_no_new_congestion() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 8 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 8 * 1024][..]).unwrap(); pair.congestion_experienced = true; pair.drive_client(); pair.congestion_experienced = false; @@ -6191,7 +6240,7 @@ fn old_ce_marks_report_no_new_congestion() { let marked = ce_events(&log); assert!(marked > 0); - pair.client_send(client_ch, s).write(&[42; 64 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 64 * 1024][..]).unwrap(); pair.drive(); assert_eq!(ce_events(&log), marked); assert!(pair.client_conn_mut(client_ch).using_ecn()); @@ -6213,7 +6262,7 @@ fn invalid_ecn_feedback_reports_no_congestion() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 16 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 16 * 1024][..]).unwrap(); pair.drive(); assert_eq!(ce_events(&log), 0, "{rewrite:?}"); if rewrite.is_some() { @@ -6250,7 +6299,7 @@ fn padded_ack_only_packets_resolve_exactly() { let round = |pair: &mut Pair, loss| { pair.to_client.loss = loss; for _ in 0..8 { - pair.client_send(client_ch, s).write(&[42; 8 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 8 * 1024][..]).unwrap(); pair.drive(); pair.server_conn_mut(server_ch).ping(); pair.drive(); @@ -6323,7 +6372,7 @@ fn check_starvation_reported_before_resumed_send(send: impl Fn(&mut Pair, Connec fn stream_starvation_reported_before_resumed_send() { check_starvation_reported_before_resumed_send(|pair, ch| { let s = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - pair.client_send(ch, s).write(&[42; 1000]).unwrap(); + pair.client_send(ch, s).write(&mut &[42; 1000][..]).unwrap(); }); } @@ -6351,7 +6400,7 @@ fn check_blocked_backlog_is_not_app_limited(factory: PacketRecorderFactory) { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&[42; 64 * 1024]).unwrap(); + pair.client_send(client_ch, s).write(&mut &[42; 64 * 1024][..]).unwrap(); pair.drive_client(); let events = log.lock().unwrap()[before..].to_vec(); @@ -6385,7 +6434,7 @@ fn partial_poll_reports_starvation() { let _guard = subscribe(); let (mut pair, ch, log) = recorded_pair(Default::default()); let stream = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - pair.client_send(ch, stream).write(&[42; 1000]).unwrap(); + pair.client_send(ch, stream).write(&mut &[42; 1000][..]).unwrap(); log.lock().unwrap().clear(); let now = pair.time; let mut buf = Vec::new(); @@ -6393,9 +6442,20 @@ fn partial_poll_reports_starvation() { assert!(matches!(log.lock().unwrap().last(), Some(PacketEvent::AppLimited { in_flight }) if *in_flight > 0)); } -/// The bytes the server has leased from `s`. -fn leased(pair: &mut Pair, server_ch: ConnectionHandle, s: StreamId) -> usize { - pair.server_conn_mut(server_ch).leased(s).map_or(0, |lease| lease.len()) +/// The bytes the server holds leased, none of them read yet. +fn leased(pair: &mut Pair, server_ch: ConnectionHandle) -> usize { + pair.server_conn_mut(server_ch).stats().credit.leased as usize +} + +/// Lease what has arrived on `s`, read `n` bytes of it, and release their credit, returning +/// whether the peer finished the stream right after the leased bytes. +fn consume(pair: &mut Pair, server_ch: ConnectionHandle, s: StreamId, n: usize) -> bool { + let mut recv = pair.server_recv(server_ch, s); + let mut lease = recv.lease(0).0.unwrap(); + let fin = lease.is_fin(); + lease.advance(n); + let _ = recv.release(lease); + fin } /// Write what the client can of `msg` up to `until`, finishing at its end, and lease what the @@ -6413,7 +6473,7 @@ fn pump_leased( pair.drive(); let mut wrote = 0; if *sent < until - && let Ok(n) = pair.client_send(client_ch, s).write(&msg[*sent..until]) + && let Ok(n) = pair.client_send(client_ch, s).write(&mut &msg[*sent..until]) { wrote = n; *sent += n; @@ -6423,9 +6483,12 @@ fn pump_leased( } pair.drive(); - let before = leased(pair, server_ch, s); - let _ = pair.server_recv(server_ch, s).lease(0); - let read = leased(pair, server_ch, s) - before; + let before = leased(pair, server_ch); + let mut recv = pair.server_recv(server_ch, s); + if let Ok(lease) = recv.lease(0).0 { + let _ = recv.restore(lease); + } + let read = leased(pair, server_ch) - before; if read == 0 && wrote == 0 { return total; } @@ -6470,22 +6533,20 @@ fn lease_spends_connection_window_only() { // Too little to announce while the peer has room, so the peer blocks at the window, and the // frame that spends the last of its room is what announces it. - let mut recv = pair.server_recv(server_ch, s); - recv.advance(100); - let _ = recv.release(); + assert!(!consume(&mut pair, server_ch, s, 100)); let all = msg.len(); assert_eq!(pump_leased(&mut pair, chs, s, (&msg, all), &mut sent), 1050); assert_eq!(sent, msg.len()); - assert_eq!(leased(&mut pair, server_ch, s), WINDOW + 50 - 100); + assert_eq!(leased(&mut pair, server_ch), WINDOW + 50 - 100); - assert!(pair.server_conn_mut(server_ch).leased(s).unwrap().is_fin()); - let all = leased(&mut pair, server_ch, s); - let mut recv = pair.server_recv(server_ch, s); - recv.advance(all); - let _ = recv.release(); - assert_eq!(pair.server_conn_mut(server_ch).stats().credit.leased, 0); + let all = leased(&mut pair, server_ch); + assert!(consume(&mut pair, server_ch, s, all), "finished after the leased bytes"); + assert_eq!(leased(&mut pair, server_ch), 0); pair.server_recv(server_ch, s).stop(0u32.into()).unwrap(); - assert!(pair.server_conn_mut(server_ch).leased(s).is_none()); + assert_eq!( + pair.server_recv(server_ch, s).lease(0).0.err(), + Some(recv::Error::Closed) + ); } #[test] @@ -6494,7 +6555,7 @@ fn sender_window_is_not_starvation() { let (mut pair, ch, log) = recorded_pair(Default::default()); pair.client_conn_mut(ch).set_send_window(1000); let stream = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - assert_eq!(pair.client_send(ch, stream).write(&[42; 2000]), Ok(1000)); + assert_eq!(pair.client_send(ch, stream).write(&mut &[42; 2000][..]), Ok(1000)); log.lock().unwrap().clear(); pair.drive_client(); assert!( @@ -6529,7 +6590,7 @@ fn receiver_credit_is_starvation() { let (ch, _) = pair.connect_with(client_cfg); let log = factory.logs.lock().unwrap()[0].clone(); let stream = pair.client_streams(ch).open(Dir::Uni, Default::default()).unwrap(); - assert_eq!(pair.client_send(ch, stream).write(&[42; 2000]), Ok(1000)); + assert_eq!(pair.client_send(ch, stream).write(&mut &[42; 2000][..]), Ok(1000)); log.lock().unwrap().clear(); pair.drive_client(); assert!( @@ -6701,9 +6762,12 @@ fn upload( let mut server_stream = None; loop { while unsent > 0 { - match pair.client_send(client_ch, s).write(&CHUNK[..unsent.min(CHUNK.len())]) { + match pair + .client_send(client_ch, s) + .write(&mut &CHUNK[..unsent.min(CHUNK.len())]) + { Ok(n) => unsent -= n, - Err(WriteError::Blocked) => break, + Err(send::Error::Blocked) => break, Err(e) => panic!("{e}"), } if unsent == 0 { diff --git a/rs/moq-quic/src/tests/probe.rs b/rs/moq-quic/src/tests/probe.rs index 20d716aab2a..79d35d29c9e 100644 --- a/rs/moq-quic/src/tests/probe.rs +++ b/rs/moq-quic/src/tests/probe.rs @@ -325,7 +325,7 @@ fn run(scenario: &Scenario) -> Outcome { if let Some((client, _, id, _)) = &competing { let mut stream = pair.client_send(*client, *id); - while stream.write(&[0; 16_384]).is_ok() {} + while stream.write(&mut &[0; 16_384][..]).is_ok() {} } pair.drive_client(); @@ -426,7 +426,7 @@ fn write_frame(pair: &mut Pair, ch: ConnectionHandle, groups: &mut Vec, k .client_streams(ch) .open( Dir::Uni, - SendOptions { + send::Options { group: None, priority: sequence, }, @@ -454,9 +454,9 @@ fn flush(pair: &mut Pair, ch: ConnectionHandle, groups: &mut [Group]) { for group in groups.iter_mut().filter(|g| g.reset.is_none() && !g.finished) { let mut stream = pair.client_send(ch, group.id); while !group.backlog.is_empty() { - match stream.write(&group.backlog) { + match stream.write(&mut &group.backlog[..]) { Ok(n) => drop(group.backlog.drain(..n)), - Err(WriteError::Blocked) => return, + Err(send::Error::Blocked) => return, Err(e) => panic!("write: {e}"), } } @@ -517,7 +517,7 @@ fn receive( reading.done = true; break; } - Err(ReadError::Blocked) => break, + Err(recv::Error::Blocked) => break, Err(_) => { reading.done = true; break; @@ -762,7 +762,7 @@ fn single_packet_writes() { let mut next_write = pair.time; while pair.time < end { if pair.time >= next_write { - pair.client_send(client_ch, id).write(&[0; 100]).unwrap(); + pair.client_send(client_ch, id).write(&mut &[0; 100][..]).unwrap(); next_write += Duration::from_millis(200); } pair.drive_client(); @@ -787,7 +787,7 @@ fn batch_within_probe_inflight() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, id).write(&[0; 1400]).unwrap(); + pair.client_send(client_ch, id).write(&mut &[0; 1400][..]).unwrap(); pair.drive_client(); let conn = pair.client_conn_mut(client_ch); diff --git a/rs/moq-quic/src/tests/reset_stream_at.rs b/rs/moq-quic/src/tests/reset_stream_at.rs index a1315537007..57ab253bb9e 100644 --- a/rs/moq-quic/src/tests/reset_stream_at.rs +++ b/rs/moq-quic/src/tests/reset_stream_at.rs @@ -18,7 +18,7 @@ fn pair(server_reset_stream_at: bool) -> Pair { } /// Read a server stream until it ends, returning the bytes and how it ended -fn read(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, Result<(), ReadError>) { +fn read(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, Result<(), recv::Error>) { let mut recv = pair.server_recv(ch, s); let mut chunks = recv.read().unwrap(); let mut data = Vec::new(); @@ -39,10 +39,10 @@ fn accept(pair: &mut Pair, ch: ConnectionHandle) -> StreamId { } /// Write `MSG` to a client stream, returning the offset after its first `RELIABLE` bytes -fn write(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> u64 { - pair.client_send(ch, s).write(&MSG[..RELIABLE]).unwrap(); +fn write(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> send::Offset { + pair.client_send(ch, s).write(&mut &MSG[..RELIABLE]).unwrap(); let offset = pair.client_send(ch, s).offset(); - pair.client_send(ch, s).write(&MSG[RELIABLE..]).unwrap(); + pair.client_send(ch, s).write(&mut &MSG[RELIABLE..]).unwrap(); offset } @@ -51,7 +51,7 @@ fn write(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> u64 { /// /// A plain RESET_STREAM lets the receiver discard data it has not read yet, so only an /// application that keeps reading sees the prefix a peer without RESET_STREAM_AT is sent. -fn drive_reading(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, Result<(), ReadError>) { +fn drive_reading(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec, Result<(), recv::Error>) { let mut accepted = false; let mut data = Vec::new(); loop { @@ -60,12 +60,12 @@ fn drive_reading(pair: &mut Pair, ch: ConnectionHandle, s: StreamId) -> (Vec if accepted { let (read, end) = read(pair, ch, s); data.extend(read); - if end != Err(ReadError::Blocked) { + if end != Err(recv::Error::Blocked) { return (data, end); } } if !busy { - return (data, Err(ReadError::Blocked)); + return (data, Err(recv::Error::Blocked)); } } } @@ -94,14 +94,14 @@ fn stop_sending_abandons_prefix(reset_stream_at: bool) { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&MSG[..1]).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..1]).unwrap(); pair.drive(); assert_eq!(accept(&mut pair, server_ch), s); // The rest of the prefix is lost, and so is any reset sent for it - pair.client_send(client_ch, s).write(&MSG[1..RELIABLE]).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[1..RELIABLE]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); - pair.client_send(client_ch, s).write(&MSG[RELIABLE..]).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[RELIABLE..]).unwrap(); lose_client_packets(&mut pair); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); pair.client.drive(pair.time, pair.server.addr); @@ -144,13 +144,13 @@ fn delivers_prefix_before_error() { assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); // Both halves let go of the stream once the prefix and the reset are through assert_eq!(pair.client_streams(client_ch).send_streams(), 0); assert!(matches!( pair.server_recv(server_ch, s).read(), - Err(ReadableError::ClosedStream) + Err(recv::Error::Closed) )); } @@ -166,7 +166,7 @@ fn delivers_unsent_prefix() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&MSG[..RELIABLE]).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..RELIABLE]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); pair.drive(); @@ -174,7 +174,7 @@ fn delivers_unsent_prefix() { assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); } @@ -196,7 +196,7 @@ fn retransmits_lost_prefix() { pair.drive_client(); pair.drive_server(); assert_eq!(accept(&mut pair, server_ch), s); - assert_eq!(read(&mut pair, server_ch, s), (Vec::new(), Err(ReadError::Blocked))); + assert_eq!(read(&mut pair, server_ch, s), (Vec::new(), Err(recv::Error::Blocked))); assert_matches!( pair.server_conn_mut(server_ch).recv_stream(s).received_reset(), Ok(None) @@ -205,7 +205,7 @@ fn retransmits_lost_prefix() { pair.drive(); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); assert_eq!(pair.client_streams(client_ch).send_streams(), 0); } @@ -229,7 +229,7 @@ fn retransmits_lost_reset() { assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); assert_eq!(pair.client_streams(client_ch).send_streams(), 0); } @@ -286,7 +286,7 @@ fn after_finish() { assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..reliable].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..reliable].to_vec(), Err(recv::Error::Reset(ERROR))) ); } assert_eq!(pair.client_streams(client_ch).send_streams(), 0); @@ -307,20 +307,23 @@ fn flow_control_blocked() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - pair.client_send(client_ch, s).write(&MSG[..RELIABLE]).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..RELIABLE]).unwrap(); let offset = pair.client_send(client_ch, s).offset(); assert_eq!( - pair.client_send(client_ch, s).write(&MSG[RELIABLE..]), + pair.client_send(client_ch, s).write(&mut &MSG[RELIABLE..]), Ok(10 - RELIABLE) ); - assert_eq!(pair.client_send(client_ch, s).write(MSG), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &MSG[..]), + Err(send::Error::Blocked) + ); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); pair.drive(); assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); assert!(!pair.server_conn_mut(server_ch).is_closed()); } @@ -345,7 +348,7 @@ fn unsupported_holds_reset_until_acked() { assert_eq!(resets_sent(&mut pair, client_ch), 0); assert_eq!( drive_reading(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); assert_eq!(resets_sent(&mut pair, client_ch), 1); pair.drive(); @@ -368,7 +371,7 @@ fn unsupported_retransmits_lost_prefix() { assert_eq!( drive_reading(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); assert_eq!(resets_sent(&mut pair, client_ch), 1); pair.drive(); @@ -389,7 +392,7 @@ fn unsupported_resets_at_once_when_acked() { let offset = write(&mut pair, client_ch, s); pair.drive(); assert_eq!(accept(&mut pair, server_ch), s); - assert_eq!(read(&mut pair, server_ch, s), (MSG.to_vec(), Err(ReadError::Blocked))); + assert_eq!(read(&mut pair, server_ch, s), (MSG.to_vec(), Err(recv::Error::Blocked))); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); pair.drive_client(); @@ -397,7 +400,7 @@ fn unsupported_resets_at_once_when_acked() { pair.drive(); assert_eq!( read(&mut pair, server_ch, s), - (Vec::new(), Err(ReadError::Reset(ERROR))) + (Vec::new(), Err(recv::Error::Reset(ERROR))) ); } @@ -415,12 +418,15 @@ fn unsupported_after_finish() { let offset = write(&mut pair, client_ch, s); pair.client_send(client_ch, s).finish().unwrap(); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); - assert_eq!(pair.client_send(client_ch, s).finish(), Err(FinishError::ClosedStream)); - assert_eq!(pair.client_send(client_ch, s).write(MSG), Err(WriteError::ClosedStream)); + assert_eq!(pair.client_send(client_ch, s).finish(), Err(send::Error::Closed)); + assert_eq!( + pair.client_send(client_ch, s).write(&mut &MSG[..]), + Err(send::Error::Closed) + ); assert_eq!( drive_reading(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); pair.drive(); assert_eq!(pair.client_streams(client_ch).send_streams(), 0); @@ -445,7 +451,7 @@ fn unsupported_repeated_reset_is_closed() { pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); assert_eq!( pair.client_send(client_ch, s).reset_at(again, ERROR), - Err(ClosedStream::default()) + Err(send::Error::Closed) ); assert!(pair.client_send(client_ch, s).reset(ERROR).is_err()); } @@ -498,14 +504,17 @@ fn unsupported_returns_flow_control() { .client_streams(client_ch) .open(Dir::Uni, Default::default()) .unwrap(); - assert_eq!(pair.client_send(client_ch, next).write(MSG), Err(WriteError::Blocked)); + assert_eq!( + pair.client_send(client_ch, next).write(&mut &MSG[..]), + Err(send::Error::Blocked) + ); assert_eq!( drive_reading(&mut pair, server_ch, s), - (MSG[..RELIABLE].to_vec(), Err(ReadError::Reset(ERROR))) + (MSG[..RELIABLE].to_vec(), Err(recv::Error::Reset(ERROR))) ); pair.drive(); - assert_eq!(pair.client_send(client_ch, next).write(MSG), Ok(MSG.len())); + assert_eq!(pair.client_send(client_ch, next).write(&mut &MSG[..]), Ok(MSG.len())); } /// A zero Reliable Size is a plain RESET_STREAM, which needs no negotiation. @@ -520,14 +529,14 @@ fn unsupported_allows_zero() { .open(Dir::Uni, Default::default()) .unwrap(); let offset = pair.client_send(client_ch, s).offset(); - pair.client_send(client_ch, s).write(MSG).unwrap(); + pair.client_send(client_ch, s).write(&mut &MSG[..]).unwrap(); pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); pair.drive(); assert_eq!(accept(&mut pair, server_ch), s); assert_eq!( read(&mut pair, server_ch, s), - (Vec::new(), Err(ReadError::Reset(ERROR))) + (Vec::new(), Err(recv::Error::Reset(ERROR))) ); } @@ -546,7 +555,7 @@ fn repeated_reset_is_closed() { pair.client_send(client_ch, s).reset_at(offset, ERROR).unwrap(); assert_eq!( pair.client_send(client_ch, s).reset_at(again, ERROR), - Err(ClosedStream::default()) + Err(send::Error::Closed) ); assert!(pair.client_send(client_ch, s).reset(ERROR).is_err()); } diff --git a/rs/moq-quic/src/tests/sim.rs b/rs/moq-quic/src/tests/sim.rs index 793d1d39f8f..6145811ecfa 100644 --- a/rs/moq-quic/src/tests/sim.rs +++ b/rs/moq-quic/src/tests/sim.rs @@ -268,9 +268,9 @@ impl Peer { while sending.written < sending.spec.len { let end = sending.spec.len.min(sending.written + 65_536); let data = payload(index, sending.written..end); - match stream.write(&data) { + match stream.write(&mut &data[..]) { Ok(written) => sending.written += written, - Err(WriteError::Blocked) => break, + Err(send::Error::Blocked) => break, Err(error) => panic!("writing stream {index}: {error}"), } } @@ -331,7 +331,7 @@ impl Peer { receiving.done = Some(now); break; } - Err(ReadError::Blocked) => break, + Err(recv::Error::Blocked) => break, Err(error) => panic!("reading stream {index}: {error}"), } } diff --git a/rs/moq-quic/src/tests/sim/workloads.rs b/rs/moq-quic/src/tests/sim/workloads.rs index 82292945db5..d2eb7267457 100644 --- a/rs/moq-quic/src/tests/sim/workloads.rs +++ b/rs/moq-quic/src/tests/sim/workloads.rs @@ -66,7 +66,7 @@ impl Transfer { } let mut stream = conn.send_stream(id); if matches!(self.end, End::Stop) - && let Some(code) = stream.stopped().unwrap() + && let Some(send::End::Stopped(code)) = stream.end().unwrap() { prop_assert_eq!(code, CODE); stream.reset(code).unwrap(); @@ -78,10 +78,10 @@ impl Transfer { _ => self.len, }; while self.written < target { - match stream.write(&payload(key, self.written..target)) { + match stream.write(&mut &payload(key, self.written..target)[..]) { Ok(written) => self.written += written, - Err(WriteError::Blocked) => return Ok(()), - Err(WriteError::Stopped(code)) if matches!(self.end, End::Stop) => { + Err(send::Error::Blocked) => return Ok(()), + Err(send::Error::Stopped(code)) if matches!(self.end, End::Stop) => { prop_assert_eq!(code, CODE); stream.reset(code).unwrap(); self.sent = true; @@ -93,7 +93,10 @@ impl Transfer { match self.end { End::Finish => stream.finish().unwrap(), End::Reset => stream.reset(CODE).unwrap(), - End::ReliableReset => stream.reset_at((self.cut / 2).max(1) as u64, CODE).unwrap(), + End::ReliableReset => { + let reliable = stream.offset().rewind((self.cut / 2).max(1) as u64); + stream.reset_at(reliable, CODE).unwrap() + } // Keep the send half open until the receiver's STOP_SENDING arrives. End::Stop => return Ok(()), } @@ -104,7 +107,7 @@ impl Transfer { fn receive(&mut self, conn: &mut Connection, id: StreamId, key: usize) -> TestCaseResult { if self.received { if matches!(self.end, End::Stop) { - prop_assert!(matches!(conn.recv_stream(id).read(), Err(ReadableError::ClosedStream))); + prop_assert!(matches!(conn.recv_stream(id).read(), Err(recv::Error::Closed))); } return Ok(()); } @@ -132,8 +135,8 @@ impl Transfer { self.received = true; break; } - Err(ReadError::Blocked) => break, - Err(ReadError::Reset(code)) => { + Err(recv::Error::Blocked) => break, + Err(recv::Error::Reset(code)) => { prop_assert_eq!(code, CODE); prop_assert!(matches!(self.end, End::Reset | End::ReliableReset)); prop_assert!(self.read <= self.cut); @@ -143,6 +146,7 @@ impl Transfer { self.received = true; break; } + Err(error) => return Err(TestCaseError::fail(format!("{id} read: {error}"))), } } let _ = chunks.finalize(); @@ -228,7 +232,7 @@ impl Run { let index = self.datagrams_sent[side]; conn(pair, self.handles, side) .datagrams() - .send(datagram(side, index).into(), Default::default()) + .send(Bytes::from(datagram(side, index)), Default::default()) .unwrap(); self.datagrams_sent[side] += 1; } diff --git a/rs/moq-quic/src/tests/util.rs b/rs/moq-quic/src/tests/util.rs index ef61060da26..9e0d73a40d0 100644 --- a/rs/moq-quic/src/tests/util.rs +++ b/rs/moq-quic/src/tests/util.rs @@ -266,11 +266,11 @@ impl Pair { self.client_conn_mut(ch).streams() } - pub(super) fn client_send(&mut self, ch: ConnectionHandle, s: StreamId) -> SendStream<'_> { + pub(super) fn client_send(&mut self, ch: ConnectionHandle, s: StreamId) -> send::Stream<'_> { self.client_conn_mut(ch).send_stream(s) } - pub(super) fn client_recv(&mut self, ch: ConnectionHandle, s: StreamId) -> RecvStream<'_> { + pub(super) fn client_recv(&mut self, ch: ConnectionHandle, s: StreamId) -> recv::Stream<'_> { self.client_conn_mut(ch).recv_stream(s) } @@ -286,11 +286,11 @@ impl Pair { self.server_conn_mut(ch).streams() } - pub(super) fn server_send(&mut self, ch: ConnectionHandle, s: StreamId) -> SendStream<'_> { + pub(super) fn server_send(&mut self, ch: ConnectionHandle, s: StreamId) -> send::Stream<'_> { self.server_conn_mut(ch).send_stream(s) } - pub(super) fn server_recv(&mut self, ch: ConnectionHandle, s: StreamId) -> RecvStream<'_> { + pub(super) fn server_recv(&mut self, ch: ConnectionHandle, s: StreamId) -> recv::Stream<'_> { self.server_conn_mut(ch).recv_stream(s) } diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 47dcbb35afb..1e7bda24ce1 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -708,8 +708,6 @@ async fn serve_connection( serve: Serve, lifecycle: &Lifecycle, ) -> anyhow::Result { - use web_transport_trait::poll::Session as _; - // The endpoint started this deadline at the connection's first packet and // enforced it through the QUIC handshake; it covers SETUP OK too. let mut deadline = Deadline::new(&handle, &conn); diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index cdd5c6e00a7..590fc9eb685 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -75,8 +75,8 @@ async fn probe(port: u16, peer: bool) -> Seen { // SETUP type, then a one-byte message holding zero parameters. let mut setup = conn.open_uni(Default::default()).await.expect("open"); - setup.write_all(&[0x01, 0x01, 0x00]).await.expect("write"); - setup.finish().expect("finish"); + setup.write_all(&[0x01, 0x01, 0x00][..]).await.expect("write"); + setup.finish(); // The relay widens a peer before it answers SETUP, so its SETUP stream // arrives behind the new credit. diff --git a/rs/moq-relay/tests/integration/refusals.rs b/rs/moq-relay/tests/integration/refusals.rs index 23fb1f771dc..bb3c52b3f06 100644 --- a/rs/moq-relay/tests/integration/refusals.rs +++ b/rs/moq-relay/tests/integration/refusals.rs @@ -502,7 +502,7 @@ async fn uri_setup_paths(io_uring: bool) { }; bytes.extend(params); let mut setup = peer.open_uni(Default::default()).await.unwrap(); - setup.write_all(&bytes).await.unwrap(); + setup.write_all(&bytes[..]).await.unwrap(); if version.is_lite() { setup.finish().unwrap(); } diff --git a/rs/moq-relay/tests/integration/runtime_uring.rs b/rs/moq-relay/tests/integration/runtime_uring.rs index f2d4219afeb..5149ab92170 100644 --- a/rs/moq-relay/tests/integration/runtime_uring.rs +++ b/rs/moq-relay/tests/integration/runtime_uring.rs @@ -906,14 +906,14 @@ async fn uring_forwards_the_setup_token() { let params = [&[0x03, encoded.len() as u8][..], &encoded].concat(); let setup = [&[0xaf, 0x00][..], &(params.len() as u16).to_be_bytes(), ¶ms].concat(); let mut send = session.open_uni(Default::default()).await.unwrap(); - send.write_all(&setup).await.unwrap(); + send.write_all(&setup[..]).await.unwrap(); let mut recv = tokio::time::timeout(TIMEOUT, session.accept_uni()) .await .unwrap() .unwrap(); let mut reply = [0u8; 1]; assert_eq!( - tokio::time::timeout(TIMEOUT, recv.read(&mut reply)) + tokio::time::timeout(TIMEOUT, recv.read(&mut &mut reply[..])) .await .unwrap() .unwrap(), diff --git a/rs/moq-tokio/src/quic/driver/connection.rs b/rs/moq-tokio/src/quic/driver/connection.rs index 265a1148ba1..1772bcb3046 100644 --- a/rs/moq-tokio/src/quic/driver/connection.rs +++ b/rs/moq-tokio/src/quic/driver/connection.rs @@ -14,10 +14,9 @@ use std::{ task::{Context, Poll, Waker, ready}, }; -use bytes::Bytes; +use bytes::{Buf, Bytes}; use pin_project_lite::pin_project; use rustc_hash::FxHashMap; -use thiserror::Error; use tokio::sync::{ Notify, futures::{Notified, OwnedNotified}, @@ -26,18 +25,14 @@ use tokio::sync::{ use tracing::{Instrument, Span, debug_span}; use crate::quic::driver::{ - ConnectionEvent, Duration, Instant, VarInt, - mutex::Mutex, - recv_stream::RecvStream, - runtime::Slot, - send_group::{self, SendGroup, SendOptions}, - send_stream::SendStream, - udp_transmit, + ConnectionEvent, Duration, Instant, VarInt, mutex::Mutex, recv, runtime::Slot, send, udp_transmit, }; use moq_congestion::Controller; pub use moq_quic::connection::{Error, Stats}; use moq_quic::{ - Dir, EndpointEvent, StreamEvent, StreamId, TransportError, TransportErrorCode, endpoint::ConnectionHandle, + EndpointEvent, StreamEvent, TransportError, TransportErrorCode, + endpoint::ConnectionHandle, + stream::{Dir, Id as StreamId}, }; /// In-progress connection attempt future @@ -268,7 +263,10 @@ impl Connection { /// /// # Panics /// - when `options.group` belongs to another connection - pub fn open_uni<'a>(&'a self, options: SendOptions<'a>) -> impl Future> + 'a { + pub fn open_uni<'a>( + &'a self, + options: send::Options<'a>, + ) -> impl Future> + 'a { OpenUni { conn: &self.0, options, @@ -276,31 +274,27 @@ impl Connection { } } - /// Open a send group at `priority`, for streams and datagrams to join with [`SendOptions`] - /// or [`SendStream::set_group`]. - /// - /// [`SendStream::set_group`]: crate::quic::driver::SendStream::set_group - pub fn send_group(&self, priority: i64) -> SendGroup { - SendGroup::new(self.0.0.clone(), priority) + /// Open a send group at `priority`, for streams and datagrams to join with [`send::Options`] + /// or [`send::Stream::set_group`]. + pub fn send_group(&self, priority: i64) -> send::Group { + send::Group::new(self.0.0.clone(), priority) } /// Initiate a new outgoing bidirectional stream, its sending half scheduled by `options`. /// /// Streams are cheap and instantaneous to open unless blocked by flow control. As a /// consequence, the peer won't be notified that a stream has been opened until the stream is - /// actually used. Calling [`open_bi()`] then waiting on the [`RecvStream`] without writing - /// anything to [`SendStream`] will never succeed. + /// actually used. Calling [`open_bi()`] then waiting on the [`recv::Stream`] without writing + /// anything to [`send::Stream`] will never succeed. /// /// [`open_bi()`]: crate::quic::driver::connection::Connection::open_bi - /// [`SendStream`]: crate::quic::driver::SendStream - /// [`RecvStream`]: crate::quic::driver::RecvStream /// /// # Panics /// - when `options.group` belongs to another connection pub fn open_bi<'a>( &'a self, - options: SendOptions<'a>, - ) -> impl Future> + 'a { + options: send::Options<'a>, + ) -> impl Future> + 'a { OpenBi { conn: &self.0, options, @@ -310,7 +304,7 @@ impl Connection { /// Open a request stream using only the credit already granted by the peer. #[cfg(feature = "net")] - pub(crate) fn open_request(&self) -> Result, Error> { + pub(crate) fn open_request(&self) -> Result, Error> { let mut state = self.0.state.lock(); if let Some(err) = &state.error { return Err(err.clone()); @@ -320,8 +314,8 @@ impl Connection { }; drop(state); Ok(Some(( - SendStream::new(self.0.clone(), id), - RecvStream::new(self.0.clone(), id), + send::Stream::new(self.0.clone(), id), + recv::Stream::new(self.0.clone(), id), ))) } @@ -330,7 +324,7 @@ impl Connection { /// # Cancel safety /// /// This method is cancellation safe. If this does not resolve, no streams were consumed. - pub fn accept_uni(&self) -> impl Future> + '_ { + pub fn accept_uni(&self) -> impl Future> + '_ { AcceptUni { conn: &self.0, notify: self.0.shared.stream_incoming[Dir::Uni as usize].notified(), @@ -339,19 +333,17 @@ impl Connection { /// Accept the next incoming bidirectional stream /// - /// **Important Note**: The `Connection` that calls [`open_bi()`] must write to its [`SendStream`] + /// **Important Note**: The `Connection` that calls [`open_bi()`] must write to its [`send::Stream`] /// before the other `Connection` is able to `accept_bi()`. Calling [`open_bi()`] then - /// waiting on the [`RecvStream`] without writing anything to [`SendStream`] will never succeed. + /// waiting on the [`recv::Stream`] without writing anything to [`send::Stream`] will never succeed. /// /// [`accept_bi()`]: crate::quic::driver::connection::Connection::accept_bi /// [`open_bi()`]: crate::quic::driver::connection::Connection::open_bi - /// [`SendStream`]: crate::quic::driver::SendStream - /// [`RecvStream`]: crate::quic::driver::RecvStream /// /// # Cancel safety /// /// This method is cancellation safe. If this does not resolve, no streams were consumed. - pub fn accept_bi(&self) -> impl Future> + '_ { + pub fn accept_bi(&self) -> impl Future> + '_ { AcceptBi { conn: &self.0, notify: self.0.shared.stream_incoming[Dir::Bi as usize].notified(), @@ -450,17 +442,22 @@ impl Connection { /// /// The datagram is scheduled like one of its group's streams at its priority, or among the /// ungrouped streams without a group. Sending never blocks: a full buffer drops the oldest - /// datagram of the lowest-priority group, see [`moq_quic::Datagrams::send`]. + /// datagram of the lowest-priority group, see [`moq_quic::Datagrams::send`]. A header chained + /// in front of a [`Bytes`] payload costs no copy of the payload. /// /// # Panics /// - when `options.group` belongs to another connection - pub fn send_datagram(&self, data: Bytes, options: SendOptions<'_>) -> Result<(), SendDatagramError> { - let options = send_group::to_quic(options, &self.0.0); + pub fn send_datagram(&self, data: impl Buf, options: send::Options<'_>) -> Result<(), send::Error> { + let options = options.to_quic(&self.0.0); let conn = &mut *self.0.state.lock(); if let Some(x) = &conn.error { - return Err(SendDatagramError::ConnectionLost(x.clone())); + return Err(send::Error::Connection(x.clone())); + } + match conn.inner.datagrams().send(data, options) { + Ok(()) => {} + Err(moq_quic::send::Error::TooLarge) => return Err(send::Error::TooLarge), + Err(_) => return Err(send::Error::Unsupported), } - conn.inner.datagrams().send(data, options)?; conn.wake(); Ok(()) } @@ -506,14 +503,13 @@ impl Connection { self.0.state.lock().inner.stats() } - /// Whether the connection has freed the send stream's state + /// How many send streams the connection still holds state for #[cfg(test)] - pub(crate) fn send_stream_freed(&self, id: StreamId) -> bool { - let mut state = self.0.state.lock(); - state.inner.send_stream(id).stopped().is_err() + pub(crate) fn send_streams(&self) -> usize { + self.0.state.lock().inner.streams().send_streams() } - /// How many streams have `SendStream::stopped` waiters registered, to check none leak + /// How many streams have `send::Stream::stopped` waiters registered, to check none leak #[cfg(test)] pub(crate) fn stopped_waiters(&self) -> usize { self.0.state.lock().stopped.len() @@ -605,18 +601,18 @@ pin_project! { /// Future produced by [`Connection::open_uni`] struct OpenUni<'a> { conn: &'a ConnectionRef, - options: SendOptions<'a>, + options: send::Options<'a>, #[pin] notify: Notified<'a>, } } impl Future for OpenUni<'_> { - type Output = Result; + type Output = Result; fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { let this = self.project(); let (conn, id) = ready!(poll_open(ctx, this.conn, *this.options, this.notify, Dir::Uni))?; - Poll::Ready(Ok(SendStream::new(conn, id))) + Poll::Ready(Ok(send::Stream::new(conn, id))) } } @@ -624,30 +620,30 @@ pin_project! { /// Future produced by [`Connection::open_bi`] struct OpenBi<'a> { conn: &'a ConnectionRef, - options: SendOptions<'a>, + options: send::Options<'a>, #[pin] notify: Notified<'a>, } } impl Future for OpenBi<'_> { - type Output = Result<(SendStream, RecvStream), Error>; + type Output = Result<(send::Stream, recv::Stream), Error>; fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { let this = self.project(); let (conn, id) = ready!(poll_open(ctx, this.conn, *this.options, this.notify, Dir::Bi))?; - Poll::Ready(Ok((SendStream::new(conn.clone(), id), RecvStream::new(conn, id)))) + Poll::Ready(Ok((send::Stream::new(conn.clone(), id), recv::Stream::new(conn, id)))) } } fn poll_open<'a>( ctx: &mut Context<'_>, conn: &'a ConnectionRef, - options: SendOptions<'_>, + options: send::Options<'_>, mut notify: Pin<&mut Notified<'a>>, dir: Dir, ) -> Poll> { - let options = send_group::to_quic(options, &conn.0); + let options = options.to_quic(&conn.0); let mut state = conn.state.lock(); if let Some(e) = &state.error { return Poll::Ready(Err(e.clone())); @@ -675,12 +671,12 @@ pin_project! { } impl Future for AcceptUni<'_> { - type Output = Result; + type Output = Result; fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { let this = self.project(); let (conn, id) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Uni))?; - Poll::Ready(Ok(RecvStream::new(conn, id))) + Poll::Ready(Ok(recv::Stream::new(conn, id))) } } @@ -694,12 +690,12 @@ pin_project! { } impl Future for AcceptBi<'_> { - type Output = Result<(SendStream, RecvStream), Error>; + type Output = Result<(send::Stream, recv::Stream), Error>; fn poll(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll { let this = self.project(); let (conn, id) = ready!(poll_accept(ctx, this.conn, this.notify, Dir::Bi))?; - Poll::Ready(Ok((SendStream::new(conn.clone(), id), RecvStream::new(conn, id)))) + Poll::Ready(Ok((send::Stream::new(conn.clone(), id), recv::Stream::new(conn, id)))) } } @@ -765,7 +761,7 @@ impl Future for ReadDatagram<'_> { pub(crate) struct ConnectionRef(pub(crate) Arc); impl ConnectionRef { - fn stable_id(&self) -> usize { + pub(crate) fn stable_id(&self) -> usize { &*self.0 as *const _ as usize } } @@ -1119,7 +1115,7 @@ fn wake_all(wakers: &mut FxHashMap) { wakers.drain().for_each(|(_, waker)| waker.wake()) } -/// The `SendStream::stopped` futures waiting on one stream +/// The `send::Stream::stopped` futures waiting on one stream #[derive(Debug, Default)] pub(crate) struct StopWaiters { pub(crate) notify: Notify, @@ -1132,18 +1128,6 @@ fn wake_all_notify(wakers: &mut FxHashMap>) { wakers.drain().for_each(|(_, waiters)| waiters.notify.notify_waiters()) } -/// Errors that can arise when sending a datagram -#[derive(Debug, Error, Clone, Eq, PartialEq)] -#[non_exhaustive] -pub enum SendDatagramError { - /// The connection refused the datagram. - #[error(transparent)] - Refused(#[from] moq_quic::SendDatagramError), - /// The connection was lost - #[error("connection lost")] - ConnectionLost(#[from] Error), -} - /// The maximum amount of datagrams which will be produced in a single `drive_transmit` call /// /// This limits the amount of CPU resources consumed by datagram generation, diff --git a/rs/moq-tokio/src/quic/driver/mod.rs b/rs/moq-tokio/src/quic/driver/mod.rs index c8746dfc433..8ea4975e592 100644 --- a/rs/moq-tokio/src/quic/driver/mod.rs +++ b/rs/moq-tokio/src/quic/driver/mod.rs @@ -1,8 +1,8 @@ //! An async QUIC driver: [`moq_quic`]'s state machine on tokio sockets and timers. //! //! It exposes moq-quic streams and datagrams with: -//! [`SendGroup`]s that schedule streams and datagrams together, [`SendStream::reset_at`] to -//! keep a prefix reliable, [`RecvStream::poll_lease`] to hold receive credit until a message is +//! [`send::Group`]s that schedule streams and datagrams together, [`send::Stream::reset_at`] to +//! keep a prefix reliable, [`recv::Stream::poll_lease`] to hold receive credit until a message is //! complete, and [`connection::Connection::set_limits`] to change flow control on a live connection. //! //! It needs no MoQ session: enable the `driver` feature (with `aws-lc-rs` or `ring`) and @@ -49,27 +49,22 @@ pub mod connection; pub mod endpoint; mod incoming; mod mutex; -mod recv_stream; +pub mod recv; mod runtime; -mod send_group; -mod send_stream; +pub mod send; mod work_limiter; pub(crate) use moq_clock::Instant; pub(crate) use std::time::Duration; pub use moq_quic::{ - ApplicationClose, Chunk, ClosedStream, ConfigError, ConnectionClose, ConnectionId, ConnectionIdGenerator, - CreditStats, Dir, EcnCodepoint, FrameStats, FrameType, InvalidCid, Limits, LimitsError, PathStats, StreamId, - Transmit, TransportErrorCode, UdpStats, VarInt, VarIntBoundsExceeded, Version, Written, client, crypto, server, - transport, + ApplicationClose, ConfigError, ConnectionClose, ConnectionId, ConnectionIdGenerator, CreditStats, EcnCodepoint, + FrameStats, FrameType, InvalidCid, Limits, LimitsError, PathStats, Transmit, TransportErrorCode, UdpStats, VarInt, + VarIntBoundsExceeded, Version, client, crypto, server, stream, transport, }; -pub use crate::quic::driver::recv_stream::{Lease, ReadError, ReadExactError, ReadToEndError, RecvStream, ResetError}; #[cfg(all(test, feature = "quic"))] pub(crate) use crate::quic::driver::runtime::wrap_udp_socket; -pub use crate::quic::driver::send_group::{SendGroup, SendOptions}; -pub use crate::quic::driver::send_stream::{SendStream, StoppedError, WriteError}; #[cfg(test)] #[expect( diff --git a/rs/moq-tokio/src/quic/driver/recv.rs b/rs/moq-tokio/src/quic/driver/recv.rs new file mode 100644 index 00000000000..6f9bdd405d1 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/recv.rs @@ -0,0 +1,526 @@ +//! Incoming streams: their handles, leases, and errors. + +use std::{ + future::poll_fn, + io, + pin::Pin, + task::{Context, Poll, ready}, +}; + +use bytes::{Buf, BufMut, Bytes}; +use moq_quic::{connection::Error as ConnectionError, recv::Chunks}; +use thiserror::Error; +use tokio::io::ReadBuf; + +use crate::quic::driver::{VarInt, connection::ConnectionRef, stream}; + +/// A stream that can only be used to receive data +/// +/// Dropped before its end, it is [`stop()`](Self::stop)ped with code 0. +/// +/// # Cancellation +/// +/// A `read` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready cannot lead to loss of stream data. This is true of methods which succeed immediately when +/// any progress is made, and is not true of methods which might need to perform multiple reads +/// internally before succeeding. Each `read` method documents whether it is cancel-safe. +/// +/// # Common issues +/// +/// ## Data never received on a locally-opened stream +/// +/// Peers are not notified of streams until they or a later-numbered stream are used to send +/// data. If a bidirectional stream is locally opened but never used to send, then the peer may +/// never see it. Application protocols should always arrange for the endpoint which will first +/// transmit on a stream to be the endpoint responsible for opening it. +/// +/// ## Data never received on a remotely-opened stream +/// +/// Verify that the stream you are receiving is the same one that the server is sending on, e.g. by +/// logging the [`id`] of each. Streams are always accepted in the same order as they are created, +/// i.e. ascending order by [`stream::Id`]. For example, even if a sender first transmits on +/// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver +/// will be bidirectional stream 0. +/// +/// [`id`]: Stream::id +/// [`Connection::accept_bi`]: crate::quic::driver::connection::Connection::accept_bi +#[derive(Debug)] +pub struct Stream { + conn: ConnectionRef, + id: stream::Id, + all_data_read: bool, + reset: Option, + /// The code a drop before the end stops with + drop_stop: VarInt, +} + +impl Stream { + pub(crate) fn new(conn: ConnectionRef, id: stream::Id) -> Self { + Self { + conn, + id, + all_data_read: false, + reset: None, + drop_stop: VarInt::from_u32(0), + } + } + + /// Read data contiguously from the stream into `buf`, advancing it + /// + /// Waits for some data, then reads as much as has arrived and `buf` can take. Yields the + /// number of bytes read, or `None` if the stream was finished. + /// + /// This operation is cancel-safe. + pub async fn read(&mut self, buf: &mut impl BufMut) -> Result, Error> { + poll_fn(|cx| self.poll_read_buf(cx, buf)).await + } + + /// Fill `buf` with data from the stream + /// + /// Fails with [`Error::FinishedEarly`] if the stream ends first. This operation is *not* + /// cancel-safe. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error> { + let len = buf.len(); + let mut rest = &mut buf[..]; + while !rest.is_empty() { + if self.read(&mut rest).await?.is_none() { + return Err(Error::FinishedEarly(len - rest.len())); + } + } + Ok(()) + } + + /// Read the next segment of data, at most `max` bytes + /// + /// Yields `None` if the stream was finished. Unlike [`Self::read()`], this does not copy. + /// Segment boundaries do not correspond to peer writes, and hence cannot be used as framing. + /// + /// This operation is cancel-safe. + pub async fn read_chunk(&mut self, max: usize) -> Result, Error> { + poll_fn(|cx| { + self.poll_read_generic(cx, |chunks| match chunks.next(max) { + Ok(Some(chunk)) => ReadStatus::Readable(chunk.bytes), + res => (None, res.err()).into(), + }) + }) + .await + } + + /// Read all remaining data + /// + /// Fails with [`Error::TooLong`] on reading more than `limit` bytes, discarding all data + /// read. `limit` bounds the returned buffer. + /// + /// This operation is *not* cancel-safe. + pub async fn read_to_end(&mut self, limit: usize) -> Result, Error> { + let mut read = Vec::new(); + while let Some(chunk) = self.read_chunk(usize::MAX).await? { + if chunk.len() > limit - read.len() { + return Err(Error::TooLong); + } + read.extend_from_slice(&chunk); + } + Ok(read) + } + + /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the + /// stream ended. See [`Self::poll_lease()`]. + pub async fn lease(&mut self, want: usize) -> Result, Error> { + let inner = poll_fn(|cx| self.poll_take(cx, want)).await?; + Ok(Lease { + stream: self, + inner: Some(inner), + }) + } + + /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the + /// stream ended. + /// + /// The lease owns the bytes and holds no lock, but they keep their connection flow control + /// credit until read through the lease as a [`Buf`] and then [`Lease::release`]d; bytes not + /// read are leased again next time. Stream credit still advances, so a stream window smaller + /// than a message keeps the message flowing. + /// + /// When the peer has spent the connection window and no reader can make progress, the + /// stream ranked lowest by [`Self::set_priority`] is evicted: its buffered bytes and their + /// credit are discarded and its next lease fails with [`Error::Evicted`]. Bytes it took + /// keep their credit until the stream is stopped. Nothing is evicted while the leased and + /// unread bytes of every stream stay below the connection receive window, nor while a lease + /// is out. + /// + /// Do not mix with the plain reads on one stream, which skip the leased bytes. + pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, Error>> { + let inner = ready!(self.poll_take(cx, want))?; + Poll::Ready(Ok(Lease { + stream: self, + inner: Some(inner), + })) + } + + /// Take the connection's lease, which [`Lease`] wraps once this returns: the borrow of `self` + /// it holds cannot start inside a poll that may return `Pending`. + fn poll_take(&mut self, cx: &mut Context<'_>, want: usize) -> Poll> { + // moq-quic freed the stream when it reported the reset. + if let Some(code) = self.reset { + return Poll::Ready(Err(Error::Reset(code))); + } + let mut conn = self.conn.state.lock(); + let (result, transmit) = conn.inner.recv_stream(self.id).lease(want); + if transmit.should_transmit() { + conn.wake(); + } + let err = match result { + Ok(lease) => return Poll::Ready(Ok(lease)), + Err(moq_quic::recv::Error::Blocked) => { + if let Some(x) = &conn.error { + return Poll::Ready(Err(Error::Connection(x.clone()))); + } + conn.blocked_readers.insert(self.id, cx.waker().clone()); + return Poll::Pending; + } + Err(err) => err, + }; + conn.blocked_readers.remove(&self.id); + let closed = conn.error.clone(); + drop(conn); + Poll::Ready(Err(lease_error(err, closed))) + } + + /// Rank this stream for eviction: when the connection window is stuck, the stream with the + /// lowest priority goes first, and the newest of those that tie. + pub fn set_priority(&mut self, priority: i64) { + let mut conn = self.conn.state.lock(); + conn.inner.recv_stream(self.id).set_priority(priority); + } + + fn poll_read_buf(&mut self, cx: &mut Context<'_>, buf: &mut impl BufMut) -> Poll, Error>> { + if !buf.has_remaining_mut() { + return Poll::Ready(Ok(Some(0))); + } + self.poll_read_generic(cx, |chunks| fill(chunks, buf)) + } + + /// Stop accepting data + /// + /// Discards unread data and notifies the peer to stop transmitting. + pub fn stop(mut self, error_code: VarInt) { + self.drop_stop = error_code; + } + + /// The identity of this stream + pub fn id(&self) -> stream::Id { + self.id + } + + /// Stop with `code`, instead of code 0, if dropped before the end + #[cfg(feature = "quic")] + pub(crate) fn set_drop_stop(&mut self, code: VarInt) { + self.drop_stop = code; + } + + /// Completes when the stream has been reset by the peer or otherwise closed + /// + /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` + /// when the stream was [`finish()`](crate::quic::driver::send::Stream::finish)ed by the peer + /// and all data has been received, after which it is no longer meaningful for the stream to + /// be reset. + /// + /// This operation is cancel-safe. + pub async fn received_reset(&mut self) -> Result, Error> { + poll_fn(|cx| { + let mut conn = self.conn.state.lock(); + + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + + match conn.inner.recv_stream(self.id).received_reset() { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(error_code)) => { + // Stream state has just now been freed, so the connection may need to issue new + // stream ID flow control credit, and later reads fail with the cached code. + self.reset = Some(error_code); + conn.wake(); + Poll::Ready(Ok(Some(error_code))) + } + Ok(None) => { + if let Some(e) = &conn.error { + return Poll::Ready(Err(e.clone().into())); + } + // Resets always notify readers, since a reset is an immediate read error. We + // could introduce a dedicated channel to reduce the risk of spurious wakeups, + // but that increased complexity is probably not justified, as an application + // that is expecting a reset is not likely to receive large amounts of data. + conn.blocked_readers.insert(self.id, cx.waker().clone()); + Poll::Pending + } + } + }) + .await + } + + /// Handle common logic related to reading out of a receive stream + /// + /// This takes an `FnMut` closure that takes care of the actual reading process, matching + /// the detailed read semantics for the calling function with a particular return type. + /// The closure can read from the passed `&mut Chunks` and has to return the status after + /// reading: the amount of data read, and the status after the final read call. + fn poll_read_generic(&mut self, cx: &mut Context<'_>, mut read_fn: T) -> Poll, Error>> + where + T: FnMut(&mut Chunks<'_>) -> ReadStatus, + { + use moq_quic::recv::Error::{Blocked, Reset}; + + if self.all_data_read { + return Poll::Ready(Ok(None)); + } + + let mut conn = self.conn.state.lock(); + + // If we stored an error during a previous call, return it now. This can happen if a + // `read_fn` both wants to return data and also returns an error in its final stream status. + let status = match self.reset { + Some(code) => ReadStatus::Failed(None, Reset(code)), + None => { + let mut recv = conn.inner.recv_stream(self.id); + let (status, transmit) = match recv.read() { + Ok(mut chunks) => { + let status = read_fn(&mut chunks); + (status, chunks.finalize().should_transmit()) + } + Err(err) => (ReadStatus::Failed(None, err), false), + }; + if transmit { + conn.wake(); + } + status + } + }; + + match status { + ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), + ReadStatus::Finished(read) => { + self.all_data_read = true; + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.id); + Poll::Ready(Ok(read)) + } + ReadStatus::Failed(Some(read), Blocked) => Poll::Ready(Ok(Some(read))), + ReadStatus::Failed(None, Blocked) => { + if let Some(x) = &conn.error { + return Poll::Ready(Err(Error::Connection(x.clone()))); + } + conn.blocked_readers.insert(self.id, cx.waker().clone()); + Poll::Pending + } + ReadStatus::Failed(read, Reset(error_code)) => { + self.reset = Some(error_code); + if read.is_some() { + return Poll::Ready(Ok(read)); + } + self.all_data_read = true; + // Stream is dead, cleanup stale waker + conn.blocked_readers.remove(&self.id); + Poll::Ready(Err(Error::Reset(error_code))) + } + ReadStatus::Failed(_, err) => { + let error = conn.error.clone(); + drop(conn); + Poll::Ready(Err(read_error(err, error))) + } + } + } +} + +/// Copy from `chunks` until `buf` is full or nothing more is readable, returning how much. +fn fill(chunks: &mut Chunks<'_>, buf: &mut impl BufMut) -> ReadStatus { + let mut read = 0; + loop { + if !buf.has_remaining_mut() { + // We know `read` is not 0 because `buf` had room before + return ReadStatus::Readable(read); + } + + match chunks.next(buf.remaining_mut()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read += chunk.bytes.len(); + } + res => return ((read > 0).then_some(read), res.err()).into(), + } + } +} + +/// The error for a lease moq-quic refused, which forgets a stream only once the connection +/// closed. +fn lease_error(err: moq_quic::recv::Error, closed: Option) -> Error { + match err { + moq_quic::recv::Error::Reset(code) => Error::Reset(code), + moq_quic::recv::Error::Evicted => Error::Evicted, + moq_quic::recv::Error::TooLarge => Error::TooLarge, + err => read_error(err, closed), + } +} + +/// The error for a read moq-quic refused on a stream the handle still holds, which it forgets +/// only once the connection closed. +fn read_error(err: moq_quic::recv::Error, closed: Option) -> Error { + match closed { + Some(closed) => Error::Connection(closed), + None => panic!("a held receive stream failed to read: {err:?}"), + } +} + +/// Unreleased bytes of a [`Stream`] from [`Stream::poll_lease`], read as a [`Buf`] +/// +/// It owns the bytes, so the connection carries on while it is held. Hand it back with +/// [`Self::release`] to return the credit of the bytes read; dropping it keeps their credit, +/// as for a message read only in part. Either way the bytes not read are leased again next +/// time. +#[derive(Debug)] +pub struct Lease<'a> { + stream: &'a mut Stream, + /// Taken only to hand it back. + inner: Option, +} + +impl Lease<'_> { + fn inner(&self) -> &moq_quic::recv::Lease { + self.inner.as_ref().expect("only taken to hand it back") + } + + fn inner_mut(&mut self) -> &mut moq_quic::recv::Lease { + self.inner.as_mut().expect("only taken to hand it back") + } + + /// Whether the peer finished the stream right after the leased bytes. + pub fn is_fin(&self) -> bool { + self.inner().is_fin() + } + + /// The bytes not yet read, in order, one contiguous chunk at a time. + pub fn chunks(&self) -> impl Iterator { + self.inner().chunks() + } + + /// Return the credit of every byte read so far, by this lease or an earlier one. + pub fn release(mut self) { + let lease = self.inner.take().expect("only taken to hand it back"); + let mut conn = self.stream.conn.state.lock(); + if conn.inner.recv_stream(self.stream.id).release(lease).should_transmit() { + conn.wake(); + } + } +} + +/// Reading moves past bytes for good, but they keep their credit until [`Lease::release`]. +impl Buf for Lease<'_> { + fn remaining(&self) -> usize { + self.inner().remaining() + } + + fn chunk(&self) -> &[u8] { + self.inner().chunk() + } + + fn advance(&mut self, n: usize) { + self.inner_mut().advance(n); + } + + fn copy_to_bytes(&mut self, n: usize) -> Bytes { + self.inner_mut().copy_to_bytes(n) + } +} + +impl Drop for Lease<'_> { + fn drop(&mut self) { + if let Some(lease) = self.inner.take() { + let mut conn = self.stream.conn.state.lock(); + if conn.inner.recv_stream(self.stream.id).restore(lease).should_transmit() { + conn.wake(); + } + } + } +} + +enum ReadStatus { + Readable(T), + Finished(Option), + Failed(Option, moq_quic::recv::Error), +} + +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { + match status { + (read, None) => Self::Finished(read), + (read, Some(e)) => Self::Failed(read, e), + } + } +} + +impl tokio::io::AsyncRead for Stream { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let mut unfilled = buf.initialize_unfilled(); + let read = ready!(self.get_mut().poll_read_buf(cx, &mut unfilled))?; + buf.advance(read.unwrap_or(0)); + Poll::Ready(Ok(())) + } +} + +impl Drop for Stream { + fn drop(&mut self) { + let mut conn = self.conn.state.lock(); + + // clean up any previously registered wakers + conn.blocked_readers.remove(&self.id); + + if self.all_data_read || conn.error.is_some() { + return; + } + + // Fails harmlessly once the stream is gone. + let _ = conn.inner.recv_stream(self.id).stop(self.drop_stop); + conn.wake(); + } +} + +/// Errors from reading or leasing a stream +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The peer abandoned transmitting data on this stream + /// + /// Carries an application-defined error code. + #[error("stream reset by peer: error {0}")] + Reset(VarInt), + /// The connection window was stuck and this stream was discarded to free it + /// + /// See [`Stream::poll_lease`]. The peer hears of it once the stream is stopped, so call + /// [`Stream::stop`] with your own code: dropping the stream sends code 0. + #[error("evicted")] + Evicted, + /// The stream finished before [`Stream::read_exact`] filled its buffer, after this many bytes + #[error("stream finished early ({0} bytes read)")] + FinishedEarly(usize), + /// The stream is longer than [`Stream::read_to_end`]'s limit + #[error("stream too long")] + TooLong, + /// A lease wanted more than the connection receive window can hold + #[error("want exceeds the receive window")] + TooLarge, + /// The connection was lost + #[error("connection lost: {0}")] + Connection(#[from] ConnectionError), +} + +impl From for io::Error { + fn from(x: Error) -> Self { + let kind = match x { + Error::Reset(_) | Error::Evicted => io::ErrorKind::ConnectionReset, + Error::FinishedEarly(_) => io::ErrorKind::UnexpectedEof, + Error::TooLong | Error::TooLarge => io::ErrorKind::InvalidInput, + Error::Connection(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} diff --git a/rs/moq-tokio/src/quic/driver/recv_stream.rs b/rs/moq-tokio/src/quic/driver/recv_stream.rs deleted file mode 100644 index 99ce60e95c6..00000000000 --- a/rs/moq-tokio/src/quic/driver/recv_stream.rs +++ /dev/null @@ -1,732 +0,0 @@ -use std::{ - future::{Future, poll_fn}, - io, - pin::Pin, - task::{Context, Poll, ready}, -}; - -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use moq_quic::{ - Chunk, Chunks, ClosedStream, LeaseError, ReadableError, StreamId, connection::Error as ConnectionError, -}; -use thiserror::Error; -use tokio::io::ReadBuf; - -use crate::quic::driver::{ - VarInt, - connection::{ConnectionRef, State}, - mutex::MutexGuard, -}; - -/// A stream that can only be used to receive data -/// -/// `stop(0)` is implicitly called on drop unless: -/// - A variant of [`ReadError`] has been yielded by a read call -/// - [`stop()`] was called explicitly -/// -/// # Cancellation -/// -/// A `read` method is said to be *cancel-safe* when dropping its future before the future becomes -/// ready cannot lead to loss of stream data. This is true of methods which succeed immediately when -/// any progress is made, and is not true of methods which might need to perform multiple reads -/// internally before succeeding. Each `read` method documents whether it is cancel-safe. -/// -/// # Common issues -/// -/// ## Data never received on a locally-opened stream -/// -/// Peers are not notified of streams until they or a later-numbered stream are used to send -/// data. If a bidirectional stream is locally opened but never used to send, then the peer may -/// never see it. Application protocols should always arrange for the endpoint which will first -/// transmit on a stream to be the endpoint responsible for opening it. -/// -/// ## Data never received on a remotely-opened stream -/// -/// Verify that the stream you are receiving is the same one that the server is sending on, e.g. by -/// logging the [`id`] of each. Streams are always accepted in the same order as they are created, -/// i.e. ascending order by [`StreamId`]. For example, even if a sender first transmits on -/// bidirectional stream 1, the first stream yielded by [`Connection::accept_bi`] on the receiver -/// will be bidirectional stream 0. -/// -/// [`ReadError`]: crate::quic::driver::ReadError -/// [`stop()`]: RecvStream::stop -/// [`SendStream::finish`]: crate::quic::driver::SendStream::finish -/// [`WriteError::Stopped`]: crate::quic::driver::WriteError::Stopped -/// [`id`]: RecvStream::id -/// [`Connection::accept_bi`]: crate::quic::driver::connection::Connection::accept_bi -#[derive(Debug)] -pub struct RecvStream { - conn: ConnectionRef, - stream: StreamId, - all_data_read: bool, - reset: Option, - /// The code a drop before the end stops with - drop_stop: VarInt, -} - -impl RecvStream { - pub(crate) fn new(conn: ConnectionRef, stream: StreamId) -> Self { - Self { - conn, - stream, - all_data_read: false, - reset: None, - drop_stop: VarInt::from_u32(0), - } - } - - /// Read data contiguously from the stream. - /// - /// Yields the number of bytes read into `buf` on success, or `None` if the stream was finished. - /// - /// This operation is cancel-safe. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - Read { - stream: self, - buf: ReadBuf::new(buf), - } - .await - } - - /// Read an exact number of bytes contiguously from the stream. - /// - /// See [`read()`] for details. This operation is *not* cancel-safe. - /// - /// [`read()`]: RecvStream::read - pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { - ReadExact { - stream: self, - buf: ReadBuf::new(buf), - } - .await - } - - /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the - /// stream ended. - /// - /// The bytes stay in the connection's receive buffer, keeping their connection flow control - /// credit, until read through the lease as a [`Buf`] and then [`Lease::release`]d; bytes not - /// read stay for the next lease. Stream credit still advances, so a stream window smaller - /// than a message keeps the message flowing. - /// - /// When the peer has spent the connection window and no reader can make progress, the - /// stream ranked lowest by [`Self::set_priority`] is evicted: its buffered bytes and their - /// credit are discarded and its next lease fails with [`ReadError::Evicted`]. Bytes it took - /// keep their credit until the stream is stopped. Nothing is evicted while the leased and - /// unread bytes of every stream stay below the connection receive window. - /// - /// The lease holds the connection's lock: drop it before touching any other stream. - /// Do not mix with the plain reads on one stream, which skip the leased bytes. - pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, ReadError>> { - let mut conn = self.conn.state.lock(); - let err = match conn.inner.recv_stream(self.stream).lease(want) { - Ok(transmit) => { - if transmit.should_transmit() { - conn.wake(); - } - return Poll::Ready(Ok(Lease { - conn, - stream: self.stream, - })); - } - Err(LeaseError::Blocked(transmit)) => { - if transmit.should_transmit() { - conn.wake(); - } - if let Some(x) = &conn.error { - return Poll::Ready(Err(ReadError::ConnectionLost(x.clone()))); - } - conn.blocked_readers.insert(self.stream, cx.waker().clone()); - return Poll::Pending; - } - Err(LeaseError::Reset(code)) => ReadError::Reset(code), - Err(LeaseError::Evicted) => ReadError::Evicted, - Err(LeaseError::TooLarge) => ReadError::TooLarge, - Err(LeaseError::ClosedStream) => match &conn.error { - Some(x) => ReadError::ConnectionLost(x.clone()), - None => ReadError::ClosedStream, - }, - }; - conn.blocked_readers.remove(&self.stream); - Poll::Ready(Err(err)) - } - - /// Rank this stream for eviction: when the connection window is stuck, the stream with the - /// lowest priority goes first, and the newest of those that tie. - pub fn set_priority(&mut self, priority: i64) { - let mut conn = self.conn.state.lock(); - conn.inner.recv_stream(self.stream).set_priority(priority); - } - - /// Attempts to read from the stream into the provided buffer - /// - /// On success, returns `Poll::Ready(Ok(num_bytes_read))` and places data into `buf`. If this - /// returns zero bytes read (and `buf` has a non-zero length), that indicates that the remote - /// side has [`finish`]ed the stream and the local side has already read all bytes. - /// - /// If no data is available for reading, this returns `Poll::Pending` and arranges for the - /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is - /// closed. - /// - /// [`finish`]: crate::quic::driver::SendStream::finish - pub fn poll_read(&mut self, cx: &mut Context<'_>, buf: &mut [u8]) -> Poll> { - let mut buf = ReadBuf::new(buf); - ready!(self.poll_read_buf(cx, &mut buf))?; - Poll::Ready(Ok(buf.filled().len())) - } - - /// Attempts to read from the stream into the provided buffer, which may be uninitialized - /// - /// On success, returns `Poll::Ready(Ok(()))` and places data into the unfilled portion of - /// `buf`. If this does not write any bytes to `buf` (and `buf.remaining()` is non-zero), that - /// indicates that the remote side has [`finish`]ed the stream and the local side has already - /// read all bytes. - /// - /// If no data is available for reading, this returns `Poll::Pending` and arranges for the - /// current task (via `cx.waker()`) to be notified when the stream becomes readable or is - /// closed. - /// - /// [`finish`]: crate::quic::driver::SendStream::finish - fn poll_read_buf(&mut self, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - if buf.remaining() == 0 { - return Poll::Ready(Ok(())); - } - - self.poll_read_generic(cx, |chunks| fill(chunks, buf)) - .map(|res| res.map(|_| ())) - } - - /// Read the next segment of data - /// - /// Yields `None` if the stream was finished, or the next contiguous segment of data. - /// Slightly more efficient than `read` due to not copying. Chunk boundaries do not correspond - /// to peer writes, and hence cannot be used as framing. - /// - /// This operation is cancel-safe. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - ReadChunk { - stream: self, - max_length, - } - .await - } - - /// Attempts to read a chunk from the stream. - /// - /// On success, returns `Poll::Ready(Ok(Some(chunk)))`. If `Poll::Ready(Ok(None))` - /// is returned, it implies that EOF has been reached. - /// - /// If no data is available for reading, the method returns `Poll::Pending` - /// and arranges for the current task (via cx.waker()) to receive a notification - /// when the stream becomes readable or is closed. - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max_length: usize) -> Poll, ReadError>> { - self.poll_read_generic(cx, |chunks| match chunks.next(max_length) { - Ok(Some(chunk)) => ReadStatus::Readable(chunk), - res => (None, res.err()).into(), - }) - } - - /// Convenience method to read all remaining data into a buffer - /// - /// Fails with [`ReadToEndError::TooLong`] on reading more than `size_limit` bytes, discarding - /// all data read. `size_limit` bounds the returned buffer. - /// - /// This operation is *not* cancel-safe. - /// - /// [`ReadToEndError::TooLong`]: crate::quic::driver::ReadToEndError::TooLong - pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { - let mut buffer = ReadToEndBuffer::new(size_limit); - while let Some(chunk) = self.read_chunk(usize::MAX).await? { - buffer.push(chunk)?; - } - Ok(buffer.finish()) - } - - /// Stop accepting data - /// - /// Discards unread data and notifies the peer to stop transmitting. Once stopped, further - /// attempts to operate on a stream will yield `ClosedStream` errors. - pub fn stop(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { - let mut conn = self.conn.state.lock(); - conn.inner.recv_stream(self.stream).stop(error_code)?; - conn.wake(); - self.all_data_read = true; - // Clean up shared state that might be left over from a cancalled read - // operation, so `drop` doesn't have to - conn.blocked_readers.remove(&self.stream); - Ok(()) - } - - /// Get the identity of this stream - pub fn id(&self) -> StreamId { - self.stream - } - - /// Stop with `code`, instead of code 0, if dropped before the end - #[cfg(feature = "quic")] - pub(crate) fn set_drop_stop(&mut self, code: VarInt) { - self.drop_stop = code; - } - - /// Completes when the stream has been reset by the peer or otherwise closed - /// - /// Yields `Some` with the reset error code when the stream is reset by the peer. Yields `None` - /// when the stream was previously [`stop()`](Self::stop)ed, or when the stream was - /// [`finish()`](crate::quic::driver::SendStream::finish)ed by the peer and all data has been received, after - /// which it is no longer meaningful for the stream to be reset. - /// - /// This operation is cancel-safe. - pub async fn received_reset(&mut self) -> Result, ResetError> { - poll_fn(|cx| { - let mut conn = self.conn.state.lock(); - - if let Some(code) = self.reset { - return Poll::Ready(Ok(Some(code))); - } - - match conn.inner.recv_stream(self.stream).received_reset() { - Err(_) => Poll::Ready(Ok(None)), - Ok(Some(error_code)) => { - // Stream state has just now been freed, so the connection may need to issue new - // stream ID flow control credit - conn.wake(); - Poll::Ready(Ok(Some(error_code))) - } - Ok(None) => { - if let Some(e) = &conn.error { - return Poll::Ready(Err(e.clone().into())); - } - // Resets always notify readers, since a reset is an immediate read error. We - // could introduce a dedicated channel to reduce the risk of spurious wakeups, - // but that increased complexity is probably not justified, as an application - // that is expecting a reset is not likely to receive large amounts of data. - conn.blocked_readers.insert(self.stream, cx.waker().clone()); - Poll::Pending - } - } - }) - .await - } - - /// Handle common logic related to reading out of a receive stream - /// - /// This takes an `FnMut` closure that takes care of the actual reading process, matching - /// the detailed read semantics for the calling function with a particular return type. - /// The closure can read from the passed `&mut Chunks` and has to return the status after - /// reading: the amount of data read, and the status after the final read call. - fn poll_read_generic(&mut self, cx: &mut Context<'_>, mut read_fn: T) -> Poll, ReadError>> - where - T: FnMut(&mut Chunks<'_>) -> ReadStatus, - { - use moq_quic::ReadError::*; - use std::collections::hash_map::Entry; - - if self.all_data_read { - return Poll::Ready(Ok(None)); - } - - let mut conn = self.conn.state.lock(); - - // If we stored an error during a previous call, return it now. This can happen if a - // `read_fn` both wants to return data and also returns an error in its final stream status. - let status = match self.reset { - Some(code) => ReadStatus::Failed(None, Reset(code)), - None => { - let mut recv = conn.inner.recv_stream(self.stream); - let mut chunks = recv.read()?; - let status = read_fn(&mut chunks); - if chunks.finalize().should_transmit() { - conn.wake(); - } - status - } - }; - - match status { - ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), - ReadStatus::Finished(read) => { - self.all_data_read = true; - // Stream is dead, cleanup stale waker - conn.blocked_readers.remove(&self.stream); - Poll::Ready(Ok(read)) - } - ReadStatus::Failed(read, Blocked) => match read { - Some(val) => Poll::Ready(Ok(Some(val))), - None => { - if let Some(x) = &conn.error { - return Poll::Ready(Err(ReadError::ConnectionLost(x.clone()))); - } - match conn.blocked_readers.entry(self.stream) { - Entry::Occupied(mut entry) => { - entry.get_mut().clone_from(cx.waker()); - } - Entry::Vacant(entry) => { - entry.insert(cx.waker().clone()); - } - } - Poll::Pending - } - }, - ReadStatus::Failed(read, Reset(error_code)) => match read { - None => { - self.all_data_read = true; - self.reset = Some(error_code); - // Stream is dead, cleanup stale waker - conn.blocked_readers.remove(&self.stream); - Poll::Ready(Err(ReadError::Reset(error_code))) - } - done => { - self.reset = Some(error_code); - Poll::Ready(Ok(done)) - } - }, - } - } -} - -/// Copy from `chunks` until `buf` is full or nothing more is readable. -fn fill(chunks: &mut Chunks<'_>, buf: &mut ReadBuf<'_>) -> ReadStatus<()> { - let mut read = false; - loop { - if buf.remaining() == 0 { - // We know `read` is `true` because `buf.remaining()` was not 0 before - return ReadStatus::Readable(()); - } - - match chunks.next(buf.remaining()) { - Ok(Some(chunk)) => { - buf.put_slice(&chunk.bytes); - read = true; - } - res => return (if read { Some(()) } else { None }, res.err()).into(), - } - } -} - -/// Unreleased bytes borrowed from a [`RecvStream`] by [`RecvStream::poll_lease`], read as a -/// [`Buf`]. -/// -/// Holds the connection's lock for as long as it lives. -pub struct Lease<'a> { - conn: MutexGuard<'a, State>, - stream: StreamId, -} - -impl Lease<'_> { - fn leased(&self) -> &moq_quic::Leased { - self.conn - .inner - .leased(self.stream) - .expect("a lease outlives nothing it borrows") - } - - /// Whether the peer finished the stream right after the leased bytes. - pub fn is_fin(&self) -> bool { - self.leased().is_fin() - } - - /// The bytes not yet read, in order, one contiguous chunk at a time. - pub fn chunks(&self) -> impl Iterator { - self.leased().chunks() - } - - /// Return the credit of every byte read so far, by this lease or an earlier one. - pub fn release(mut self) { - if self.conn.inner.recv_stream(self.stream).release().should_transmit() { - self.conn.wake(); - } - } -} - -/// Reading moves past bytes for good, but they keep their credit until [`Lease::release`]. -impl Buf for Lease<'_> { - fn remaining(&self) -> usize { - self.leased().len() - } - - fn chunk(&self) -> &[u8] { - self.leased().chunks().next().unwrap_or_default() - } - - fn advance(&mut self, n: usize) { - self.conn.inner.recv_stream(self.stream).advance(n); - } - - fn copy_to_bytes(&mut self, n: usize) -> Bytes { - match n <= self.chunk().len() { - true => self.conn.inner.recv_stream(self.stream).take(n), - false => { - let mut bytes = BytesMut::with_capacity(n); - bytes.put((&mut *self).take(n)); - bytes.freeze() - } - } - } -} - -enum ReadStatus { - Readable(T), - Finished(Option), - Failed(Option, moq_quic::ReadError), -} - -impl From<(Option, Option)> for ReadStatus { - fn from(status: (Option, Option)) -> Self { - match status { - (read, None) => Self::Finished(read), - (read, Some(e)) => Self::Failed(read, e), - } - } -} - -struct ReadToEndBuffer { - read: Vec, - size_limit: usize, -} - -impl ReadToEndBuffer { - fn new(size_limit: usize) -> Self { - Self { - read: Vec::new(), - size_limit, - } - } - - fn push(&mut self, chunk: Chunk) -> Result<(), ReadToEndError> { - if chunk.bytes.len() > self.size_limit - self.read.len() { - return Err(ReadToEndError::TooLong); - } - self.read.extend_from_slice(&chunk.bytes); - Ok(()) - } - - fn finish(self) -> Vec { - self.read - } -} - -/// Errors from [`RecvStream::read_to_end`] -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum ReadToEndError { - /// An error occurred during reading - #[error("read error: {0}")] - Read(#[from] ReadError), - /// The stream is larger than the user-supplied limit - #[error("stream too long")] - TooLong, -} - -impl tokio::io::AsyncRead for RecvStream { - fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - ready!(Self::poll_read_buf(self.get_mut(), cx, buf))?; - Poll::Ready(Ok(())) - } -} - -impl Drop for RecvStream { - fn drop(&mut self) { - if self.all_data_read { - debug_assert!( - !self.conn.state.lock().blocked_readers.contains_key(&self.stream), - "Stream {} should not have a blocked reader when all data read is true", - self.stream - ); - return; - } - let mut conn = self.conn.state.lock(); - - // clean up any previously registered wakers - conn.blocked_readers.remove(&self.stream); - - if conn.error.is_some() { - return; - } - - // Ignore ClosedStream errors - let _ = conn.inner.recv_stream(self.stream).stop(self.drop_stop); - conn.wake(); - } -} - -/// Errors that arise from reading from a stream. -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum ReadError { - /// The peer abandoned transmitting data on this stream - /// - /// Carries an application-defined error code. - #[error("stream reset by peer: error {0}")] - Reset(VarInt), - /// The connection was lost - #[error("connection lost")] - ConnectionLost(#[from] ConnectionError), - /// The stream has already been stopped, finished, or reset - #[error("closed stream")] - ClosedStream, - /// The connection window was stuck and this stream was discarded to free it - /// - /// See [`RecvStream::poll_lease`]. The peer hears of it once the stream is stopped, so call - /// [`RecvStream::stop`] with your own code: dropping the stream sends code 0. - #[error("evicted")] - Evicted, - /// A lease wanted more than the connection receive window can hold - #[error("want exceeds the receive window")] - TooLarge, -} - -impl From for ReadError { - fn from(e: ReadableError) -> Self { - match e { - ReadableError::ClosedStream => Self::ClosedStream, - } - } -} - -impl From for ReadError { - fn from(e: ResetError) -> Self { - match e { - ResetError::ConnectionLost(e) => Self::ConnectionLost(e), - } - } -} - -impl From for io::Error { - fn from(x: ReadError) -> Self { - use ReadError::*; - let kind = match x { - Reset { .. } | Evicted => io::ErrorKind::ConnectionReset, - ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, - TooLarge => io::ErrorKind::InvalidInput, - }; - Self::new(kind, x) - } -} - -/// Errors that arise while waiting for a stream to be reset -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum ResetError { - /// The connection was lost - #[error("connection lost")] - ConnectionLost(#[from] ConnectionError), -} - -impl From for io::Error { - fn from(x: ResetError) -> Self { - use ResetError::*; - let kind = match x { - ConnectionLost(_) => io::ErrorKind::NotConnected, - }; - Self::new(kind, x) - } -} - -/// Future produced by [`RecvStream::read()`]. -/// -/// [`RecvStream::read()`]: crate::quic::driver::RecvStream::read -struct Read<'a> { - stream: &'a mut RecvStream, - buf: ReadBuf<'a>, -} - -impl Future for Read<'_> { - type Output = Result, ReadError>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; - match this.buf.filled().len() { - 0 if this.buf.capacity() != 0 => Poll::Ready(Ok(None)), - n => Poll::Ready(Ok(Some(n))), - } - } -} - -/// Future produced by [`RecvStream::read_exact()`]. -/// -/// [`RecvStream::read_exact()`]: crate::quic::driver::RecvStream::read_exact -struct ReadExact<'a> { - stream: &'a mut RecvStream, - buf: ReadBuf<'a>, -} - -impl Future for ReadExact<'_> { - type Output = Result<(), ReadExactError>; - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.get_mut(); - let mut remaining = this.buf.remaining(); - while remaining > 0 { - ready!(this.stream.poll_read_buf(cx, &mut this.buf))?; - let new = this.buf.remaining(); - if new == remaining { - return Poll::Ready(Err(ReadExactError::FinishedEarly(this.buf.filled().len()))); - } - remaining = new; - } - Poll::Ready(Ok(())) - } -} - -/// Errors that arise from reading from a stream. -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum ReadExactError { - /// The stream finished before all bytes were read - #[error("stream finished early ({0} bytes read)")] - FinishedEarly(usize), - /// A read error occurred - #[error(transparent)] - ReadError(#[from] ReadError), -} - -/// Future produced by [`RecvStream::read_chunk()`]. -/// -/// [`RecvStream::read_chunk()`]: crate::quic::driver::RecvStream::read_chunk -struct ReadChunk<'a> { - stream: &'a mut RecvStream, - max_length: usize, -} - -impl Future for ReadChunk<'_> { - type Output = Result, ReadError>; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let max_length = self.max_length; - self.stream.poll_read_chunk(cx, max_length) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn read_to_end_limit_covers_all_chunks() { - let mut buffer = ReadToEndBuffer::new(4); - buffer - .push(Chunk { - bytes: Bytes::from_static(b"ab"), - }) - .unwrap(); - buffer - .push(Chunk { - bytes: Bytes::from_static(b"cd"), - }) - .unwrap(); - assert_eq!( - buffer.push(Chunk { - bytes: Bytes::from_static(b"e") - }), - Err(ReadToEndError::TooLong) - ); - assert_eq!(buffer.finish(), b"abcd"); - let mut empty = ReadToEndBuffer::new(0); - assert_eq!( - empty.push(Chunk { - bytes: Bytes::from_static(b"x") - }), - Err(ReadToEndError::TooLong) - ); - assert!(empty.finish().is_empty()); - } -} diff --git a/rs/moq-tokio/src/quic/driver/send.rs b/rs/moq-tokio/src/quic/driver/send.rs new file mode 100644 index 00000000000..04810905214 --- /dev/null +++ b/rs/moq-tokio/src/quic/driver/send.rs @@ -0,0 +1,567 @@ +//! Outgoing streams and datagrams: their handles, scheduling, and errors. + +use std::{ + future::{Future, poll_fn}, + io, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use bytes::Buf; +use moq_quic::{connection::Error as ConnectionError, send::End}; +use thiserror::Error; + +use crate::quic::driver::{ + VarInt, + connection::{ConnectionInner, ConnectionRef, State}, + stream, +}; + +/// A stream that can only be used to send data +/// +/// [`finish()`](Self::finish) it to end it cleanly, which hands back a [`Finished`] that can +/// no longer be written. Dropped unfinished, it is [`reset()`](Self::reset), with code 0 unless +/// the peer stopped it, so the peer can't mistake a truncated stream for a complete one. +/// +/// # Cancellation +/// +/// A `write` method is said to be *cancel-safe* when dropping its future before the future becomes +/// ready will always result in no data being written to the stream. This is true of methods which +/// succeed immediately when any progress is made, and is not true of methods which might need to +/// perform multiple writes internally before succeeding. Each `write` method documents whether it is +/// cancel-safe. +#[derive(Debug)] +pub struct Stream(Half); + +/// A stream [`Stream::finish`]ed, which keeps (re)transmitting its data until the peer +/// acknowledges all of it or the connection closes +/// +/// Dropping it leaves the stream to finish, unless the peer stopped it, in which case it is +/// reset with the peer's code. +#[derive(Debug)] +pub struct Finished(Half); + +/// What [`Stream`] and [`Finished`] share. +#[derive(Debug)] +struct Half { + conn: ConnectionRef, + id: stream::Id, + state: HalfState, +} + +#[derive(Debug)] +enum HalfState { + /// Still open: a drop resets at the offset with the code. + Open(Option, VarInt), + /// Finished, by [`Stream::finish`] or an [`AsyncWrite`](tokio::io::AsyncWrite) shutdown. + Finished, + Reset, +} + +impl Stream { + pub(crate) fn new(conn: ConnectionRef, id: stream::Id) -> Self { + Self(Half { + conn, + id, + state: HalfState::Open(None, VarInt::from_u32(0)), + }) + } + + /// Write from `buf` into this stream, returning how many bytes were written + /// + /// Unless this method errors, it waits until some of `buf` can be written into this stream, + /// and then writes as much as it can without waiting again, advancing `buf` past it. Due to + /// congestion and flow control, this may be less than `buf.remaining()`. A [`Bytes`](bytes::Bytes) + /// source is taken without copying. + /// + /// # Cancel safety + /// + /// This method is cancellation safe. If this does not resolve, no bytes were written. + pub async fn write(&mut self, buf: &mut impl Buf) -> Result { + poll_fn(|cx| self.poll_write(cx, buf)).await + } + + /// Write all of `buf` into this stream + /// + /// # Cancel safety + /// + /// This method is *not* cancellation safe. Even if this does not resolve, some prefix of `buf` + /// may have been written when previously polled. + pub async fn write_all(&mut self, mut buf: impl Buf) -> Result<(), Error> { + while buf.has_remaining() { + self.write(&mut buf).await?; + } + Ok(()) + } + + fn poll_write(&mut self, cx: &mut Context<'_>, buf: &mut impl Buf) -> Poll> { + assert!( + matches!(self.0.state, HalfState::Open(..)), + "wrote to a stream after shutting it down" + ); + let mut conn = self.0.conn.state.lock(); + if let Some(x) = &conn.error { + return Poll::Ready(Err(Error::Connection(x.clone()))); + } + match conn.inner.send_stream(self.0.id).write(buf) { + Ok(written) => { + conn.wake(); + Poll::Ready(Ok(written)) + } + Err(moq_quic::send::Error::Blocked) => { + conn.blocked_writers.insert(self.0.id, cx.waker().clone()); + Poll::Pending + } + Err(moq_quic::send::Error::Stopped(code)) => Poll::Ready(Err(Error::Stopped(code))), + Err(err) => unreachable!("an open stream failed to write: {err:?}"), + } + } + + /// Notify the peer that no more data will ever be written to this stream + /// + /// [`Finished::stopped()`] waits for the peer to receive all of it, or learns that it + /// stopped the stream first. + pub fn finish(mut self) -> Finished { + self.0.finish(); + Finished(self.0) + } + + /// Close the send stream immediately + /// + /// Locally buffered data is dropped, and previously transmitted data will no longer be + /// retransmitted if lost. + pub fn reset(mut self, error_code: VarInt) { + self.0.reset(None, error_code); + } + + /// Close the send stream immediately, but still deliver the data before `offset` + /// + /// Like [`Self::reset()`], except the data before `offset` still reaches the peer. A peer + /// that accepts RESET_STREAM_AT reads it before the error. Otherwise the reset waits a round + /// trip until the peer acknowledges the data, and the peer's application may never read it: + /// see [`moq_quic::send::Stream::reset_at`]. + /// + /// # Panics + /// - when `offset` came from another stream + pub fn reset_at(mut self, offset: Offset, error_code: VarInt) { + self.0.reset(Some(offset), error_code); + } + + /// The offset just past the data written so far + pub fn offset(&self) -> Offset { + let mut conn = self.0.conn.state.lock(); + Offset { + inner: conn.inner.send_stream(self.0.id).offset(), + conn: self.0.conn.stable_id(), + id: self.0.id, + } + } + + /// Reset at `offset` with `code`, instead of at 0 with code 0, if dropped before `finish` + #[cfg(feature = "quic")] + pub(crate) fn set_drop_reset(&mut self, offset: Option, code: VarInt) { + if let Some(offset) = &offset { + self.0.own(offset); + } + self.0.state = HalfState::Open(offset, code); + } + + /// Set the priority within the stream's send group, or session-wide when ungrouped + /// + /// Every send stream has an initial priority of 0. Locally buffered data from streams with + /// higher priority will be transmitted before data from streams with lower priority. Changing + /// the priority of a stream with pending data may only take effect after that data has been + /// transmitted. Using many different priority levels per connection may have a negative + /// impact on performance. + pub fn set_priority(&mut self, priority: i64) { + self.0.set_priority(priority); + } + + /// The priority of the send stream + pub fn priority(&self) -> i64 { + self.0.priority() + } + + /// Move the stream into a send group, or with `None` back to the ungrouped streams, which + /// go before every group + /// + /// # Panics + /// - when `group` belongs to another connection + pub fn set_group(&mut self, group: Option<&Group>) { + self.0.set_group(group); + } + + /// Completes when the peer stops the stream, or the stream ends otherwise + /// + /// See [`Finished::stopped()`]. A stream reset or dropped before the peer stopped it + /// resolves with `None`. + pub fn stopped(&self) -> impl Future, Error>> + Send + Sync + 'static { + self.0.stopped() + } + + /// The identity of this stream + pub fn id(&self) -> stream::Id { + self.0.id + } +} + +impl Finished { + /// Abandon the stream's data the peer has not acknowledged yet. See [`Stream::reset()`]. + pub fn reset(mut self, error_code: VarInt) { + self.0.reset(None, error_code); + } + + /// Abandon the stream's data past `offset`. See [`Stream::reset_at()`]. + /// + /// # Panics + /// - when `offset` came from another stream + pub fn reset_at(mut self, offset: Offset, error_code: VarInt) { + self.0.reset(Some(offset), error_code); + } + + /// Set the priority its data is retransmitted at. See [`Stream::set_priority()`]. + pub fn set_priority(&mut self, priority: i64) { + self.0.set_priority(priority); + } + + /// The priority of the send stream + pub fn priority(&self) -> i64 { + self.0.priority() + } + + /// Move the stream into a send group. See [`Stream::set_group()`]. + /// + /// # Panics + /// - when `group` belongs to another connection + pub fn set_group(&mut self, group: Option<&Group>) { + self.0.set_group(group); + } + + /// Completes when the peer stops the stream or acknowledges all of its data + /// + /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` once + /// the peer acknowledges receipt of all stream data (although not necessarily the processing + /// of it), or once the stream is reset, after which the peer closing it is no longer + /// meaningful. A peer's stop code still wins over a local reset if it arrived first, but not + /// if it arrives after. + /// + /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving + /// data. As such, relying on `stopped` to know when the peer has read a stream to completion + /// may introduce more latency than using an application-level response of some sort. + pub fn stopped(&self) -> impl Future, Error>> + Send + Sync + 'static { + self.0.stopped() + } + + /// The identity of this stream + pub fn id(&self) -> stream::Id { + self.0.id + } +} + +impl Half { + /// Check `offset` before taking the lock: moq-quic makes the same check, but panicking + /// there poisons the lock, so every handle's drop panics again and aborts the process. + fn own(&self, offset: &Offset) { + assert!( + offset.conn == self.conn.stable_id() && offset.id == self.id, + "offset used on another stream" + ); + } + + fn finish(&mut self) { + let mut conn = self.conn.state.lock(); + // A STOP_SENDING may have beaten us here, which `stopped` reports. + if conn.inner.send_stream(self.id).finish().is_ok() { + conn.wake(); + } + self.state = HalfState::Finished; + } + + fn reset(&mut self, offset: Option, code: VarInt) { + if let Some(offset) = &offset { + self.own(offset); + } + self.state = HalfState::Reset; + let mut conn = self.conn.state.lock(); + let mut stream = conn.inner.send_stream(self.id); + let stopped = match stream.end() { + Ok(Some(End::Stopped(code))) => Some(code), + _ => None, + }; + let reset = match offset { + Some(offset) => stream.reset_at(offset.inner, code), + None => stream.reset(code), + }; + // Fails harmlessly once the connection is gone. + if reset.is_ok() { + conn.resolve_stopped(self.id, stopped); + conn.wake(); + } + } + + fn set_priority(&mut self, priority: i64) { + let mut conn = self.conn.state.lock(); + let _ = conn.inner.send_stream(self.id).set_priority(priority); + } + + fn priority(&self) -> i64 { + let mut conn = self.conn.state.lock(); + conn.inner.send_stream(self.id).priority().unwrap_or_default() + } + + fn set_group(&mut self, group: Option<&Group>) { + let group = group.map(|group| group.id_on(&self.conn.0)); + let mut conn = self.conn.state.lock(); + let _ = conn.inner.send_stream(self.id).set_group(group); + } + + fn stopped(&self) -> impl Future, Error>> + Send + Sync + 'static { + let conn = self.conn.clone(); + let id = self.id; + // Join the waiters now rather than on the first poll, so a reset in between, which no + // stream event reports, still reaches this future. + let waiters = { + let mut conn = conn.state.lock(); + match ended(&mut conn, id) { + Some(output) => Err(output), + None => Ok(conn.stopped.entry(id).or_default().clone()), + } + }; + async move { + let mut waiters = match waiters { + Ok(waiters) => waiters, + Err(output) => return output, + }; + loop { + // The `Notify::notified` future needs to be created while the lock is being held, + // otherwise a wakeup could be missed if triggered inbetween releasing the lock + // and creating the future. + // The lock may only be held in a block without `await`s, otherwise the future + // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore + // we need to declare `waiters` outside of the block, and update it inside. + { + let mut conn = conn.state.lock(); + if let Some(&stopped) = waiters.outcome.get() { + return Ok(stopped); + } + if let Some(output) = ended(&mut conn, id) { + return output; + } + + // A wakeup that resolved nothing removed the waiters; join the current ones + waiters = conn.stopped.entry(id).or_default().clone(); + waiters.notify.notified() + } + .await + } + } + } +} + +/// How a send stream ended, or `None` while it goes on +/// +/// A released stream's end is gone, but its handle settled the waiters when it let go. +fn ended(conn: &mut State, id: stream::Id) -> Option, Error>> { + match conn.inner.send_stream(id).end() { + Ok(Some(End::Stopped(code))) => Some(Ok(Some(code))), + Ok(Some(_)) => Some(Ok(None)), + Ok(None) | Err(_) => conn.error.clone().map(|error| Err(error.into())), + } +} + +impl Drop for Half { + fn drop(&mut self) { + let mut conn = self.conn.state.lock(); + + // clean up any previously registered wakers + conn.blocked_writers.remove(&self.id); + + if conn.error.is_none() { + let mut stream = conn.inner.send_stream(self.id); + let stopped = match stream.end() { + Ok(Some(End::Stopped(code))) => Some(code), + _ => None, + }; + let reset = match (&self.state, stopped) { + (HalfState::Reset, _) => None, + // Echo the code, as RFC 9000 section 3.5 asks, even after `finish`: the peer reads + // nothing more, so retransmitting the rest is wasted. + (_, Some(code)) => Some(stream.reset(code)), + (HalfState::Finished, None) => None, + (HalfState::Open(offset, code), None) => Some(match offset { + Some(offset) => stream.reset_at(offset.inner, *code), + None => stream.reset(*code), + }), + }; + if reset.is_some_and(|reset| reset.is_ok()) { + conn.resolve_stopped(self.id, stopped); + conn.wake(); + } + } + conn.inner.send_stream(self.id).release(); + } +} + +impl tokio::io::AsyncWrite for Stream { + fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: &[u8]) -> Poll> { + let this = self.get_mut(); + if !matches!(this.0.state, HalfState::Open(..)) { + return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())); + } + this.poll_write(cx, &mut buf).map_err(Into::into) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + if matches!(this.0.state, HalfState::Open(..)) { + this.0.finish(); + } + Poll::Ready(Ok(())) + } +} + +/// A position in a [`Stream`]'s data, minted by [`Stream::offset()`] and good only on that +/// stream, so it is never past the data written +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Offset { + inner: moq_quic::send::Offset, + /// The connection's [`ConnectionRef::stable_id`], checked before taking its lock. + conn: usize, + id: stream::Id, +} + +impl From for u64 { + fn from(offset: Offset) -> Self { + offset.inner.into() + } +} + +/// How an outgoing stream or datagram is scheduled: its [`Group`] and priority +#[derive(Debug, Default, Clone, Copy)] +pub struct Options<'a> { + /// The group it joins, or `None` to go before every group + pub group: Option<&'a Group>, + /// Its priority within the group, or among the ungrouped streams and datagrams + pub priority: i64, +} + +impl<'a> Options<'a> { + /// The options as `conn` knows them. + /// + /// # Panics + /// - when the group belongs to another connection + pub(crate) fn to_quic(self, conn: &ConnectionInner) -> moq_quic::send::Options<'a> { + moq_quic::send::Options { + group: self.group.map(|group| group.id_on(conn)), + priority: self.priority, + } + } +} + +/// A send group on a [`Connection`](crate::quic::driver::connection::Connection) +/// +/// Streams that [join](Stream::set_group) a group share bytes with the connection's other +/// groups: a higher priority group preempts every lower one, and groups of equal priority get an +/// equal share of bytes however many streams each has queued. A stream's own priority orders it +/// only against the group's other streams, and a datagram sent in the group is ordered among +/// them by its priority. Streams and datagrams outside any group go before every group. See +/// [`moq_quic::send::Groups`]. +/// +/// Clones share one group, released once the last clone is dropped and its streams finish +/// and its datagrams are sent. A group does not keep its connection open. +#[derive(Debug, Clone)] +pub struct Group(Arc); + +#[derive(Debug)] +struct GroupInner { + conn: Arc, + group: Option, +} + +impl Group { + pub(crate) fn new(conn: Arc, priority: i64) -> Self { + let group = conn.state.lock().inner.send_groups().open(priority); + Self(Arc::new(GroupInner { + conn, + group: Some(group), + })) + } + + /// Change the group's priority, moving any data it has queued along with it + pub fn set_priority(&self, priority: i64) { + let mut conn = self.0.conn.state.lock(); + conn.inner.send_groups().set_priority(self.id(), priority); + } + + /// The group's priority + pub fn priority(&self) -> i64 { + let mut conn = self.0.conn.state.lock(); + conn.inner.send_groups().priority(self.id()) + } + + /// The group as the connection knows it, which refuses it on any other connection. + fn id(&self) -> &moq_quic::send::Group { + self.0.group.as_ref().expect("only taken on drop") + } + + /// The group as `conn` knows it. + /// + /// Call it before taking `conn`'s lock: moq-quic makes the same check, but panicking + /// there poisons the lock, so every handle's drop panics again and aborts the process. + /// + /// # Panics + /// - when the group belongs to another connection + pub(crate) fn id_on(&self, conn: &ConnectionInner) -> &moq_quic::send::Group { + assert!( + std::ptr::eq(&*self.0.conn, conn), + "send group used on another connection" + ); + self.id() + } +} + +impl Drop for GroupInner { + fn drop(&mut self) { + if let Some(group) = self.group.take() { + let mut conn = self.conn.state.lock(); + conn.inner.send_groups().close(group); + } + } +} + +/// Errors from sending on a stream or as a datagram +#[derive(Debug, Error, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The peer is no longer accepting data on this stream + /// + /// Carries an application-defined error code. + #[error("sending stopped by peer: error {0}")] + Stopped(VarInt), + /// The datagram is larger than the connection can send + #[error("datagram too large")] + TooLarge, + /// Datagrams are disabled locally or unsupported by the peer + #[error("datagrams unsupported")] + Unsupported, + /// The connection was lost + #[error("connection lost: {0}")] + Connection(#[from] ConnectionError), +} + +impl From for io::Error { + fn from(x: Error) -> Self { + let kind = match x { + Error::Stopped(_) => io::ErrorKind::ConnectionReset, + Error::TooLarge => io::ErrorKind::InvalidInput, + Error::Unsupported => io::ErrorKind::Unsupported, + Error::Connection(_) => io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} diff --git a/rs/moq-tokio/src/quic/driver/send_group.rs b/rs/moq-tokio/src/quic/driver/send_group.rs deleted file mode 100644 index 1654166d87c..00000000000 --- a/rs/moq-tokio/src/quic/driver/send_group.rs +++ /dev/null @@ -1,88 +0,0 @@ -use std::sync::Arc; - -use crate::quic::driver::connection::ConnectionInner; - -/// A send group on a [`Connection`](crate::quic::driver::connection::Connection) -/// -/// Streams that [join](crate::quic::driver::SendStream::set_group) a group share bytes with -/// the connection's other groups: a higher priority group preempts every lower one, and groups -/// of equal priority get an equal share of bytes however many streams each has queued. A -/// stream's own priority orders it only against the group's other streams, and a datagram sent -/// in the group is ordered among them by its priority. Streams and datagrams outside any group -/// go before every group. See [`moq_quic::SendGroups`]. -/// -/// Clones share one group, released once the last clone is dropped and its streams finish -/// and its datagrams are sent. A group does not keep its connection open. -#[derive(Debug, Clone)] -pub struct SendGroup(Arc); - -/// How an outgoing stream or datagram is scheduled: its [`SendGroup`] and priority -pub type SendOptions<'a> = moq_quic::SendOptions<'a, SendGroup>; - -/// The options as `conn` knows them. -/// -/// # Panics -/// - when `options.group` belongs to another connection -pub(crate) fn to_quic<'a>(options: SendOptions<'a>, conn: &ConnectionInner) -> moq_quic::SendOptions<'a> { - moq_quic::SendOptions { - group: options.group.map(|group| group.id_on(conn)), - priority: options.priority, - } -} - -#[derive(Debug)] -struct Inner { - conn: Arc, - group: Option, -} - -impl SendGroup { - pub(crate) fn new(conn: Arc, priority: i64) -> Self { - let group = conn.state.lock().inner.send_groups().open(priority); - Self(Arc::new(Inner { - conn, - group: Some(group), - })) - } - - /// Change the group's priority, moving any data it has queued along with it - pub fn set_priority(&self, priority: i64) { - let mut conn = self.0.conn.state.lock(); - conn.inner.send_groups().set_priority(self.id(), priority); - } - - /// Get the group's priority - pub fn priority(&self) -> i64 { - let mut conn = self.0.conn.state.lock(); - conn.inner.send_groups().priority(self.id()) - } - - /// The group as the connection knows it, which refuses it on any other connection. - pub(crate) fn id(&self) -> &moq_quic::SendGroup { - self.0.group.as_ref().expect("only taken on drop") - } - - /// The group as `conn` knows it. - /// - /// Call it before taking `conn`'s lock: moq-quic makes the same check, but panicking - /// there poisons the lock, so every handle's drop panics again and aborts the process. - /// - /// # Panics - /// - when the group belongs to another connection - pub(crate) fn id_on(&self, conn: &ConnectionInner) -> &moq_quic::SendGroup { - assert!( - std::ptr::eq(&*self.0.conn, conn), - "send group used on another connection" - ); - self.id() - } -} - -impl Drop for Inner { - fn drop(&mut self) { - if let Some(group) = self.group.take() { - let mut conn = self.conn.state.lock(); - conn.inner.send_groups().close(group); - } - } -} diff --git a/rs/moq-tokio/src/quic/driver/send_stream.rs b/rs/moq-tokio/src/quic/driver/send_stream.rs deleted file mode 100644 index 28ebfebf5a9..00000000000 --- a/rs/moq-tokio/src/quic/driver/send_stream.rs +++ /dev/null @@ -1,468 +0,0 @@ -use std::{ - future::{Future, poll_fn}, - io, - pin::{Pin, pin}, - task::{Context, Poll}, -}; - -use bytes::Bytes; -use moq_quic::{ClosedStream, FinishError, StreamId, Written, connection::Error as ConnectionError}; -use thiserror::Error; - -use crate::quic::driver::{ - SendGroup, VarInt, - connection::{ConnectionRef, State}, -}; - -/// A stream that can only be used to send data -/// -/// If dropped, streams that haven't been explicitly [`finish()`]ed are [`reset()`], with code 0 -/// unless the peer stopped them, so the peer can't mistake a truncated stream for a complete one. -/// A finished stream keeps (re)transmitting previously written data until it has been fully -/// acknowledged or the connection is closed. -/// -/// # Cancellation -/// -/// A `write` method is said to be *cancel-safe* when dropping its future before the future becomes -/// ready will always result in no data being written to the stream. This is true of methods which -/// succeed immediately when any progress is made, and is not true of methods which might need to -/// perform multiple writes internally before succeeding. Each `write` method documents whether it is -/// cancel-safe. -/// -/// [`reset()`]: SendStream::reset -/// [`finish()`]: SendStream::finish -#[derive(Debug)] -pub struct SendStream { - conn: ConnectionRef, - stream: StreamId, - finished: bool, - /// The offset and code a drop before `finish` resets at - drop_reset: (u64, VarInt), - /// `reset` or `reset_at` succeeded, with the peer's STOP_SENDING code if it arrived first, so - /// `stopped` has nothing left to wait for - reset: Option>, -} - -impl SendStream { - pub(crate) fn new(conn: ConnectionRef, stream: StreamId) -> Self { - Self { - conn, - stream, - finished: false, - drop_reset: (0, VarInt::from_u32(0)), - reset: None, - } - } - - /// Write a buffer into this stream, returning how many bytes were written - /// - /// Unless this method errors, it waits until some amount of `buf` can be written into this - /// stream, and then writes as much as it can without waiting again. Due to congestion and flow - /// control, this may be shorter than `buf.len()`. On success this yields the length of the - /// prefix that was written. - /// - /// # Cancel safety - /// - /// This method is cancellation safe. If this does not resolve, no bytes were written. - pub async fn write(&mut self, buf: &[u8]) -> Result { - poll_fn(|cx| self.execute_poll(cx, |s| s.write(buf))).await - } - - /// Write a buffer into this stream in its entirety - /// - /// This method repeatedly calls [`write`](Self::write) until all bytes are written, or an - /// error occurs. - /// - /// # Cancel safety - /// - /// This method is *not* cancellation safe. Even if this does not resolve, some prefix of `buf` - /// may have been written when previously polled. - pub async fn write_all(&mut self, mut buf: &[u8]) -> Result<(), WriteError> { - while !buf.is_empty() { - let written = self.write(buf).await?; - buf = &buf[written..]; - } - Ok(()) - } - - /// Write a slice of [`Bytes`] into this stream, returning how much was written - /// - /// Bytes to try to write are provided to this method as an array of cheaply cloneable chunks. - /// Unless this method errors, it waits until some amount of those bytes can be written into - /// this stream, and then writes as much as it can without waiting again. Due to congestion and - /// flow control, this may be less than the total number of bytes. - /// - /// On success, this method both mutates `bufs` and yields an informative [`Written`] struct - /// indicating how much was written: - /// - /// - [`Bytes`] chunks that were fully written are mutated to be [empty](Bytes::is_empty). - /// - If a [`Bytes`] chunk was partially written, it is [split to](Bytes::split_to) contain - /// only the suffix of bytes that were not written. - /// - The yielded [`Written`] struct indicates how many chunks were fully written as well as - /// how many bytes were written. - /// - /// # Cancel safety - /// - /// This method is cancellation safe. If this does not resolve, no bytes were written. - pub async fn write_chunks(&mut self, bufs: &mut [Bytes]) -> Result { - poll_fn(|cx| self.execute_poll(cx, |s| s.write_chunks(bufs))).await - } - - /// Write a single [`Bytes`] into this stream in its entirety - /// - /// Bytes to write are provided to this method as an single cheaply cloneable chunk. This - /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, - /// or an error occurs. - /// - /// # Cancel safety - /// - /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have - /// been written when previously polled. - pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { - self.write_all_chunks(&mut [buf]).await?; - Ok(()) - } - - /// Write a slice of [`Bytes`] into this stream in its entirety - /// - /// Bytes to write are provided to this method as an array of cheaply cloneable chunks. This - /// method repeatedly calls [`write_chunks`](Self::write_chunks) until all bytes are written, - /// or an error occurs. This method mutates `bufs` by mutating all chunks to be - /// [empty](Bytes::is_empty). - /// - /// # Cancel safety - /// - /// This method is *not* cancellation safe. Even if this does not resolve, some bytes may have - /// been written when previously polled. - pub async fn write_all_chunks(&mut self, mut bufs: &mut [Bytes]) -> Result<(), WriteError> { - while !bufs.is_empty() { - let written = self.write_chunks(bufs).await?; - bufs = &mut bufs[written.chunks..]; - } - Ok(()) - } - - fn execute_poll(&mut self, cx: &mut Context<'_>, write_fn: F) -> Poll> - where - F: FnOnce(&mut moq_quic::SendStream<'_>) -> Result, - { - use moq_quic::WriteError::*; - let mut conn = self.conn.state.lock(); - if let Some(x) = &conn.error { - return Poll::Ready(Err(WriteError::ConnectionLost(x.clone()))); - } - - let result = match write_fn(&mut conn.inner.send_stream(self.stream)) { - Ok(result) => result, - Err(Blocked) => { - conn.blocked_writers.insert(self.stream, cx.waker().clone()); - return Poll::Pending; - } - Err(Stopped(error_code)) => { - return Poll::Ready(Err(WriteError::Stopped(error_code))); - } - Err(ClosedStream) => { - return Poll::Ready(Err(WriteError::ClosedStream)); - } - }; - - conn.wake(); - Poll::Ready(Ok(result)) - } - - /// Notify the peer that no more data will ever be written to this stream - /// - /// It is an error to write to a [`SendStream`] after `finish()`ing it. [`reset()`](Self::reset) - /// may still be called after `finish` to abandon transmission of any stream data that might - /// still be buffered. - /// - /// To wait for the peer to receive all buffered stream data, see [`stopped()`](Self::stopped). - /// - /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously - /// called. This error is harmless and serves only to indicate that the caller may have - /// incorrect assumptions about the stream's state. - pub fn finish(&mut self) -> Result<(), ClosedStream> { - let mut conn = self.conn.state.lock(); - match conn.inner.send_stream(self.stream).finish() { - Ok(()) => { - self.finished = true; - conn.wake(); - Ok(()) - } - Err(FinishError::ClosedStream) => Err(ClosedStream::default()), - // Harmless. If the application needs to know about stopped streams at this point, it - // should call `stopped`. - Err(FinishError::Stopped(_)) => Ok(()), - } - } - - /// Close the send stream immediately. - /// - /// No new data can be written after calling this method. Locally buffered data is dropped, and - /// previously transmitted data will no longer be retransmitted if lost. If an attempt has - /// already been made to finish the stream, the peer may still receive all written data. - /// - /// May fail if [`finish()`](Self::finish) or [`reset()`](Self::reset) was previously - /// called. This error is harmless and serves only to indicate that the caller may have - /// incorrect assumptions about the stream's state. - pub fn reset(&mut self, error_code: VarInt) -> Result<(), ClosedStream> { - let mut conn = self.conn.state.lock(); - let mut stream = conn.inner.send_stream(self.stream); - let stopped = stream.stopped().ok().flatten(); - stream.reset(error_code)?; - self.reset = Some(stopped); - conn.resolve_stopped(self.stream, stopped); - conn.wake(); - Ok(()) - } - - /// Close the send stream immediately, but still deliver its first `offset` bytes. - /// - /// Like [`SendStream::reset()`], except the data before `offset` still reaches the peer. - /// A peer that accepts RESET_STREAM_AT reads it before the error. Otherwise the reset waits a - /// round trip until the peer acknowledges the data, and the peer's application may never read - /// it: see [`moq_quic::SendStream::reset_at`]. - /// - /// # Panics - /// When `offset` is past the data written. - pub fn reset_at(&mut self, offset: u64, error_code: VarInt) -> Result<(), ClosedStream> { - let mut conn = self.conn.state.lock(); - let mut stream = conn.inner.send_stream(self.stream); - let stopped = stream.stopped().ok().flatten(); - stream.reset_at(offset, error_code)?; - self.reset = Some(stopped); - conn.resolve_stopped(self.stream, stopped); - conn.wake(); - Ok(()) - } - - /// Set the priority within the stream's send group, or session-wide when ungrouped - /// - /// Every send stream has an initial priority of 0. Locally buffered data from streams with - /// higher priority will be transmitted before data from streams with lower priority. Changing - /// the priority of a stream with pending data may only take effect after that data has been - /// transmitted. Using many different priority levels per connection may have a negative - /// impact on performance. - pub fn set_priority(&self, priority: i64) -> Result<(), ClosedStream> { - let mut conn = self.conn.state.lock(); - conn.inner.send_stream(self.stream).set_priority(priority)?; - Ok(()) - } - - /// Get the priority of the send stream - pub fn priority(&self) -> Result { - let mut conn = self.conn.state.lock(); - conn.inner.send_stream(self.stream).priority() - } - - /// Move the stream into a send group, or with `None` back to the ungrouped streams, which - /// go before every group - /// - /// # Panics - /// - when `group` belongs to another connection - pub fn set_group(&self, group: Option<&SendGroup>) -> Result<(), ClosedStream> { - let group = group.map(|group| group.id_on(&self.conn.0)); - let mut conn = self.conn.state.lock(); - conn.inner.send_stream(self.stream).set_group(group) - } - - /// Completes when the peer stops the stream or reads the stream to completion - /// - /// Yields `Some` with the stop error code if the peer stops the stream. Yields `None` if the - /// local side [`finish()`](Self::finish)es the stream and then the peer acknowledges receipt - /// of all stream data (although not necessarily the processing of it), or once the local side - /// resets it with [`reset()`](Self::reset) or [`reset_at()`](Self::reset_at), or by dropping - /// it unfinished, after which the peer closing the stream is no longer meaningful. A peer's - /// stop code still wins over a local reset if it arrived first, but not if it arrives after. - /// - /// For a variety of reasons, the peer may not send acknowledgements immediately upon receiving - /// data. As such, relying on `stopped` to know when the peer has read a stream to completion - /// may introduce more latency than using an application-level response of some sort. - /// - pub fn stopped(&self) -> impl Future, StoppedError>> + Send + Sync + 'static { - let conn = self.conn.clone(); - let stream = self.stream; - // Join the waiters now rather than on the first poll, so a reset in between, which no - // stream event reports, still reaches this future. - let waiters = { - let mut conn = conn.state.lock(); - // A reset settles it: a STOP_SENDING after it is recorded but no longer counts. - if let Some(stopped) = self.reset { - Err(Ok(stopped)) - } else if let Some(output) = send_stream_stopped(&mut conn, stream) { - Err(output) - } else { - Ok(conn.stopped.entry(stream).or_default().clone()) - } - }; - async move { - let mut waiters = match waiters { - Ok(waiters) => waiters, - Err(output) => return output, - }; - loop { - // The `Notify::notified` future needs to be created while the lock is being held, - // otherwise a wakeup could be missed if triggered inbetween releasing the lock - // and creating the future. - // The lock may only be held in a block without `await`s, otherwise the future - // becomes `!Send`. `Notify::notified` is lifetime-bound to `Notify`, therefore - // we need to declare `waiters` outside of the block, and update it inside. - { - let mut conn = conn.state.lock(); - if let Some(&stopped) = waiters.outcome.get() { - return Ok(stopped); - } - if let Some(output) = send_stream_stopped(&mut conn, stream) { - return output; - } - - // A wakeup that resolved nothing removed the waiters; join the current ones - waiters = conn.stopped.entry(stream).or_default().clone(); - waiters.notify.notified() - } - .await - } - } - } - - /// Get the identity of this stream - pub fn id(&self) -> StreamId { - self.stream - } - - /// Reset at `offset` with `code`, instead of at 0 with code 0, if dropped before `finish` - #[cfg(feature = "quic")] - pub(crate) fn set_drop_reset(&mut self, offset: u64, code: VarInt) { - self.drop_reset = (offset, code); - } - - /// Attempt to write bytes from buf into the stream. - /// - /// On success, returns Poll::Ready(Ok(num_bytes_written)). - /// - /// If the stream is not ready for writing, the method returns Poll::Pending and arranges - /// for the current task (via cx.waker().wake_by_ref()) to receive a notification when the - /// stream becomes writable or is closed. - pub fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - pin!(self.get_mut().write(buf)).as_mut().poll(cx) - } -} - -/// Check if a send stream is stopped. -/// -/// Returns `Some` if the stream is stopped or the connection is closed. -/// Returns `None` if the stream is not stopped. -fn send_stream_stopped(conn: &mut State, stream: StreamId) -> Option, StoppedError>> { - match conn.inner.send_stream(stream).stopped() { - Err(ClosedStream { .. }) => Some(Ok(None)), - Ok(Some(error_code)) => Some(Ok(Some(error_code))), - Ok(None) => conn.error.clone().map(|error| Err(error.into())), - } -} - -impl tokio::io::AsyncWrite for SendStream { - fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - self.poll_write(cx, buf).map_err(Into::into) - } - - fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(self.get_mut().finish().map_err(Into::into)) - } -} - -impl Drop for SendStream { - fn drop(&mut self) { - let mut conn = self.conn.state.lock(); - - // clean up any previously registered wakers - conn.blocked_writers.remove(&self.stream); - - if conn.error.is_some() { - return; - } - // Fails harmlessly when already reset. - let mut stream = conn.inner.send_stream(self.stream); - let stopped = stream.stopped().ok().flatten(); - let reset = match stopped { - // Echo the code, as RFC 9000 section 3.5 asks, even after `finish`: the peer reads - // nothing more, so retransmitting the rest is wasted. - Some(code) => stream.reset(code), - None if self.finished => return, - None => { - let (offset, code) = self.drop_reset; - let offset = offset.min(stream.offset()); - stream.reset_at(offset, code) - } - }; - if reset.is_ok() { - conn.resolve_stopped(self.stream, stopped); - conn.wake(); - } - } -} - -/// Errors that arise from writing to a stream -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum WriteError { - /// The peer is no longer accepting data on this stream - /// - /// Carries an application-defined error code. - #[error("sending stopped by peer: error {0}")] - Stopped(VarInt), - /// The connection was lost - #[error("connection lost")] - ConnectionLost(#[from] ConnectionError), - /// The stream has already been finished or reset - #[error("closed stream")] - ClosedStream, -} - -impl From for WriteError { - #[inline] - fn from(_: ClosedStream) -> Self { - Self::ClosedStream - } -} - -impl From for WriteError { - fn from(x: StoppedError) -> Self { - match x { - StoppedError::ConnectionLost(e) => Self::ConnectionLost(e), - } - } -} - -impl From for io::Error { - fn from(x: WriteError) -> Self { - use WriteError::*; - let kind = match x { - Stopped(_) => io::ErrorKind::ConnectionReset, - ConnectionLost(_) | ClosedStream => io::ErrorKind::NotConnected, - }; - Self::new(kind, x) - } -} - -/// Errors that arise while monitoring for a send stream stop from the peer -#[derive(Debug, Error, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub enum StoppedError { - /// The connection was lost - #[error("connection lost")] - ConnectionLost(#[from] ConnectionError), -} - -impl From for io::Error { - fn from(x: StoppedError) -> Self { - use StoppedError::*; - let kind = match x { - ConnectionLost(_) => io::ErrorKind::NotConnected, - }; - Self::new(kind, x) - } -} diff --git a/rs/moq-tokio/src/quic/driver/tests.rs b/rs/moq-tokio/src/quic/driver/tests.rs index a801caac8bc..e18eda54d6f 100755 --- a/rs/moq-tokio/src/quic/driver/tests.rs +++ b/rs/moq-tokio/src/quic/driver/tests.rs @@ -44,10 +44,10 @@ use tracing::{error_span, info}; use tracing_subscriber::EnvFilter; use super::{ - ReadError, ReadToEndError, RecvStream, SendStream, VarInt, client, + VarInt, client, connection::Connection, endpoint::{self, Endpoint}, - server, transport, + recv, send, server, transport, }; #[test] @@ -159,7 +159,7 @@ fn read_after_close() { .expect("connection"); let mut s = new_conn.open_uni(Default::default()).await.unwrap(); s.write_all(MSG).await.unwrap(); - s.finish().unwrap(); + let s = s.finish(); // Wait for the stream to be closed, one way or another. _ = s.stopped().await; }); @@ -851,8 +851,8 @@ fn run_echo(args: EchoArgs) { let msg = gen_data(args.stream_size, SEED); let send_task = async { - send.write_all(&msg).await.expect("write"); - send.finish().unwrap(); + send.write_all(&msg[..]).await.expect("write"); + send.finish(); }; let recv_task = async { recv.read_to_end(usize::MAX).await.expect("read") }; @@ -879,9 +879,9 @@ struct EchoArgs { stream_receive_window: Option, } -async fn echo((mut send, mut recv): (SendStream, RecvStream)) { +async fn echo((mut send, mut recv): (send::Stream, recv::Stream)) { while let Some(chunk) = recv.read_chunk(32 << 10).await.expect("read chunk") { - send.write_chunk(chunk.bytes).await.expect("write chunk"); + send.write_all(chunk).await.expect("write chunk"); } let _ = send.finish(); @@ -985,8 +985,8 @@ async fn rebind_recv() { connected_send.notify_one(); write_recv.notified().await; let mut stream = connection.open_uni(Default::default()).await.unwrap(); - stream.write_all(MSG).await.unwrap(); - stream.finish().unwrap(); + stream.write_all(MSG.as_slice()).await.unwrap(); + let stream = stream.finish(); // Wait for the stream to be closed, one way or another. _ = stream.stopped().await; }); @@ -1309,8 +1309,8 @@ async fn remote_address_follows_migration() { // send data from the new socket to trigger migration on the server let mut stream = client_conn.open_uni(Default::default()).await.unwrap(); - stream.write_all(b"hello").await.unwrap(); - stream.finish().unwrap(); + stream.write_all(&b"hello"[..]).await.unwrap(); + stream.finish(); }; trigger_migration.notified().await; @@ -1341,7 +1341,7 @@ async fn foreign_send_group_panics_cleanly() { let (conn, server_conn) = connect().await; let (other, _other_server) = connect().await; let group = other.send_group(0); - let options = super::SendOptions { + let options = super::send::Options { group: Some(&group), priority: 0, }; @@ -1359,8 +1359,8 @@ async fn foreign_send_group_panics_cleanly() { assert!(set.is_err(), "set_group accepted a foreign group"); // The connection still works, and its handles drop without panicking. - send.write_all(b"after").await.unwrap(); - send.finish().unwrap(); + send.write_all(&b"after"[..]).await.unwrap(); + send.finish(); let mut recv = server_conn.accept_uni().await.unwrap(); assert_eq!(recv.read_to_end(usize::MAX).await.unwrap(), b"after"); } @@ -1420,9 +1420,9 @@ async fn two_datagram_readers() { x }, async { - server.send_datagram(b"one"[..].into(), Default::default()).unwrap(); + server.send_datagram(&b"one"[..], Default::default()).unwrap(); done.notified().await; - server.send_datagram(b"two"[..].into(), Default::default()).unwrap(); + server.send_datagram(&b"two"[..], Default::default()).unwrap(); } ); assert!(*a == *b"one" || *b == *b"one"); @@ -1492,7 +1492,7 @@ async fn stream_stopped() { let stopped2 = stream.stopped(); let stopped3 = stream.stopped(); - stream.write_all(b"hi").await.unwrap(); + stream.write_all(&b"hi"[..]).await.unwrap(); // spawn one of the futures into a task let stopped1 = tokio::task::spawn(stopped1); // verify that both futures resolved @@ -1511,7 +1511,7 @@ async fn stream_stopped() { let mut stream = conn.accept_uni().await.unwrap(); let mut buf = [0u8; 2]; stream.read_exact(&mut buf).await.unwrap(); - stream.stop(42u32.into()).unwrap(); + stream.stop(42u32.into()); conn } .instrument(error_span!("server")); @@ -1529,7 +1529,7 @@ async fn stream_stopped_2() { endpoint.accept().await.unwrap().accept().unwrap().await }) .unwrap(); - let mut send_stream = conn.open_uni(Default::default()).await.unwrap(); + let send_stream = conn.open_uni(Default::default()).await.unwrap(); let stopped = timeout(Duration::from_millis(100), send_stream.stopped()).instrument(error_span!("stopped")); tokio::pin!(stopped); // poll the future once so that the waker is registered. @@ -1539,7 +1539,7 @@ async fn stream_stopped_2() { _x = std::future::ready(()) => {} } // finish and drop the send stream, since dropping it unfinished would reset it - send_stream.finish().unwrap(); + let send_stream = send_stream.finish(); drop(send_stream); // make sure the stopped future still resolves let res = stopped.await; @@ -1558,13 +1558,13 @@ async fn dropped_send_stream_resets() { }) .unwrap(); let mut send_stream = conn.open_uni(Default::default()).await.unwrap(); - send_stream.write_all(b"truncated").await.unwrap(); + send_stream.write_all(&b"truncated"[..]).await.unwrap(); drop(send_stream); let mut recv_stream = server_conn.accept_uni().await.unwrap(); let res = recv_stream.read_to_end(usize::MAX).await; assert!( - matches!(res, Err(ReadToEndError::Read(ReadError::Reset(code))) if code == VarInt::from_u32(0)), + matches!(res, Err(recv::Error::Reset(code)) if code == VarInt::from_u32(0)), "{res:?}" ); } @@ -1580,8 +1580,8 @@ async fn finished_send_stream_survives_drop() { }) .unwrap(); let mut send_stream = conn.open_uni(Default::default()).await.unwrap(); - send_stream.write_all(b"complete").await.unwrap(); - send_stream.finish().unwrap(); + send_stream.write_all(&b"complete"[..]).await.unwrap(); + let send_stream = send_stream.finish(); drop(send_stream); let mut recv_stream = server_conn.accept_uni().await.unwrap(); @@ -1615,13 +1615,14 @@ async fn stream_stop_before_reset_wins() { let endpoint = endpoint(); let (conn, server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); - let mut recv = server.accept_uni().await.unwrap(); - recv.stop(5u32.into()).unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); + let recv = server.accept_uni().await.unwrap(); + recv.stop(5u32.into()); stop_sending_received(&conn).await; - send.reset(7u32.into()).unwrap(); - let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + let stopped = send.stopped(); + send.reset(7u32.into()); + let stopped = timeout(Duration::from_secs(10), stopped).await; assert_eq!(stopped, Ok(Ok(Some(5u32.into())))); } @@ -1632,14 +1633,15 @@ async fn stream_stop_after_reset_is_ignored() { let endpoint = endpoint(); let (conn, server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); - let mut recv = server.accept_uni().await.unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); + let recv = server.accept_uni().await.unwrap(); // Neither side runs in between, so the peer stops a stream it hasn't seen reset yet. - send.reset(7u32.into()).unwrap(); - recv.stop(5u32.into()).unwrap(); + let stopped = send.stopped(); + send.reset(7u32.into()); + recv.stop(5u32.into()); stop_sending_received(&conn).await; - let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + let stopped = timeout(Duration::from_secs(10), stopped).await; assert_eq!(stopped, Ok(Ok(None))); } @@ -1650,16 +1652,15 @@ async fn stream_stop_survives_freed_reset() { let endpoint = endpoint(); let (conn, server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); let stopped = send.stopped(); - let mut recv = server.accept_uni().await.unwrap(); - recv.stop(5u32.into()).unwrap(); + let recv = server.accept_uni().await.unwrap(); + recv.stop(5u32.into()); stop_sending_received(&conn).await; - send.reset(7u32.into()).unwrap(); - let id = send.id(); + send.reset(7u32.into()); timeout(Duration::from_secs(10), async { - while !conn.send_stream_freed(id) { + while conn.send_streams() > 0 { tokio::task::yield_now().await; } }) @@ -1676,9 +1677,10 @@ async fn stream_reset_resolves_stopped() { let endpoint = endpoint(); let (conn, _server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); - send.reset(7u32.into()).unwrap(); - let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + send.write_all(&b"hello"[..]).await.unwrap(); + let stopped = send.stopped(); + send.reset(7u32.into()); + let stopped = timeout(Duration::from_secs(10), stopped).await; assert_eq!(stopped, Ok(Ok(None))); } @@ -1688,9 +1690,12 @@ async fn stream_reset_at_resolves_stopped() { let endpoint = endpoint(); let (conn, _server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); - send.reset_at(2, 7u32.into()).unwrap(); - let stopped = timeout(Duration::from_secs(10), send.stopped()).await; + send.write_all(&b"he"[..]).await.unwrap(); + let offset = send.offset(); + send.write_all(&b"llo"[..]).await.unwrap(); + let stopped = send.stopped(); + send.reset_at(offset, 7u32.into()); + let stopped = timeout(Duration::from_secs(10), stopped).await; assert_eq!(stopped, Ok(Ok(None))); } @@ -1701,12 +1706,12 @@ async fn stream_stopped_pending_across_reset() { let endpoint = endpoint(); let (conn, _server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); let unpolled = send.stopped(); let polled = tokio::spawn(send.stopped()); tokio::task::yield_now().await; assert!(!polled.is_finished()); - send.reset(7u32.into()).unwrap(); + send.reset(7u32.into()); let polled = timeout(Duration::from_secs(10), polled).await; assert_eq!(polled.expect("polled never resolved").unwrap(), Ok(None)); let unpolled = timeout(Duration::from_secs(10), unpolled).await; @@ -1720,7 +1725,7 @@ async fn dropped_stream_resolves_stopped() { let endpoint = endpoint(); let (conn, _server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"truncated").await.unwrap(); + send.write_all(&b"truncated"[..]).await.unwrap(); let stopped = send.stopped(); drop(send); let stopped = timeout(Duration::from_secs(10), stopped).await; @@ -1735,17 +1740,14 @@ async fn stream_reset_releases_stopped() { let endpoint = endpoint(); let (conn, _server) = connect_self(&endpoint).await; let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); { let mut stopped = pin!(send.stopped()); let poll = std::future::poll_fn(|cx| Poll::Ready(stopped.as_mut().poll(cx))).await; assert!(poll.is_pending()); } assert_eq!(conn.stopped_waiters(), 1); - send.reset(7u32.into()).unwrap(); - assert_eq!(conn.stopped_waiters(), 0); - // Answered at once, without registering a waiter for a stream that will never report - drop(send.stopped()); + send.reset(7u32.into()); assert_eq!(conn.stopped_waiters(), 0); } @@ -1772,6 +1774,7 @@ async fn stream_drop_removes_blocked_reader() { // do a blocking read which will add the stream in conn.blocked_readers { let mut buf = [0u8; 64]; + let mut buf = &mut buf[..]; let read_fut = stream.read(&mut buf); tokio::pin!(read_fut); assert!(matches!(read_fut.as_mut().poll(&mut cx), Poll::Pending)); @@ -1796,7 +1799,7 @@ async fn stream_drop_removes_blocked_reader() { let conn = client.dial(server_address, "localhost").unwrap().await.unwrap(); let mut stream = conn.open_uni(Default::default()).await.unwrap(); // need to send some data to actually start the stream - stream.write_all(b"hello").await.unwrap(); + stream.write_all(&b"hello"[..]).await.unwrap(); server_task.await.unwrap(); } @@ -1830,12 +1833,12 @@ async fn recv_stream_cancel_stop_drop() { assert!(fut.poll(&mut cx).is_pending()); } recv_dropped.set(()).unwrap(); - recv.stop(0u32.into()).unwrap(); + recv.stop(0u32.into()); }, async { let conn = client.dial(server_addr, "localhost").unwrap().await.unwrap(); let mut send = conn.open_uni(Default::default()).await.unwrap(); - _ = send.write_all(b"hello").await; + _ = send.write_all(&b"hello"[..]).await; // Don't drop (finish) the send stream until the read has been // cancelled by the server, ensuring that read_to_end can't complete // immediately. @@ -1919,8 +1922,8 @@ async fn leases_spend_the_connection_window() { .unwrap(); let mut send = client.open_uni(Default::default()).await.unwrap(); let writer = tokio::spawn(async move { - send.write_all(&[7; WINDOW + 500]).await.unwrap(); - send.finish().unwrap(); + send.write_all(&[7; WINDOW + 500][..]).await.unwrap(); + send.finish(); }); let mut recv = server.accept_uni().await.unwrap(); @@ -1968,10 +1971,11 @@ async fn stuck_window_evicts_the_lower_priority() { let id = send.id(); let writer = tokio::spawn(async move { // Keep the low stream open: an acknowledged FIN wins over a later STOP_SENDING. - if send.write_all(&[7; WINDOW]).await.is_ok() && id != low_id { - send.finish().unwrap(); + if send.write_all(&[7; WINDOW][..]).await.is_ok() && id != low_id { + send.finish().stopped().await.unwrap() + } else { + send.stopped().await.unwrap() } - send.stopped().await.unwrap() }); (id, writer) }); @@ -1992,7 +1996,7 @@ async fn stuck_window_evicts_the_lower_priority() { len })), ); - assert_eq!(leased, (Err(ReadError::Evicted), Ok(WINDOW))); + assert_eq!(leased, (Err(recv::Error::Evicted), Ok(WINDOW))); // The peer hears of the eviction once the stream is stopped. Drain the survivor to FIN // instead of dropping unread data, which would stop that stream too. @@ -2008,14 +2012,14 @@ async fn stuck_window_evicts_the_lower_priority() { } /// Wait until `recv` leases at least `want` bytes, returning how many it leased. -async fn leased(recv: &mut RecvStream, want: usize) -> usize { +async fn leased(recv: &mut recv::Stream, want: usize) -> usize { poll_fn(|cx| recv.poll_lease(cx, want).map_ok(|lease| lease.remaining())) .await .unwrap() } /// Consume the first `n` leased bytes of `recv`. -async fn consume(recv: &mut RecvStream, n: usize) { +async fn consume(recv: &mut recv::Stream, n: usize) { poll_fn(|cx| { recv.poll_lease(cx, n).map_ok(|mut lease| { lease.advance(n); @@ -2026,6 +2030,55 @@ async fn consume(recv: &mut RecvStream, n: usize) { .unwrap() } +/// A lease owns its bytes and holds no lock, so the connection keeps moving its other streams +/// while one is held, and the lease may cross an await in a spawned task. +#[tokio::test] +async fn held_lease_does_not_stall_the_connection() { + fn assert_send() {} + assert_send::>(); + + let _guard = subscribe(); + let endpoint = endpoint(); + let (client, server) = connect_self(&endpoint).await; + let mut held = client.open_uni(Default::default()).await.unwrap(); + held.write_all(&b"held"[..]).await.unwrap(); + let mut recv = server.accept_uni().await.unwrap(); + let lease = recv.lease(4).await.unwrap(); + + let mut other = client.open_uni(Default::default()).await.unwrap(); + other.write_all(&b"other"[..]).await.unwrap(); + other.finish(); + let mut other = server.accept_uni().await.unwrap(); + let read = timeout(Duration::from_secs(10), other.read_to_end(64)).await; + assert_eq!(read.unwrap().unwrap(), b"other"); + + assert_eq!(lease.chunk(), b"held"); + lease.release(); +} + +/// Once `received_reset` reports a reset, moq-quic has freed the stream, so later reads and +/// leases fail with the same reset instead of finding no stream. +#[tokio::test] +async fn reads_after_a_received_reset_fail_with_it() { + let _guard = subscribe(); + let endpoint = endpoint(); + let (client, server) = connect_self(&endpoint).await; + let mut send = client.open_uni(Default::default()).await.unwrap(); + send.write_all(&b"x"[..]).await.unwrap(); + let mut recv = server.accept_uni().await.unwrap(); + send.reset(VarInt::from_u32(7)); + + let reset = timeout(Duration::from_secs(10), recv.received_reset()).await.unwrap(); + assert_eq!(reset.unwrap(), Some(VarInt::from_u32(7))); + let lease = recv.lease(1).await.err(); + assert!(matches!(lease, Some(recv::Error::Reset(code)) if code == VarInt::from_u32(7))); + let read = recv.read(&mut Vec::new()).await; + assert!( + matches!(read, Err(recv::Error::Reset(code)) if code == VarInt::from_u32(7)), + "{read:?}" + ); +} + /// A tiny datagram, and a tiny stream frame taken from a lease, each reach the application in /// an allocation of their own rather than as a slice of the receive batch they arrived in, which /// their credit does not pay for. @@ -2041,7 +2094,7 @@ async fn tiny_frames_do_not_pin_receive_batches() { assert!(owns_exactly(server.read_datagram().await.unwrap())); let mut send = client.open_uni(Default::default()).await.unwrap(); - send.write_all(b"y").await.unwrap(); + send.write_all(&b"y"[..]).await.unwrap(); let mut recv = server.accept_uni().await.unwrap(); let taken = poll_fn(|cx| { recv.poll_lease(cx, 1).map_ok(|mut lease| { @@ -2187,8 +2240,8 @@ async fn unreachable_sends_are_loss_once_established() { outage.active.store(true, Ordering::SeqCst); let mut send = client_conn.open_uni(Default::default()).await.unwrap(); - send.write_all(b"hello").await.unwrap(); - send.finish().unwrap(); + send.write_all(&b"hello"[..]).await.unwrap(); + send.finish(); outage.dropped.notified().await; outage.active.store(false, Ordering::SeqCst); diff --git a/rs/moq-tokio/src/quic/web/connect.rs b/rs/moq-tokio/src/quic/web/connect.rs index 4c95b820ce0..a73f696d472 100644 --- a/rs/moq-tokio/src/quic/web/connect.rs +++ b/rs/moq-tokio/src/quic/web/connect.rs @@ -17,10 +17,10 @@ pub enum ConnectError { ConnectionError(#[from] crate::quic::driver::connection::Error), #[error("read error")] - ReadError(#[from] crate::quic::driver::ReadError), + ReadError(#[from] crate::quic::driver::recv::Error), #[error("write error")] - WriteError(#[from] crate::quic::driver::WriteError), + WriteError(#[from] crate::quic::driver::send::Error), #[error("http error status: {0}")] ErrorStatus(http::StatusCode), @@ -35,8 +35,8 @@ pub struct Connecting { pub request: ConnectRequest, // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: crate::quic::driver::SendStream, - pub(crate) recv: crate::quic::driver::RecvStream, + pub(crate) send: crate::quic::driver::send::Stream, + pub(crate) recv: crate::quic::driver::recv::Stream, } impl Connecting { @@ -75,8 +75,8 @@ impl Connecting { } pub async fn reject(self, status: http::StatusCode) -> Result<(), ConnectError> { - let mut connect = self.respond(status).await?; - connect.send.finish().ok(); + let connect = self.respond(status).await?; + connect.send.finish(); Ok(()) } } @@ -97,8 +97,8 @@ pub struct Connected { pub response: ConnectResponse, // A reference to the send/recv stream, so we don't close it until dropped. - pub(crate) send: crate::quic::driver::SendStream, - pub(crate) recv: crate::quic::driver::RecvStream, + pub(crate) send: crate::quic::driver::send::Stream, + pub(crate) recv: crate::quic::driver::recv::Stream, } impl Connected { diff --git a/rs/moq-tokio/src/quic/web/error.rs b/rs/moq-tokio/src/quic/web/error.rs index 1b40075be99..a23b5b505cb 100644 --- a/rs/moq-tokio/src/quic/web/error.rs +++ b/rs/moq-tokio/src/quic/web/error.rs @@ -15,10 +15,10 @@ pub enum ClientError { Connection(#[from] crate::quic::driver::connection::Error), #[error("failed to write: {0}")] - WriteError(#[from] crate::quic::driver::WriteError), + WriteError(#[from] crate::quic::driver::send::Error), #[error("failed to read: {0}")] - ReadError(#[from] crate::quic::driver::ReadError), + ReadError(#[from] crate::quic::driver::recv::Error), #[error("failed to exchange h3 settings: {0}")] SettingsError(#[from] SettingsError), @@ -42,7 +42,7 @@ pub enum SessionError { WebTransportError(#[from] WebTransportError), #[error("send datagram error: {0}")] - SendDatagramError(#[from] crate::quic::driver::connection::SendDatagramError), + SendDatagramError(#[from] crate::quic::driver::send::Error), } impl From for SessionError { @@ -129,9 +129,7 @@ impl CloseReason { pub(crate) fn map(&self, err: SessionError) -> SessionError { let conn = match &err { SessionError::ConnectionError(conn) - | SessionError::SendDatagramError(crate::quic::driver::connection::SendDatagramError::ConnectionLost( - conn, - )) => Some(conn), + | SessionError::SendDatagramError(crate::quic::driver::send::Error::Connection(conn)) => Some(conn), SessionError::WebTransportError(WebTransportError::Closed(..) | WebTransportError::SessionGone) => None, _ => return err, }; @@ -167,13 +165,13 @@ pub enum WebTransportError { SessionGone, #[error("read error: {0}")] - ReadError(#[from] crate::quic::driver::ReadExactError), + ReadError(#[from] crate::quic::driver::recv::Error), #[error("write error: {0}")] - WriteError(#[from] crate::quic::driver::WriteError), + WriteError(#[from] crate::quic::driver::send::Error), } -/// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::WriteError`]. +/// An error when writing to [`crate::quic::web::SendStream`]. Similar to [`crate::quic::driver::send::Error`]. #[derive(Clone, Error, Debug)] #[non_exhaustive] pub enum WriteError { @@ -186,26 +184,28 @@ pub enum WriteError { #[error("session error: {0}")] SessionError(#[from] SessionError), + /// The stream was already finished or reset. #[error("stream closed")] ClosedStream, } -impl From for WriteError { - fn from(e: crate::quic::driver::WriteError) -> Self { +impl From for WriteError { + fn from(e: crate::quic::driver::send::Error) -> Self { match e { - crate::quic::driver::WriteError::Stopped(code) => { + crate::quic::driver::send::Error::Stopped(code) => { match web_transport_proto::error_from_http3(code.into_inner()) { Some(code) => WriteError::Stopped(code), None => WriteError::InvalidStopped(code), } } - crate::quic::driver::WriteError::ClosedStream => WriteError::ClosedStream, - crate::quic::driver::WriteError::ConnectionLost(e) => WriteError::SessionError(e.into()), + crate::quic::driver::send::Error::Connection(e) => WriteError::SessionError(e.into()), + // Only datagrams fail otherwise, and they report a session error. + e => WriteError::SessionError(e.into()), } } } -/// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::ReadError`]. +/// An error when reading from [`crate::quic::web::RecvStream`]. Similar to [`crate::quic::driver::recv::Error`]. #[derive(Clone, Error, Debug)] #[non_exhaustive] pub enum ReadError { @@ -218,6 +218,7 @@ pub enum ReadError { #[error("invalid RESET_STREAM: {0}")] InvalidReset(crate::quic::driver::VarInt), + /// The stream was already stopped. #[error("stream already closed")] ClosedStream, @@ -225,79 +226,37 @@ pub enum ReadError { #[error("evicted")] Evicted, - /// A lease wanted more than the connection receive window can hold. - #[error("want exceeds the receive window")] - TooLarge, -} - -impl From for ReadError { - fn from(value: crate::quic::driver::ReadError) -> Self { - match value { - crate::quic::driver::ReadError::Reset(code) => { - match web_transport_proto::error_from_http3(code.into_inner()) { - Some(code) => ReadError::Reset(code), - None => ReadError::InvalidReset(code), - } - } - crate::quic::driver::ReadError::ConnectionLost(e) => ReadError::SessionError(e.into()), - crate::quic::driver::ReadError::ClosedStream => ReadError::ClosedStream, - crate::quic::driver::ReadError::Evicted => ReadError::Evicted, - crate::quic::driver::ReadError::TooLarge => ReadError::TooLarge, - } - } -} - -/// An error returned by [`crate::quic::web::RecvStream::read_exact`]. Similar to [`crate::quic::driver::ReadExactError`]. -#[derive(Clone, Error, Debug)] -#[non_exhaustive] -pub enum ReadExactError { + /// The stream finished before [`read_exact`](crate::quic::web::RecvStream::read_exact) + /// filled its buffer, after this many bytes. #[error("finished early")] FinishedEarly(usize), - #[error("read error: {0}")] - ReadError(#[from] ReadError), -} - -impl From for ReadExactError { - fn from(e: crate::quic::driver::ReadExactError) -> Self { - match e { - crate::quic::driver::ReadExactError::FinishedEarly(size) => ReadExactError::FinishedEarly(size), - crate::quic::driver::ReadExactError::ReadError(e) => ReadExactError::ReadError(e.into()), - } - } -} - -/// An error returned by [`crate::quic::web::RecvStream::read_to_end`]. Similar to [`crate::quic::driver::ReadToEndError`]. -#[derive(Clone, Error, Debug)] -#[non_exhaustive] -pub enum ReadToEndError { + /// The stream is longer than [`read_to_end`](crate::quic::web::RecvStream::read_to_end)'s limit. #[error("too long")] TooLong, - #[error("read error: {0}")] - ReadError(#[from] ReadError), + /// A lease wanted more than the connection receive window can hold. + #[error("want exceeds the receive window")] + TooLarge, } -impl From for ReadToEndError { - fn from(e: crate::quic::driver::ReadToEndError) -> Self { - match e { - crate::quic::driver::ReadToEndError::TooLong => ReadToEndError::TooLong, - crate::quic::driver::ReadToEndError::Read(e) => ReadToEndError::ReadError(e.into()), +impl From for ReadError { + fn from(value: crate::quic::driver::recv::Error) -> Self { + use crate::quic::driver::recv::Error; + match value { + Error::Reset(code) => match web_transport_proto::error_from_http3(code.into_inner()) { + Some(code) => ReadError::Reset(code), + None => ReadError::InvalidReset(code), + }, + Error::Connection(e) => ReadError::SessionError(e.into()), + Error::Evicted => ReadError::Evicted, + Error::FinishedEarly(n) => ReadError::FinishedEarly(n), + Error::TooLong => ReadError::TooLong, + Error::TooLarge => ReadError::TooLarge, } } } -/// An error indicating the stream was already closed. -#[derive(Clone, Error, Debug)] -#[error("stream closed")] -pub struct ClosedStream; - -impl From for ClosedStream { - fn from(_: crate::quic::driver::ClosedStream) -> Self { - ClosedStream - } -} - /// An error returned when receiving a new WebTransport session. #[derive(Error, Debug, Clone)] #[non_exhaustive] @@ -309,10 +268,10 @@ pub enum ServerError { Connection(#[from] crate::quic::driver::connection::Error), #[error("failed to write")] - WriteError(#[from] crate::quic::driver::WriteError), + WriteError(#[from] crate::quic::driver::send::Error), #[error("failed to read")] - ReadError(#[from] crate::quic::driver::ReadError), + ReadError(#[from] crate::quic::driver::recv::Error), #[error("failed to exchange h3 settings")] SettingsError(#[from] SettingsError), diff --git a/rs/moq-tokio/src/quic/web/recv.rs b/rs/moq-tokio/src/quic/web/recv.rs index fd7c4c144dd..541774131fb 100644 --- a/rs/moq-tokio/src/quic/web/recv.rs +++ b/rs/moq-tokio/src/quic/web/recv.rs @@ -5,97 +5,104 @@ use std::{ task::{Context, Poll}, }; -use bytes::Bytes; +use bytes::{BufMut, Bytes}; -use crate::quic::web::{CloseReason, ReadError, ReadExactError, ReadToEndError, SessionError}; +use crate::quic::driver::recv; +use crate::quic::web::{CloseReason, ReadError, SessionError}; -/// A stream that can be used to receive bytes. See [`crate::quic::driver::RecvStream`]. +/// A stream that can be used to receive bytes. See [`crate::quic::driver::recv::Stream`]. /// /// Dropping it before the end stops it with code 0. #[derive(Debug)] pub struct RecvStream { - inner: crate::quic::driver::RecvStream, + /// Taken by [`Self::stop`], since `web_transport_trait` stops through `&mut self`. + inner: Option, close: Arc, + id: crate::quic::driver::stream::Id, } impl RecvStream { - pub(crate) fn new(mut stream: crate::quic::driver::RecvStream, close: Arc) -> Self { + pub(crate) fn new(mut stream: recv::Stream, close: Arc) -> Self { // The driver's raw 0 is outside the HTTP/3 range, so a WebTransport peer can't decode it. stream.set_drop_stop(close.encode_stream_code(0)); - Self { inner: stream, close } + Self { + id: stream.id(), + inner: Some(stream), + close, + } } - fn map_error(&self, e: crate::quic::driver::ReadError) -> ReadError { - map_error(&self.close, e) + fn inner(&mut self) -> Result<&mut recv::Stream, ReadError> { + self.inner.as_mut().ok_or(ReadError::ClosedStream) } - /// Lease the stream's unreleased bytes. See [`crate::quic::driver::RecvStream::poll_lease`]. - pub fn poll_lease( - &mut self, - cx: &mut Context<'_>, - want: usize, - ) -> Poll, ReadError>> { + /// Lease the stream's unreleased bytes. See [`recv::Stream::poll_lease`]. + pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, ReadError>> { let close = &self.close; - self.inner.poll_lease(cx, want).map_err(|e| map_error(close, e)) + match &mut self.inner { + Some(inner) => inner.poll_lease(cx, want).map_err(|e| map_error(close, e)), + None => Poll::Ready(Err(ReadError::ClosedStream)), + } } - /// Rank this stream for eviction. See [`crate::quic::driver::RecvStream::set_priority`]. + /// Rank this stream for eviction. See [`recv::Stream::set_priority`]. pub fn set_priority(&mut self, priority: i64) { - self.inner.set_priority(priority); + if let Some(inner) = &mut self.inner { + inner.set_priority(priority); + } } /// Tell the other end to stop sending data with the given error code. See - /// [`crate::quic::driver::RecvStream::stop`]. This is a u32 with WebTransport since it shares the error - /// space with HTTP/3. A raw QUIC session sends the code as is. - pub fn stop(&mut self, code: u32) -> Result<(), crate::quic::driver::ClosedStream> { - let code = self.close.encode_stream_code(code); - self.inner.stop(code) + /// [`recv::Stream::stop`]. This is a u32 with WebTransport since it shares the error + /// space with HTTP/3. A raw QUIC session sends the code as is. Does nothing once stopped. + pub fn stop(&mut self, code: u32) { + if let Some(inner) = self.inner.take() { + inner.stop(self.close.encode_stream_code(code)); + } } - // Unfortunately, we have to wrap ReadError for a bunch of functions. - - /// Read some data into the buffer and return the amount read. See [`crate::quic::driver::RecvStream::read`]. - pub async fn read(&mut self, buf: &mut [u8]) -> Result, ReadError> { - self.inner.read(buf).await.map_err(|e| self.map_error(e)) + /// Read some data into `buf` and return the amount read. See [`recv::Stream::read`]. + pub async fn read(&mut self, buf: &mut impl BufMut) -> Result, ReadError> { + let result = self.inner()?.read(buf).await; + result.map_err(|e| map_error(&self.close, e)) } - /// Fill the entire buffer with data. See [`crate::quic::driver::RecvStream::read_exact`]. - pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadExactError> { - self.inner.read_exact(buf).await.map_err(|e| match e { - crate::quic::driver::ReadExactError::ReadError(e) => self.map_error(e).into(), - e => e.into(), - }) + /// Fill the entire buffer with data. See [`recv::Stream::read_exact`]. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), ReadError> { + let result = self.inner()?.read_exact(buf).await; + result.map_err(|e| map_error(&self.close, e)) } - /// Read a chunk of data from the stream. See [`crate::quic::driver::RecvStream::read_chunk`]. - pub async fn read_chunk(&mut self, max_length: usize) -> Result, ReadError> { - self.inner - .read_chunk(max_length) - .await - .map(|chunk| chunk.map(|chunk| chunk.bytes)) - .map_err(|e| self.map_error(e)) + /// Read a chunk of data from the stream. See [`recv::Stream::read_chunk`]. + pub async fn read_chunk(&mut self, max: usize) -> Result, ReadError> { + let result = self.inner()?.read_chunk(max).await; + result.map_err(|e| map_error(&self.close, e)) } - /// Read until the end of the stream or the limit is hit. See [`crate::quic::driver::RecvStream::read_to_end`]. - pub async fn read_to_end(&mut self, size_limit: usize) -> Result, ReadToEndError> { - self.inner.read_to_end(size_limit).await.map_err(|e| match e { - crate::quic::driver::ReadToEndError::Read(e) => self.map_error(e).into(), - e => e.into(), - }) + /// Read until the end of the stream or the limit is hit. See [`recv::Stream::read_to_end`]. + pub async fn read_to_end(&mut self, limit: usize) -> Result, ReadError> { + let result = self.inner()?.read_to_end(limit).await; + result.map_err(|e| map_error(&self.close, e)) } /// Block until the stream has been reset and return the error code. See - /// [`crate::quic::driver::RecvStream::received_reset`]. + /// [`recv::Stream::received_reset`]. /// /// Returns the stream's reset code or its session error. pub async fn received_reset(&mut self) -> Result, SessionError> { - match self.inner.received_reset().await { + let Some(inner) = &mut self.inner else { + return Ok(None); + }; + match inner.received_reset().await { Ok(None) => Ok(None), Ok(Some(code)) => match self.close.session_gone(code) { Some(err) => Err(err), None => Ok(self.close.decode_stream_code(code)), }, - Err(crate::quic::driver::ResetError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(err) => match map_error(&self.close, err) { + ReadError::SessionError(err) => Err(err), + err => unreachable!("only the connection fails a reset wait: {err}"), + }, } } @@ -105,10 +112,10 @@ impl RecvStream { /// > /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other /// > sessions. - /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > The [crate::quic::driver::stream::Id::index] might not increment by 1 like expected when using raw QUIC. /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> crate::quic::driver::StreamId { - self.inner.id() + pub fn quic_id(&self) -> crate::quic::driver::stream::Id { + self.id } // We purposely don't expose the 0RTT because it's not valid with WebTransport @@ -116,9 +123,9 @@ impl RecvStream { /// Decode the peer's reset code, and report a session error, or a reset for a session /// that is gone, as the session's close reason. -fn map_error(close: &CloseReason, e: crate::quic::driver::ReadError) -> ReadError { +fn map_error(close: &CloseReason, e: recv::Error) -> ReadError { match e { - crate::quic::driver::ReadError::Reset(code) => match close.decode_stream_code(code) { + recv::Error::Reset(code) => match close.decode_stream_code(code) { Some(code) => ReadError::Reset(code), None => match close.session_gone(code) { Some(err) => ReadError::SessionError(err), @@ -134,7 +141,10 @@ fn map_error(close: &CloseReason, e: crate::quic::driver::ReadError) -> ReadErro impl tokio::io::AsyncRead for RecvStream { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut tokio::io::ReadBuf) -> Poll> { - Pin::new(&mut self.inner).poll_read(cx, buf) + match &mut self.inner { + Some(inner) => Pin::new(inner).poll_read(cx, buf), + None => Poll::Ready(Err(io::ErrorKind::NotConnected.into())), + } } } @@ -142,11 +152,11 @@ impl web_transport_trait::RecvStream for RecvStream { type Error = ReadError; fn stop(&mut self, code: u32) { - Self::stop(self, code).ok(); + Self::stop(self, code); } - async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { - self.read(dst).await + async fn read(&mut self, mut dst: &mut [u8]) -> Result, Self::Error> { + self.read(&mut dst).await } async fn read_chunk(&mut self, max: usize) -> Result, Self::Error> { diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 29d7af960f1..c7eaaa7f80e 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -6,10 +6,12 @@ use std::{ }; use bytes::{Buf, Bytes}; +use futures::FutureExt; -use crate::quic::web::{CloseReason, ClosedStream, SessionError, WriteError}; +use crate::quic::driver::send; +use crate::quic::web::{CloseReason, SessionError, WriteError}; -/// A stream that can be used to send bytes. See [`crate::quic::driver::SendStream`]. +/// A stream that can be used to send bytes. See [`crate::quic::driver::send::Stream`]. /// /// Dropping it before [`finish`](Self::finish) resets it with code 0, still delivering the /// WebTransport header. @@ -18,27 +20,38 @@ use crate::quic::web::{CloseReason, ClosedStream, SessionError, WriteError}; /// WebTransport uses u32 error codes and they're mapped in a reserved HTTP/3 error space. #[derive(Debug)] pub struct SendStream { - stream: crate::quic::driver::SendStream, + state: State, close: Arc, /// The end of the WebTransport header this side wrote first, which a reset must still deliver. - header: Option, + header: Option, + id: crate::quic::driver::stream::Id, +} + +/// Where the stream is, since `web_transport_trait` finishes and resets through `&mut self`. +#[derive(Debug)] +enum State { + Open(send::Stream), + Finished(send::Finished), + /// Reset, with the peer's stop code if it arrived first. + Reset(Result, send::Error>), } impl SendStream { - pub(crate) fn new( - mut stream: crate::quic::driver::SendStream, - close: Arc, - header: Option, - ) -> Self { - stream.set_drop_reset(header.unwrap_or(0), close.encode_stream_code(0)); - Self { stream, close, header } + pub(crate) fn new(mut stream: send::Stream, close: Arc, header: Option) -> Self { + stream.set_drop_reset(header, close.encode_stream_code(0)); + Self { + id: stream.id(), + state: State::Open(stream), + close, + header, + } } /// Decode the peer's stop code, and report a session error, or a stop for a session /// that is gone, as the session's close reason. - fn map_error(&self, e: crate::quic::driver::WriteError) -> WriteError { + fn map_error(&self, e: send::Error) -> WriteError { match e { - crate::quic::driver::WriteError::Stopped(code) => match self.close.decode_stream_code(code) { + send::Error::Stopped(code) => match self.close.decode_stream_code(code) { Some(code) => WriteError::Stopped(code), None => match self.close.session_gone(code) { Some(err) => WriteError::SessionError(err), @@ -52,77 +65,120 @@ impl SendStream { } } - /// Abruptly reset the stream with the provided error code. See [`crate::quic::driver::SendStream::reset`]. + fn open(&mut self) -> Result<&mut send::Stream, WriteError> { + match &mut self.state { + State::Open(stream) => Ok(stream), + _ => Err(WriteError::ClosedStream), + } + } + + /// Abruptly reset the stream with the provided error code. See [`send::Stream::reset`]. /// This is a u32 with WebTransport because we share the error space with HTTP/3. - /// A raw QUIC session sends the code as is. + /// A raw QUIC session sends the code as is. Does nothing once already reset. /// /// The WebTransport header still reaches the peer, so it can tell which session the reset /// belongs to. - pub fn reset(&mut self, code: u32) -> Result<(), ClosedStream> { + pub fn reset(&mut self, code: u32) { let code = self.close.encode_stream_code(code); - match self.header.take() { - Some(header) => self.stream.reset_at(header, code).map_err(Into::into), - None => self.stream.reset(code).map_err(Into::into), - } + let header = self.header; + // The peer's stop code still wins if it arrived first, which only a `stopped` future + // taken before the reset learns, and the reset settles it. + let stopped = match std::mem::replace(&mut self.state, State::Reset(Ok(None))) { + State::Open(stream) => { + let stopped = stream.stopped(); + match header { + Some(header) => stream.reset_at(header, code), + None => stream.reset(code), + } + stopped.now_or_never() + } + State::Finished(stream) => { + let stopped = stream.stopped(); + match header { + Some(header) => stream.reset_at(header, code), + None => stream.reset(code), + } + stopped.now_or_never() + } + state @ State::Reset(_) => { + self.state = state; + return; + } + }; + self.state = State::Reset(stopped.expect("a reset settles its stopped future")); } /// Wait until the stream has been stopped and return the error code. See - /// [`crate::quic::driver::SendStream::stopped`]. + /// [`send::Finished::stopped`]. /// /// Unlike the driver, this returns None if the code is not a valid WebTransport error code. pub async fn stopped(&self) -> Result, SessionError> { - match self.stream.stopped().await { + let stopped = match &self.state { + State::Open(stream) => stream.stopped().await, + State::Finished(stream) => stream.stopped().await, + State::Reset(stopped) => stopped.clone(), + }; + match stopped { Ok(Some(code)) => match self.close.session_gone(code) { Some(err) => Err(err), None => Ok(self.close.decode_stream_code(code)), }, Ok(None) => Ok(None), - Err(crate::quic::driver::StoppedError::ConnectionLost(conn_err)) => Err(self.close.map(conn_err.into())), + Err(err) => match WriteError::from(err) { + WriteError::SessionError(err) => Err(self.close.map(err)), + err => unreachable!("only the connection fails a stopped wait: {err}"), + }, } } - // Unfortunately, we have to wrap WriteError for a bunch of functions. - - /// Write some data to the stream, returning the size written. See [`crate::quic::driver::SendStream::write`]. - pub async fn write(&mut self, buf: &[u8]) -> Result { - self.stream.write(buf).await.map_err(|e| self.map_error(e)) - } - - /// Write all of the data to the stream. See [`crate::quic::driver::SendStream::write_all`]. - pub async fn write_all(&mut self, buf: &[u8]) -> Result<(), WriteError> { - self.stream.write_all(buf).await.map_err(|e| self.map_error(e)) + /// Write from `buf`, returning the size written. See [`send::Stream::write`]. + pub async fn write(&mut self, buf: &mut impl Buf) -> Result { + let result = self.open()?.write(buf).await; + result.map_err(|e| self.map_error(e)) } - /// Write a chunk of data to the stream. See [`crate::quic::driver::SendStream::write_chunk`]. - pub async fn write_chunk(&mut self, buf: Bytes) -> Result<(), WriteError> { - self.stream.write_chunk(buf).await.map_err(|e| self.map_error(e)) - } - - /// Write all of the chunks of data to the stream. See [`crate::quic::driver::SendStream::write_all_chunks`]. - pub async fn write_all_chunks(&mut self, bufs: &mut [Bytes]) -> Result<(), WriteError> { - self.stream.write_all_chunks(bufs).await.map_err(|e| self.map_error(e)) + /// Write all of `buf` to the stream. See [`send::Stream::write_all`]. + pub async fn write_all(&mut self, buf: impl Buf) -> Result<(), WriteError> { + let result = self.open()?.write_all(buf).await; + result.map_err(|e| self.map_error(e)) } /// Mark the stream as finished, such that no more data can be written. See - /// [`crate::quic::driver::SendStream::finish`]. - pub fn finish(&mut self) -> Result<(), ClosedStream> { - self.stream.finish().map_err(Into::into) + /// [`send::Stream::finish`]. Fails if it was already finished or reset. + pub fn finish(&mut self) -> Result<(), WriteError> { + self.open()?; + let State::Open(stream) = std::mem::replace(&mut self.state, State::Reset(Ok(None))) else { + unreachable!("checked above"); + }; + self.state = State::Finished(stream.finish()); + Ok(()) } /// Set the priority within the stream's send group, or session-wide when ungrouped. - pub fn set_priority(&self, priority: i64) -> Result<(), ClosedStream> { - self.stream.set_priority(priority).map_err(Into::into) + pub fn set_priority(&mut self, priority: i64) { + match &mut self.state { + State::Open(stream) => stream.set_priority(priority), + State::Finished(stream) => stream.set_priority(priority), + State::Reset(_) => {} + } } - /// Get the stream's priority. See [`crate::quic::driver::SendStream::priority`]. - pub fn priority(&self) -> Result { - self.stream.priority().map_err(Into::into) + /// The stream's priority. See [`send::Stream::priority`]. + pub fn priority(&self) -> i64 { + match &self.state { + State::Open(stream) => stream.priority(), + State::Finished(stream) => stream.priority(), + State::Reset(_) => 0, + } } - /// Move the stream into a send group, or with `None` back out. See - /// [`crate::quic::driver::SendStream::set_group`]. - pub fn set_group(&self, group: Option<&crate::quic::driver::SendGroup>) -> Result<(), ClosedStream> { - self.stream.set_group(group).map_err(Into::into) + /// Move the stream into a send group, or with `None` back out. See [`send::Stream::set_group`]. + pub fn set_group(&mut self, group: Option<&send::Group>) { + match &mut self.state { + State::Open(stream) => stream.set_group(group), + State::Finished(stream) => stream.set_group(group), + State::Reset(_) => {} + } } /// Return the underlying QUIC stream ID. @@ -131,25 +187,29 @@ impl SendStream { /// > /// > WebTransport sessions share the QUIC connection with HTTP/3 and potentially other /// > sessions. - /// > The [crate::quic::driver::StreamId::index] might not increment by 1 like expected when using raw QUIC. + /// > The [crate::quic::driver::stream::Id::index] might not increment by 1 like expected when using raw QUIC. /// > This is why the Javascript WebTransport API does not expose the Stream ID. - pub fn quic_id(&self) -> crate::quic::driver::StreamId { - self.stream.id() + pub fn quic_id(&self) -> crate::quic::driver::stream::Id { + self.id } } impl tokio::io::AsyncWrite for SendStream { fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - // We have to use this syntax because the driver has its own poll_write method. - tokio::io::AsyncWrite::poll_write(Pin::new(&mut self.stream), cx, buf) + match &mut self.state { + State::Open(stream) => Pin::new(stream).poll_write(cx, buf), + _ => Poll::Ready(Err(io::ErrorKind::BrokenPipe.into())), + } } - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_flush(cx) + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Ready(Ok(())) } - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - Pin::new(&mut self.stream).poll_shutdown(cx) + fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + // Already finished or reset, so there is nothing left to shut. + let _ = self.finish(); + Poll::Ready(Ok(())) } } @@ -157,31 +217,27 @@ impl web_transport_trait::SendStream for SendStream { type Error = WriteError; fn set_priority(&mut self, order: i32) { - Self::set_priority(self, order.into()).ok(); + Self::set_priority(self, order.into()); } fn reset(&mut self, code: u32) { - Self::reset(self, code).ok(); + Self::reset(self, code); } fn finish(&mut self) -> Result<(), Self::Error> { - Self::finish(self).map_err(|_| WriteError::ClosedStream) + Self::finish(self) } - async fn write(&mut self, buf: &[u8]) -> Result { - Self::write(self, buf).await + async fn write(&mut self, mut buf: &[u8]) -> Result { + Self::write(self, &mut buf).await } async fn write_buf(&mut self, buf: &mut B) -> Result { - // This can avoid making a copy when Buf is Bytes, as the driver will allocate anyway. - let size = buf.chunk().len(); - let chunk = buf.copy_to_bytes(size); - self.write_chunk(chunk).await?; - Ok(size) + Self::write(self, buf).await } async fn write_chunk(&mut self, chunk: Bytes) -> Result<(), Self::Error> { - self.write_chunk(chunk).await + self.write_all(chunk).await } async fn closed(&mut self) -> Result<(), Self::Error> { diff --git a/rs/moq-tokio/src/quic/web/session.rs b/rs/moq-tokio/src/quic/web/session.rs index 43fca747aaa..e3ae440a7dc 100644 --- a/rs/moq-tokio/src/quic/web/session.rs +++ b/rs/moq-tokio/src/quic/web/session.rs @@ -9,11 +9,11 @@ use std::{ time::Duration, }; -use bytes::{Bytes, BytesMut}; +use bytes::{Buf, Bytes}; use futures::stream::{FuturesUnordered, Stream, StreamExt}; use kio::Waiter; -use crate::quic::driver::SendOptions; +use crate::quic::driver::send; use crate::quic::web::fan::Fan; use crate::quic::web::{ ClientError, CloseReason, Connected, RecvStream, SendStream, SessionError, Settings, WebTransportError, @@ -81,7 +81,7 @@ pub struct Session { // The send side of the CONNECT stream, used to write the CloseWebTransportSession capsule. // Wrapped in Arc>> so close() can take it exactly once. - connect_send: Arc>>, + connect_send: Arc>>, // Why the session closed: set once by a local close() or the background task // when a remote CloseWebTransportSession capsule is received. @@ -159,7 +159,7 @@ impl Session { // then record the close error and tear down the connection. async fn run_recv( conn: crate::quic::driver::connection::Connection, - recv: crate::quic::driver::RecvStream, + recv: crate::quic::driver::recv::Stream, close: Arc, ) { let close_info = Self::read_capsules(&conn, recv).await; @@ -195,7 +195,7 @@ impl Session { // or None if the stream closed without a capsule. async fn read_capsules( conn: &crate::quic::driver::connection::Connection, - recv: crate::quic::driver::RecvStream, + recv: crate::quic::driver::recv::Stream, ) -> Option<(u32, String)> { let mut reader = web_transport_proto::Http3CapsuleReader::new(recv); loop { @@ -246,7 +246,7 @@ impl Session { .await } - /// Accept a new unidirectional stream. See [`crate::quic::driver::Connection::accept_uni`]. + /// Accept a new unidirectional stream. See [`crate::quic::driver::connection::Connection::accept_uni`]. pub async fn accept_uni(&self) -> Result { if let Some(accept) = &self.accept { // `kio::wait` owns the waiter, so dropping this future — a `timeout` that @@ -260,7 +260,7 @@ impl Session { } } - /// Accept a new bidirectional stream. See [`crate::quic::driver::Connection::accept_bi`]. + /// Accept a new bidirectional stream. See [`crate::quic::driver::connection::Connection::accept_bi`]. pub async fn accept_bi(&self) -> Result<(SendStream, RecvStream), SessionError> { if let Some(accept) = &self.accept { kio::wait(|waiter| poll_accept_bi_shared(accept, waiter)) @@ -276,13 +276,13 @@ impl Session { } /// Open a new unidirectional stream, its sending half scheduled by `options`. - /// See [`crate::quic::driver::Connection::open_uni`]. - pub async fn open_uni(&self, options: SendOptions<'_>) -> Result { + /// See [`crate::quic::driver::connection::Connection::open_uni`]. + pub async fn open_uni(&self, options: send::Options<'_>) -> Result { // Write the stream header ungrouped at the highest priority, before scheduling the stream. // Otherwise the application could write data with lower priority than the header, resulting // in queuing. Also the header is very important for determining the session ID // without reliable reset. - let header = SendOptions { + let header = send::Options { group: None, priority: i64::MAX, }; @@ -291,22 +291,22 @@ impl Session { .await .map_err(|e| self.map_error(e))?; - send.set_priority(options.priority).ok(); + send.set_priority(options.priority); if let Some(group) = options.group { - send.set_group(Some(group)).ok(); + send.set_group(Some(group)); } - let header = self.header_uni.len() as u64; + let header = send.offset(); Ok(SendStream::new(send, self.close.clone(), Some(header))) } /// Open a new bidirectional stream, its sending half scheduled by `options`. - /// See [`crate::quic::driver::Connection::open_bi`]. - pub async fn open_bi(&self, options: SendOptions<'_>) -> Result<(SendStream, RecvStream), SessionError> { + /// See [`crate::quic::driver::connection::Connection::open_bi`]. + pub async fn open_bi(&self, options: send::Options<'_>) -> Result<(SendStream, RecvStream), SessionError> { // Write the stream header ungrouped at the highest priority, before scheduling the stream. // Otherwise the application could write data with lower priority than the header, resulting // in queuing. Also the header is very important for determining the session ID // without reliable reset. - let header = SendOptions { + let header = send::Options { group: None, priority: i64::MAX, }; @@ -315,11 +315,11 @@ impl Session { .await .map_err(|e| self.map_error(e))?; - send.set_priority(options.priority).ok(); + send.set_priority(options.priority); if let Some(group) = options.group { - send.set_group(Some(group)).ok(); + send.set_group(Some(group)); } - let header = self.header_bi.len() as u64; + let header = send.offset(); Ok(( SendStream::new(send, self.close.clone(), Some(header)), RecvStream::new(recv, self.close.clone()), @@ -331,12 +331,13 @@ impl Session { let Some((mut send, recv)) = self.conn.open_request().map_err(|err| self.map_error(err))? else { return Ok(None); }; - send.set_priority(i64::MAX).ok(); + send.set_priority(i64::MAX); Self::write_full(&mut send, &self.header_bi) .await .map_err(|err| self.map_error(err))?; + let header = send.offset(); Ok(Some(( - SendStream::new(send, self.close.clone(), Some(self.header_bi.len() as u64)), + SendStream::new(send, self.close.clone(), Some(header)), RecvStream::new(recv, self.close.clone()), ))) } @@ -369,32 +370,24 @@ impl Session { /// /// Datagrams are unreliable and may be dropped or delivered out of order. /// The data must be smaller than [`max_datagram_size`](Self::max_datagram_size). - /// See [`Connection::send_datagram`](crate::quic::driver::Connection::send_datagram) for how + /// See [`Connection::send_datagram`](crate::quic::driver::connection::Connection::send_datagram) for how /// `options` schedule it. - pub fn send_datagram(&self, data: Bytes, options: SendOptions<'_>) -> Result<(), SessionError> { - let result = if !self.header_datagram.is_empty() { - // The driver sends a single contiguous buffer, including the session header. - let mut buf = BytesMut::with_capacity(self.header_datagram.len() + data.len()); - - // Prepend the datagram with the header indicating the session ID. - buf.extend_from_slice(&self.header_datagram); - buf.extend_from_slice(&data); - - self.conn.send_datagram(buf.into(), options) - } else { - self.conn.send_datagram(data, options) - }; - - result.map_err(|e| self.map_error(e))?; + pub fn send_datagram(&self, data: Bytes, options: send::Options<'_>) -> Result<(), SessionError> { + // Prepend the header naming the session, which is empty for a raw QUIC session. The + // payload is chained rather than copied. + let datagram = (&self.header_datagram[..]).chain(data); + self.conn + .send_datagram(datagram, options) + .map_err(|e| self.map_error(e))?; Ok(()) } - /// Open a send group at `priority`, for streams and datagrams to join with [`SendOptions`] + /// Open a send group at `priority`, for streams and datagrams to join with [`send::Options`] /// or [`SendStream::set_group`]. /// - /// See [`crate::quic::driver::SendGroup`]. The HTTP/3 streams stay outside every group, + /// See [`crate::quic::driver::send::Group`]. The HTTP/3 streams stay outside every group, /// so they go first. - pub fn send_group(&self, priority: i64) -> crate::quic::driver::SendGroup { + pub fn send_group(&self, priority: i64) -> crate::quic::driver::send::Group { self.conn.send_group(priority) } @@ -470,7 +463,7 @@ impl Session { /// Deliver the close capsule and FIN, then give the peer its teardown grace. async fn close_with_capsule( &self, - mut send: crate::quic::driver::SendStream, + mut send: crate::quic::driver::send::Stream, capsule: web_transport_proto::Capsule, code: u32, timeout: std::time::Duration, @@ -494,18 +487,14 @@ impl Session { frame.extend_from_slice(&capsule_bytes); // Write the DATA frame to the CONNECT send stream. - if let Err(e) = send.write_all(&frame).await { + if let Err(e) = send.write_all(&frame[..]).await { tracing::warn!(?e, "failed to write CloseWebTransportSession capsule"); self.conn.close(http3_code, ""); return; } // FIN the send stream so the peer knows no more capsules are coming. - if let Err(e) = send.finish() { - tracing::warn!(?e, "failed to finish CONNECT send stream"); - self.conn.close(http3_code, ""); - return; - } + let send = send.finish(); // A queued write is not delivery: tearing down QUIC before the DATA and FIN // are acknowledged can discard the capsule during loss recovery. Transport @@ -530,12 +519,12 @@ impl Session { } } - /// Wait until the session is closed, returning the error. See [`crate::quic::driver::Connection::closed`]. + /// Wait until the session is closed, returning the error. See [`crate::quic::driver::connection::Connection::closed`]. /// /// If the peer sent a `CloseWebTransportSession` capsule, the returned error will be /// [`WebTransportError::Closed`] with the code and reason from the capsule. /// - /// Unlike [`crate::quic::driver::Connection::closed`], this does **not** return early when + /// Unlike [`crate::quic::driver::connection::Connection::closed`], this does **not** return early when /// [`close()`](Self::close) has been called. It waits for the underlying QUIC /// connection to shut down, ensuring the `CloseWebTransportSession` capsule has /// been delivered. Use [`close_reason()`](Self::close_reason) for a non-blocking check. @@ -544,7 +533,7 @@ impl Session { } /// Return why the session was closed, or None if it's not closed. See - /// [`crate::quic::driver::Connection::close_reason`]. + /// [`crate::quic::driver::connection::Connection::close_reason`]. pub fn close_reason(&self) -> Option { self.conn.close_reason().map(|e| self.map_error(e)) } @@ -554,10 +543,10 @@ impl Session { self.close.map(e.into()) } - async fn write_full(send: &mut crate::quic::driver::SendStream, buf: &[u8]) -> Result<(), SessionError> { + async fn write_full(send: &mut crate::quic::driver::send::Stream, buf: &[u8]) -> Result<(), SessionError> { match send.write_all(buf).await { Ok(_) => Ok(()), - Err(crate::quic::driver::WriteError::ConnectionLost(err)) => Err(err.into()), + Err(crate::quic::driver::send::Error::Connection(err)) => Err(err.into()), Err(err) => Err(WebTransportError::WriteError(err).into()), } } @@ -708,20 +697,21 @@ fn poll_accept_bi_shared( // Type aliases just so clippy doesn't complain about the complexity. type AcceptUni = - dyn Stream> + Send; + dyn Stream> + Send; type AcceptBi = dyn Stream< Item = Result< - (crate::quic::driver::SendStream, crate::quic::driver::RecvStream), + (crate::quic::driver::send::Stream, crate::quic::driver::recv::Stream), crate::quic::driver::connection::Error, >, > + Send; /// A unidirectional stream's header: its type, the session a WebTransport stream names, and the /// stream positioned after it. -pub(crate) type UniHeader = (StreamUni, Option, crate::quic::driver::RecvStream); +pub(crate) type UniHeader = (StreamUni, Option, crate::quic::driver::recv::Stream); /// A unidirectional stream whose header is being read. pub(crate) type PendingUni = dyn Future> + Send; -type PendingBi = dyn Future, SessionError>> - + Send; +type PendingBi = dyn Future< + Output = Result, SessionError>, + > + Send; // Logic just for accepting streams, which is annoying because of the stream header. pub struct SessionAccept { @@ -732,8 +722,8 @@ pub struct SessionAccept { // We also need to keep a reference to the qpack streams if the endpoint (incorrectly) creates // them. Again, this is just so they don't get closed until we drop the session. - qpack_encoder: Option, - qpack_decoder: Option, + qpack_encoder: Option, + qpack_decoder: Option, accept_uni: Pin>, // Pending WebTransport headers must be decoded before returning this error. @@ -927,10 +917,10 @@ impl SessionAccept { // Reads the stream header, returning Some if it's a WebTransport stream. async fn decode_bi( - send: crate::quic::driver::SendStream, - mut recv: crate::quic::driver::RecvStream, + send: crate::quic::driver::send::Stream, + mut recv: crate::quic::driver::recv::Stream, expected_session: VarInt, - ) -> Result, SessionError> { + ) -> Result, SessionError> { let typ = read_varint(&mut recv).await?; if Frame(typ) != Frame::WEBTRANSPORT { tracing::debug!(?typ, "ignoring unknown bidirectional stream"); @@ -949,7 +939,7 @@ impl SessionAccept { /// Read a unidirectional stream's header: its type, then the session ID if it is a WebTransport /// stream. Which session is the caller's to check. -pub(crate) async fn read_uni_header(mut recv: crate::quic::driver::RecvStream) -> Result { +pub(crate) async fn read_uni_header(mut recv: crate::quic::driver::recv::Stream) -> Result { let typ = StreamUni(read_varint(&mut recv).await?); let session = match typ { StreamUni::WEBTRANSPORT => Some(read_varint(&mut recv).await?), @@ -961,7 +951,7 @@ pub(crate) async fn read_uni_header(mut recv: crate::quic::driver::RecvStream) - // Read a stream header VarInt, keeping the read's real cause. `VarInt::read` reports any // failure as a truncation, which would hide a stream the peer reset before its header // arrived. -async fn read_varint(recv: &mut crate::quic::driver::RecvStream) -> Result { +async fn read_varint(recv: &mut crate::quic::driver::recv::Stream) -> Result { let mut buf = [0u8; 8]; recv.read_exact(&mut buf[..1]).await?; @@ -977,9 +967,7 @@ async fn read_varint(recv: &mut crate::quic::driver::RecvStream) -> Result tracing::debug!(?err, direction, "stream closed before its header"), _ => tracing::warn!(?err, direction, "failed to decode stream header"), } @@ -1036,11 +1024,11 @@ impl web_transport_trait::Session for Session { } async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { - Self::open_bi(self, SendOptions::default()).await + Self::open_bi(self, send::Options::default()).await } async fn open_uni(&self) -> Result { - Self::open_uni(self, SendOptions::default()).await + Self::open_uni(self, send::Options::default()).await } fn close(&self, code: u32, reason: &str) { @@ -1052,7 +1040,7 @@ impl web_transport_trait::Session for Session { } fn send_datagram(&self, data: Bytes) -> Result<(), Self::Error> { - Self::send_datagram(self, data, SendOptions::default()) + Self::send_datagram(self, data, send::Options::default()) } async fn recv_datagram(&self) -> Result { @@ -1227,9 +1215,9 @@ mod tests { Poll::Ready(Ok(None)) )); } - send.finish().unwrap(); + let send = send.finish(); let (mut peer_send, mut peer_recv) = server.accept_bi().await.unwrap(); - peer_send.finish().unwrap(); + let peer_send = peer_send.finish(); assert!(peer_recv.read_chunk(1).await.unwrap().is_none()); assert!(recv.read_chunk(1).await.unwrap().is_none()); drop((send, recv, peer_send, peer_recv)); @@ -1324,7 +1312,7 @@ mod tests { matches!( &err, SessionError::WebTransportError(WebTransportError::ReadError( - crate::quic::driver::ReadExactError::ReadError(crate::quic::driver::ReadError::Reset(code)) + crate::quic::driver::recv::Error::Reset(code) )) if code.into_inner() == 7 ), "{err:?}" @@ -1339,7 +1327,7 @@ mod tests { // A zero-length reserved frame, so the server sees the stream without a capsule. let (mut send, _recv) = client.open_bi(Default::default()).await.unwrap(); - send.write_all(&[0x21, 0x00]).await.unwrap(); + send.write_all(&[0x21, 0x00][..]).await.unwrap(); let (_send, recv) = server.accept_bi().await.unwrap(); client.close(0u32.into(), "done"); @@ -1355,8 +1343,8 @@ mod tests { // A DATA frame that promises 5 bytes and carries 1. let (mut send, _recv) = client.open_bi(Default::default()).await.unwrap(); - send.write_all(&[0x00, 0x05, 0x01]).await.unwrap(); - send.finish().unwrap(); + send.write_all(&[0x00, 0x05, 0x01][..]).await.unwrap(); + send.finish(); let (_send, recv) = server.accept_bi().await.unwrap(); assert!(Session::read_capsules(&server, recv).await.is_none()); @@ -1371,10 +1359,10 @@ mod tests { let (client, server, _endpoints) = pair().await; let mut send = client.open_uni(Default::default()).await.unwrap(); // WebTransport stream type, session ID 0, then the application payload. - send.write_all(&[0x40, 0x54, 0x00, b'l', b'a', b's', b't']) + send.write_all(&[0x40, 0x54, 0x00, b'l', b'a', b's', b't'][..]) .await .unwrap(); - send.finish().unwrap(); + let send = send.finish(); assert!(send.stopped().await.unwrap().is_none()); client.close(0u32.into(), "done"); server.closed().await; @@ -1393,8 +1381,8 @@ mod tests { // The stream type alone, so the reset also cuts off the session ID. let mut send = client.open_uni(Default::default()).await.unwrap(); - send.write_all(&[0x40, 0x54]).await.unwrap(); - send.reset(7u32.into()).unwrap(); + send.write_all(&[0x40, 0x54][..]).await.unwrap(); + send.reset(7u32.into()); let recv = server.accept_uni().await.unwrap(); let err = read_uni_header(recv).await.err().unwrap(); @@ -1406,8 +1394,8 @@ mod tests { async fn bi_reset_before_header() { let (client, server, _endpoints) = pair().await; - let (mut send, _recv) = client.open_bi(Default::default()).await.unwrap(); - send.reset(7u32.into()).unwrap(); + let (send, _recv) = client.open_bi(Default::default()).await.unwrap(); + send.reset(7u32.into()); let (send, recv) = server.accept_bi().await.unwrap(); let err = SessionAccept::decode_bi(send, recv, SESSION).await.err().unwrap(); @@ -1425,9 +1413,9 @@ mod tests { ); let (client, server) = (client.unwrap(), server.unwrap()); - client.open_uni(Default::default()).await.unwrap().reset(7).unwrap(); + client.open_uni(Default::default()).await.unwrap().reset(7); let (mut send, _recv) = client.open_bi(Default::default()).await.unwrap(); - send.reset(8).unwrap(); + send.reset(8); let read = async { let mut uni = server.accept_uni().await.unwrap(); @@ -1455,10 +1443,10 @@ mod tests { let (client, server) = (client.unwrap(), server.unwrap()); let mut uni = client.open_uni(Default::default()).await.unwrap(); - uni.write_all(b"truncated").await.unwrap(); + uni.write_all(&b"truncated"[..]).await.unwrap(); drop(uni); let (mut send, mut client_bi) = client.open_bi(Default::default()).await.unwrap(); - send.write_all(b"request").await.unwrap(); + send.write_all(&b"request"[..]).await.unwrap(); send.finish().unwrap(); let read = async { @@ -1493,9 +1481,9 @@ mod tests { let (client, server) = (client.unwrap(), server.unwrap()); let mut uni = client.open_uni(Default::default()).await.unwrap(); - uni.write_all(b"unread").await.unwrap(); + uni.write_all(&b"unread"[..]).await.unwrap(); let (mut bi, _client_recv) = client.open_bi(Default::default()).await.unwrap(); - bi.write_all(b"unread").await.unwrap(); + bi.write_all(&b"unread"[..]).await.unwrap(); let mut recv = server.accept_uni().await.unwrap(); recv.read_chunk(usize::MAX).await.unwrap(); @@ -1506,7 +1494,7 @@ mod tests { let stopped = |mut send: SendStream| async move { loop { - if let Err(err) = send.write(b"more").await { + if let Err(err) = send.write(&mut &b"more"[..]).await { return err; } } @@ -1528,9 +1516,9 @@ mod tests { for header in [[0x40, 0x54, 0x04], [0x40, 0x54, 0x00]] { let mut send = client.open_uni(Default::default()).await.unwrap(); - send.write_all(&header).await.unwrap(); + send.write_all(&header[..]).await.unwrap(); send.write_all(&header[2..]).await.unwrap(); - send.finish().unwrap(); + send.finish(); } let mut accept = SessionAccept::new(server, SESSION, Arc::new(CloseReason::new(false)), Vec::new()); @@ -1541,11 +1529,11 @@ mod tests { /// Run the HTTP/3 handshake on both ends once `before` has opened the client's streams that /// precede its control stream. async fn handshake_after( - before: impl AsyncFnOnce(&crate::quic::driver::connection::Connection) -> Vec, + before: impl AsyncFnOnce(&crate::quic::driver::connection::Connection) -> Vec, ) -> ( Session, Session, - Vec, + Vec, [crate::quic::driver::endpoint::Endpoint; 2], ) { let (client_conn, server_conn, endpoints) = pair().await; @@ -1574,7 +1562,7 @@ mod tests { .await; let mut send = client.open_uni(Default::default()).await.unwrap(); - send.write_all(b"after").await.unwrap(); + send.write_all(&b"after"[..]).await.unwrap(); send.finish().unwrap(); let mut recv = server.accept_uni().await.unwrap(); assert_eq!(recv.read_to_end(64).await.unwrap(), b"after"); @@ -1591,14 +1579,14 @@ mod tests { let (_client, server, mut early, _endpoints) = handshake_after(async |conn| { let mut send = conn.open_uni(Default::default()).await.unwrap(); // The first byte of the two-byte WebTransport stream type. - send.write_all(&[0x40]).await.unwrap(); + send.write_all(&[0x40][..]).await.unwrap(); vec![send] }) .await; let mut send = early.remove(0); - send.write_all(&[0x54, 0x00, b'l', b'a', b't', b'e']).await.unwrap(); - send.finish().unwrap(); + send.write_all(&[0x54, 0x00, b'l', b'a', b't', b'e'][..]).await.unwrap(); + send.finish(); let mut recv = server.accept_uni().await.unwrap(); assert_eq!(recv.read_to_end(64).await.unwrap(), b"late"); } @@ -1608,11 +1596,12 @@ mod tests { async fn handshake_keeps_an_early_webtransport_stream() { let (_client, server, _early, _endpoints) = handshake_after(async |conn| { let mut send = conn.open_uni(Default::default()).await.unwrap(); - send.write_all(&[0x40, 0x54, 0x00, b'e', b'a', b'r', b'l', b'y']) + send.write_all(&[0x40, 0x54, 0x00, b'e', b'a', b'r', b'l', b'y'][..]) .await .unwrap(); - send.finish().unwrap(); - vec![send] + // A finished stream delivers its data and FIN without its handle. + send.finish(); + Vec::new() }) .await; @@ -1626,7 +1615,7 @@ mod tests { async fn handshake_ends_with_the_connection() { let (client, server, _endpoints) = pair().await; let mut send = client.open_uni(Default::default()).await.unwrap(); - send.write_all(&[0x02]).await.unwrap(); + send.write_all(&[0x02][..]).await.unwrap(); client.close(0u32.into(), "done"); assert!(crate::quic::web::Request::accept(server).await.is_err()); } diff --git a/rs/moq-tokio/src/quic/web/settings.rs b/rs/moq-tokio/src/quic/web/settings.rs index 60729ced983..70f5d7d6f9b 100644 --- a/rs/moq-tokio/src/quic/web/settings.rs +++ b/rs/moq-tokio/src/quic/web/settings.rs @@ -24,10 +24,10 @@ pub enum SettingsError { ConnectionError(#[from] crate::quic::driver::connection::Error), #[error("read error")] - ReadError(#[from] crate::quic::driver::ReadError), + ReadError(#[from] crate::quic::driver::recv::Error), #[error("write error")] - WriteError(#[from] crate::quic::driver::WriteError), + WriteError(#[from] crate::quic::driver::send::Error), } pub struct Settings { @@ -35,10 +35,10 @@ pub struct Settings { dead_code, reason = "a reference to the send/recv stream, so we don't close it until dropped" )] - send: crate::quic::driver::SendStream, + send: crate::quic::driver::send::Stream, #[allow(dead_code, reason = "held so the stream stays open until dropped")] - recv: crate::quic::driver::RecvStream, + recv: crate::quic::driver::recv::Stream, /// The peer's other unidirectional streams that arrived before its control stream, their /// headers read or still reading, for the session to take over. Behind a lock only so the @@ -70,7 +70,7 @@ impl Settings { /// one kept holds the peer's stream credit, which bounds how many it can open first. async fn accept( conn: &crate::quic::driver::connection::Connection, - ) -> Result<(crate::quic::driver::RecvStream, Vec>>), SettingsError> { + ) -> Result<(crate::quic::driver::recv::Stream, Vec>>), SettingsError> { let mut accept = std::pin::pin!(futures::stream::unfold(conn.clone(), |conn| async { Some((conn.accept_uni().await, conn)) })); @@ -103,7 +103,7 @@ impl Settings { } async fn open( conn: &crate::quic::driver::connection::Connection, - ) -> Result { + ) -> Result { let mut settings = web_transport_proto::Settings::default(); settings.enable_webtransport(1); diff --git a/rs/moq-tokio/src/transport.rs b/rs/moq-tokio/src/transport.rs index 14e8fc1fe1a..4d1ca90fd27 100644 --- a/rs/moq-tokio/src/transport.rs +++ b/rs/moq-tokio/src/transport.rs @@ -828,16 +828,16 @@ impl Grouped for crate::quic::web::Session { Some(crate::quic::driver::connection::Connection::stats(self).credit.evicted) } - fn send_group(&self, priority: i64) -> Option { + fn send_group(&self, priority: i64) -> Option { Some(Self::send_group(self, priority)) } fn send_datagram( &self, payload: Bytes, - options: moq_net::transport::poll::SendOptions<'_, crate::quic::driver::SendGroup>, + options: moq_net::transport::poll::SendOptions<'_, crate::quic::driver::send::Group>, ) -> Result<(), Self::Error> { - let options = crate::quic::driver::SendOptions { + let options = crate::quic::driver::send::Options { group: options.group, priority: options.priority, }; @@ -847,20 +847,19 @@ impl Grouped for crate::quic::web::Session { #[cfg(feature = "quic")] impl GroupedSend for crate::quic::web::SendStream { - type Group = crate::quic::driver::SendGroup; + type Group = crate::quic::driver::send::Group; fn set_group(&mut self, group: &Self::Group) { - // A closed stream has nothing left to schedule. - Self::set_group(self, Some(group)).ok(); + Self::set_group(self, Some(group)); } fn set_priority(&mut self, priority: i64) { - crate::quic::web::SendStream::set_priority(self, priority).ok(); + crate::quic::web::SendStream::set_priority(self, priority); } } #[cfg(feature = "quic")] -impl moq_net::transport::poll::SendGroup for crate::quic::driver::SendGroup { +impl moq_net::transport::poll::SendGroup for crate::quic::driver::send::Group { fn set_priority(&mut self, priority: i64) { Self::set_priority(self, priority) } diff --git a/rs/moq-tokio/src/transport/owned.rs b/rs/moq-tokio/src/transport/owned.rs index 9972fb70cfa..9ded0bbba8b 100644 --- a/rs/moq-tokio/src/transport/owned.rs +++ b/rs/moq-tokio/src/transport/owned.rs @@ -247,7 +247,7 @@ impl poll::Chunks for super::RecvStream { #[cfg(feature = "quic")] impl poll::RecvStream for crate::quic::web::RecvStream { type Error = Error; - type Lease<'a> = crate::quic::driver::Lease<'a>; + type Lease<'a> = crate::quic::driver::recv::Lease<'a>; fn poll_lease(&mut self, waiter: &moq_net::kio::Waiter, want: usize) -> Poll, Self::Error>> { crate::quic::web::RecvStream::poll_lease(self, &mut waiter.context(), want).map_err(Error) @@ -257,20 +257,20 @@ impl poll::RecvStream for crate::quic::web::RecvStream { } fn stop(&mut self, code: u32) { // A stream already closed has nobody left to tell. - crate::quic::web::RecvStream::stop(self, code).ok(); + crate::quic::web::RecvStream::stop(self, code); } } #[cfg(feature = "quic")] -impl poll::Lease for crate::quic::driver::Lease<'_> { +impl poll::Lease for crate::quic::driver::recv::Lease<'_> { fn is_fin(&self) -> bool { - crate::quic::driver::Lease::is_fin(self) + crate::quic::driver::recv::Lease::is_fin(self) } fn chunks(&self) -> impl Iterator { - crate::quic::driver::Lease::chunks(self) + crate::quic::driver::recv::Lease::chunks(self) } fn release(self) { - crate::quic::driver::Lease::release(self) + crate::quic::driver::recv::Lease::release(self) } } diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 9e85edee36e..0dfde1f9e8d 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -522,7 +522,7 @@ impl Reversed { .expect("open"); // The credit an eviction frees can reach the writer before the stop does, letting // the whole write queue, so only the stop itself proves the stream was stopped. - let _ = send.write_all(&bytes).await; + let _ = send.write_all(&bytes[..]).await; if let Ok(Some(_)) = send.stopped().await { let _ = tx.send(()); } @@ -693,12 +693,12 @@ async fn javascript_partial_frame_gives_way(version: &str) { let mut high = subscribe(&remote, "high", 100).await; // Wait until three quarters of the window holds a frame that can never complete. - commands.write_all(b"low\n").await.unwrap(); + commands.write_all(&b"low\n"[..]).await.unwrap(); let mut partial = partial_frame(&mut low).await; read_partial(&mut partial, WINDOW * 3 / 4).await; // Half the window, which only fits once the low frame lets go. - commands.write_all(b"high\n").await.unwrap(); + commands.write_all(&b"high\n"[..]).await.unwrap(); assert_eq!(next_frame(&mut high).await, vec![2u8; WINDOW / 2], "{version}"); loop { let chunk = tokio::time::timeout(TIMEOUT, partial.read_chunk()) @@ -712,7 +712,7 @@ async fn javascript_partial_frame_gives_way(version: &str) { } // The session survives, and the low track keeps delivering what fits. - commands.write_all(b"after\n").await.unwrap(); + commands.write_all(&b"after\n"[..]).await.unwrap(); assert_eq!(next_frame(&mut low).await, &b"after"[..], "{version}"); drop(commands); diff --git a/rs/moq-tokio/tests/integration/reset_secret.rs b/rs/moq-tokio/tests/integration/reset_secret.rs index 7a414aa1ff9..52e1dc1179d 100644 --- a/rs/moq-tokio/tests/integration/reset_secret.rs +++ b/rs/moq-tokio/tests/integration/reset_secret.rs @@ -120,8 +120,8 @@ fn restart(reset_secret: bool) -> Result Option { // SETUP type, then a one-byte message holding zero parameters. let mut setup = conn.open_uni(Default::default()).await.expect("open"); - setup.write_all(&[0x01, 0x01, 0x00]).await.expect("write"); - setup.finish().expect("finish"); + setup.write_all(&[0x01, 0x01, 0x00][..]).await.expect("write"); + setup.finish(); // GROUP type, then a three-byte header: subscribe 7, sequence 0, frame start 0. let mut send = conn.open_uni(Default::default()).await.expect("open"); - send.write_all(&[0x00, 0x03, 0x07, 0x00, 0x00]).await.expect("write"); + send.write_all(&[0x00, 0x03, 0x07, 0x00, 0x00][..]) + .await + .expect("write"); tokio::time::timeout(Duration::from_secs(10), send.stopped()) .await diff --git a/rs/moq-uring/benches/echo_quic.rs b/rs/moq-uring/benches/echo_quic.rs index c75cf003f9e..960de178c05 100644 --- a/rs/moq-uring/benches/echo_quic.rs +++ b/rs/moq-uring/benches/echo_quic.rs @@ -85,7 +85,7 @@ mod linux { let payload = vec![0x5a; PAYLOAD]; while let Ok(done) = start_rx.recv() { let (mut send, mut recv) = session.open_bi(Default::default()).await.expect("open stream"); - send.write_all(&payload).await.expect("write"); + send.write_all(&payload[..]).await.expect("write"); send.finish().expect("finish"); assert_eq!(recv.read_to_end(PAYLOAD + 1).await.expect("read").len(), PAYLOAD); done.send(()).expect("notify server"); diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index d645c2f33c4..0ac224dcdbd 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -6,12 +6,13 @@ use std::net::SocketAddr; use std::rc::{Rc, Weak}; use std::task::{Context, Poll}; -use bytes::Bytes; -use moq_quic::{Dir, StreamId, VarInt, endpoint::ConnectionHandle}; +use bytes::{Buf, Bytes}; +use moq_quic::endpoint::ConnectionHandle; +use moq_quic::stream::{Dir, Id as StreamId}; use rustc_hash::FxHashMap; -use super::super::Error; -use super::endpoint; +use super::super::{Error, VarInt}; +use super::{endpoint, recv, send}; use crate::udp; use crate::worker::Owner; @@ -66,15 +67,6 @@ pub(crate) struct State { /// Send streams waiting for their end: a FIN the peer acknowledged, or a /// `STOP_SENDING`. finishing: FxHashMap, - /// Every send stream a handle still holds, and how it ended once the - /// driver has seen that happen. That is what makes a later close mean - /// "already delivered" rather than "we never found out". - /// - /// Keyed on the live handles, because the verdict arrives as an event: a - /// stream finished and then dropped before its FIN was acknowledged would - /// otherwise leave an entry nobody can ever remove, one per group, for as - /// long as the connection lives. - sends: FxHashMap>, datagram_recv_waiters: kio::WaiterList, @@ -104,7 +96,6 @@ impl State { readable: FxHashMap::default(), writable: FxHashMap::default(), finishing: FxHashMap::default(), - sends: FxHashMap::default(), datagram_recv_waiters: kio::WaiterList::new(), closed: None, closed_waiters: kio::WaiterList::new(), @@ -115,7 +106,6 @@ impl State { /// Drop send stream `id`'s bookkeeping, parking included. fn forget_send(&mut self, id: StreamId) { - self.sends.remove(&id); self.finishing.remove(&id); self.writable.remove(&id); } @@ -190,29 +180,6 @@ impl Inner { waiter.register(state.finishing.entry(id).or_default()); } - /// Start tracking send stream `id`, which a handle now owns. - /// - /// Seeded from moq-quic rather than empty: a peer can open a bidirectional - /// stream and stop it before the application ever accepts it, and that - /// event arrives with no handle to record it against. moq-quic still knows, - /// so a `poll_closed` on the fresh handle reports the stop instead of - /// waiting for an event that has already been and gone. - pub(crate) fn track(&self, id: StreamId) { - // Err means moq-quic has no send half for the id, which is nothing to - // report either way. - let stopped = self.conn.borrow_mut().send_stream(id).stopped().ok().flatten(); - self.state - .borrow_mut() - .sends - .insert(id, stopped.map(|code| End::Stopped(code.into_inner()))); - } - - /// How send stream `id` ended, if the driver saw it end while a handle - /// still held it. - pub(crate) fn ended(&self, id: StreamId) -> Option { - self.state.borrow().sends.get(&id).copied().flatten() - } - /// Forget send stream `id`'s bookkeeping; called when its handle drops. /// /// The parking table goes with it. A stream reset on the way out is never @@ -290,34 +257,64 @@ impl Connection { } /// Open a send group at `priority`, for streams to join with - /// [`SendStream::set_group`](super::SendStream::set_group) and datagrams with - /// [`SendOptions`](super::SendOptions). - pub fn send_group(&self, priority: i64) -> super::SendGroup { - super::SendGroup::new(self.shared.clone(), priority) + /// [`send::Stream::set_group`] and datagrams with [`send::Options`]. + pub fn send_group(&self, priority: i64) -> send::Group { + send::Group::new(self.shared.clone(), priority) } /// Send a datagram scheduled by `options`, like one of its group's streams at its priority, /// or among the ungrouped streams without a group. /// /// Never blocks: a full buffer drops the oldest datagram of the lowest-priority group, - /// see [`moq_quic::Datagrams::send`]. + /// see [`moq_quic::Datagrams::send`]. A [`Bytes`] payload, or a chain of them, is queued + /// without copying. /// /// # Panics /// - when `options.group` belongs to another connection - pub fn send_datagram(&self, payload: Bytes, options: super::SendOptions<'_>) -> Result<(), Error> { + pub fn send_datagram(&self, data: impl Buf, options: send::Options<'_>) -> Result<(), send::Error> { if let Some(err) = self.shared.closed() { - return Err(err); + return Err(send::Error::Connection(err)); + } + let sent = self.shared.conn.borrow_mut().datagrams().send(data, options.to_quic()); + match sent { + Ok(()) => {} + Err(moq_quic::send::Error::TooLarge) => return Err(send::Error::TooLarge), + Err(_) => return Err(send::Error::Unsupported), } - self.shared - .conn - .borrow_mut() - .datagrams() - .send(payload, super::group::to_quic(options)) - .map_err(|err| Error::Quic(err.to_string()))?; self.shared.kick(); Ok(()) } + /// The largest datagram [`Self::send_datagram`] takes, or `None` if datagrams are + /// unsupported by the peer or disabled locally. + pub fn max_datagram_size(&self) -> Option { + self.shared.conn.borrow_mut().datagrams().max_size() + } + + /// The negotiated ALPN protocol, if any. + pub fn protocol(&self) -> Option<&str> { + self.alpn.as_deref() + } + + /// Close the connection immediately with `error_code` and `reason`. + /// + /// Only records the close: the driver task frames the CONNECTION_CLOSE, so keep the worker + /// running until [`Self::closed`] resolves. + pub fn close(&self, error_code: VarInt, reason: &[u8]) { + self.shared + .close_code(error_code.into_inner(), &String::from_utf8_lossy(reason)); + } + + /// Wait for the connection to close, returning why. + pub async fn closed(&self) -> Error { + kio::wait(|waiter| poll_closed(&self.shared, waiter)).await + } + + /// The connection's statistics. + pub fn stats(&self) -> moq_quic::connection::Stats { + self.shared.conn.borrow().stats() + } + /// The peer's certificate chain in DER, leaf first, or `None` if it /// presented none. /// @@ -516,7 +513,7 @@ pub(crate) async fn establish(shared: Shared, deadline: Option) -> Resu } impl Connection { - pub(crate) fn open_request(&self) -> Result, Error> { + pub(crate) fn open_request(&self) -> Result, Error> { if let Some(err) = self.shared.closed() { return Err(err); } @@ -528,148 +525,93 @@ impl Connection { .open(Dir::Bi, Default::default()); Ok(id.map(|id| { ( - super::SendStream::new(self.shared.clone(), id), - super::RecvStream::new(self.shared.clone(), id), + send::Stream::new(self.shared.clone(), id), + recv::Stream::new(self.shared.clone(), id), ) })) } } -impl web_transport_trait::poll::Session for Connection { - type SendStream = super::SendStream; - type RecvStream = super::RecvStream; - type Error = Error; +impl Connection { + /// Accept the next unidirectional stream the peer opens. + pub async fn accept_uni(&self) -> Result { + kio::wait(|waiter| accept_uni(&self.shared, waiter)).await + } - fn poll_accept_uni(&mut self, cx: &mut Context<'_>) -> Poll> { - let waiter = self.park.hold(cx); - if let Some(id) = self.shared.conn.borrow_mut().streams().accept(Dir::Uni) { - return Poll::Ready(Ok(super::RecvStream::new(self.shared.clone(), id))); - } - let mut state = self.shared.state.borrow_mut(); - if let Some(err) = &state.closed { - return Poll::Ready(Err(err.clone())); - } - waiter.register(&mut state.accept_uni_waiters); - Poll::Pending + /// Accept the next bidirectional stream the peer opens. + pub async fn accept_bi(&self) -> Result<(send::Stream, recv::Stream), Error> { + kio::wait(|waiter| accept_bi(&self.shared, waiter)).await } - fn poll_accept_bi( - &mut self, - cx: &mut Context<'_>, - ) -> Poll> { - let waiter = self.park.hold(cx); - // The borrow ends before the handles are built: constructing a send - // stream reaches back into the connection. - let accepted = self.shared.conn.borrow_mut().streams().accept(Dir::Bi); - if let Some(id) = accepted { - return Poll::Ready(Ok(( - super::SendStream::new(self.shared.clone(), id), - super::RecvStream::new(self.shared.clone(), id), - ))); - } - let mut state = self.shared.state.borrow_mut(); - if let Some(err) = &state.closed { - return Poll::Ready(Err(err.clone())); - } - waiter.register(&mut state.accept_bi_waiters); - Poll::Pending + /// Open a unidirectional stream scheduled by `options`, once the peer's stream credit + /// allows. + /// + /// # Panics + /// - when `options.group` belongs to another connection + pub async fn open_uni(&self, options: send::Options<'_>) -> Result { + kio::wait(|waiter| open(&self.shared, Dir::Uni, options, waiter)) + .await + .map(|(send, _)| send) } - fn poll_open_uni(&mut self, cx: &mut Context<'_>) -> Poll> { - let waiter = self.park.hold(cx); - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - // The peer's MAX_STREAMS credit is what gates us: `open` hands back - // nothing while it is spent. - let opened = self - .shared - .conn - .borrow_mut() - .streams() - .open(Dir::Uni, Default::default()); - match opened { - Some(id) => Poll::Ready(Ok(super::SendStream::new(self.shared.clone(), id))), - None => { - let mut state = self.shared.state.borrow_mut(); - waiter.register(&mut state.open_waiters); - Poll::Pending - } - } + /// Open a bidirectional stream scheduled by `options`, once the peer's stream credit + /// allows. + /// + /// # Panics + /// - when `options.group` belongs to another connection + pub async fn open_bi(&self, options: send::Options<'_>) -> Result<(send::Stream, recv::Stream), Error> { + kio::wait(|waiter| open(&self.shared, Dir::Bi, options, waiter)) + .await + .map(|(send, recv)| (send, recv.expect("bidirectional"))) } - fn poll_open_bi( - &mut self, - cx: &mut Context<'_>, - ) -> Poll> { - let waiter = self.park.hold(cx); - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - let opened = self - .shared - .conn - .borrow_mut() - .streams() - .open(Dir::Bi, Default::default()); - match opened { - Some(id) => Poll::Ready(Ok(( - super::SendStream::new(self.shared.clone(), id), - super::RecvStream::new(self.shared.clone(), id), - ))), - None => { - let mut state = self.shared.state.borrow_mut(); - waiter.register(&mut state.open_waiters); - Poll::Pending - } - } + /// Receive the next datagram. + pub async fn read_datagram(&self) -> Result { + kio::wait(|waiter| recv_datagram(&self.shared, waiter)).await } - fn poll_send_datagram(&mut self, _: &mut Context<'_>, payload: &[u8]) -> Poll> { - Poll::Ready(Self::send_datagram( - self, - Bytes::copy_from_slice(payload), - Default::default(), - )) + /// [`Self::accept_uni`], as a poll. + pub(crate) fn poll_accept_uni(&mut self, cx: &Context<'_>) -> Poll> { + accept_uni(&self.shared, self.park.hold(cx)) } - fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { - let waiter = self.park.hold(cx); - if let Some(datagram) = self.shared.conn.borrow_mut().datagrams().recv() { - return Poll::Ready(Ok(datagram)); - } - let mut state = self.shared.state.borrow_mut(); - if let Some(err) = &state.closed { - return Poll::Ready(Err(err.clone())); - } - waiter.register(&mut state.datagram_recv_waiters); - Poll::Pending + /// [`Self::accept_bi`], as a poll. + pub(crate) fn poll_accept_bi(&mut self, cx: &Context<'_>) -> Poll> { + accept_bi(&self.shared, self.park.hold(cx)) } - fn max_datagram_size(&self) -> usize { - self.shared.conn.borrow_mut().datagrams().max_size().unwrap_or(0) + /// [`Self::open_uni`], as a poll. + pub(crate) fn poll_open_uni( + &mut self, + cx: &Context<'_>, + options: send::Options<'_>, + ) -> Poll> { + open(&self.shared, Dir::Uni, options, self.park.hold(cx)).map_ok(|(send, _)| send) } - fn protocol(&self) -> Option<&str> { - self.alpn.as_deref() + /// [`Self::open_bi`], as a poll. + pub(crate) fn poll_open_bi( + &mut self, + cx: &Context<'_>, + options: send::Options<'_>, + ) -> Poll> { + open(&self.shared, Dir::Bi, options, self.park.hold(cx)) + .map_ok(|(send, recv)| (send, recv.expect("bidirectional"))) } - fn close(&mut self, code: u32, reason: &str) { - self.shared.close_code(u64::from(code), reason); + /// [`Self::read_datagram`], as a poll. + pub(crate) fn poll_recv_datagram(&mut self, cx: &Context<'_>) -> Poll> { + recv_datagram(&self.shared, self.park.hold(cx)) } - fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { - let waiter = self.park.hold(cx); - let mut state = self.shared.state.borrow_mut(); - if let Some(err) = &state.closed { - return Poll::Ready(err.clone()); - } - waiter.register(&mut state.closed_waiters); - Poll::Pending + /// [`Self::closed`], as a poll. + pub(crate) fn poll_closed(&mut self, cx: &Context<'_>) -> Poll { + poll_closed(&self.shared, self.park.hold(cx)) } - fn stats(&self) -> impl web_transport_trait::Stats { - let stats = self.shared.conn.borrow().stats(); + /// The connection's counters in [`web_transport_trait::Stats`] shape. + pub(crate) fn transport_stats(&self) -> impl web_transport_trait::Stats + use<> { + let stats = self.stats(); Stats { bytes_sent: stats.udp_tx.bytes, bytes_received: stats.udp_rx.bytes, @@ -683,6 +625,76 @@ impl web_transport_trait::poll::Session for Connection { } } +fn accept_uni(shared: &Shared, waiter: &kio::Waiter) -> Poll> { + if let Some(id) = shared.conn.borrow_mut().streams().accept(Dir::Uni) { + return Poll::Ready(Ok(recv::Stream::new(shared.clone(), id))); + } + let mut state = shared.state.borrow_mut(); + if let Some(err) = &state.closed { + return Poll::Ready(Err(err.clone())); + } + waiter.register(&mut state.accept_uni_waiters); + Poll::Pending +} + +fn accept_bi(shared: &Shared, waiter: &kio::Waiter) -> Poll> { + let accepted = shared.conn.borrow_mut().streams().accept(Dir::Bi); + if let Some(id) = accepted { + return Poll::Ready(Ok(( + send::Stream::new(shared.clone(), id), + recv::Stream::new(shared.clone(), id), + ))); + } + let mut state = shared.state.borrow_mut(); + if let Some(err) = &state.closed { + return Poll::Ready(Err(err.clone())); + } + waiter.register(&mut state.accept_bi_waiters); + Poll::Pending +} + +/// Open a stream in `dir`, with its receive half when bidirectional. +fn open( + shared: &Shared, + dir: Dir, + options: send::Options<'_>, + waiter: &kio::Waiter, +) -> Poll), Error>> { + if let Some(err) = shared.closed() { + return Poll::Ready(Err(err)); + } + // The peer's MAX_STREAMS credit is what gates us: `open` hands back nothing while it is + // spent. + let opened = shared.conn.borrow_mut().streams().open(dir, options.to_quic()); + let Some(id) = opened else { + waiter.register(&mut shared.state.borrow_mut().open_waiters); + return Poll::Pending; + }; + let recv = (dir == Dir::Bi).then(|| recv::Stream::new(shared.clone(), id)); + Poll::Ready(Ok((send::Stream::new(shared.clone(), id), recv))) +} + +fn recv_datagram(shared: &Shared, waiter: &kio::Waiter) -> Poll> { + if let Some(datagram) = shared.conn.borrow_mut().datagrams().recv() { + return Poll::Ready(Ok(datagram)); + } + let mut state = shared.state.borrow_mut(); + if let Some(err) = &state.closed { + return Poll::Ready(Err(err.clone())); + } + waiter.register(&mut state.datagram_recv_waiters); + Poll::Pending +} + +fn poll_closed(shared: &Shared, waiter: &kio::Waiter) -> Poll { + let mut state = shared.state.borrow_mut(); + if let Some(err) = &state.closed { + return Poll::Ready(err.clone()); + } + waiter.register(&mut state.closed_waiters); + Poll::Pending +} + impl std::fmt::Debug for Connection { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Connection").field("alpn", &self.alpn).finish() @@ -952,23 +964,8 @@ impl Driver { } } -/// How a send stream ended, once the driver has seen it happen. -#[derive(Clone, Copy, Debug)] -pub(crate) enum End { - /// The peer acknowledged the FIN. - Delivered, - /// The peer sent `STOP_SENDING` with this code. - Stopped(u64), -} - -/// Record how a send stream ended, and wake whoever is watching it. -/// -/// A stream whose handle has already dropped records nothing: nobody is left -/// to read the verdict, and the entry would outlive every use for it. -fn end(state: &mut State, id: StreamId, end: End) { - if let Some(slot) = state.sends.get_mut(&id) { - *slot = Some(end); - } +/// Wake whoever waits for send stream `id` to end. +fn ended(state: &mut State, id: StreamId) { if let Some(mut waiters) = state.finishing.remove(&id) { waiters.wake(); } @@ -992,9 +989,9 @@ fn sweep_stream(state: &mut State, event: moq_quic::StreamEvent) { waiters.wake(); } } - moq_quic::StreamEvent::Finished { id } => end(state, id, End::Delivered), - moq_quic::StreamEvent::Stopped { id, error_code } => { - end(state, id, End::Stopped(error_code.into_inner())); + moq_quic::StreamEvent::Finished { id } => ended(state, id), + moq_quic::StreamEvent::Stopped { id, .. } => { + ended(state, id); // A writer blocked on capacity has to learn it will never come. if let Some(mut waiters) = state.writable.remove(&id) { waiters.wake(); @@ -1023,12 +1020,10 @@ mod tests { state.writable.entry(id).or_default(); state.readable.entry(id).or_default(); state.finishing.entry(id).or_default(); - state.sends.insert(id, None); state.forget_send(id); assert!(state.writable.is_empty(), "the write half's parking"); assert!(state.finishing.is_empty(), "the finish watch"); - assert!(state.sends.is_empty(), "the end bookkeeping"); // The read half is a separate handle, which may still be parked. assert!(!state.readable.is_empty(), "the read half's parking survives"); diff --git a/rs/moq-uring/src/quic/driver/endpoint.rs b/rs/moq-uring/src/quic/driver/endpoint.rs index 684fc089630..cae5314287e 100644 --- a/rs/moq-uring/src/quic/driver/endpoint.rs +++ b/rs/moq-uring/src/quic/driver/endpoint.rs @@ -215,8 +215,8 @@ impl Drop for Endpoint { // Nobody will ever claim the queued connections; close them rather // than leaving their drivers to idle out. if let Some(accepting) = self.inner.accepting.borrow_mut().as_mut() { - for mut conn in accepting.queue.drain(..) { - web_transport_trait::poll::Session::close(&mut conn, 0, "endpoint closed"); + for conn in accepting.queue.drain(..) { + conn.close_code(0, "endpoint closed"); } } self.inner.task_waiters.borrow_mut().wake(); @@ -401,7 +401,7 @@ impl Inner { self.owner.spawn(async move { let outcome = connection::establish(shared, deadline).await; inner.pending.set(inner.pending.get() - 1); - let mut conn = match outcome { + let conn = match outcome { Ok(conn) => conn, Err(err) => { tracing::debug!(%err, "incoming handshake failed"); @@ -410,7 +410,7 @@ impl Inner { }; // The last handle may have left while we handshook; see Drop. if inner.handles.get() == 0 { - web_transport_trait::poll::Session::close(&mut conn, 0, "endpoint closed"); + conn.close_code(0, "endpoint closed"); return; } { diff --git a/rs/moq-uring/src/quic/driver/group.rs b/rs/moq-uring/src/quic/driver/group.rs deleted file mode 100644 index 54861086322..00000000000 --- a/rs/moq-uring/src/quic/driver/group.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Send groups: a shared handle around one of moq-quic's. - -use std::rc::Rc; - -use super::Shared; - -/// A send group on a [`Connection`](super::Connection). -/// -/// Streams that [join](super::SendStream::set_group) a group share bytes with the -/// connection's other groups: a higher priority group preempts every lower one, and groups of -/// equal priority get an equal share of bytes however many streams each has queued. A stream's -/// own priority orders it only against the group's other streams, and a datagram sent in the -/// group is ordered among them by its priority. Streams and datagrams outside any group go -/// before every group. See [`moq_quic::SendGroups`]. -/// -/// Clones share one group, released once the last clone is dropped and its streams finish -/// and its datagrams are sent. -#[derive(Clone)] -pub struct SendGroup(Rc); - -/// How an outgoing datagram is scheduled: its [`SendGroup`] and priority. -pub type SendOptions<'a> = moq_quic::SendOptions<'a, SendGroup>; - -/// The options as the connection knows them. -pub(crate) fn to_quic(options: SendOptions<'_>) -> moq_quic::SendOptions<'_> { - moq_quic::SendOptions { - group: options.group.map(SendGroup::id), - priority: options.priority, - } -} - -struct Inner { - shared: Shared, - group: Option, -} - -impl SendGroup { - pub(crate) fn new(shared: Shared, priority: i64) -> Self { - let group = shared.conn.borrow_mut().send_groups().open(priority); - Self(Rc::new(Inner { - shared, - group: Some(group), - })) - } - - /// Change the group's priority, moving any data it has queued along with it. - pub fn set_priority(&self, priority: i64) { - self.0 - .shared - .conn - .borrow_mut() - .send_groups() - .set_priority(self.id(), priority); - } - - /// The group's priority. - pub fn priority(&self) -> i64 { - self.0.shared.conn.borrow_mut().send_groups().priority(self.id()) - } - - /// The group as the connection knows it, which refuses it on any other connection. - pub(crate) fn id(&self) -> &moq_quic::SendGroup { - self.0.group.as_ref().expect("only taken on drop") - } -} - -impl Drop for Inner { - fn drop(&mut self) { - if let Some(group) = self.group.take() { - self.shared.conn.borrow_mut().send_groups().close(group); - } - } -} - -impl std::fmt::Debug for SendGroup { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SendGroup").finish_non_exhaustive() - } -} diff --git a/rs/moq-uring/src/quic/driver/mod.rs b/rs/moq-uring/src/quic/driver/mod.rs index a55ed4c0004..140178dfc81 100644 --- a/rs/moq-uring/src/quic/driver/mod.rs +++ b/rs/moq-uring/src/quic/driver/mod.rs @@ -11,15 +11,13 @@ mod connection; mod endpoint; -mod group; -mod stream; +pub mod recv; +pub mod send; pub use connection::Connection; pub use endpoint::Endpoint; -pub use group::{SendGroup, SendOptions}; -pub use stream::{Lease, RecvStream, SendStream}; -pub(crate) use connection::{End, Shared}; +pub(crate) use connection::Shared; use std::{num::NonZeroU64, sync::Arc}; diff --git a/rs/moq-uring/src/quic/driver/recv.rs b/rs/moq-uring/src/quic/driver/recv.rs new file mode 100644 index 00000000000..99056d71e0b --- /dev/null +++ b/rs/moq-uring/src/quic/driver/recv.rs @@ -0,0 +1,523 @@ +//! Incoming streams: their handles, leases, and errors. +//! +//! A read comes straight out of moq-quic's reassembly buffer; the handles kick the driver when +//! a read owes the peer credit and park on the per-stream waiter lists the driver wakes. + +use std::future::poll_fn; +use std::task::{Context, Poll, ready}; + +use bytes::{Buf, BufMut, Bytes}; +use moq_quic::recv::Chunks; + +use super::Shared; +use crate::quic::{Error as ConnectionError, VarInt, stream}; + +/// A stream that can only be used to receive data. +/// +/// Dropped before its end, it is [`stop()`](Self::stop)ped with code 0. +pub struct Stream { + shared: Shared, + id: stream::Id, + /// Retains the waiter registrations of the polls across calls. + park: kio::Park, + all_data_read: bool, + reset: Option, + /// The code a drop before the end stops with + drop_stop: VarInt, +} + +impl Stream { + pub(crate) fn new(shared: Shared, id: stream::Id) -> Self { + Self { + shared, + id, + park: kio::Park::default(), + all_data_read: false, + reset: None, + drop_stop: VarInt::from_u32(0), + } + } + + /// Read data contiguously from the stream into `buf`, advancing it. + /// + /// Waits for some data, then reads as much as has arrived and `buf` can take. Yields the + /// number of bytes read, or `None` if the stream was finished. + /// + /// Cancel-safe. + pub async fn read(&mut self, buf: &mut impl BufMut) -> Result, Error> { + poll_fn(|cx| self.poll_read(cx, buf)).await + } + + /// [`Self::read`], as a poll. + pub(crate) fn poll_read(&mut self, cx: &Context<'_>, buf: &mut impl BufMut) -> Poll, Error>> { + if !buf.has_remaining_mut() { + return Poll::Ready(Ok(Some(0))); + } + self.parked(cx, |this, waiter| { + this.poll_read_generic(waiter, |chunks| fill(chunks, buf)) + }) + } + + /// Fill `buf` with data from the stream. + /// + /// Fails with [`Error::FinishedEarly`] if the stream ends first. Not cancel-safe. + pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<(), Error> { + let len = buf.len(); + let mut rest = &mut buf[..]; + while !rest.is_empty() { + if self.read(&mut rest).await?.is_none() { + return Err(Error::FinishedEarly(len - rest.len())); + } + } + Ok(()) + } + + /// Read the next segment of data, at most `max` bytes. + /// + /// Yields `None` if the stream was finished. Unlike [`Self::read()`], this does not copy. + /// Segment boundaries do not correspond to peer writes, and hence cannot be used as framing. + /// + /// Cancel-safe. + pub async fn read_chunk(&mut self, max: usize) -> Result, Error> { + poll_fn(|cx| self.poll_read_chunk(cx, max)).await + } + + /// [`Self::read_chunk`], as a poll. + pub(crate) fn poll_read_chunk(&mut self, cx: &Context<'_>, max: usize) -> Poll, Error>> { + self.parked(cx, |this, waiter| { + this.poll_read_generic(waiter, |chunks| match chunks.next(max) { + Ok(Some(chunk)) => ReadStatus::Readable(chunk.bytes), + res => (None, res.err()).into(), + }) + }) + } + + /// Run `poll` with a waiter held in [`Self::park`], which outlives the call. + fn parked(&mut self, cx: &Context<'_>, poll: impl FnOnce(&mut Self, &kio::Waiter) -> R) -> R { + let mut park = std::mem::take(&mut self.park); + let result = poll(self, park.hold(cx)); + self.park = park; + result + } + + /// Read all remaining data. + /// + /// Fails with [`Error::TooLong`] on reading more than `limit` bytes, discarding all data + /// read. `limit` bounds the returned buffer. + /// + /// Not cancel-safe. + pub async fn read_to_end(&mut self, limit: usize) -> Result, Error> { + let mut read = Vec::new(); + while let Some(chunk) = self.read_chunk(usize::MAX).await? { + if chunk.len() > limit - read.len() { + return Err(Error::TooLong); + } + read.extend_from_slice(&chunk); + } + Ok(read) + } + + /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the + /// stream ended. See [`Self::poll_lease()`]. + pub async fn lease(&mut self, want: usize) -> Result, Error> { + let inner = poll_fn(|cx| self.parked(cx, |this, waiter| this.poll_take(waiter, want))).await?; + Ok(Lease { + stream: self, + inner: Some(inner), + }) + } + + /// Lease the stream's unreleased bytes once at least `want` have arrived in order, or the + /// stream ended. + /// + /// The lease owns the bytes and borrows no connection state, but they keep their connection + /// flow control credit until read through the lease as a [`Buf`] and then + /// [`Lease::release`]d; bytes not read are leased again next time. Stream credit still + /// advances, so a stream window smaller than a message keeps the message flowing. + /// + /// When the peer has spent the connection window and no reader can make progress, the + /// stream ranked lowest by [`Self::set_priority`] is evicted: its buffered bytes and their + /// credit are discarded and its next lease fails with [`Error::Evicted`]. Bytes it took + /// keep their credit until the stream is stopped. Nothing is evicted while the leased and + /// unread bytes of every stream stay below the connection receive window, nor while a lease + /// is out. + /// + /// Do not mix with the plain reads on one stream, which skip the leased bytes. + pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, Error>> { + let inner = ready!(self.parked(cx, |this, waiter| this.poll_take(waiter, want)))?; + Poll::Ready(Ok(Lease { + stream: self, + inner: Some(inner), + })) + } + + /// Take the connection's lease, which [`Lease`] wraps once this returns: the borrow of `self` + /// it holds cannot start inside a poll that may return `Pending`. + fn poll_take(&mut self, waiter: &kio::Waiter, want: usize) -> Poll> { + // moq-quic freed the stream when it reported the reset. + if let Some(code) = self.reset { + return Poll::Ready(Err(Error::Reset(code))); + } + let (result, transmit) = self.shared.conn.borrow_mut().recv_stream(self.id).lease(want); + if transmit.should_transmit() { + self.shared.kick(); + } + let err = match result { + Ok(lease) => return Poll::Ready(Ok(lease)), + Err(moq_quic::recv::Error::Blocked) => { + if let Some(err) = self.shared.closed() { + return Poll::Ready(Err(Error::Connection(err))); + } + self.shared.park_readable(self.id, waiter); + return Poll::Pending; + } + Err(moq_quic::recv::Error::Reset(code)) => Error::Reset(code), + Err(moq_quic::recv::Error::Evicted) => Error::Evicted, + Err(moq_quic::recv::Error::TooLarge) => Error::TooLarge, + Err(err) => self.read_error(err), + }; + Poll::Ready(Err(err)) + } + + /// Rank this stream for eviction: when the connection window is stuck, the stream with the + /// lowest priority goes first, and the newest of those that tie. + pub fn set_priority(&mut self, priority: i64) { + self.shared + .conn + .borrow_mut() + .recv_stream(self.id) + .set_priority(priority); + } + + /// Stop accepting data. + /// + /// Discards unread data and notifies the peer to stop transmitting. + pub fn stop(mut self, error_code: VarInt) { + self.drop_stop = error_code; + } + + /// The identity of this stream. + pub fn id(&self) -> stream::Id { + self.id + } + + /// Stop with `code`, instead of code 0, if dropped before the end. + pub(crate) fn set_drop_stop(&mut self, code: VarInt) { + self.drop_stop = code; + } + + /// Completes when the stream has been reset by the peer or otherwise closed. + /// + /// Yields `Some` with the reset error code when the stream is reset by the peer, and `None` + /// once every byte the peer finished it with has been read, after which it is no longer + /// meaningful for the stream to be reset. + /// + /// Cancel-safe. + pub async fn received_reset(&mut self) -> Result, Error> { + poll_fn(|cx| self.parked(cx, Self::wait_received_reset)).await + } + + fn wait_received_reset(&mut self, waiter: &kio::Waiter) -> Poll, Error>> { + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + let reset = self.shared.conn.borrow_mut().recv_stream(self.id).received_reset(); + match reset { + Err(_) => Poll::Ready(Ok(None)), + Ok(Some(code)) => { + // The stream's state was just freed, so the peer may be owed stream credit, and + // later reads fail with the cached code. + self.reset = Some(code); + self.shared.kick(); + Poll::Ready(Ok(Some(code))) + } + Ok(None) => match self.shared.closed() { + Some(err) => Poll::Ready(Err(Error::Connection(err))), + // A reset is a read error, so it always wakes readers. + None => { + self.shared.park_readable(self.id, waiter); + Poll::Pending + } + }, + } + } + + /// Completes once the stream ends: `None` once its finish is read, or `Some` with a reset's + /// code. + /// + /// Unlike [`Self::received_reset`], it observes a finish whose bytes were all taken but + /// whose end was not read yet. An empty read reaches the end without taking the bytes + /// before it, so it keeps waiting while any are unread. + pub(crate) fn poll_closed(&mut self, cx: &Context<'_>) -> Poll, Error>> { + if let Some(code) = self.reset { + return Poll::Ready(Ok(Some(code))); + } + let read = self.parked(cx, |this, waiter| { + this.poll_read_generic(waiter, |chunks| match chunks.next(0) { + // Bytes are still unread, so the end is not reached. + Ok(Some(_)) => ReadStatus::Failed(None, moq_quic::recv::Error::Blocked), + res => (None::<()>, res.err()).into(), + }) + }); + Poll::Ready(match ready!(read) { + Ok(_) => Ok(None), + Err(Error::Reset(code)) => Ok(Some(code)), + Err(err) => Err(err), + }) + } + + /// Handle the logic common to every read out of a receive stream. + /// + /// `read_fn` reads from the passed `Chunks` and returns the status after reading: the + /// amount of data read, and the status after the final read call. + fn poll_read_generic(&mut self, waiter: &kio::Waiter, mut read_fn: T) -> Poll, Error>> + where + T: FnMut(&mut Chunks<'_>) -> ReadStatus, + { + use moq_quic::recv::Error::{Blocked, Reset}; + + if self.all_data_read { + return Poll::Ready(Ok(None)); + } + + // A stored reset is returned now: a `read_fn` can both return data and end in a reset. + let status = match self.reset { + Some(code) => ReadStatus::Failed(None, Reset(code)), + None => { + let mut conn = self.shared.conn.borrow_mut(); + let (status, transmit) = match conn.recv_stream(self.id).read() { + Ok(mut chunks) => { + let status = read_fn(&mut chunks); + (status, chunks.finalize().should_transmit()) + } + Err(err) => (ReadStatus::Failed(None, err), false), + }; + drop(conn); + // Reading is what returns the peer's flow control credit. + if transmit { + self.shared.kick(); + } + status + } + }; + + match status { + ReadStatus::Readable(read) => Poll::Ready(Ok(Some(read))), + ReadStatus::Finished(read) => { + self.all_data_read = true; + Poll::Ready(Ok(read)) + } + ReadStatus::Failed(Some(read), Blocked) => Poll::Ready(Ok(Some(read))), + ReadStatus::Failed(None, Blocked) => { + if let Some(err) = self.shared.closed() { + return Poll::Ready(Err(Error::Connection(err))); + } + self.shared.park_readable(self.id, waiter); + Poll::Pending + } + ReadStatus::Failed(read, Reset(code)) => { + self.reset = Some(code); + if read.is_some() { + return Poll::Ready(Ok(read)); + } + self.all_data_read = true; + Poll::Ready(Err(Error::Reset(code))) + } + ReadStatus::Failed(_, err) => Poll::Ready(Err(self.read_error(err))), + } + } + + /// The error for a read moq-quic refused on a stream the handle still holds, which it forgets + /// only once the connection closed. + fn read_error(&self, err: moq_quic::recv::Error) -> Error { + match self.shared.closed() { + Some(closed) => Error::Connection(closed), + None => panic!("a held receive stream failed to read: {err:?}"), + } + } +} + +/// Copy from `chunks` until `buf` is full or nothing more is readable, returning how much. +fn fill(chunks: &mut Chunks<'_>, buf: &mut impl BufMut) -> ReadStatus { + let mut read = 0; + loop { + if !buf.has_remaining_mut() { + // `read` is not 0, because `buf` had room before. + return ReadStatus::Readable(read); + } + + match chunks.next(buf.remaining_mut()) { + Ok(Some(chunk)) => { + buf.put_slice(&chunk.bytes); + read += chunk.bytes.len(); + } + res => return ((read > 0).then_some(read), res.err()).into(), + } + } +} + +impl Drop for Stream { + fn drop(&mut self) { + self.shared.forget_recv(self.id); + if self.all_data_read || self.shared.closed().is_some() { + return; + } + // Fails harmlessly once the stream is gone. + let _ = self.shared.conn.borrow_mut().recv_stream(self.id).stop(self.drop_stop); + self.shared.kick(); + } +} + +impl std::fmt::Debug for Stream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Stream").field("id", &self.id).finish() + } +} + +/// Unreleased bytes of a [`Stream`] from [`Stream::poll_lease`], read as a [`Buf`]. +/// +/// It owns the bytes, so the connection carries on while it is held. Hand it back with +/// [`Self::release`] to return the credit of the bytes read; dropping it keeps their credit, +/// as for a message read only in part. Either way the bytes not read are leased again next +/// time. +pub struct Lease<'a> { + stream: &'a mut Stream, + /// Taken only to hand it back. + inner: Option, +} + +impl Lease<'_> { + fn inner(&self) -> &moq_quic::recv::Lease { + self.inner.as_ref().expect("only taken to hand it back") + } + + fn inner_mut(&mut self) -> &mut moq_quic::recv::Lease { + self.inner.as_mut().expect("only taken to hand it back") + } + + /// Whether the peer finished the stream right after the leased bytes. + pub fn is_fin(&self) -> bool { + self.inner().is_fin() + } + + /// The bytes not yet read, in order, one contiguous chunk at a time. + pub fn chunks(&self) -> impl Iterator { + self.inner().chunks() + } + + /// Return the credit of every byte read so far, by this lease or an earlier one. + pub fn release(mut self) { + let lease = self.inner.take().expect("only taken to hand it back"); + let stream = &*self.stream; + let transmit = stream.shared.conn.borrow_mut().recv_stream(stream.id).release(lease); + if transmit.should_transmit() { + stream.shared.kick(); + } + } +} + +/// Reading moves past bytes for good, but they keep their credit until [`Lease::release`]. +impl Buf for Lease<'_> { + fn remaining(&self) -> usize { + self.inner().remaining() + } + + fn chunk(&self) -> &[u8] { + self.inner().chunk() + } + + fn advance(&mut self, n: usize) { + self.inner_mut().advance(n); + } + + fn copy_to_bytes(&mut self, n: usize) -> Bytes { + self.inner_mut().copy_to_bytes(n) + } +} + +impl Drop for Lease<'_> { + fn drop(&mut self) { + if let Some(lease) = self.inner.take() { + let stream = &*self.stream; + let transmit = stream.shared.conn.borrow_mut().recv_stream(stream.id).restore(lease); + if transmit.should_transmit() { + stream.shared.kick(); + } + } + } +} + +impl std::fmt::Debug for Lease<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Lease") + .field("stream", &self.stream.id) + .finish_non_exhaustive() + } +} + +enum ReadStatus { + Readable(T), + Finished(Option), + Failed(Option, moq_quic::recv::Error), +} + +impl From<(Option, Option)> for ReadStatus { + fn from(status: (Option, Option)) -> Self { + match status { + (read, None) => Self::Finished(read), + (read, Some(e)) => Self::Failed(read, e), + } + } +} + +/// Errors from reading or leasing a stream. +#[derive(Debug, thiserror::Error, Clone)] +#[non_exhaustive] +pub enum Error { + /// The peer abandoned transmitting data on this stream, with its application error code. + #[error("stream reset by peer: error {0}")] + Reset(VarInt), + /// The connection window was stuck and this stream was discarded to free it. + /// + /// See [`Stream::poll_lease`]. The peer hears of it once the stream is stopped, so call + /// [`Stream::stop`] with your own code: dropping the stream sends code 0. + #[error("evicted")] + Evicted, + /// The stream finished before [`Stream::read_exact`] filled its buffer, after this many bytes. + #[error("stream finished early ({0} bytes read)")] + FinishedEarly(usize), + /// The stream is longer than [`Stream::read_to_end`]'s limit. + #[error("stream too long")] + TooLong, + /// A lease wanted more than the connection receive window can hold. + #[error("want exceeds the receive window")] + TooLarge, + /// The connection was lost. + #[error("connection lost: {0}")] + Connection(#[from] ConnectionError), +} + +impl From for std::io::Error { + fn from(x: Error) -> Self { + let kind = match x { + Error::Reset(_) | Error::Evicted => std::io::ErrorKind::ConnectionReset, + Error::FinishedEarly(_) => std::io::ErrorKind::UnexpectedEof, + Error::TooLong | Error::TooLarge => std::io::ErrorKind::InvalidInput, + Error::Connection(_) => std::io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// What the `web_transport_trait` impls report, which have one error type per session. +impl From for ConnectionError { + fn from(err: Error) -> Self { + match err { + Error::Reset(code) => Self::Reset(code.into_inner()), + Error::Evicted => Self::Evicted, + Error::TooLarge => Self::TooLarge, + Error::Connection(err) => err, + err => Self::Quic(err.to_string()), + } + } +} diff --git a/rs/moq-uring/src/quic/driver/send.rs b/rs/moq-uring/src/quic/driver/send.rs new file mode 100644 index 00000000000..d0ccae1d0bd --- /dev/null +++ b/rs/moq-uring/src/quic/driver/send.rs @@ -0,0 +1,466 @@ +//! Outgoing streams and datagrams: their handles, scheduling, and errors. +//! +//! Single-threaded sans-IO means a write goes straight into moq-quic's send queue (no staging +//! copy); the handles just kick the driver so egress reaches the wire and park on the +//! per-stream waiter lists the driver wakes. + +use std::future::poll_fn; +use std::rc::Rc; +use std::task::{Context, Poll}; + +use bytes::Buf; +use moq_quic::send::End; + +use super::Shared; +use crate::quic::{Error as ConnectionError, VarInt, stream}; + +/// A stream that can only be used to send data. +/// +/// [`finish()`](Self::finish) it to end it cleanly, which hands back a [`Finished`] that can no +/// longer be written. Dropped unfinished, it is [`reset()`](Self::reset), with code 0 unless the +/// peer stopped it, so the peer can't mistake a truncated stream for a complete one. +pub struct Stream(Half); + +/// A stream [`Stream::finish`]ed, which keeps (re)transmitting its data until the peer +/// acknowledges all of it or the connection closes. +/// +/// Dropping it leaves the stream to finish, unless the peer stopped it, in which case it is +/// reset with the peer's code. +pub struct Finished(Half); + +/// A position in a [`Stream`]'s data, minted by [`Stream::offset()`] and good only on that +/// stream, so it is never past the data written. +pub use moq_quic::send::Offset; + +/// What [`Stream`] and [`Finished`] share. +struct Half { + shared: Shared, + id: stream::Id, + /// Retains the waiter registrations of the polls across calls. + park: kio::Park, + state: HalfState, +} + +enum HalfState { + /// Still open: a drop resets at the offset with the code. + Open(Option, VarInt), + Finished, + Reset, +} + +impl Stream { + pub(crate) fn new(shared: Shared, id: stream::Id) -> Self { + Self(Half { + shared, + id, + park: kio::Park::default(), + state: HalfState::Open(None, VarInt::from_u32(0)), + }) + } + + /// Write from `buf` into this stream, returning how many bytes were written. + /// + /// Waits until some of `buf` can be written, then writes as much as it can without waiting + /// again, advancing `buf` past it. Due to congestion and flow control, this may be less than + /// `buf.remaining()`. A [`Bytes`](bytes::Bytes) source is taken without copying. + /// + /// Cancel-safe: if this does not resolve, no bytes were written. + pub async fn write(&mut self, buf: &mut impl Buf) -> Result { + poll_fn(|cx| self.poll_write(cx, buf)).await + } + + /// Write all of `buf` into this stream. + /// + /// Not cancel-safe: some prefix of `buf` may have been written when previously polled. + pub async fn write_all(&mut self, mut buf: impl Buf) -> Result<(), Error> { + while buf.has_remaining() { + self.write(&mut buf).await?; + } + Ok(()) + } + + /// [`Self::write`], as a poll. + pub(crate) fn poll_write(&mut self, cx: &Context<'_>, buf: &mut impl Buf) -> Poll> { + let waiter = self.0.park.hold(cx); + let id = self.0.id; + let shared = &self.0.shared; + if let Some(err) = shared.closed() { + return Poll::Ready(Err(Error::Connection(err))); + } + let written = shared.conn.borrow_mut().send_stream(id).write(buf); + match written { + Ok(written) => { + shared.kick(); + Poll::Ready(Ok(written)) + } + // The driver wakes us when moq-quic reports the stream writable, or stopped. + Err(moq_quic::send::Error::Blocked) => { + shared.park_writable(id, waiter); + Poll::Pending + } + Err(moq_quic::send::Error::Stopped(code)) => Poll::Ready(Err(Error::Stopped(code))), + Err(err) => unreachable!("an open stream failed to write: {err:?}"), + } + } + + /// Notify the peer that no more data will ever be written to this stream. + /// + /// [`Finished::stopped()`] waits for the peer to receive all of it, or learns that it stopped + /// the stream first. + pub fn finish(mut self) -> Finished { + self.0.finish(); + Finished(self.0) + } + + /// Close the send stream immediately. + /// + /// Locally buffered data is dropped, and previously transmitted data will no longer be + /// retransmitted if lost. + pub fn reset(mut self, error_code: VarInt) { + self.0.reset(None, error_code); + } + + /// Close the send stream immediately, but still deliver the data before `offset`. + /// + /// See [`moq_quic::send::Stream::reset_at`]. + /// + /// # Panics + /// - when `offset` came from another stream + pub fn reset_at(mut self, offset: Offset, error_code: VarInt) { + self.0.reset(Some(offset), error_code); + } + + /// The offset just past the data written so far. + pub fn offset(&self) -> Offset { + self.0.shared.conn.borrow_mut().send_stream(self.0.id).offset() + } + + /// Reset at `offset` with `code`, instead of at 0 with code 0, if dropped before `finish`. + pub(crate) fn set_drop_reset(&mut self, offset: Option, code: VarInt) { + self.0.state = HalfState::Open(offset, code); + } + + /// The peer's STOP_SENDING code, if it already stopped the stream. + pub(crate) fn stop_code(&self) -> Option { + match self.0.shared.conn.borrow_mut().send_stream(self.0.id).end() { + Ok(Some(End::Stopped(code))) => Some(code), + _ => None, + } + } + + /// Set the priority within the stream's send group, or session-wide when ungrouped. + /// + /// Higher goes first. + pub fn set_priority(&mut self, priority: i64) { + self.0.set_priority(priority); + } + + /// The priority of the send stream. + pub fn priority(&self) -> i64 { + self.0.priority() + } + + /// Move the stream into a send group, or with `None` back to the ungrouped streams, which + /// go before every group. + /// + /// # Panics + /// - when `group` belongs to another connection + pub fn set_group(&mut self, group: Option<&Group>) { + self.0.set_group(group); + } + + /// Completes when the peer stops the stream. See [`Finished::stopped()`]. + pub async fn stopped(&self) -> Result, Error> { + kio::wait(|waiter| poll_stopped(&self.0.shared, self.0.id, waiter)).await + } + + /// [`Self::stopped`], as a poll. + pub(crate) fn poll_stopped(&mut self, cx: &Context<'_>) -> Poll, Error>> { + poll_stopped(&self.0.shared, self.0.id, self.0.park.hold(cx)) + } + + /// The identity of this stream. + pub fn id(&self) -> stream::Id { + self.0.id + } +} + +impl Finished { + /// Abandon the stream's data the peer has not acknowledged yet. See [`Stream::reset()`]. + pub fn reset(mut self, error_code: VarInt) { + self.0.reset(None, error_code); + } + + /// Abandon the stream's data past `offset`. See [`Stream::reset_at()`]. + /// + /// # Panics + /// - when `offset` came from another stream + pub fn reset_at(mut self, offset: Offset, error_code: VarInt) { + self.0.reset(Some(offset), error_code); + } + + /// Set the priority its data is retransmitted at. See [`Stream::set_priority()`]. + pub fn set_priority(&mut self, priority: i64) { + self.0.set_priority(priority); + } + + /// The priority of the send stream. + pub fn priority(&self) -> i64 { + self.0.priority() + } + + /// Move the stream into a send group. See [`Stream::set_group()`]. + /// + /// # Panics + /// - when `group` belongs to another connection + pub fn set_group(&mut self, group: Option<&Group>) { + self.0.set_group(group); + } + + /// Completes when the peer stops the stream or acknowledges all of its data. + /// + /// Yields `Some` with the stop error code if the peer stops the stream, and `None` once the + /// peer acknowledges receipt of all stream data (although not necessarily the processing of + /// it). Fails only when the connection ends first, since then the caller cannot read success + /// as "every byte arrived". + pub async fn stopped(&self) -> Result, Error> { + kio::wait(|waiter| poll_stopped(&self.0.shared, self.0.id, waiter)).await + } + + /// [`Self::stopped`], as a poll. + pub(crate) fn poll_stopped(&mut self, cx: &Context<'_>) -> Poll, Error>> { + poll_stopped(&self.0.shared, self.0.id, self.0.park.hold(cx)) + } + + /// The identity of this stream. + pub fn id(&self) -> stream::Id { + self.0.id + } +} + +impl Half { + fn finish(&mut self) { + // A STOP_SENDING may have beaten us here, which `stopped` reports. + if self.shared.conn.borrow_mut().send_stream(self.id).finish().is_ok() { + self.shared.kick(); + } + self.state = HalfState::Finished; + } + + fn reset(&mut self, offset: Option, code: VarInt) { + self.state = HalfState::Reset; + let mut conn = self.shared.conn.borrow_mut(); + let mut stream = conn.send_stream(self.id); + // Fails harmlessly once the connection is gone. + let reset = match offset { + Some(offset) => stream.reset_at(offset, code), + None => stream.reset(code), + }; + drop(conn); + if reset.is_ok() { + self.shared.kick(); + } + } + + fn set_priority(&mut self, priority: i64) { + let _ = self + .shared + .conn + .borrow_mut() + .send_stream(self.id) + .set_priority(priority); + } + + fn priority(&self) -> i64 { + self.shared + .conn + .borrow_mut() + .send_stream(self.id) + .priority() + .unwrap_or_default() + } + + fn set_group(&mut self, group: Option<&Group>) { + let group = group.map(Group::id); + let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_group(group); + } +} + +/// Wait for send stream `id` to end: moq-quic keeps how it ended until its handle releases it. +fn poll_stopped(shared: &Shared, id: stream::Id, waiter: &kio::Waiter) -> Poll, Error>> { + let end = shared.conn.borrow_mut().send_stream(id).end(); + match end { + Ok(Some(End::Stopped(code))) => Poll::Ready(Ok(Some(code))), + Ok(Some(_)) => Poll::Ready(Ok(None)), + Ok(None) | Err(_) => match shared.closed() { + Some(err) => Poll::Ready(Err(Error::Connection(err))), + None => { + shared.park_finishing(id, waiter); + Poll::Pending + } + }, + } +} + +impl Drop for Half { + fn drop(&mut self) { + self.shared.forget_send(self.id); + let mut conn = self.shared.conn.borrow_mut(); + let mut stream = conn.send_stream(self.id); + let stopped = match stream.end() { + Ok(Some(End::Stopped(code))) => Some(code), + _ => None, + }; + let reset = match (&self.state, stopped) { + (HalfState::Reset, _) => None, + // Echo the code, as RFC 9000 section 3.5 asks, even after `finish`: the peer reads + // nothing more, so retransmitting the rest is wasted. + (_, Some(code)) => Some(stream.reset(code)), + (HalfState::Finished, None) => None, + (HalfState::Open(offset, code), None) => Some(match offset { + Some(offset) => stream.reset_at(*offset, *code), + None => stream.reset(*code), + }), + }; + stream.release(); + drop(conn); + if reset.is_some_and(|reset| reset.is_ok()) { + self.shared.kick(); + } + } +} + +impl std::fmt::Debug for Stream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Stream").field("id", &self.0.id).finish() + } +} + +impl std::fmt::Debug for Finished { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Finished").field("id", &self.0.id).finish() + } +} + +/// How an outgoing stream or datagram is scheduled: its [`Group`] and priority. +#[derive(Debug, Default, Clone, Copy)] +pub struct Options<'a> { + /// The group it joins, or `None` to go before every group + pub group: Option<&'a Group>, + /// Its priority within the group, or among the ungrouped streams and datagrams + pub priority: i64, +} + +impl<'a> Options<'a> { + /// The options as the connection knows them. + pub(crate) fn to_quic(self) -> moq_quic::send::Options<'a> { + moq_quic::send::Options { + group: self.group.map(Group::id), + priority: self.priority, + } + } +} + +/// A send group on a [`Connection`](super::Connection). +/// +/// Streams that [join](Stream::set_group) a group share bytes with the connection's other +/// groups: a higher priority group preempts every lower one, and groups of equal priority get an +/// equal share of bytes however many streams each has queued. A stream's own priority orders it +/// only against the group's other streams, and a datagram sent in the group is ordered among +/// them by its priority. Streams and datagrams outside any group go before every group. See +/// [`moq_quic::send::Groups`]. +/// +/// Clones share one group, released once the last clone is dropped and its streams finish +/// and its datagrams are sent. +#[derive(Clone)] +pub struct Group(Rc); + +struct GroupInner { + shared: Shared, + group: Option, +} + +impl Group { + pub(crate) fn new(shared: Shared, priority: i64) -> Self { + let group = shared.conn.borrow_mut().send_groups().open(priority); + Self(Rc::new(GroupInner { + shared, + group: Some(group), + })) + } + + /// Change the group's priority, moving any data it has queued along with it. + pub fn set_priority(&self, priority: i64) { + self.0 + .shared + .conn + .borrow_mut() + .send_groups() + .set_priority(self.id(), priority); + } + + /// The group's priority. + pub fn priority(&self) -> i64 { + self.0.shared.conn.borrow_mut().send_groups().priority(self.id()) + } + + /// The group as the connection knows it, which refuses it on any other connection. + fn id(&self) -> &moq_quic::send::Group { + self.0.group.as_ref().expect("only taken on drop") + } +} + +impl Drop for GroupInner { + fn drop(&mut self) { + if let Some(group) = self.group.take() { + self.shared.conn.borrow_mut().send_groups().close(group); + } + } +} + +impl std::fmt::Debug for Group { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Group").finish_non_exhaustive() + } +} + +/// Errors from sending on a stream or as a datagram. +#[derive(Debug, thiserror::Error, Clone)] +#[non_exhaustive] +pub enum Error { + /// The peer is no longer accepting data on this stream, with its application error code. + #[error("sending stopped by peer: error {0}")] + Stopped(VarInt), + /// The datagram is larger than the connection can send. + #[error("datagram too large")] + TooLarge, + /// Datagrams are disabled locally or unsupported by the peer. + #[error("datagrams unsupported")] + Unsupported, + /// The connection was lost. + #[error("connection lost: {0}")] + Connection(#[from] ConnectionError), +} + +impl From for std::io::Error { + fn from(x: Error) -> Self { + let kind = match x { + Error::Stopped(_) => std::io::ErrorKind::ConnectionReset, + Error::TooLarge => std::io::ErrorKind::InvalidInput, + Error::Unsupported => std::io::ErrorKind::Unsupported, + Error::Connection(_) => std::io::ErrorKind::NotConnected, + }; + Self::new(kind, x) + } +} + +/// What the `web_transport_trait` impls report, which have one error type per session. +impl From for ConnectionError { + fn from(err: Error) -> Self { + match err { + Error::Stopped(code) => Self::Stop(code.into_inner()), + Error::Connection(err) => err, + err => Self::Quic(err.to_string()), + } + } +} diff --git a/rs/moq-uring/src/quic/driver/stream.rs b/rs/moq-uring/src/quic/driver/stream.rs deleted file mode 100644 index 6374959a73e..00000000000 --- a/rs/moq-uring/src/quic/driver/stream.rs +++ /dev/null @@ -1,537 +0,0 @@ -//! Stream handles: thin, direct calls into the shared moq-quic connection. -//! -//! Single-threaded sans-IO means a write goes straight into moq-quic's send -//! queue (no staging copy) and a read comes straight out of its reassembly -//! buffer; the handles just kick the driver so egress reaches the wire and -//! park on the per-stream waiter lists the driver wakes. - -use std::task::{Context, Poll, ready}; - -use std::cell::RefMut; - -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use moq_quic::{LeaseError, StreamId, VarInt}; - -use super::super::Error; -use super::{End, Shared}; - -/// An outgoing stream. Dropping it unfinished resets it with code 0. -pub struct SendStream { - shared: Shared, - id: StreamId, - park: kio::Park, - /// The FIN went out; further writes are refused and `poll_closed` waits - /// for the acknowledgement. - fin: bool, - /// We reset the stream; it is as closed as it will ever be. - reset: bool, -} - -impl SendStream { - pub(crate) fn new(shared: Shared, id: StreamId) -> Self { - // The driver records how this stream ends only while a handle holds - // it, so the handle is what announces itself. - shared.track(id); - Self { - shared, - id, - park: kio::Park::default(), - fin: false, - reset: false, - } - } - - /// The QUIC stream id, which the WebTransport layer uses as the session id. - pub(crate) fn id(&self) -> u64 { - self.id.into() - } - - /// Whether the send side is already terminated, so [`Drop`] would do - /// nothing. The WebTransport wrapper asks before mapping a reset of its - /// own, since a finished stream must not be reset instead. - pub(crate) fn ended(&self) -> bool { - self.fin || self.reset - } - - /// How many bytes were written so far, for a later [`reset_code`](Self::reset_code) to deliver. - pub(crate) fn offset(&self) -> u64 { - self.shared.conn.borrow_mut().send_stream(self.id).offset() - } - - /// Set the priority within the stream's send group, or session-wide when ungrouped. - /// - /// Higher goes first. - pub fn set_priority(&mut self, priority: i64) { - let _ = self - .shared - .conn - .borrow_mut() - .send_stream(self.id) - .set_priority(priority); - } - - /// Move the stream into a send group, or with `None` back to the - /// ungrouped streams, which go before every group. - /// - /// # Panics - /// When `group` belongs to another connection. - pub fn set_group(&mut self, group: Option<&super::SendGroup>) { - let group = group.map(super::SendGroup::id); - let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_group(group); - } - - /// [`reset`](web_transport_trait::poll::SendStream::reset) with a - /// full-width code, for the WebTransport HTTP/3 error mapping, that still - /// delivers the first `offset` bytes. - pub(crate) fn reset_code(&mut self, code: u64, offset: Option) { - if self.reset { - return; - } - let code = VarInt::from_u64(code).unwrap_or(VarInt::MAX); - let mut conn = self.shared.conn.borrow_mut(); - // Err means the stream is already gone, which is what we wanted. - let _ = match offset { - Some(offset) => conn.send_stream(self.id).reset_at(offset, code), - None => conn.send_stream(self.id).reset(code), - }; - drop(conn); - self.reset = true; - self.shared.kick(); - } -} - -impl web_transport_trait::poll::SendStream for SendStream { - type Error = Error; - - fn poll_write(&mut self, cx: &mut Context<'_>, mut buf: &[u8]) -> Poll> { - self.poll_write_buf(cx, &mut buf) - } - - fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { - let waiter = self.park.hold(cx); - if self.fin || self.reset { - return Poll::Ready(Err(Error::Quic("stream already finished".to_string()))); - } - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - let result = { - let mut conn = self.shared.conn.borrow_mut(); - let mut stream = conn.send_stream(self.id); - // Bytes taken from `buf` cannot go back, so take only what moq-quic - // accepts. It keeps the chunk by reference, which for a `Bytes` - // source means no copy at all. - match stream.write_limit().min(buf.chunk().len()) { - // Blocked, failing, or nothing to write; the empty write says which. - 0 => stream.write(&[]), - take => stream.write_chunks(&mut [buf.copy_to_bytes(take)]).map(|written| { - debug_assert_eq!(written.bytes, take, "moq-quic accepted less than its write limit"); - written.bytes - }), - } - }; - match result { - Ok(n) => { - self.shared.kick(); - Poll::Ready(Ok(n)) - } - // No capacity right now; the driver wakes us when moq-quic reports - // the stream writable. - Err(moq_quic::WriteError::Blocked) => { - self.shared.park_writable(self.id, waiter); - Poll::Pending - } - Err(moq_quic::WriteError::Stopped(code)) => Poll::Ready(Err(Error::Stop(code.into_inner()))), - Err(moq_quic::WriteError::ClosedStream) => { - Poll::Ready(Err(Error::Quic("stream already finished".to_string()))) - } - } - } - - fn set_priority(&mut self, order: i32) { - // The trait (like W3C sendOrder) sends HIGHER values first, and so - // does moq-quic. - SendStream::set_priority(self, order.into()); - } - - fn finish(&mut self) -> Result<(), Self::Error> { - if self.fin || self.reset { - return Ok(()); - } - match self.shared.conn.borrow_mut().send_stream(self.id).finish() { - Ok(()) => {} - // A STOP_SENDING beat us here. Carry the code like `poll_write` - // does, or `moq_net::Error::from_transport` cannot decode a - // routine cancellation. - Err(moq_quic::FinishError::Stopped(code)) => { - self.reset = true; - return Err(Error::Stop(code.into_inner())); - } - // Already finished or reset, so the FIN it wanted is out. - Err(moq_quic::FinishError::ClosedStream) => {} - } - self.fin = true; - self.shared.kick(); - Ok(()) - } - - fn reset(&mut self, code: u32) { - self.reset_code(u64::from(code), None); - } - - fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { - let waiter = self.park.hold(cx); - if self.reset { - return Poll::Ready(Ok(())); - } - // moq-quic reports a send stream's end as an event, which the driver - // records: an acknowledged FIN, or the peer's STOP_SENDING. - match self.shared.ended(self.id) { - Some(End::Stopped(code)) => Poll::Ready(Err(Error::Stop(code))), - Some(End::Delivered) => Poll::Ready(Ok(())), - // The end never came. If the connection died first the caller - // cannot read success as "every byte arrived". - None => match self.shared.closed() { - Some(err) => Poll::Ready(Err(err)), - None => { - self.shared.park_finishing(self.id, waiter); - Poll::Pending - } - }, - } - } -} - -impl Drop for SendStream { - fn drop(&mut self) { - self.shared.forget_send(self.id); - if !self.fin && !self.reset { - let _ = self - .shared - .conn - .borrow_mut() - .send_stream(self.id) - .reset(VarInt::from_u32(0)); - self.shared.kick(); - } - } -} - -impl std::fmt::Debug for SendStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SendStream").field("id", &self.id).finish() - } -} - -/// One read out of moq-quic's reassembly buffer. -enum Read { - /// Bytes, at most as many as were asked for. - Chunk(Bytes), - /// Every byte up to the FIN has been handed over. - Finished, - /// Nothing buffered; the driver wakes us when that changes. - Blocked, - /// The peer reset the stream with this code. - Reset(u64), -} - -/// An incoming stream. Dropping it unfinished sends STOP_SENDING with code 0. -pub struct RecvStream { - shared: Shared, - id: StreamId, - park: kio::Park, - /// Every byte up to the FIN was read out of moq-quic. - finished: bool, - /// We stopped the stream; no more reads matter. - stopped: bool, -} - -impl RecvStream { - pub(crate) fn new(shared: Shared, id: StreamId) -> Self { - Self { - shared, - id, - park: kio::Park::default(), - finished: false, - stopped: false, - } - } - - /// Lease the stream's unreleased bytes once at least `want` have arrived - /// in order, or the stream ended. - /// - /// The bytes stay in moq-quic's receive buffer, keeping their connection - /// flow control credit, until read through the lease as a [`Buf`] and - /// then [`Lease::release`]d; bytes not read stay for the next lease. - /// Stream credit still advances, so a stream window smaller than a - /// message keeps the message flowing. - /// - /// When the peer has spent the connection window and no reader can make - /// progress, the stream ranked lowest by [`Self::set_priority`] is - /// evicted: its buffered bytes and their credit are discarded and its next - /// lease fails with [`Error::Evicted`]. Bytes it took keep their credit - /// until the stream is stopped. Nothing is evicted while the leased and - /// unread bytes of every stream stay below the connection receive window. - /// - /// Do not mix with the plain reads on one stream, which skip the leased - /// bytes. - pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, Error>> { - let waiter = self.park.hold(cx); - let mut conn = self.shared.conn.borrow_mut(); - let err = match conn.recv_stream(self.id).lease(want) { - Ok(transmit) => { - if transmit.should_transmit() { - self.shared.kick(); - } - return Poll::Ready(Ok(Lease { - conn, - shared: &self.shared, - id: self.id, - })); - } - Err(LeaseError::Blocked(transmit)) => { - drop(conn); - if transmit.should_transmit() { - self.shared.kick(); - } - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - self.shared.park_readable(self.id, waiter); - return Poll::Pending; - } - Err(LeaseError::Reset(code)) => Error::Reset(code.into_inner()), - Err(LeaseError::Evicted) => Error::Evicted, - Err(LeaseError::TooLarge) => Error::TooLarge, - Err(LeaseError::ClosedStream) => { - drop(conn); - self.shared.closed().unwrap_or(Error::Closed) - } - }; - Poll::Ready(Err(err)) - } - - /// Rank this stream for eviction: when the connection window is stuck, - /// the stream with the lowest priority goes first, and the newest of - /// those that tie. - pub fn set_priority(&mut self, priority: i64) { - self.shared - .conn - .borrow_mut() - .recv_stream(self.id) - .set_priority(priority); - } - - /// [`poll_read`](web_transport_trait::poll::RecvStream::poll_read). - fn poll_read_into(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Error>> { - let Some(chunk) = ready!(self.poll_chunk(cx, dst.len()))? else { - return Poll::Ready(Ok(None)); - }; - dst[..chunk.len()].copy_from_slice(&chunk); - Poll::Ready(Ok(Some(chunk.len()))) - } - - /// [`poll_read_chunk`](web_transport_trait::poll::RecvStream::poll_read_chunk). - fn poll_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Error>> { - let waiter = self.park.hold(cx); - if max == 0 { - return Poll::Ready(Ok(Some(Bytes::new()))); - } - if self.finished { - return Poll::Ready(Ok(None)); - } - match Self::read(&self.shared, self.id, max) { - // moq-quic hands out at most `max`. - Read::Chunk(bytes) => Poll::Ready(Ok(Some(bytes))), - Read::Finished => { - self.finished = true; - Poll::Ready(Ok(None)) - } - Read::Blocked => { - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - self.shared.park_readable(self.id, waiter); - Poll::Pending - } - Read::Reset(code) => Poll::Ready(Err(Error::Reset(code))), - } - } - - /// Whether the read side is already terminated, so [`Drop`] would do - /// nothing. - pub(crate) fn ended(&self) -> bool { - self.finished || self.stopped - } - - /// [`stop`](web_transport_trait::poll::RecvStream::stop) with a - /// full-width code, for the WebTransport HTTP/3 error mapping. - pub(crate) fn stop_code(&mut self, code: u64) { - if self.stopped || self.finished { - return; - } - // Err means the stream is already gone, which is what we wanted. - let _ = self - .shared - .conn - .borrow_mut() - .recv_stream(self.id) - .stop(VarInt::from_u64(code).unwrap_or(VarInt::MAX)); - self.stopped = true; - self.shared.kick(); - } - - /// Take up to `max` bytes out of moq-quic's reassembly buffer. - /// - /// Reading is what returns the peer's flow control credit, so a read that - /// owes it a frame kicks the driver. A `max` of 0 takes nothing, and still - /// reports the FIN or a reset once nothing is left before it. - /// - /// Takes the shared state rather than `&mut self` so a caller can hold a - /// waiter from `self.park` across the read. - fn read(shared: &Shared, id: StreamId, max: usize) -> Read { - let mut conn = shared.conn.borrow_mut(); - let mut recv = conn.recv_stream(id); - let mut chunks = match recv.read() { - Ok(chunks) => chunks, - // The stream is gone, so everything it held is already ours. - Err(_) => return Read::Finished, - }; - let read = match chunks.next(max) { - Ok(Some(chunk)) => Read::Chunk(chunk.bytes), - Ok(None) => Read::Finished, - Err(moq_quic::ReadError::Blocked) => Read::Blocked, - Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), - }; - let transmit = chunks.finalize().should_transmit(); - drop(conn); - if transmit { - shared.kick(); - } - read - } -} - -impl web_transport_trait::poll::RecvStream for RecvStream { - type Error = Error; - - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - self.poll_read_into(cx, dst) - } - - fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - self.poll_chunk(cx, max) - } - - fn stop(&mut self, code: u32) { - self.stop_code(u64::from(code)); - } - - fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { - let waiter = self.park.hold(cx); - if self.finished || self.stopped { - return Poll::Ready(Ok(())); - } - // An empty read observes the end without taking the bytes before it, so - // the stream closes once those are read. - match Self::read(&self.shared, self.id, 0) { - Read::Finished => { - self.finished = true; - Poll::Ready(Ok(())) - } - Read::Reset(code) => Poll::Ready(Err(Error::Reset(code))), - Read::Chunk(_) | Read::Blocked => { - if let Some(err) = self.shared.closed() { - return Poll::Ready(Err(err)); - } - self.shared.park_readable(self.id, waiter); - Poll::Pending - } - } - } -} - -impl Drop for RecvStream { - fn drop(&mut self) { - self.shared.forget_recv(self.id); - if !self.finished && !self.stopped { - let _ = self - .shared - .conn - .borrow_mut() - .recv_stream(self.id) - .stop(VarInt::from_u32(0)); - self.shared.kick(); - } - } -} - -impl std::fmt::Debug for RecvStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RecvStream").field("id", &self.id).finish() - } -} - -/// Unreleased bytes borrowed from a [`RecvStream`] by [`RecvStream::poll_lease`], -/// read as a [`Buf`]. -/// -/// Borrows the connection for as long as it lives. -pub struct Lease<'a> { - conn: RefMut<'a, moq_quic::connection::Connection>, - shared: &'a Shared, - id: StreamId, -} - -impl Lease<'_> { - fn leased(&self) -> &moq_quic::Leased { - self.conn.leased(self.id).expect("a lease outlives nothing it borrows") - } - - /// Whether the peer finished the stream right after the leased bytes. - pub fn is_fin(&self) -> bool { - self.leased().is_fin() - } - - /// The bytes not yet read, in order, one contiguous chunk at a time. - pub fn chunks(&self) -> impl Iterator { - self.leased().chunks() - } - - /// Return the credit of every byte read so far, by this lease or an - /// earlier one. - pub fn release(self) { - let Self { mut conn, shared, id } = self; - let transmit = conn.recv_stream(id).release().should_transmit(); - drop(conn); - if transmit { - shared.kick(); - } - } -} - -/// Reading moves past bytes for good, but they keep their credit until -/// [`Lease::release`]. -impl Buf for Lease<'_> { - fn remaining(&self) -> usize { - self.leased().len() - } - - fn chunk(&self) -> &[u8] { - self.leased().chunks().next().unwrap_or_default() - } - - fn advance(&mut self, n: usize) { - self.conn.recv_stream(self.id).advance(n); - } - - fn copy_to_bytes(&mut self, n: usize) -> Bytes { - match n <= self.chunk().len() { - true => self.conn.recv_stream(self.id).take(n), - false => { - let mut bytes = BytesMut::with_capacity(n); - bytes.put((&mut *self).take(n)); - bytes.freeze() - } - } - } -} diff --git a/rs/moq-uring/src/quic/mod.rs b/rs/moq-uring/src/quic/mod.rs index 778c76b550a..c98b15d0b68 100644 --- a/rs/moq-uring/src/quic/mod.rs +++ b/rs/moq-uring/src/quic/mod.rs @@ -31,10 +31,12 @@ pub mod web; mod driver; -pub use driver::{Connection, Lease, RecvStream, SendGroup, SendOptions, SendStream}; +pub use driver::{Connection, recv, send}; pub use endpoint::Endpoint; /// The flow-control limits a live connection can change; see [`Connection::set_limits`]. pub use moq_quic::{Limits, LimitsError}; +/// Stream identity, and the varint every QUIC code is. +pub use moq_quic::{VarInt, stream}; /// A TLS certificate chain and the private key that signs for it, as PEM. /// One value, so neither half can be configured alone. @@ -155,7 +157,7 @@ pub enum Error { #[error("stream stopped: {0}")] Stop(u64), /// The connection window was stuck and this stream was discarded to free - /// it. See [`RecvStream::poll_lease`]. The peer hears of it once the + /// it. See [`recv::Stream::poll_lease`]. The peer hears of it once the /// stream is stopped, so call `stop` with your own code: dropping the /// stream sends code 0. #[error("stream evicted")] diff --git a/rs/moq-uring/src/quic/web.rs b/rs/moq-uring/src/quic/web.rs index 801d48d1a5c..8970f1ab407 100644 --- a/rs/moq-uring/src/quic/web.rs +++ b/rs/moq-uring/src/quic/web.rs @@ -23,7 +23,7 @@ use std::task::{Context, Poll, ready}; use bytes::{Buf, Bytes, BytesMut}; use web_transport_proto as proto; -use super::{Connection, Error}; +use super::{Connection, Error, VarInt, recv, send}; /// The frame type WebTransport bidirectional streams lead with. const FRAME_WEBTRANSPORT: u64 = 0x41; @@ -77,16 +77,16 @@ pub struct Request { /// Closes the connection on every way out of here that is not an answer. guard: Guard, request: proto::ConnectRequest, - send: super::SendStream, - recv: super::RecvStream, + send: send::Stream, + recv: recv::Stream, /// The peer's control and QPACK streams, held open for the session's /// life: closing them reads as tearing the H3 connection down. - held: Vec, + held: Vec, /// Our control stream, same deal. - control: super::SendStream, + control: send::Stream, /// WebTransport streams a pipelining peer opened before its CONNECT was /// answered, headers consumed, keyed by the session id they claimed. - early: Vec<(u64, super::RecvStream)>, + early: Vec<(u64, recv::Stream)>, /// Streams that arrived during the handshake and are still mid-header; /// the session goes on classifying them. pending: Vec, @@ -159,7 +159,7 @@ impl Request { async fn handshake(mut conn: Connection) -> Result { // Our control stream: the SETTINGS advertising WebTransport support. - let mut control = open_uni(&mut conn).await?; + let mut control = conn.open_uni(Default::default()).await?; let mut settings = proto::Settings::default(); settings.enable_webtransport(1); let mut buf = Vec::new(); @@ -191,7 +191,7 @@ impl Request { // cancel a legitimate stream on a handshake that then succeeds. let mut over = false; while !over { - match web_transport_trait::poll::Session::poll_accept_uni(&mut conn, cx) { + match conn.poll_accept_uni(cx) { Poll::Ready(Ok(recv)) => { arrivals += 1; pending.push(PendingUni::new(recv)); @@ -248,7 +248,7 @@ impl Request { // The CONNECT request rides the client's first bidirectional stream, // which arrives first: QUIC creates lower-numbered streams // implicitly, and the accept queue is in id order. - let (send, mut recv) = accept_bi(&mut conn).await?; + let (send, mut recv) = conn.accept_bi().await?; let request = read_connect(&mut recv).await?; Ok(Self { @@ -315,16 +315,15 @@ impl Request { let mut buf = Vec::new(); response.encode(&mut buf).map_err(|err| Error::Web(err.to_string()))?; write_all(&mut self.send, &buf).await?; - web_transport_trait::poll::SendStream::finish(&mut self.send)?; + let mut send = self.send.finish(); // The guard stays armed across the wait below. Cancelling this future // mid-grace would otherwise skip the deliberate close and leak the // connection, which is the very thing the guard is here to prevent. let mut deadline = self.conn.owner().after(CLOSE_GRACE); - let send = &mut self.send; kio::wait(|waiter| { - let mut cx = waiter.context(); - if web_transport_trait::poll::SendStream::poll_closed(send, &mut cx).is_ready() { + let cx = waiter.context(); + if send.poll_stopped(&cx).is_ready() { return Poll::Ready(()); } deadline.poll(waiter) @@ -416,13 +415,13 @@ struct State { pending_uni: Vec, pending_bi: Vec, /// Classified application streams nobody has accepted yet. - ready_uni: VecDeque, - ready_bi: VecDeque<(super::SendStream, super::RecvStream)>, + ready_uni: VecDeque, + ready_bi: VecDeque<(send::Stream, recv::Stream)>, /// Streams held open, never read: control and QPACK, ours and theirs. - held_recv: Vec, - _control_send: Option, + held_recv: Vec, + _control_send: Option, /// The CONNECT stream's send half; taken exactly once by the close path. - connect_send: Option, + connect_send: Option, } /// A MoQ transport over the worker: raw QUIC, or a WebTransport session. @@ -463,27 +462,28 @@ impl Session { /// Open a send group at `priority`, for streams to join with /// [`SendStream::set_group`]. The HTTP/3 streams stay outside every /// group, so they go first. - pub fn send_group(&self, priority: i64) -> super::SendGroup { + pub fn send_group(&self, priority: i64) -> send::Group { self.conn.send_group(priority) } /// Send a datagram scheduled by `options`, like one of its group's streams. /// /// See [`Connection::send_datagram`]. - pub fn send_datagram(&self, payload: Bytes, options: super::SendOptions<'_>) -> Result<(), Error> { + pub fn send_datagram(&self, payload: Bytes, options: send::Options<'_>) -> Result<(), Error> { let Some(web) = &self.web else { - return self.conn.send_datagram(payload, options); + return Ok(self.conn.send_datagram(payload, options)?); }; - let mut framed = BytesMut::with_capacity(web.header_datagram.len() + payload.len()); - framed.extend_from_slice(&web.header_datagram); - framed.extend_from_slice(&payload); - self.conn.send_datagram(framed.freeze(), options).map_err(unmap_err) + // The session id goes ahead of the payload as its own chunk, so neither is copied. + let framed = web.header_datagram.clone().chain(payload); + self.conn + .send_datagram(framed, options) + .map_err(|err| unmap_err(err.into())) } /// Wrap a raw-QUIC connection in a session, with no /// WebTransport layering: streams and datagrams pass through untouched. pub fn raw(conn: Connection) -> Self { - let protocol = web_transport_trait::poll::Session::protocol(&conn).map(str::to_owned); + let protocol = conn.protocol().map(str::to_owned); Self { handles: Rc::new(()), conn, @@ -514,7 +514,7 @@ impl Session { pending, } = request; - let session_id = send.id(); + let session_id = u64::from(send.id()); let mut header_uni = Vec::new(); encode_varint(STREAM_WEBTRANSPORT, &mut header_uni); encode_varint(session_id, &mut header_uni); @@ -618,7 +618,7 @@ impl Session { }; let web = self.web.is_some(); let header = self.web.as_ref().map(|web| web.header_bi.clone()).unwrap_or_default(); - self.opening_bi = Some((SendStream::new(send, web), RecvStream { inner: recv, web }, header)); + self.opening_bi = Some((SendStream::new(send, web), RecvStream::new(recv, web), header)); } web_transport_trait::poll::Session::poll_open_bi(self, cx).map_ok(Some) } @@ -631,20 +631,20 @@ impl web_transport_trait::poll::Session for Session { fn poll_accept_uni(&mut self, cx: &mut Context<'_>) -> Poll> { let Some(web) = self.web.clone() else { - let inner = ready!(web_transport_trait::poll::Session::poll_accept_uni(&mut self.conn, cx))?; - return Poll::Ready(Ok(RecvStream { inner, web: false })); + let inner = ready!(self.conn.poll_accept_uni(cx))?; + return Poll::Ready(Ok(RecvStream::new(inner, false))); }; loop { if let Some(inner) = web.state.borrow_mut().ready_uni.pop_front() { - return Poll::Ready(Ok(RecvStream { inner, web: true })); + return Poll::Ready(Ok(RecvStream::new(inner, true))); } // Adopt every new arrival, then try to classify the pending set. // Each mid-header stream parks the caller in its own stream's // waiters, so progress on any of them re-polls us. let ended = loop { - match web_transport_trait::poll::Session::poll_accept_uni(&mut self.conn, cx) { + match self.conn.poll_accept_uni(cx) { Poll::Ready(Ok(recv)) => web.state.borrow_mut().pending_uni.push(PendingUni::new(recv)), Poll::Ready(Err(err)) => break Some(err), Poll::Pending => break None, @@ -669,23 +669,17 @@ impl web_transport_trait::poll::Session for Session { cx: &mut Context<'_>, ) -> Poll> { let Some(web) = self.web.clone() else { - let (send, recv) = ready!(web_transport_trait::poll::Session::poll_accept_bi(&mut self.conn, cx))?; - return Poll::Ready(Ok(( - SendStream::new(send, false), - RecvStream { - inner: recv, - web: false, - }, - ))); + let (send, recv) = ready!(self.conn.poll_accept_bi(cx))?; + return Poll::Ready(Ok((SendStream::new(send, false), RecvStream::new(recv, false)))); }; loop { if let Some((send, recv)) = web.state.borrow_mut().ready_bi.pop_front() { - return Poll::Ready(Ok((SendStream::new(send, true), RecvStream { inner: recv, web: true }))); + return Poll::Ready(Ok((SendStream::new(send, true), RecvStream::new(recv, true)))); } loop { - match web_transport_trait::poll::Session::poll_accept_bi(&mut self.conn, cx) { + match self.conn.poll_accept_bi(cx) { Poll::Ready(Ok((send, recv))) => web.state.borrow_mut().pending_bi.push(PendingBi::new(send, recv)), Poll::Ready(Err(err)) => return Poll::Ready(Err(unmap_err(err))), Poll::Pending => break, @@ -700,8 +694,7 @@ impl web_transport_trait::poll::Session for Session { fn poll_open_uni(&mut self, cx: &mut Context<'_>) -> Poll> { if self.opening_uni.is_none() { - let inner = ready!(web_transport_trait::poll::Session::poll_open_uni(&mut self.conn, cx)) - .map_err(|err| self.map_err(err))?; + let inner = ready!(self.conn.poll_open_uni(cx, Default::default())).map_err(|err| self.map_err(err))?; let header = self.web.as_ref().map(|web| web.header_uni.clone()).unwrap_or_default(); self.opening_uni = Some((SendStream::new(inner, self.web.is_some()), header)); } @@ -716,11 +709,11 @@ impl web_transport_trait::poll::Session for Session { cx: &mut Context<'_>, ) -> Poll> { if self.opening_bi.is_none() { - let (send, recv) = ready!(web_transport_trait::poll::Session::poll_open_bi(&mut self.conn, cx)) - .map_err(|err| self.map_err(err))?; + let (send, recv) = + ready!(self.conn.poll_open_bi(cx, Default::default())).map_err(|err| self.map_err(err))?; let web = self.web.is_some(); let header = self.web.as_ref().map(|web| web.header_bi.clone()).unwrap_or_default(); - self.opening_bi = Some((SendStream::new(send, web), RecvStream { inner: recv, web }, header)); + self.opening_bi = Some((SendStream::new(send, web), RecvStream::new(recv, web), header)); } let (send, _, header) = self.opening_bi.as_mut().expect("opening"); let result = ready!(send.poll_header(cx, header)); @@ -738,14 +731,10 @@ impl web_transport_trait::poll::Session for Session { fn poll_recv_datagram(&mut self, cx: &mut Context<'_>) -> Poll> { let Some(web) = self.web.clone() else { - return web_transport_trait::poll::Session::poll_recv_datagram(&mut self.conn, cx); + return self.conn.poll_recv_datagram(cx); }; loop { - let datagram = ready!(web_transport_trait::poll::Session::poll_recv_datagram( - &mut self.conn, - cx - )) - .map_err(unmap_err)?; + let datagram = ready!(self.conn.poll_recv_datagram(cx)).map_err(unmap_err)?; // The prefix names the session; anything else is not ours. let mut peek: &[u8] = &datagram; match decode_varint(&mut peek) { @@ -759,7 +748,7 @@ impl web_transport_trait::poll::Session for Session { } fn max_datagram_size(&self) -> usize { - let inner = web_transport_trait::poll::Session::max_datagram_size(&self.conn); + let inner = self.conn.max_datagram_size().unwrap_or(0); match &self.web { Some(web) => inner.saturating_sub(web.header_datagram.len()), None => inner, @@ -772,7 +761,7 @@ impl web_transport_trait::poll::Session for Session { fn close(&mut self, code: u32, reason: &str) { let Some(web) = &self.web else { - return web_transport_trait::poll::Session::close(&mut self.conn, code, reason); + return self.conn.close_code(u64::from(code), reason); }; { @@ -788,7 +777,7 @@ impl web_transport_trait::poll::Session for Session { let connect_send = web.state.borrow_mut().connect_send.take(); let http3 = proto::error_to_http3(code); - let Some(mut send) = connect_send else { + let Some(send) = connect_send else { self.conn.close_code(http3, reason); return; }; @@ -820,25 +809,30 @@ impl web_transport_trait::poll::Session for Session { // connection error, which a browser reports instead of the capsule. let web = web.clone(); self.conn.owner().spawn(async move { - let mut offset = 0; + let mut frame = Bytes::from(frame); + // Taken to finish once the frame is all written, and kept until the close. + let mut send = Some(send); + let mut finished = None; kio::wait(|waiter| { - let mut cx = waiter.context(); + let cx = waiter.context(); // Stop writing once the connection is gone; the deadline is the // only other thing that ends this. - if web_transport_trait::poll::Session::poll_closed(&mut conn, &mut cx).is_ready() { + if conn.poll_closed(&cx).is_ready() { return Poll::Ready(()); } - while offset < frame.len() { - match web_transport_trait::poll::SendStream::poll_write(&mut send, &mut cx, &frame[offset..]) { - Poll::Ready(Ok(n)) => offset += n, - // The stream is unusable, so the connection close below - // is all the peer is going to get. - Poll::Ready(Err(_)) => return Poll::Ready(()), - Poll::Pending => break, + if let Some(stream) = &mut send { + while frame.has_remaining() { + match stream.poll_write(&cx, &mut frame) { + Poll::Ready(Ok(_)) => {} + // The stream is unusable, so the connection close below + // is all the peer is going to get. + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } } - if offset == frame.len() { - let _ = web_transport_trait::poll::SendStream::finish(&mut send); + if !frame.has_remaining() { + finished = send.take().map(send::Stream::finish); } } @@ -847,12 +841,12 @@ impl web_transport_trait::poll::Session for Session { .await; conn.close_code(http3, &reason); // Only now, so the critical streams end with the connection. - drop(web); + drop((web, finished)); }); } fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll { - let err = ready!(web_transport_trait::poll::Session::poll_closed(&mut self.conn, cx)); + let err = ready!(self.conn.poll_closed(cx)); let Some(web) = &self.web else { return Poll::Ready(err); }; @@ -865,7 +859,7 @@ impl web_transport_trait::poll::Session for Session { } fn stats(&self) -> impl web_transport_trait::Stats { - web_transport_trait::poll::Session::stats(&self.conn) + self.conn.transport_stats() } } @@ -875,38 +869,60 @@ impl web_transport_trait::poll::Session for Session { /// reset still delivers the header and the peer can tell which session the /// reset belongs to. pub struct SendStream { - inner: super::SendStream, + state: SendState, /// The end of the WebTransport header on a stream we opened, for a reset /// to deliver. - header: Option, + header: Option, web: bool, } +/// Where the stream is, since `web_transport_trait` finishes and resets through `&mut self`. +enum SendState { + Open(send::Stream), + Finished(send::Finished), + /// Reset, by us or by echoing the peer's stop. + Reset, +} + impl SendStream { - fn new(inner: super::SendStream, web: bool) -> Self { + fn new(mut inner: send::Stream, web: bool) -> Self { + // The driver's raw 0 reads to a browser as an HTTP/3 stream error rather than the + // WebTransport cancellation that dropping a stream means. moq cancels subscriptions by + // dropping, so this is the ordinary path. + if web { + inner.set_drop_reset(None, http3_code(proto::error_to_http3(0))); + } Self { - inner, + state: SendState::Open(inner), header: None, web, } } - fn map(&self, err: Error) -> Error { + fn map(&self, err: impl Into) -> Error { match self.web { - true => unmap_err(err), - false => unmap_legacy(err), + true => unmap_err(err.into()), + false => unmap_legacy(err.into()), } } /// Set the priority within the stream's send group, or session-wide when ungrouped. pub fn set_priority(&mut self, priority: i64) { - self.inner.set_priority(priority); + match &mut self.state { + SendState::Open(stream) => stream.set_priority(priority), + SendState::Finished(stream) => stream.set_priority(priority), + SendState::Reset => {} + } } /// Move the stream into a send group, or with `None` back out. See - /// [`super::SendStream::set_group`]. - pub fn set_group(&mut self, group: Option<&super::SendGroup>) { - self.inner.set_group(group); + /// [`send::Stream::set_group`]. + pub fn set_group(&mut self, group: Option<&send::Group>) { + match &mut self.state { + SendState::Open(stream) => stream.set_group(group), + SendState::Finished(stream) => stream.set_group(group), + SendState::Reset => {} + } } /// Write the WebTransport header of a stream we opened, noting where it @@ -915,104 +931,147 @@ impl SendStream { if !self.web { return Poll::Ready(Ok(())); } - while !header.is_empty() { - let n = ready!(web_transport_trait::poll::SendStream::poll_write( - &mut self.inner, - cx, - header - )) - .map_err(|err| self.map(err))?; - header.advance(n); + let SendState::Open(stream) = &mut self.state else { + unreachable!("the header goes first"); + }; + while header.has_remaining() { + if let Err(err) = ready!(stream.poll_write(cx, header)) { + return Poll::Ready(Err(self.map(err))); + } } - self.header = Some(self.inner.offset()); + let offset = stream.offset(); + stream.set_drop_reset(Some(offset), http3_code(proto::error_to_http3(0))); + self.header = Some(offset); Poll::Ready(Ok(())) } - /// Reset a WebTransport stream, still delivering the header of one we - /// opened. - fn reset_web(&mut self, code: u64) { - self.inner.reset_code(code, self.header.take()); + fn poll_write_any(&mut self, cx: &mut Context<'_>, buf: &mut impl Buf) -> Poll> { + let SendState::Open(stream) = &mut self.state else { + return Poll::Ready(Err(Error::Quic("stream already finished".to_string()))); + }; + match ready!(stream.poll_write(cx, buf)) { + Ok(n) => Poll::Ready(Ok(n)), + Err(err) => Poll::Ready(Err(self.map(err))), + } } } impl web_transport_trait::poll::SendStream for SendStream { type Error = Error; - fn poll_write(&mut self, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - match web_transport_trait::poll::SendStream::poll_write(&mut self.inner, cx, buf) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, - } + fn poll_write(&mut self, cx: &mut Context<'_>, mut buf: &[u8]) -> Poll> { + self.poll_write_any(cx, &mut buf) } fn poll_write_buf(&mut self, cx: &mut Context<'_>, buf: &mut B) -> Poll> { - match web_transport_trait::poll::SendStream::poll_write_buf(&mut self.inner, cx, buf) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, - } + self.poll_write_any(cx, buf) } fn set_priority(&mut self, order: i32) { - self.inner.set_priority(order.into()); + SendStream::set_priority(self, order.into()); } fn finish(&mut self) -> Result<(), Self::Error> { - web_transport_trait::poll::SendStream::finish(&mut self.inner).map_err(|err| self.map(err)) + let stream = match std::mem::replace(&mut self.state, SendState::Reset) { + SendState::Open(stream) => stream, + // Already finished or reset, so the FIN it wanted is out. A finished stream keeps + // its handle, which `poll_closed` and `reset` still need. + done => { + self.state = done; + return Ok(()); + } + }; + // A STOP_SENDING beat us here. Carry the code like `poll_write` does, or + // `moq_net::Error::from_transport` cannot decode a routine cancellation. Dropping the + // stream echoes the code. + if let Some(code) = stream.stop_code() { + return Err(self.map(send::Error::Stopped(code))); + } + self.state = SendState::Finished(stream.finish()); + Ok(()) } fn reset(&mut self, code: u32) { - match self.web { - true => self.reset_web(proto::error_to_http3(code)), - false => web_transport_trait::poll::SendStream::reset(&mut self.inner, code), + let code = match self.web { + true => http3_code(proto::error_to_http3(code)), + false => VarInt::from_u32(code), + }; + match std::mem::replace(&mut self.state, SendState::Reset) { + SendState::Open(stream) => match self.header { + Some(header) => stream.reset_at(header, code), + None => stream.reset(code), + }, + SendState::Finished(stream) => match self.header { + Some(header) => stream.reset_at(header, code), + None => stream.reset(code), + }, + SendState::Reset => {} } } fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { - match web_transport_trait::poll::SendStream::poll_closed(&mut self.inner, cx) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, - } + let stopped = match &mut self.state { + SendState::Open(stream) => ready!(stream.poll_stopped(cx)), + SendState::Finished(stream) => ready!(stream.poll_stopped(cx)), + SendState::Reset => return Poll::Ready(Ok(())), + }; + Poll::Ready(match stopped { + Ok(None) => Ok(()), + Ok(Some(code)) => Err(self.map(send::Error::Stopped(code))), + Err(err) => Err(self.map(err)), + }) } } -impl Drop for SendStream { - fn drop(&mut self) { - // The inner `Drop` resets with a raw 0, which reads to a browser as an - // HTTP/3 stream error rather than the WebTransport cancellation that - // dropping a stream means. moq cancels subscriptions by dropping, so - // this is the ordinary path. - if self.web && !self.inner.ended() { - self.reset_web(proto::error_to_http3(0)); +impl std::fmt::Debug for SendStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.state { + SendState::Open(stream) => stream.fmt(f), + SendState::Finished(stream) => stream.fmt(f), + SendState::Reset => f.write_str("Reset"), } } } -impl std::fmt::Debug for SendStream { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.inner.fmt(f) - } +/// An HTTP/3 error code as a QUIC one; every code HTTP/3 maps to fits. +fn http3_code(code: u64) -> VarInt { + VarInt::from_u64(code).expect("HTTP/3 codes are varints") } /// An incoming stream; its WebTransport header was consumed on accept. pub struct RecvStream { - inner: super::RecvStream, + /// Taken by [`stop`](web_transport_trait::poll::RecvStream::stop), since + /// `web_transport_trait` stops through `&mut self`. + inner: Option, web: bool, } impl RecvStream { - fn map(&self, err: Error) -> Error { - map(self.web, err) + fn new(mut inner: recv::Stream, web: bool) -> Self { + // Same as the send side: the driver stops with a raw 0. + if web { + inner.set_drop_stop(http3_code(proto::error_to_http3(0))); + } + Self { + inner: Some(inner), + web, + } } - /// Lease the stream's unreleased bytes. See [`super::RecvStream::poll_lease`]. - pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, Error>> { + /// Lease the stream's unreleased bytes. See [`recv::Stream::poll_lease`]. + pub fn poll_lease(&mut self, cx: &mut Context<'_>, want: usize) -> Poll, Error>> { let web = self.web; - self.inner.poll_lease(cx, want).map_err(|err| map(web, err)) + match &mut self.inner { + Some(inner) => inner.poll_lease(cx, want).map_err(|err| map(web, err.into())), + None => Poll::Ready(Err(Error::Closed)), + } } - /// Rank this stream for eviction. See [`super::RecvStream::set_priority`]. + /// Rank this stream for eviction. See [`recv::Stream::set_priority`]. pub fn set_priority(&mut self, priority: i64) { - self.inner.set_priority(priority); + if let Some(inner) = &mut self.inner { + inner.set_priority(priority); + } } } @@ -1027,47 +1086,54 @@ fn map(web: bool, err: Error) -> Error { impl web_transport_trait::poll::RecvStream for RecvStream { type Error = Error; - fn poll_read(&mut self, cx: &mut Context<'_>, dst: &mut [u8]) -> Poll, Self::Error>> { - match web_transport_trait::poll::RecvStream::poll_read(&mut self.inner, cx, dst) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, - } + fn poll_read(&mut self, cx: &mut Context<'_>, mut dst: &mut [u8]) -> Poll, Self::Error>> { + let web = self.web; + let Some(inner) = &mut self.inner else { + return Poll::Ready(Err(Error::Closed)); + }; + inner.poll_read(cx, &mut dst).map_err(|err| map(web, err.into())) } fn poll_read_chunk(&mut self, cx: &mut Context<'_>, max: usize) -> Poll, Self::Error>> { - match web_transport_trait::poll::RecvStream::poll_read_chunk(&mut self.inner, cx, max) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, + let web = self.web; + let Some(inner) = &mut self.inner else { + return Poll::Ready(Err(Error::Closed)); + }; + if max == 0 { + return Poll::Ready(Ok(Some(Bytes::new()))); } + inner.poll_read_chunk(cx, max).map_err(|err| map(web, err.into())) } fn stop(&mut self, code: u32) { - match self.web { - true => self.inner.stop_code(proto::error_to_http3(code)), - false => web_transport_trait::poll::RecvStream::stop(&mut self.inner, code), + let code = match self.web { + true => http3_code(proto::error_to_http3(code)), + false => VarInt::from_u32(code), + }; + if let Some(inner) = self.inner.take() { + inner.stop(code); } } fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll> { - match web_transport_trait::poll::RecvStream::poll_closed(&mut self.inner, cx) { - Poll::Ready(Err(err)) => Poll::Ready(Err(self.map(err))), - other => other, - } - } -} - -impl Drop for RecvStream { - fn drop(&mut self) { - // Same as the send side: the inner `Drop` stops with a raw 0. - if self.web && !self.inner.ended() { - self.inner.stop_code(proto::error_to_http3(0)); - } + let web = self.web; + let Some(inner) = &mut self.inner else { + return Poll::Ready(Ok(())); + }; + Poll::Ready(match ready!(inner.poll_closed(cx)) { + Ok(None) => Ok(()), + Ok(Some(code)) => Err(map(web, Error::Reset(code.into_inner()))), + Err(err) => Err(map(web, err.into())), + }) } } impl std::fmt::Debug for RecvStream { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.inner.fmt(f) + match &self.inner { + Some(inner) => inner.fmt(f), + None => f.write_str("Stopped"), + } } } @@ -1136,7 +1202,7 @@ enum VarPoll { } impl VarRead { - fn poll(&mut self, cx: &mut Context<'_>, recv: &mut super::RecvStream) -> Poll { + fn poll(&mut self, cx: &mut Context<'_>, recv: &mut recv::Stream) -> Poll { loop { let need = match self.have { 0 => 1, @@ -1149,11 +1215,7 @@ impl VarRead { } return Poll::Ready(VarPoll::Value(value)); } - match ready!(web_transport_trait::poll::RecvStream::poll_read( - recv, - cx, - &mut self.buf[self.have..need] - )) { + match ready!(recv.poll_read(cx, &mut &mut self.buf[self.have..need])) { Ok(Some(n)) => self.have += n, Ok(None) | Err(_) => return Poll::Ready(VarPoll::End), } @@ -1203,14 +1265,14 @@ enum UniClass { /// An incoming unidirectional stream mid-header. struct PendingUni { - recv: super::RecvStream, + recv: recv::Stream, typ: VarRead, session: VarRead, typ_value: Option, } impl PendingUni { - fn new(recv: super::RecvStream) -> Self { + fn new(recv: recv::Stream) -> Self { Self { recv, typ: VarRead::default(), @@ -1282,15 +1344,15 @@ fn classify_uni(web: &Rc, cx: &mut Context<'_>) -> bool { /// An incoming bidirectional stream mid-header: frame type, then session id. struct PendingBi { - send: super::SendStream, - recv: super::RecvStream, + send: send::Stream, + recv: recv::Stream, typ: VarRead, session: VarRead, typ_value: Option, } impl PendingBi { - fn new(send: super::SendStream, recv: super::RecvStream) -> Self { + fn new(send: send::Stream, recv: recv::Stream) -> Self { Self { send, recv, @@ -1347,29 +1409,17 @@ fn classify_bi(web: &Rc, cx: &mut Context<'_>) -> bool { // ── Handshake plumbing ────────────────────────────────────────────── -async fn open_uni(conn: &mut Connection) -> Result { - std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_open_uni(conn, cx)).await -} - -async fn accept_bi(conn: &mut Connection) -> Result<(super::SendStream, super::RecvStream), Error> { - std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_accept_bi(conn, cx)).await -} - -async fn write_all(send: &mut super::SendStream, mut buf: &[u8]) -> Result<(), Error> { - while !buf.is_empty() { - let n = std::future::poll_fn(|cx| web_transport_trait::poll::SendStream::poll_write(send, cx, buf)).await?; - buf = &buf[n..]; - } - Ok(()) +async fn write_all(send: &mut send::Stream, buf: &[u8]) -> Result<(), Error> { + Ok(send.write_all(buf).await?) } /// Read another chunk into `buf`; `false` once the stream has ended. /// /// Fails rather than letting `buf` grow past [`MESSAGE_LIMIT`], which is what /// bounds a peer that declares a huge frame and never finishes it. -async fn read_some(recv: &mut super::RecvStream, buf: &mut BytesMut) -> Result { +async fn read_some(recv: &mut recv::Stream, buf: &mut BytesMut) -> Result { let mut chunk = [0u8; 4096]; - let n = std::future::poll_fn(|cx| web_transport_trait::poll::RecvStream::poll_read(recv, cx, &mut chunk)).await?; + let n = recv.read(&mut &mut chunk[..]).await?; match n { Some(n) => { if buf.len() + n > MESSAGE_LIMIT { @@ -1384,7 +1434,7 @@ async fn read_some(recv: &mut super::RecvStream, buf: &mut BytesMut) -> Result Result { +async fn read_settings(recv: &mut recv::Stream) -> Result { let mut buf = BytesMut::new(); // The decoder expects the stream to start with its type; re-seed the one // byte classification took. @@ -1403,7 +1453,7 @@ async fn read_settings(recv: &mut super::RecvStream) -> Result Result { +async fn read_connect(recv: &mut recv::Stream) -> Result { let mut buf = BytesMut::new(); loop { let mut peek: &[u8] = &buf; @@ -1423,7 +1473,7 @@ async fn read_connect(recv: &mut super::RecvStream) -> Result, conn: Connection, mut recv: super::RecvStream) { +async fn read_capsules(web: Rc, conn: Connection, mut recv: recv::Stream) { let mut capsules = Capsules::default(); let capsule = loop { match capsules.take() { diff --git a/rs/moq-uring/src/transport.rs b/rs/moq-uring/src/transport.rs index b2430821d64..6b043479fd7 100644 --- a/rs/moq-uring/src/transport.rs +++ b/rs/moq-uring/src/transport.rs @@ -37,16 +37,16 @@ impl Grouped for quic::web::Session { Some(quic::web::Session::streams_evicted(self)) } - fn send_group(&mut self, priority: i64) -> Option { + fn send_group(&mut self, priority: i64) -> Option { Some(quic::web::Session::send_group(self, priority)) } fn send_datagram( &mut self, payload: bytes::Bytes, - options: moq_net::transport::poll::SendOptions<'_, quic::SendGroup>, + options: moq_net::transport::poll::SendOptions<'_, quic::send::Group>, ) -> Result<(), Self::Error> { - let options = quic::SendOptions { + let options = quic::send::Options { group: options.group, priority: options.priority, }; @@ -56,7 +56,7 @@ impl Grouped for quic::web::Session { #[cfg(feature = "quic")] impl GroupedSend for quic::web::SendStream { - type Group = quic::SendGroup; + type Group = quic::send::Group; fn set_group(&mut self, group: &Self::Group) { quic::web::SendStream::set_group(self, Some(group)); @@ -68,16 +68,16 @@ impl GroupedSend for quic::web::SendStream { } #[cfg(feature = "quic")] -impl moq_net::transport::poll::SendGroup for quic::SendGroup { +impl moq_net::transport::poll::SendGroup for quic::send::Group { fn set_priority(&mut self, priority: i64) { - quic::SendGroup::set_priority(self, priority) + quic::send::Group::set_priority(self, priority) } } #[cfg(feature = "quic")] impl moq_net::transport::poll::RecvStream for quic::web::RecvStream { type Error = adapter::Error; - type Lease<'a> = quic::Lease<'a>; + type Lease<'a> = quic::recv::Lease<'a>; fn poll_lease( &mut self, @@ -95,14 +95,14 @@ impl moq_net::transport::poll::RecvStream for quic::web::RecvStream { } #[cfg(feature = "quic")] -impl moq_net::transport::poll::Lease for quic::Lease<'_> { +impl moq_net::transport::poll::Lease for quic::recv::Lease<'_> { fn is_fin(&self) -> bool { - quic::Lease::is_fin(self) + quic::recv::Lease::is_fin(self) } fn chunks(&self) -> impl Iterator { - quic::Lease::chunks(self) + quic::recv::Lease::chunks(self) } fn release(self) { - quic::Lease::release(self) + quic::recv::Lease::release(self) } } diff --git a/rs/moq-uring/tests/integration/credit.rs b/rs/moq-uring/tests/integration/credit.rs index 01b34a1cddb..99f2b03cd68 100644 --- a/rs/moq-uring/tests/integration/credit.rs +++ b/rs/moq-uring/tests/integration/credit.rs @@ -16,7 +16,6 @@ use std::time::Duration; use bytes::{Buf, Bytes}; use moq_uring::{Config, Error, Worker, quic, udp}; -use web_transport_trait::poll::{RecvStream as _, SendStream as _, Session as _}; const ALPN: &str = "moq-uring-credit"; /// The server's connection window. @@ -50,7 +49,7 @@ async fn within(handle: &moq_uring::Handle, what: &str, future: impl Future (usize, bool) { +async fn leased(handle: &moq_uring::Handle, recv: &mut quic::recv::Stream, want: usize) -> (usize, bool) { within( handle, "stream data", @@ -64,7 +63,7 @@ async fn leased(handle: &moq_uring::Handle, recv: &mut quic::RecvStream, want: u } /// Consume the first `n` leased bytes. -async fn consume(handle: &moq_uring::Handle, recv: &mut quic::RecvStream, n: usize) { +async fn consume(handle: &moq_uring::Handle, recv: &mut quic::recv::Stream, n: usize) { within( handle, "stream data", @@ -113,21 +112,19 @@ fn endpoints( } /// Write `len` bytes and the FIN, giving up quietly once the receiver stops. -fn write(handle: &moq_uring::Handle, mut send: quic::SendStream, len: usize) { +fn write(handle: &moq_uring::Handle, mut send: quic::send::Stream, len: usize) { handle.spawn(async move { - let payload = vec![7u8; len]; - let mut buf = &payload[..]; - while !buf.is_empty() { - match std::future::poll_fn(|cx| send.poll_write(cx, buf)).await { - Ok(n) => buf = &buf[n..], - Err(_) => return, - } + if send.write_all(Bytes::from(vec![7u8; len])).await.is_ok() { + let _ = send.finish().stopped().await; } - let _ = send.finish(); - let _ = std::future::poll_fn(|cx| send.poll_closed(cx)).await; }); } +/// Poll `future` once, without a waker. +fn now(future: F) -> Poll { + std::pin::pin!(future).poll(&mut Context::from_waker(Waker::noop())) +} + #[test] fn leases_spend_the_connection_window() { let Some(mut worker) = worker() else { return }; @@ -137,29 +134,23 @@ fn leases_spend_the_connection_window() { worker .block_on(async move { - let mut dialed = client + let dialed = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); - let mut accepted = server.accept().await.expect("accept"); - let send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) - .await - .expect("open_uni"); + let accepted = server.accept().await.expect("accept"); + let send = dialed.open_uni(Default::default()).await.expect("open_uni"); write(&handle, send, WINDOW + 500); - let mut recv = within( - &handle, - "the stream", - std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)), - ) - .await - .expect("accept_uni"); + let mut recv = within(&handle, "the stream", accepted.accept_uni()) + .await + .expect("accept_uni"); // A lease returns nothing, so the peer stops at the window. assert_eq!(leased(&handle, &mut recv, WINDOW).await, (WINDOW, false)); let mut cx = Context::from_waker(Waker::noop()); assert!(matches!( recv.poll_lease(&mut cx, WINDOW + 1).map_ok(|lease| lease.remaining()), - Poll::Ready(Err(quic::Error::TooLarge)) + Poll::Ready(Err(quic::recv::Error::TooLarge)) )); // Far below the usual update threshold, and still enough to unblock. @@ -181,20 +172,18 @@ fn stuck_window_evicts_the_lower_priority() { worker .block_on(async move { - let mut dialed = client + let dialed = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); - let mut accepted = server.accept().await.expect("accept"); + let accepted = server.accept().await.expect("accept"); for _ in 0..2 { - let send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) - .await - .expect("open_uni"); + let send = dialed.open_uni(Default::default()).await.expect("open_uni"); write(&handle, send, WINDOW); } let mut streams = Vec::new(); for _ in 0..2 { - let accept = std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)); + let accept = accepted.accept_uni(); streams.push(within(&handle, "the stream", accept).await.expect("accept_uni")); } let [mut low, mut high] = streams.try_into().expect("two streams"); @@ -225,7 +214,10 @@ fn stuck_window_evicts_the_lower_priority() { }), ) .await; - assert!(matches!(low_done, Some(Err(quic::Error::Evicted))), "{low_done:?}"); + assert!( + matches!(low_done, Some(Err(quic::recv::Error::Evicted))), + "{low_done:?}" + ); assert_eq!(high_done.expect("polled").expect("lease"), WINDOW); }) .expect("worker"); @@ -265,74 +257,50 @@ fn chunked_writes_stop_at_the_window() { worker .block_on(async move { - let mut dialed = client + let dialed = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); - let mut accepted = server.accept().await.expect("accept"); + let accepted = server.accept().await.expect("accept"); - let mut send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) - .await - .expect("open_uni"); + let mut send = dialed.open_uni(Default::default()).await.expect("open_uni"); // The bytes past the window stay in the caller's buffer, untaken, // and moq-quic keeps the ones it took by reference, not as a copy. - let mut cx = Context::from_waker(Waker::noop()); let mut buf = Bytes::copy_from_slice(&payload); assert!(buf.is_unique()); - assert!(matches!( - send.poll_write_buf(&mut cx, &mut buf), - Poll::Ready(Ok(STREAM_WINDOW)) - )); + assert!(matches!(now(send.write(&mut buf)), Poll::Ready(Ok(STREAM_WINDOW)))); assert_eq!(buf, payload.slice(STREAM_WINDOW..)); assert!(!buf.is_unique(), "moq-quic copied the chunk"); - assert!(send.poll_write_buf(&mut cx, &mut buf).is_pending()); + assert!(now(send.write(&mut buf)).is_pending()); assert_eq!(buf, payload.slice(STREAM_WINDOW..)); handle.spawn(async move { - while !buf.is_empty() { - std::future::poll_fn(|cx| send.poll_write_buf(cx, &mut buf)) - .await - .expect("write"); - } - send.finish().expect("finish"); - let _ = std::future::poll_fn(|cx| send.poll_closed(cx)).await; + send.write_all(buf).await.expect("write"); + let _ = send.finish().stopped().await; }); - let mut recv = within( - &handle, - "the stream", - std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)), - ) - .await - .expect("accept_uni"); + let mut recv = within(&handle, "the stream", accepted.accept_uni()) + .await + .expect("accept_uni"); // Straight out of moq-quic, capped at `max`. - let first = within( - &handle, - "stream data", - std::future::poll_fn(|cx| recv.poll_read_chunk(cx, 1000)), - ) - .await - .expect("read") - .expect("not finished"); + let first = within(&handle, "stream data", recv.read_chunk(1000)) + .await + .expect("read") + .expect("not finished"); assert!(!first.is_empty() && first.len() <= 1000); let mut received = first.to_vec(); - while let Some(chunk) = within( - &handle, - "stream data", - std::future::poll_fn(|cx| recv.poll_read_chunk(cx, usize::MAX)), - ) - .await - .expect("read") + while let Some(chunk) = within(&handle, "stream data", recv.read_chunk(usize::MAX)) + .await + .expect("read") { received.extend_from_slice(&chunk); } assert_eq!(received, payload); - within(&handle, "the FIN", std::future::poll_fn(|cx| recv.poll_closed(cx))) - .await - .expect("finished"); + let reset = within(&handle, "the FIN", recv.received_reset()).await; + assert!(matches!(reset, Ok(None)), "{reset:?}"); }) .expect("worker"); } @@ -348,35 +316,25 @@ fn tiny_frames_do_not_pin_receive_buffers() { worker .block_on(async move { - let mut dialed = client + let dialed = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); - let mut accepted = server.accept().await.expect("accept"); + let accepted = server.accept().await.expect("accept"); dialed .send_datagram(Bytes::from_static(b"x"), Default::default()) .expect("send_datagram"); - let datagram = within( - &handle, - "the datagram", - std::future::poll_fn(|cx| accepted.poll_recv_datagram(cx)), - ) - .await - .expect("recv_datagram"); + let datagram = within(&handle, "the datagram", accepted.read_datagram()) + .await + .expect("recv_datagram"); assert!(owns_exactly(datagram)); - let send = std::future::poll_fn(|cx| dialed.poll_open_uni(cx)) - .await - .expect("open_uni"); + let send = dialed.open_uni(Default::default()).await.expect("open_uni"); write(&handle, send, 1); - let mut recv = within( - &handle, - "the stream", - std::future::poll_fn(|cx| accepted.poll_accept_uni(cx)), - ) - .await - .expect("accept_uni"); + let mut recv = within(&handle, "the stream", accepted.accept_uni()) + .await + .expect("accept_uni"); let taken = within( &handle, "stream data", @@ -400,3 +358,48 @@ fn owns_exactly(bytes: Bytes) -> bool { let len = bytes.len(); bytes.try_into_mut().is_ok_and(|bytes| bytes.capacity() == len) } + +/// A held lease borrows nothing from the connection, so the driver and the +/// other streams keep moving while it is out. +#[test] +fn a_held_lease_does_not_stall_the_connection() { + let Some(mut worker) = worker() else { return }; + let handle = worker.handle(); + let (server, client, dial) = endpoints(&handle, WINDOW, WINDOW); + + worker + .block_on(async move { + let dialed = client + .connect(&dial, server.local_addr(), "localhost") + .await + .expect("dial"); + let accepted = server.accept().await.expect("accept"); + + write( + &handle, + dialed.open_uni(Default::default()).await.expect("open_uni"), + 100, + ); + let mut held = within(&handle, "the first stream", accepted.accept_uni()) + .await + .expect("accept_uni"); + let lease = within(&handle, "stream data", held.lease(100)).await.expect("lease"); + + write( + &handle, + dialed.open_uni(Default::default()).await.expect("open_uni"), + WINDOW / 2, + ); + let mut other = within(&handle, "the second stream", accepted.accept_uni()) + .await + .expect("accept_uni"); + let read = within(&handle, "the second stream's data", other.read_to_end(WINDOW)) + .await + .expect("read"); + assert_eq!(read.len(), WINDOW / 2); + + assert_eq!(lease.remaining(), 100); + lease.release(); + }) + .expect("worker"); +} diff --git a/rs/moq-uring/tests/integration/echo.rs b/rs/moq-uring/tests/integration/echo.rs index d084ff1c156..24ccf757bca 100644 --- a/rs/moq-uring/tests/integration/echo.rs +++ b/rs/moq-uring/tests/integration/echo.rs @@ -84,7 +84,7 @@ fn echo_tokio_peer() { .await .expect("connect"); let (mut send, mut recv) = session.open_bi(Default::default()).await.expect("open stream"); - send.write_all(&payload).await.expect("write"); + send.write_all(&payload[..]).await.expect("write"); send.finish().expect("finish"); let echoed = recv.read_to_end(PAYLOAD + 1).await.expect("read"); done_tx.send(()).expect("notify server"); diff --git a/rs/moq-uring/tests/integration/endpoint.rs b/rs/moq-uring/tests/integration/endpoint.rs index 4afc405f08e..3c9f1a5833f 100644 --- a/rs/moq-uring/tests/integration/endpoint.rs +++ b/rs/moq-uring/tests/integration/endpoint.rs @@ -14,7 +14,7 @@ use std::task::Poll; use std::time::Duration; use moq_uring::{Config, Error, Worker, quic, udp}; -use web_transport_trait::poll::{RecvStream as _, SendStream as _, Session as _}; +use web_transport_trait::poll::{SendStream as _, Session as _}; fn worker() -> Option { match Worker::new(Config::default()) { @@ -86,11 +86,7 @@ fn dial_and_accept_share_one_socket() { let a_in = a.accept().await.expect("a accepts b"); for conn in [&a_to_b, &b_in, &b_to_a, &a_in] { - assert_eq!( - web_transport_trait::poll::Session::protocol(conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); } for (conn, peer) in [(&a_to_b, b_addr), (&b_in, a_addr), (&b_to_a, a_addr), (&a_in, b_addr)] { assert_eq!(conn.remote_addr(), peer, "peer address"); @@ -137,7 +133,7 @@ fn dropped_raw_session_closes(explicit: bool) { .connect(&dial_config(), server.local_addr(), "localhost") .await .expect("dial"); - let mut peer = server.accept().await.expect("accept"); + let peer = server.accept().await.expect("accept"); let session = quic::web::Session::raw(conn); let mut remaining = session.clone(); drop(session); @@ -147,13 +143,9 @@ fn dropped_raw_session_closes(explicit: bool) { std::future::poll_fn(|cx| send.poll_write(cx, b"alive")) .await .expect("write"); - let mut recv = std::future::poll_fn(|cx| peer.poll_accept_uni(cx)) - .await - .expect("accept stream"); + let mut recv = peer.accept_uni().await.expect("accept stream"); let mut payload = [0; 5]; - let read = std::future::poll_fn(|cx| recv.poll_read(cx, &mut payload)) - .await - .expect("read"); + let read = recv.read(&mut &mut payload[..]).await.expect("read"); assert_eq!(read, Some(5)); assert_eq!(&payload, b"alive"); let code = if explicit { @@ -164,7 +156,7 @@ fn dropped_raw_session_closes(explicit: bool) { }; drop(remaining); // Stream handles intentionally survive the final session handle. - let error = std::future::poll_fn(|cx| peer.poll_closed(cx)).await; + let error = peer.closed().await; assert!( matches!(error, quic::Error::App { code: actual, .. } if actual == code), "peer saw {error:?}" @@ -215,10 +207,10 @@ fn a_close_reaches_both_ends_with_its_code() { assert!(closes(&outgoing), "outbound dials are not admissions"); let mut accepted = server.accept().await.expect("accept"); - dialed.close(CLOSE_CODE, CLOSE_REASON); + dialed.close(CLOSE_CODE.into(), CLOSE_REASON.as_bytes()); for (end, conn) in [("closer", &mut dialed), ("peer", &mut accepted)] { - match std::future::poll_fn(|cx| conn.poll_closed(cx)).await { + match conn.closed().await { quic::Error::App { code, reason } => { assert_eq!(code, u64::from(CLOSE_CODE), "{end} close code"); assert_eq!(reason, CLOSE_REASON, "{end} close reason"); @@ -398,40 +390,26 @@ fn a_peer_stop_reaches_the_accepted_stream() { worker .block_on(async move { - let mut dialed = client + let dialed = client .connect(&dial_config(), server.local_addr(), "localhost") .await .expect("dial"); - let mut accepted = server.accept().await.expect("accept"); + let accepted = server.accept().await.expect("accept"); // Held, not dropped: dropping the send half would reset it, and // the peer would then see a reset rather than the stop. - let (mut send, mut recv) = std::future::poll_fn(|cx| dialed.poll_open_bi(cx)) - .await - .expect("open_bi"); + let (mut send, recv) = dialed.open_bi(Default::default()).await.expect("open_bi"); // One byte makes the stream readable before the stop arrives. Nothing // is awaited between them, so both frames leave in the same flush. - std::future::poll_fn(|cx| send.poll_write(cx, b"x")) - .await - .expect("write"); - recv.stop(CLOSE_CODE); + send.write_all(&b"x"[..]).await.expect("write"); + recv.stop(CLOSE_CODE.into()); - let (mut send, _recv) = within( - &handle, - "the stopped stream to arrive", - std::future::poll_fn(|cx| accepted.poll_accept_bi(cx)), - ) - .await - .expect("accept_bi"); + let (send, _recv) = within(&handle, "the stopped stream to arrive", accepted.accept_bi()) + .await + .expect("accept_bi"); - match within( - &handle, - "the stop to be reported", - std::future::poll_fn(|cx| send.poll_closed(cx)), - ) - .await - { - Err(quic::Error::Stop(code)) => assert_eq!(code, u64::from(CLOSE_CODE), "stop code"), + match within(&handle, "the stop to be reported", send.stopped()).await { + Ok(Some(code)) => assert_eq!(code, CLOSE_CODE.into(), "stop code"), other => panic!("expected the peer's stop, got {other:?}"), } }) @@ -637,9 +615,10 @@ fn connections_share_one_tx_buffer_fairly() { let first_client = quic::client::connect(first_sock, &dial_config(), server, "localhost") .await .expect("first connection"); - let mut first_server = endpoint.accept().await.expect("first accepted connection"); + let first_server = endpoint.accept().await.expect("first accepted connection"); - let mut send = std::future::poll_fn(|cx| first_server.poll_open_uni(cx)) + let mut send = first_server + .open_uni(Default::default()) .await .expect("open busy stream"); let running = std::rc::Rc::new(std::cell::Cell::new(true)); @@ -656,7 +635,7 @@ fn connections_share_one_tx_buffer_fairly() { while send_running.get() { let mut offset = 0; while offset < chunk.len() { - match std::future::poll_fn(|cx| send.poll_write(cx, &chunk[offset..])).await { + match send.write(&mut &chunk[offset..]).await { Ok(n) => offset += n, // The reader drops its half as the test winds down, which // stops this stream mid-write. Only a stop *before* that is @@ -674,13 +653,12 @@ fn connections_share_one_tx_buffer_fairly() { }); let recv_running = running.clone(); handle.spawn(async move { - let mut first_client = first_client; - let mut recv = std::future::poll_fn(|cx| first_client.poll_accept_uni(cx)) - .await - .expect("accept busy stream"); + let first_client = first_client; + let mut recv = first_client.accept_uni().await.expect("accept busy stream"); let mut chunk = vec![0; 64 * 1024]; while recv_running.get() { - if std::future::poll_fn(|cx| recv.poll_read(cx, &mut chunk)) + if recv + .read(&mut &mut chunk[..]) .await .expect("read busy stream") .is_none() diff --git a/rs/moq-uring/tests/integration/identity.rs b/rs/moq-uring/tests/integration/identity.rs index d6ba92aa8df..6a0b387c8b7 100644 --- a/rs/moq-uring/tests/integration/identity.rs +++ b/rs/moq-uring/tests/integration/identity.rs @@ -126,10 +126,10 @@ fn an_endpoint_runs_on_the_worker_that_adopted_its_socket() { ready.send(()).expect("test alive"); worker .block_on(async move { - let mut conn = quic::client::connect(sock, &dial_config(), addr, "localhost") + let conn = quic::client::connect(sock, &dial_config(), addr, "localhost") .await .expect("dial"); - std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_closed(&mut conn, cx)).await + conn.closed().await }) .expect("client loop") }); @@ -156,14 +156,10 @@ fn an_endpoint_runs_on_the_worker_that_adopted_its_socket() { owner .block_on(async { - let mut accepted = endpoint.accept().await.expect("accept on the owner"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&accepted), - Some(ALPN), - "negotiated ALPN" - ); - web_transport_trait::poll::Session::close(&mut accepted, 0, "done"); - std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_closed(&mut accepted, cx)).await; + let accepted = endpoint.accept().await.expect("accept on the owner"); + assert_eq!(accepted.protocol(), Some(ALPN), "negotiated ALPN"); + accepted.close(moq_uring::quic::VarInt::from_u32(0), b"done"); + accepted.closed().await; }) .expect("owner loop"); diff --git a/rs/moq-uring/tests/integration/mtu.rs b/rs/moq-uring/tests/integration/mtu.rs index 734ef16d97b..32648b505c1 100644 --- a/rs/moq-uring/tests/integration/mtu.rs +++ b/rs/moq-uring/tests/integration/mtu.rs @@ -20,7 +20,6 @@ use std::task::Poll; use std::time::Duration; use moq_uring::{Config, Error, Worker, quic, udp}; -use web_transport_trait::poll::{RecvStream as _, SendStream as _, Session as _}; const ALPN: &str = "moq-uring-mtu"; /// The UDP payload every QUIC path must carry, and moq-quic's floor. @@ -234,32 +233,18 @@ async fn connect( /// Send `len` bytes on a fresh stream from `from` and read them all at `to`. async fn transfer(handle: &moq_uring::Handle, from: &mut quic::Connection, to: &mut quic::Connection, len: usize) { - let mut send = std::future::poll_fn(|cx| from.poll_open_uni(cx)) - .await - .expect("open_uni"); + let mut send = from.open_uni(Default::default()).await.expect("open_uni"); handle.spawn(async move { - let payload = vec![7u8; len]; - let mut buf = &payload[..]; - while !buf.is_empty() { - match std::future::poll_fn(|cx| send.poll_write(cx, buf)).await { - Ok(n) => buf = &buf[n..], - Err(_) => return, - } + if send.write_all(bytes::Bytes::from(vec![7u8; len])).await.is_ok() { + let _ = send.finish().stopped().await; } - let _ = send.finish(); - let _ = std::future::poll_fn(|cx| send.poll_closed(cx)).await; }); let read = within(handle, "the transfer", async { - let mut recv = std::future::poll_fn(|cx| to.poll_accept_uni(cx)) - .await - .expect("accept_uni"); + let mut recv = to.accept_uni().await.expect("accept_uni"); let mut buf = vec![0u8; 64 * 1024]; let mut read = 0; - while let Some(n) = std::future::poll_fn(|cx| recv.poll_read(cx, &mut buf)) - .await - .expect("read") - { + while let Some(n) = recv.read(&mut &mut buf[..]).await.expect("read") { read += n; } read diff --git a/rs/moq-uring/tests/integration/qlog.rs b/rs/moq-uring/tests/integration/qlog.rs index f9145ce9297..ea643d959d5 100644 --- a/rs/moq-uring/tests/integration/qlog.rs +++ b/rs/moq-uring/tests/integration/qlog.rs @@ -83,11 +83,7 @@ fn a_connection_writes_a_trace() { let conn = quic::client::connect(client_sock, &dial, server_addr, "localhost") .await .expect("quic connect"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); }) .expect("worker"); diff --git a/rs/moq-uring/tests/integration/session.rs b/rs/moq-uring/tests/integration/session.rs index 0759d7813ed..7bf2fb10e08 100644 --- a/rs/moq-uring/tests/integration/session.rs +++ b/rs/moq-uring/tests/integration/session.rs @@ -113,11 +113,7 @@ fn lite_session_over_the_worker() { let conn = quic::client::connect(client_sock, &dial, server_addr, "localhost") .await .expect("quic connect"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); let (session, driver) = moq_net::Client::new() .with_subscriber(sub.clone()) @@ -356,11 +352,7 @@ fn configured_roots_verify_the_server() { let conn = quic::client::connect(client_sock, &dial, server_addr, "localhost") .await .expect("quic connect"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); }) .expect("worker"); } @@ -424,8 +416,8 @@ fn required_client_auth_refuses_an_anonymous_client() { // The dial itself was refused; done. Err(_) => {} // Established locally, but the server's refusal closes it. - Ok(mut conn) => { - std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_closed(&mut conn, cx)).await; + Ok(conn) => { + conn.closed().await; } } }) @@ -479,11 +471,7 @@ fn a_root_bundle_is_loaded_whole() { let conn = quic::client::connect(client_sock, &dial, server_addr, "localhost") .await .expect("quic connect"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); }) .expect("worker"); } diff --git a/rs/moq-uring/tests/integration/teardown.rs b/rs/moq-uring/tests/integration/teardown.rs index 7bd13b7ab78..548e9c0f503 100644 --- a/rs/moq-uring/tests/integration/teardown.rs +++ b/rs/moq-uring/tests/integration/teardown.rs @@ -18,7 +18,6 @@ use std::net::UdpSocket; use std::time::Duration; use moq_uring::{Config, Error, Worker, quic, udp}; -use web_transport_trait::poll::Session as _; const ALPN: &str = "moq-uring-teardown"; const CLOSE_CODE: u32 = 42; @@ -113,8 +112,8 @@ fn a_published_close_has_already_left_the_client() { ready.send(()).expect("test alive"); worker .block_on(async move { - let mut conn = endpoint.accept().await.expect("accept"); - std::future::poll_fn(|cx| conn.poll_closed(cx)).await + let conn = endpoint.accept().await.expect("accept"); + conn.closed().await }) .expect("server worker") }); @@ -131,12 +130,12 @@ fn a_published_close_has_already_left_the_client() { client_worker .block_on(async move { - let mut conn = endpoint + let conn = endpoint .connect(&dial_config(), server_addr, "localhost") .await .expect("dial"); - conn.close(CLOSE_CODE, CLOSE_REASON); - std::future::poll_fn(|cx| conn.poll_closed(cx)).await; + conn.close(CLOSE_CODE.into(), CLOSE_REASON.as_bytes()); + conn.closed().await; }) .expect("client worker"); drop(client_worker); diff --git a/rs/moq-uring/tests/integration/web.rs b/rs/moq-uring/tests/integration/web.rs index efbaa0a89be..e22f0a9d3f0 100644 --- a/rs/moq-uring/tests/integration/web.rs +++ b/rs/moq-uring/tests/integration/web.rs @@ -13,7 +13,6 @@ use std::net::UdpSocket; use moq_net::origin; use moq_uring::{Config, Error, Worker, quic, udp}; -use web_transport_trait::Stats as _; use web_transport_trait::poll::{RecvStream as _, SendStream as _, Session as _}; fn worker() -> Option { @@ -81,7 +80,7 @@ type ClientFuture = std::pin::Pin + Send>>; async fn read_all(recv: &mut moq_tokio::quic::web::RecvStream) -> Vec { let mut out = Vec::new(); let mut buf = [0u8; 4096]; - while let Some(n) = recv.read(&mut buf).await.expect("read") { + while let Some(n) = recv.read(&mut &mut buf[..]).await.expect("read") { out.extend_from_slice(&buf[..n]); } out @@ -243,6 +242,51 @@ fn webtransport_accepts_a_buffered_uni_after_the_peer_close() { client.join().expect("client thread"); } +/// An empty stream's FIN closes it with no read, and a second `finish` keeps +/// the first one's stream. +#[test] +fn a_clean_finish_closes_a_stream() { + let Some(mut worker) = worker() else { return }; + let handle = worker.handle(); + let certs = support::certs().expect("certificates"); + let endpoint = h3_endpoint(&handle, &certs); + let client = tokio_client(format!("https://{}/fin", endpoint.local_addr()), |session| { + Box::pin(async move { + let mut send = session.open_uni(Default::default()).await.expect("open uni"); + send.finish().expect("finish"); + let mut recv = session.accept_uni().await.expect("accept uni"); + assert_eq!(read_all(&mut recv).await, PAYLOAD); + session.close(0, "done"); + session.closed().await; + }) + }); + worker + .block_on(async move { + let conn = endpoint.accept().await.expect("accept"); + let request = quic::web::Request::accept(conn).await.expect("handshake"); + let mut session = request + .respond(quic::web::Response::default().with_protocol(PROTO)) + .await + .expect("respond"); + let mut recv = std::future::poll_fn(|cx| session.poll_accept_uni(cx)) + .await + .expect("accept uni"); + let closed = within(&handle, "the FIN", std::future::poll_fn(|cx| recv.poll_closed(cx))).await; + assert!(closed.is_ok(), "{closed:?}"); + + let mut send = std::future::poll_fn(|cx| session.poll_open_uni(cx)) + .await + .expect("open uni"); + write_finish(&mut send, PAYLOAD).await; + send.finish().expect("a second finish"); + let closed = within(&handle, "the ACK", std::future::poll_fn(|cx| send.poll_closed(cx))).await; + assert!(closed.is_ok(), "{closed:?}"); + std::future::poll_fn(|cx| session.poll_closed(cx)).await; + }) + .expect("worker"); + client.join().expect("client thread"); +} + /// A whole moq-lite session over WebTransport: subprotocol negotiation, SETUP /// on the bidirectional stream, announce, subscribe, and a group on a /// unidirectional stream, with the tokio stack as the subscriber. This is the @@ -426,7 +470,7 @@ fn an_abandoned_handshake_closes_the_connection() { worker .block_on(async move { let conn = endpoint.accept().await.expect("accept"); - let mut watch = conn.clone(); + let watch = conn.clone(); let request = quic::web::Request::accept(conn).await.expect("handshake"); let err = request .respond(quic::web::Response::default().with_protocol("never-offered")) @@ -436,12 +480,7 @@ fn an_abandoned_handshake_closes_the_connection() { // Nothing here closed it by hand; the guard on the dropped request // is what does. - within( - &handle, - "the abandoned connection to close", - std::future::poll_fn(|cx| watch.poll_closed(cx)), - ) - .await; + within(&handle, "the abandoned connection to close", watch.closed()).await; }) .expect("worker"); @@ -463,14 +502,9 @@ fn an_unpolled_handshake_closes_the_connection() { worker .block_on(async move { let conn = endpoint.accept().await.expect("accept"); - let mut watch = conn.clone(); + let watch = conn.clone(); drop(quic::web::Request::accept(conn)); - within( - &handle, - "the unpolled handshake's connection to close", - std::future::poll_fn(|cx| watch.poll_closed(cx)), - ) - .await; + within(&handle, "the unpolled handshake's connection to close", watch.closed()).await; }) .expect("worker"); @@ -536,15 +570,15 @@ fn a_dropped_stream_carries_a_webtransport_code() { // The server writes this and then drops the stream unfinished. let mut recv = session.accept_uni().await.expect("accept_uni"); let mut buf = [0u8; 4096]; - let n = recv.read(&mut buf).await.expect("read").expect("payload"); + let n = recv.read(&mut &mut buf[..]).await.expect("read").expect("payload"); assert_eq!(&buf[..n], PAYLOAD); // Tell it we have the payload, so the reset below cannot race it. let mut ack = session.open_uni(Default::default()).await.expect("open_uni"); - ack.write_all(b"ack").await.expect("write"); + ack.write_all(&b"ack"[..]).await.expect("write"); ack.finish().expect("finish"); - let err = recv.read(&mut buf).await.expect_err("the server dropped it"); + let err = recv.read(&mut &mut buf[..]).await.expect_err("the server dropped it"); assert!(matches!(err, moq_tokio::quic::web::ReadError::Reset(0)), "got {err:?}"); // And the other direction: the server drops the read half of this @@ -610,19 +644,15 @@ fn a_dropped_stream_carries_a_webtransport_code() { } /// Write all of `buf` to a raw QUIC stream. -async fn write_raw(send: &mut quic::SendStream, mut buf: &[u8]) { - while !buf.is_empty() { - let n = std::future::poll_fn(|cx| send.poll_write(cx, buf)) - .await - .expect("write"); - buf = &buf[n..]; - } +async fn write_raw(send: &mut quic::send::Stream, buf: &[u8]) { + send.write_all(buf).await.expect("write"); } /// Append whatever arrives next on a raw QUIC stream, failing on its end. -async fn read_more(recv: &mut quic::RecvStream, out: &mut Vec) { +async fn read_more(recv: &mut quic::recv::Stream, out: &mut Vec) { let mut buf = [0u8; 4096]; - let n = std::future::poll_fn(|cx| recv.poll_read(cx, &mut buf)) + let n = recv + .read(&mut &mut buf[..]) .await .expect("read") .expect("the stream ended early"); @@ -646,12 +676,7 @@ fn a_reset_stream_keeps_its_header() { let err = recv.read_chunk(usize::MAX).await.expect_err("the server reset it"); assert!(matches!(err, moq_tokio::quic::web::ReadError::Reset(7)), "got {err:?}"); - session - .open_uni(Default::default()) - .await - .expect("open_uni") - .reset(8) - .expect("reset"); + session.open_uni(Default::default()).await.expect("open_uni").reset(8); session.closed().await; }) }); @@ -799,26 +824,22 @@ fn dropped_session_delivers_close(explicit: bool) { worker .block_on(async { - let mut peer = client + let peer = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); let conn = server.accept().await.expect("accept"); - let mut watch = conn.clone(); + let watch = conn.clone(); // The client half of the handshake: SETTINGS, then the CONNECT. - let mut control = std::future::poll_fn(|cx| peer.poll_open_uni(cx)) - .await - .expect("control stream"); + let mut control = peer.open_uni(Default::default()).await.expect("control stream"); let mut settings = web_transport_proto::Settings::default(); settings.enable_webtransport(1); let mut bytes = Vec::new(); settings.encode(&mut bytes); write_raw(&mut control, &bytes).await; - let (mut connect_send, mut connect_recv) = std::future::poll_fn(|cx| peer.poll_open_bi(cx)) - .await - .expect("CONNECT stream"); + let (mut connect_send, mut connect_recv) = peer.open_bi(Default::default()).await.expect("CONNECT stream"); let url = url::Url::parse(&format!("https://localhost:{}/", server.local_addr().port())).expect("url"); let mut bytes = Vec::new(); web_transport_proto::ConnectRequest::new(url) @@ -850,9 +871,7 @@ fn dropped_session_delivers_close(explicit: bool) { // The server's control stream, read past its SETTINGS so that // anything it reports from here on is the stream ending. - let mut server_control = std::future::poll_fn(|cx| peer.poll_accept_uni(cx)) - .await - .expect("server control stream"); + let mut server_control = peer.accept_uni().await.expect("server control stream"); let mut received = Vec::new(); loop { read_more(&mut server_control, &mut received).await; @@ -894,19 +913,14 @@ fn dropped_session_delivers_close(explicit: bool) { // including a reset of the control stream had the drop caused one. let mut buf = [0u8; 64]; let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); - match server_control.poll_read(&mut cx, &mut buf) { + match std::pin::pin!(server_control.read(&mut &mut buf[..])).poll(&mut cx) { std::task::Poll::Pending => {} other => panic!("the server's control stream ended under the capsule: {other:?}"), } // The peer acts on the capsule, as a browser would. - peer.close(0, ""); - within( - &handle, - "the connection to close", - std::future::poll_fn(|cx| watch.poll_closed(cx)), - ) - .await; + peer.close(0u32.into(), b""); + within(&handle, "the connection to close", watch.closed()).await; drop(control); }) .expect("worker"); @@ -929,38 +943,25 @@ fn cancelling_a_pending_web_handshake(send_settings: bool) { worker .block_on(async { - let mut peer = client + let peer = client .connect(&dial, server.local_addr(), "localhost") .await .expect("dial"); let conn = server.accept().await.expect("accept"); - let mut watch = conn.clone(); + let watch = conn.clone(); let mut control = None; if send_settings { - let before = web_transport_trait::poll::Session::stats(&watch) - .bytes_received() - .expect("counted"); - let mut stream = std::future::poll_fn(|cx| peer.poll_open_uni(cx)) - .await - .expect("control stream"); + let before = watch.stats().udp_rx.bytes; + let mut stream = peer.open_uni(Default::default()).await.expect("control stream"); let mut settings = web_transport_proto::Settings::default(); settings.enable_webtransport(1); let mut bytes = Vec::new(); settings.encode(&mut bytes); - let mut remaining = bytes.as_slice(); - while !remaining.is_empty() { - let n = std::future::poll_fn(|cx| stream.poll_write(cx, remaining)) - .await - .expect("write settings"); - remaining = &remaining[n..]; - } + stream.write_all(bytes.as_slice()).await.expect("write settings"); control = Some(stream); within(&handle, "peer SETTINGS to arrive", async { loop { - if web_transport_trait::poll::Session::stats(&watch) - .bytes_received() - .expect("counted") > before - { + if watch.stats().udp_rx.bytes > before { break; } let mut tick = moq_uring::Timer::after(&handle, std::time::Duration::from_millis(10)); @@ -981,19 +982,9 @@ fn cancelling_a_pending_web_handshake(send_settings: bool) { .await; drop(handshake); - let err = within( - &handle, - "cancelled WebTransport connection to close", - std::future::poll_fn(|cx| watch.poll_closed(cx)), - ) - .await; + let err = within(&handle, "cancelled WebTransport connection to close", watch.closed()).await; assert!(matches!(err, quic::Error::App { code: 0x101, .. }), "got {err:?}"); - within( - &handle, - "peer to receive the close", - std::future::poll_fn(|cx| peer.poll_closed(cx)), - ) - .await; + within(&handle, "peer to receive the close", peer.closed()).await; drop(control); let sibling = within( diff --git a/rs/moq-uring/tests/integration/workers.rs b/rs/moq-uring/tests/integration/workers.rs index d0b14145ed5..0b9938706ae 100644 --- a/rs/moq-uring/tests/integration/workers.rs +++ b/rs/moq-uring/tests/integration/workers.rs @@ -121,9 +121,9 @@ fn a_steered_group_serves_a_shared_port() { handle.spawn(async move { // Each accepted connection is counted, then closed: that // close is how the client learns this side accepted it. - while let Ok(mut conn) = endpoint.accept().await { + while let Ok(conn) = endpoint.accept().await { accepted[usize::from(shard.index())].fetch_add(1, Ordering::AcqRel); - web_transport_trait::poll::Session::close(&mut conn, 0, "done"); + conn.close(moq_uring::quic::VarInt::from_u32(0), b"done"); } }); ready.send(()).expect("test alive"); @@ -150,19 +150,15 @@ fn a_steered_group_serves_a_shared_port() { let socket = handle .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) .expect("client socket"); - let mut conn = quic::client::connect(socket, &dial, addr, "localhost") + let conn = quic::client::connect(socket, &dial, addr, "localhost") .await .expect("connect"); - assert_eq!( - web_transport_trait::poll::Session::protocol(&conn), - Some(ALPN), - "negotiated ALPN" - ); + assert_eq!(conn.protocol(), Some(ALPN), "negotiated ALPN"); // The client's handshake completes a flight before the // server's, so closing here could discard the client's Finished // (still behind the pacer, say) and the server would never // accept. The server's close is the proof it did. - match std::future::poll_fn(|cx| web_transport_trait::poll::Session::poll_closed(&mut conn, cx)).await { + match conn.closed().await { quic::Error::App { code: 0, .. } => {} other => panic!("the client saw {other:?} instead of the server's close"), } diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 5c56c81d02a..536816a1f4d 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -14,8 +14,9 @@ use std::time::Duration; use bytes::{Buf, Bytes}; use moq_clock::Instant; use moq_net::transport::{self as net, Address, poll}; +use moq_quic::stream::{Dir, Id as StreamId}; use moq_quic::{ - Dir, Side, StreamId, TransportError, TransportErrorCode, VarInt, connection::Error as ConnectionError, mux, + Side, TransportError, TransportErrorCode, VarInt, connection::Error as ConnectionError, mux, recv, send, }; use rustc_hash::FxHashMap; @@ -59,8 +60,6 @@ struct State { writable: FxHashMap, /// Send streams waiting for their end: a FIN serialized, or a `STOP_SENDING`. finishing: FxHashMap, - /// Every send stream a handle still holds, and how it ended once that happened. - sends: FxHashMap>, datagram_waiters: kio::WaiterList, @@ -76,15 +75,6 @@ struct State { received_bytes: u64, } -/// How a send stream ended, once the session has seen it happen. -#[derive(Clone, Copy, Debug)] -enum End { - /// The FIN went to the transport, which on a reliable transport is delivery. - Delivered, - /// The peer sent `STOP_SENDING` with this code. - Stopped(u64), -} - impl State { /// Terminate with `err` (the first one wins) and wake absolutely everyone. fn fail(&mut self, err: Error) { @@ -146,9 +136,9 @@ impl State { StreamEvent::Available { .. } => self.open_waiters.wake(), StreamEvent::Readable { id } => wake(&mut self.readable, id), StreamEvent::Writable { id } => wake(&mut self.writable, id), - StreamEvent::Finished { id } => self.end(id, End::Delivered), - StreamEvent::Stopped { id, error_code } => { - self.end(id, End::Stopped(error_code.into_inner())); + StreamEvent::Finished { id } => wake(&mut self.finishing, id), + StreamEvent::Stopped { id, .. } => { + wake(&mut self.finishing, id); // A writer blocked on capacity has to learn it will never come. wake(&mut self.writable, id); } @@ -157,22 +147,11 @@ impl State { } } - /// Record how a send stream ended, and wake whoever is watching it. - /// - /// A stream whose handle already dropped records nothing: nobody is left to read the - /// verdict, and the entry would outlive every use for it. - fn end(&mut self, id: StreamId, end: End) { - if let Some(slot) = self.sends.get_mut(&id) { - *slot = Some(end); - } - wake(&mut self.finishing, id); - } - - /// Drop send stream `id`'s bookkeeping, parking included. + /// Drop send stream `id`'s bookkeeping, parking and its end included. fn forget_send(&mut self, id: StreamId) { - self.sends.remove(&id); self.finishing.remove(&id); self.writable.remove(&id); + self.conn.send_stream(id).release(); } } @@ -491,7 +470,6 @@ impl Session { readable: FxHashMap::default(), writable: FxHashMap::default(), finishing: FxHashMap::default(), - sends: FxHashMap::default(), datagram_waiters: kio::WaiterList::new(), closed: None, closed_waiters: kio::WaiterList::new(), @@ -603,8 +581,8 @@ impl std::fmt::Debug for Session { } /// The options as the connection knows them. -fn to_quic<'a>(options: poll::SendOptions<'a, SendGroup>) -> moq_quic::SendOptions<'a> { - moq_quic::SendOptions { +fn to_quic<'a>(options: poll::SendOptions<'a, SendGroup>) -> send::Options<'a> { + send::Options { group: options.group.map(SendGroup::id), priority: options.priority, } @@ -690,7 +668,7 @@ impl poll::Session for Session { } match state.conn.datagrams().send(payload, to_quic(options)) { Ok(()) => {} - Err(moq_quic::SendDatagramError::TooLarge) => return Err(Error::DatagramTooLarge), + Err(send::Error::TooLarge) => return Err(Error::DatagramTooLarge), Err(_) => return Err(Error::DatagramsUnsupported), } state.driver.wake(); @@ -778,11 +756,11 @@ pub struct SendGroup(Arc); struct GroupInner { inner: Arc, /// Taken only on drop. - id: Option, + id: Option, } impl SendGroup { - fn id(&self) -> &moq_quic::SendGroup { + fn id(&self) -> &send::Group { self.0.id.as_ref().expect("only taken on drop") } } @@ -822,15 +800,6 @@ pub struct SendStream { impl SendStream { fn new(inner: Arc, id: StreamId) -> Self { - { - // Seeded from the connection rather than empty: a peer can stop a stream before - // the application accepts it, and that event had no handle to land on. - let mut state = inner.lock(); - let stopped = state.conn.send_stream(id).stopped().ok().flatten(); - state - .sends - .insert(id, stopped.map(|code| End::Stopped(code.into_inner()))); - } Self { inner, id, @@ -868,27 +837,18 @@ impl poll::SendStream for SendStream { if let Some(err) = &state.closed { return Poll::Ready(Err(err.clone())); } - let mut stream = state.conn.send_stream(self.id); - // Bytes taken from `buf` cannot go back, so take only what the connection accepts. - // It keeps the chunk by reference, which for a `Bytes` source means no copy at all. - let result = match stream.write_limit().min(buf.chunk().len()) { - // Blocked, failing, or nothing to write; the empty write says which. - 0 => stream.write(&[]), - take => stream - .write_chunks(&mut [buf.copy_to_bytes(take)]) - .map(|written| written.bytes), - }; - match result { + // Takes only what the connection accepts, which for a `Bytes` source is no copy at all. + match state.conn.send_stream(self.id).write(buf) { Ok(n) => { state.driver.wake(); Poll::Ready(Ok(n)) } - Err(moq_quic::WriteError::Blocked) => { + Err(send::Error::Blocked) => { waiter.register(state.writable.entry(self.id).or_default()); Poll::Pending } - Err(moq_quic::WriteError::Stopped(code)) => Poll::Ready(Err(Error::StreamStop(code.into_inner()))), - Err(moq_quic::WriteError::ClosedStream) => Poll::Ready(Err(Error::StreamClosed)), + Err(send::Error::Stopped(code)) => Poll::Ready(Err(Error::StreamStop(code.into_inner()))), + Err(_) => Poll::Ready(Err(Error::StreamClosed)), } } @@ -911,12 +871,12 @@ impl poll::SendStream for SendStream { Ok(()) => {} // A STOP_SENDING beat us here; carry its code like `poll_write` does. The stream is // over without our reset, so `poll_closed` reports the stop the sweep recorded. - Err(moq_quic::FinishError::Stopped(code)) => { + Err(send::Error::Stopped(code)) => { self.fin = true; return Err(Error::StreamStop(code.into_inner())); } // Already finished or reset, so the FIN it wanted is out. - Err(moq_quic::FinishError::ClosedStream) => {} + Err(_) => {} } state.driver.wake(); self.fin = true; @@ -932,12 +892,14 @@ impl poll::SendStream for SendStream { return Poll::Ready(Ok(())); } let mut state = self.inner.lock(); - match state.sends.get(&self.id).copied().flatten() { - Some(End::Stopped(code)) => Poll::Ready(Err(Error::StreamStop(code))), - Some(End::Delivered) => Poll::Ready(Ok(())), + // A finish is acknowledged once its FIN goes to the transport, which on a reliable + // transport is delivery. + match state.conn.send_stream(self.id).end() { + Ok(Some(send::End::Stopped(code))) => Poll::Ready(Err(Error::StreamStop(code.into_inner()))), + Ok(Some(_)) => Poll::Ready(Ok(())), // The end never came. If the session died first, the caller cannot read success // as "every byte arrived". - None => match &state.closed { + Ok(None) | Err(_) => match &state.closed { Some(err) => Poll::Ready(Err(err.clone())), None => { waiter.register(state.finishing.entry(self.id).or_default()); @@ -951,11 +913,12 @@ impl poll::SendStream for SendStream { impl Drop for SendStream { fn drop(&mut self) { let mut state = self.inner.lock(); - state.forget_send(self.id); + // Reset before giving up the ID, which refuses every later call on it. if !self.fin && !self.reset { let _ = state.conn.send_stream(self.id).reset(VarInt::from_u32(0)); state.driver.wake(); } + state.forget_send(self.id); } } @@ -1091,8 +1054,9 @@ fn read(state: &mut State, id: StreamId, max: usize) -> Read { let read = match chunks.next(max) { Ok(Some(chunk)) => Read::Chunk(chunk.bytes), Ok(None) => Read::Finished, - Err(moq_quic::ReadError::Blocked) => Read::Blocked, - Err(moq_quic::ReadError::Reset(code)) => Read::Reset(code.into_inner()), + Err(recv::Error::Reset(code)) => Read::Reset(code.into_inner()), + // Blocked, or the connection closed, which the caller reports. + Err(_) => Read::Blocked, }; if chunks.finalize().should_transmit() { state.driver.wake(); diff --git a/rs/qmux/src/session/tests.rs b/rs/qmux/src/session/tests.rs index 2f416b54b2d..4101c332489 100644 --- a/rs/qmux/src/session/tests.rs +++ b/rs/qmux/src/session/tests.rs @@ -197,6 +197,19 @@ async fn resets_and_stops_carry_their_codes() { assert!(matches!(send.closed().await, Err(Error::StreamStop(11)))); } +/// A send stream dropped before `finish` resets with code 0, so its reader learns it ended. +#[tokio::test(start_paused = true)] +async fn dropping_an_unfinished_send_resets_it() { + let (client, server) = pair(config(Version::QMux02), config(Version::QMux02)).await; + + let mut send = client.open_uni(SendOptions::default()).await.unwrap(); + write_all(&mut send, Bytes::from_static(b"partial")).await; + drop(send); + let mut recv = server.accept_uni().await.unwrap(); + let read = moq_clock::timeout(Duration::from_secs(5), read_all(&mut recv)).await; + assert!(matches!(read, Ok(Err(Error::StreamReset(0)))), "{read:?}"); +} + #[tokio::test(start_paused = true)] async fn the_peer_limits_concurrent_streams() { let mut server = config(Version::QMux02); From 00116628c60b85fb6ae4f4dcf24647ec0d79e63b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 3/8] fix(smoke): hold the Cloudflare probe's track reader; gate the runner on moq-dev pairs (#7073) * chore(quest): claim quest/m0/smoke-released.md Quest-Claim: 1a9318c095d636e0 * fix(smoke): hold the Cloudflare probe's track reader The probe dropped its TrackReader, so when the relay rejected an early SUBSCRIBE with DoesNotExist, cloudflare/moq-rs failed to close the track and resolved the subscription as done. The harness then failed the cell instead of retrying the readiness race. Also rewrite the smoke-released quest to the classified runner failures. * fix(smoke): gate the upstream runner on moq-dev pairs only Only moq-dev-rs and moq-dev-js against this checkout's relay fail the Upstream Docker interoperability job; every other pair is reported, as relays.sh treats non-moq-dev relays. A required pair the registry stops pairing fails too. With the Cloudflare probe fix this finishes the quest, so delete it. * fix(smoke): annotate optional runner failures; record the moq-rs close bug Co-authored-by: Claude Opus 5.5 --- .github/workflows/smoke.yml | 5 ++-- quest/m0/README.md | 1 - quest/m0/smoke-released.md | 21 ------------- quest/m2/audit-interop.md | 7 +++++ test/smoke/AGENTS.md | 1 + test/smoke/README.md | 2 +- test/smoke/clients/cloudflare/src/main.rs | 5 +++- test/smoke/runner.py | 20 +++++++++---- test/smoke/runner.test.py | 36 +++++++++++++++++++---- 9 files changed, 61 insertions(+), 37 deletions(-) delete mode 100644 quest/m0/smoke-released.md diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index c3e66ca17a0..c9f9d56228b 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -559,8 +559,9 @@ jobs: path: runner-upstream persist-credentials: false - # Any failed pair or setup step fails the job. A complete run still writes - # its report first, since a failing run is the one worth looking at. + # Only moq-dev clients against this checkout's relay gate the job, plus any + # setup step; the other pairs are reported only. A complete run still + # writes its report first, since a failing run is the one worth looking at. - name: Run upstream Docker matrices run: just runner "$GITHUB_WORKSPACE/runner-upstream" "$GITHUB_WORKSPACE/interop-runner-report" diff --git a/quest/m0/README.md b/quest/m0/README.md index ff21e43c9f7..10981fb27c3 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -26,6 +26,5 @@ edge, which embeds moq-relay; it moved there on 2026-09-28. - [Release shutdown backport](/quest/m0/ffi-shutdown-backport.md) - `release` publishes a non-breaking moq-ffi with an async `drain()` and qmux sessions on the QUIC stream limit - [Backport @moq/flate bounds](/quest/m0/backport-flate-budget.md) - `@moq/flate` on `release` stops inflating past its cap and stops pinning pako scratch - [Backport audio groupDuration](/quest/m0/backport-audio-group-duration.md) - `@moq/publish` 0.5.4 on `release` can pack several Opus frames per group, making Cloudflare's relay drop audio less often -- [Resolve the remaining external Smoke interoperability failures](/quest/m0/smoke-released.md) - every external Docker interop cell in Smoke passes, fixed at its cause - [CI cost and latency](/quest/m0/ci/README.md) - cloud CI runs only in a batched merge gate as one `just ci` pass per batch, expensive checks run nightly and gate `release`, caches move to R2, and runners are chosen last - [More tests under load](/quest/m0/test-flakes-2/README.md) - load-only and queue failures, one quest per flake, fixed at the cause, since each one fails a whole batch diff --git a/quest/m0/smoke-released.md b/quest/m0/smoke-released.md deleted file mode 100644 index ec0167de07c..00000000000 --- a/quest/m0/smoke-released.md +++ /dev/null @@ -1,21 +0,0 @@ -# [S] Resolve the remaining external Smoke interoperability failures - -## Goal - -The Smoke workflow passes on main, with every red cell fixed at its cause. -Smoke clients continue to use the interfaces their installed artifacts expose. - -## Plan - -The Go, C, and Cloudflare clients call their installed interfaces -([moq-dev/moq#6981](https://github.com/moq-dev/moq/pull/6981)), and -`freshness.sh` enforces Go `@latest` and the newest `libmoq`. - -What remains is external Docker interoperability. The -[Smoke run](https://github.com/moq-dev/moq/actions/runs/40608138308) builds -the Docker relay, then its second runner matrix fails 14 of 20 cells. Read -the uploaded reports of the latest normal Smoke run and establish each cause -before changing any protocol behavior. -Do not weaken a lane or retry media exchange to make it pass. - -Public API: none. Wire: none. diff --git a/quest/m2/audit-interop.md b/quest/m2/audit-interop.md index 76584de55fa..ba7f1cb7bc8 100644 --- a/quest/m2/audit-interop.md +++ b/quest/m2/audit-interop.md @@ -30,6 +30,8 @@ Decided with the maintainer on 2026-10-21: through every registered public relay), plus every cell of the runner's report with a moq.dev side: our clients, our docker relay, or `cdn.moq.dev`, including third-party clients failing against our relay. + Smoke's upstream runner job fails only on our clients against our relay, + so this audit is what catches every other red cell. - Sync new registry entries. The smoke workflow's upstream runner job runs the runner's HEAD, so check that newly registered clients and docker-only relays show up in its [report](https://git.moq.dev/interop-runner/), @@ -125,6 +127,11 @@ Theirs, reported or waiting on others: IPv6 address of `cdn.moq.dev` and needs an explicit port in an `https://` URL; and the runner lists moqxr as not run for lack of a published image. Search before filing again. +- To file: cloudflare/moq-rs `SubscribeRecv::error` returns before setting + `closed` when `TrackWriter::close` fails because the track has no reader, + so a REQUEST_ERROR reaches the caller as `done` instead of not-found. + Smoke's Cloudflare probe holds its reader to avoid it + ([moq-dev/moq#7073](https://github.com/moq-dev/moq/pull/7073)). - Not reported, lower value: red5 subscribes the published prefix instead of the requested tuple namespace (no public repo); moqlivemock's relay refuses a SUBSCRIBE under an announced prefix; libquicr answers diff --git a/test/smoke/AGENTS.md b/test/smoke/AGENTS.md index a13bbde01d9..029e926bdcb 100644 --- a/test/smoke/AGENTS.md +++ b/test/smoke/AGENTS.md @@ -11,6 +11,7 @@ Cross-language interop smoke for the published MoQ packages. - A published-package bug is fixed at its source, then goes green here on the next release. - Hang media (`smoke.sh`), Cloudflare, and moxygen stay on separate lanes. - In `relays.sh`, only `--required` relays fail the run. The default required relay is moq-dev's own. +- In `runner.py`, only moq-dev's clients against this checkout's relay fail the run. Every other upstream Docker pair is reported only. - Wire new lanes into `.github/workflows/smoke.yml`. It runs on pull requests that touch `test/smoke/`, and nightly. - New dependencies use the newest stable version, except the pins `freshness.sh` enforces. diff --git a/test/smoke/README.md b/test/smoke/README.md index 7caa4435370..7e9986713bb 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -12,7 +12,7 @@ This suite installs the published moq packages and checks that they exchange med | --- | --- | --- | | Packages | `just smoke`, `just full` | Published clients through a local `moq-relay`. `smoke` is Rust only. `full` is the cross-language matrix. | | Public relays | `just relays` | Rust publisher and subscriber through every relay in the [moq-interop-runner](https://github.com/englishm/moq-interop-runner) registry. CI publishes with Rust and the browser (`js-web`) and subscribes with Rust, `js-web`, and `js-bun`. On those relays `js-web` and `js-bun` run only on WebTransport (`@moq/web-transport` has no raw QUIC mode); a client that does not belong is omitted. WebSocket is its own column for the relays in `WEBSOCKET_KEYS` (`moq-dev-rs` and `stitcher-moq`): each of their `http(s)` endpoints also runs as `ws(s)`, and every other relay keeps that fallback off. A relay that refuses anonymous root broadcasts is dialed on the path `publish_path` names (`stitcher-moq` publishes under `/moq-test`). That run is what the [results page](https://moq-dev.github.io/moq/relays/) shows. Only `--required` relays fail the run; the default is moq-dev's `cdn.moq.dev`, including its WebSocket endpoint. | -| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. Any failed pair or setup step fails the run, after the report is written. CI publishes it to [interop-runner/](https://git.moq.dev/interop-runner/). | +| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. Only a failed pair of a moq-dev client and this checkout's relay, or a failed setup step, fails the run, after the report is written; every other pair is reported only. CI publishes it to [interop-runner/](https://git.moq.dev/interop-runner/). | | Cloudflare | `just cloudflare` | Relays built from the heads of [cloudflare/moq-rs](https://github.com/cloudflare/moq-rs)'s `draft-18-dev` branch and moq-dev/moq's default branch, driven by a Cloudflare client over WebTransport and raw QUIC. | | moxygen | `just moxygen` | Meta's [`moxygen`](https://github.com/facebookexperimental/moxygen) interop client through the latest moq-dev relay. Needs Linux Docker. | | Tokens | `just token`, `just token-full` | `moq auth` and npm [`@moq/auth`](https://www.npmjs.com/package/@moq/auth) mint and verify each other's JWTs, and each verifier rejects a tampered token. | diff --git a/test/smoke/clients/cloudflare/src/main.rs b/test/smoke/clients/cloudflare/src/main.rs index f0fe44201f1..ef0cc67e70c 100644 --- a/test/smoke/clients/cloudflare/src/main.rs +++ b/test/smoke/clients/cloudflare/src/main.rs @@ -267,7 +267,10 @@ async fn probe( .await .context("probe SETUP failed")?; let namespace = TrackNamespace::from_utf8_path(&args.namespace); - let (track_writer, _) = Track::new(namespace, args.track).produce(); + // Hold the reader until the probe ends. Without one, moq-rs cannot close + // the track on REQUEST_ERROR, so a not-found rejection surfaces as `done` + // and the readiness race fails instead of retrying. + let (track_writer, _track_reader) = Track::new(namespace, args.track).produce(); let accept_namespaces = accept_namespaces(subscriber.clone()); tokio::select! { diff --git a/test/smoke/runner.py b/test/smoke/runner.py index 0225cc0809b..37e7ea69358 100644 --- a/test/smoke/runner.py +++ b/test/smoke/runner.py @@ -13,6 +13,10 @@ OWN = ('moq-dev-rs', 'moq-dev-js') RELAY_IMAGE = 'moq-interop-relay:local' +# Only our clients against this checkout's relay gate the run. Every other pair +# depends on a third party or a feature we don't serve, so it is reported but +# optional, as relays.sh treats every relay but moq-dev's. +REQUIRED = {(client, 'moq-dev-rs') for client in OWN} def matrices(registry): @@ -92,15 +96,14 @@ def run(upstream, output): raise ValueError(f'runner checkout already contains results: {results}') combined = None target = None - failed = [] try: for index, plan in enumerate(plans): registry_file.write_text(json.dumps(plan)) expected = expected_pairs(plan) if not expected: raise ValueError(f'runner matrix {index} has no compatible pairs') - if subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream).returncode: - failed.append(index) + # The runner exits nonzero on any red pair; the validated summary decides instead. + subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream) summaries = list(results.glob('*/summary.json')) if len(summaries) != 1: raise ValueError('runner must produce exactly one summary per matrix') @@ -134,9 +137,16 @@ def run(upstream, output): registry_file.write_text(original) subprocess.run(['docker', 'compose', '-f', 'docker-compose.test.yml', 'down', '--timeout', '1'], cwd=upstream, check=True) - # Any failure fails the run, but only after the report shows which pairs failed. + # Fail only after the report shows every pair. A required pair the registry + # no longer pairs fails too, rather than passing unchecked. + status = {(run['client'], run['relay']): run['status'] for run in combined['runs']} + optional = sorted(pair for pair, verdict in status.items() if verdict != 'pass' and pair not in REQUIRED) + if optional: + # A workflow annotation, so a green job still shows them. + print(f'::warning::optional pairs not passing (reported only): {optional}') + failed = sorted(pair for pair in REQUIRED if status.get(pair) != 'pass') if failed: - raise SystemExit(f'runner matrices {failed} exited nonzero; see the report') + raise SystemExit(f'required pairs failed: {failed}; see the report') def main(): diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py index 3f509af9916..2e0dcac0991 100644 --- a/test/smoke/runner.test.py +++ b/test/smoke/runner.test.py @@ -53,7 +53,8 @@ def test_missing_skipped_duplicate_or_unexpected_cells_fail(self): with self.subTest(cells=cells), self.assertRaises(ValueError): runner.validate_summary({'runs': cells}, {('a', 'b')}) - def test_failed_matrix_still_reports_every_matrix_then_fails(self): + def run_matrices(self, status): + """Run both matrices against a fake runner that gives each pair `status(client, relay)`.""" with tempfile.TemporaryDirectory() as temporary: upstream = Path(temporary) registry = upstream / 'implementations.json' @@ -67,24 +68,47 @@ def command(args, **kwargs): pairs = runner.expected_pairs(json.loads(registry.read_text())) result = results / str(len(matrices)) result.mkdir(parents=True) - (result / 'summary.json').write_text(json.dumps({'runs': [ - {'client': client, 'relay': relay, 'mode': 'docker', 'status': 'fail'} - for client, relay in pairs]})) + runs = [{'client': client, 'relay': relay, 'mode': 'docker', + 'status': status(client, relay)} for client, relay in pairs] + (result / 'summary.json').write_text(json.dumps({'runs': runs})) matrices.append(pairs) - return subprocess.CompletedProcess(args, 1 if len(matrices) == 1 else 0) + failed = any(run['status'] != 'pass' for run in runs) + return subprocess.CompletedProcess(args, int(failed)) if args[:2] == ['bash', 'generate-report.sh']: (results / 'index.html').write_text('') (results / 'nightly' / 'report.html').write_text('') return subprocess.CompletedProcess(args, 0) + failure = None with patch.object(runner.subprocess, 'run', side_effect=command): - with self.assertRaises(SystemExit): + try: runner.run(upstream, upstream / 'report') + except SystemExit as error: + failure = error self.assertEqual(len(matrices), 2) self.assertEqual(registry.read_text(), original) runs = json.loads((upstream / 'report' / 'nightly' / 'summary.json').read_text())['runs'] self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices)) self.assertIn('local-adapter', (upstream / 'report' / 'index.html').read_text()) + return failure + + def test_failed_required_pair_reports_every_matrix_then_fails(self): + required = ('moq-dev-js', 'moq-dev-rs') + failure = self.run_matrices(lambda client, relay: 'timeout' if (client, relay) == required else 'pass') + self.assertIn(str(required), str(failure)) + + def test_failed_optional_pairs_are_reported_but_pass(self): + def status(client, relay): + return 'pass' if (client, relay) in runner.REQUIRED else 'fail' + with patch('builtins.print') as report: + failure = self.run_matrices(status) + self.assertIsNone(failure) + self.assertIn("('new', 'moq-dev-rs')", report.call_args.args[0]) + + def test_required_pair_the_registry_drops_fails(self): + self.registry['implementations']['moq-dev-js']['draft_versions'] = ['draft-22'] + self.registry['implementations']['new']['draft_versions'] = ['draft-18', 'draft-22'] + self.assertIn("('moq-dev-js', 'moq-dev-rs')", str(self.run_matrices(lambda client, relay: 'pass'))) def test_drafts_without_overlap_have_no_pair(self): self.registry['implementations']['new']['draft_versions'] = ['draft-22'] From 869836ffa53424308191cb039b72b5024bcfce5b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 4/8] fix(quic): unbreak moq-quic's all-features rustdoc (#7069) * chore(quest): claim quest/m0/nightly/quic-rustdoc.md Quest-Claim: 0a0d799823f82f7f * fix(quic): unbreak moq-quic's all-features rustdoc The fuzz-gated bench map linked to the private SentPackets type, which fails rustdoc under -D warnings. Nightly's features lane already builds workspace rustdoc with --all-features, so no new lane is needed. Co-authored-by: Claude Opus 5.5 --- quest/m0/nightly/README.md | 1 - quest/m0/nightly/quic-rustdoc.md | 16 ---------------- 2 files changed, 17 deletions(-) delete mode 100644 quest/m0/nightly/quic-rustdoc.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index c99f5d9fd5b..a94e14047d5 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -41,4 +41,3 @@ Public API: none. Wire: none. - [Paired bench regressions bisected](/quest/m0/nightly/bench-regressions.md) - retirement/selection's +21-24% and two 10% rows fixed or shown to be noise - [Firefox's player closes every VideoFrame](/quest/m0/nightly/firefox-videoframe.md) - the Media job's Firefox player stops leaking an unclosed VideoFrame - [Gate stops hanging](/quest/m0/nightly/gate-hang.md) - `just gate` finishes instead of hanging to the timeout, and a hang names its module -- [moq-quic rustdoc](/quest/m0/nightly/quic-rustdoc.md) - `--all-features` rustdoc for moq-quic builds and a lane checks it diff --git a/quest/m0/nightly/quic-rustdoc.md b/quest/m0/nightly/quic-rustdoc.md deleted file mode 100644 index 837ceba890f..00000000000 --- a/quest/m0/nightly/quic-rustdoc.md +++ /dev/null @@ -1,16 +0,0 @@ -# [XS] moq-quic's all-features rustdoc builds - -## Goal - -`cargo doc -p moq-quic --all-features` succeeds; today it fails on -`SentPackets`, already broken on `main` before -[moq-dev/moq#6950](https://github.com/moq-dev/moq/pull/6950). - -## Plan - -Fix the doc build. The gate's Rust lint builds workspace rustdoc without -`--all-features`, and `rs doctest` runs `cargo test --doc`, so nothing builds -this. Add `cargo doc -p moq-quic --all-features` to Nightly's feature lane -(`just rs features`), and name the lane in the PR. - -Public API: none. From 0ab2441d1cad3c998de25132884934bb0e68c992 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 5/8] test: stop tracing-test forcing TRACE across the build (#7071) * chore(quest): claim quest/m0/ci/tracing-filter.md Quest-Claim: a26a0b73687228f9 * wip: stop forcing tracing-test's no-env-filter * chore(quest): finish quest/m0/ci/tracing-filter.md * test: keep captured spans enabled beside parallel tests With one dispatcher registered, tracing-core caches a callsite's interest from the thread that first hits it. A parallel test without a subscriber then disabled the `srt` ingest span for the capturing test, so `publisher_reports_a_silent_pid` failed about two runs in three under `cargo test`. A second, no-op dispatcher makes every callsite ask the current thread's subscriber. Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 2 +- quest/m0/ci/README.md | 1 - quest/m0/ci/tracing-filter.md | 16 ---- rs/moq-hls/Cargo.toml | 2 +- rs/moq-mux/Cargo.toml | 2 +- rs/moq-rtmp/Cargo.toml | 2 +- rs/moq-rtsp/Cargo.toml | 2 +- rs/moq-srt/Cargo.toml | 2 +- rs/moq-srt/src/ts.rs | 67 ++++++++++----- rs/moq-tokio/Cargo.toml | 2 +- rs/moq-tokio/tests/integration/backend.rs | 23 +----- rs/moq-tokio/tests/integration/broadcast.rs | 91 +++------------------ rs/moq-tokio/tests/integration/main.rs | 48 +++++++++++ 13 files changed, 114 insertions(+), 146 deletions(-) delete mode 100644 quest/m0/ci/tracing-filter.md diff --git a/Cargo.lock b/Cargo.lock index 54f44dab4d4..220f432804f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5919,7 +5919,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", - "tracing-test", + "tracing-subscriber", ] [[package]] diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 464888ab79b..e650572c619 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -135,7 +135,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Unify dependency features](/quest/m0/ci/hakari.md) - a cargo-hakari workspace-hack so every job compiles each dependency alike and mbx hits across jobs - [Fewer BTreeMap instantiations](/quest/m0/ci/btree-keys.md) - moq-net shares BTreeMap key and value types, kept only if its CPU time drops - [Halve the slowest tests' CPU](/quest/m0/ci/slow-tests.md) - the slowest 1% of tests take under half their 289 CPU-seconds, with coverage kept -- [tracing-test filter](/quest/m0/ci/tracing-filter.md) - moq-tokio's tests stop forcing every crate to format at TRACE - [Stats oversize](/quest/m0/ci/stats-oversize.md) - moq-stats refuses an oversized plain snapshot before parsing it back - [Gate send-depth](/quest/m0/ci/send-depth-gate.md) - the merge gate runs send-depth on a pinned nightly when moq-net changes, after #6890 fixes main - [Cargo aliases in nested checkouts](/quest/m0/ci/cargo-alias-strings.md) - `cargo build-ffi` and `cargo moq` work from a worktree nested inside another checkout diff --git a/quest/m0/ci/tracing-filter.md b/quest/m0/ci/tracing-filter.md deleted file mode 100644 index 876c1778c7e..00000000000 --- a/quest/m0/ci/tracing-filter.md +++ /dev/null @@ -1,16 +0,0 @@ -# [XS] tracing-test stops forcing TRACE everywhere - -## Goal - -moq-tokio's tests no longer turn on `tracing-test`'s `no-env-filter` feature -for the whole build, so other crates' tests stop formatting every event at -TRACE, which costs 11 to 17% of the moq-tokio integration tests' CPU. - -## Plan - -Found by [moq-dev/moq#6987](https://github.com/moq-dev/moq/pull/6987)'s -profile. Cargo feature unification enables the feature for every crate that -shares the build. Narrow it to the tests that need it, or filter per test, -and measure the integration tests before and after. - -Public API: none. diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index 85d2c4c6acf..143ee06b55a 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -46,7 +46,7 @@ moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "quic", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-test = { workspace = true } [lints] workspace = true diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index b4616bc7462..ceaabe8420b 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -50,7 +50,7 @@ futures = { workspace = true } moq-clock = { workspace = true, features = ["sim"] } proptest = { workspace = true } tokio = { workspace = true, features = ["test-util"] } -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-test = { workspace = true } # The ts_schedule bench pins glibc's heap. [target.'cfg(all(target_os = "linux", target_env = "gnu"))'.dev-dependencies] diff --git a/rs/moq-rtmp/Cargo.toml b/rs/moq-rtmp/Cargo.toml index 5e296e4b6fa..052203b6f0c 100644 --- a/rs/moq-rtmp/Cargo.toml +++ b/rs/moq-rtmp/Cargo.toml @@ -61,7 +61,7 @@ moq-clock = { path = "../moq-clock", features = ["sim"] } # what hid moq-tokio's backend-less build from the nightly feature matrix. moq-tokio = { path = "../moq-tokio", default-features = false, features = ["aws-lc-rs", "net"] } tokio = { workspace = true, features = ["test-util"] } -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-test = { workspace = true } [[bench]] name = "chunk_interleaving" diff --git a/rs/moq-rtsp/Cargo.toml b/rs/moq-rtsp/Cargo.toml index 29ffbee6b2f..a7f78c1ee9a 100644 --- a/rs/moq-rtsp/Cargo.toml +++ b/rs/moq-rtsp/Cargo.toml @@ -40,7 +40,7 @@ base64 = { workspace = true } # A clock only the stall tests move, so a loaded runner never fires a deadline early. moq-clock = { path = "../moq-clock", features = ["manual"] } tokio = { workspace = true, features = ["full", "test-util"] } -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-test = { workspace = true } [lints] workspace = true diff --git a/rs/moq-srt/Cargo.toml b/rs/moq-srt/Cargo.toml index 265cef7bf79..105b0699f81 100644 --- a/rs/moq-srt/Cargo.toml +++ b/rs/moq-srt/Cargo.toml @@ -29,7 +29,7 @@ tracing = { workspace = true } # Builds the synthetic broadcasts the egress tests mux back to TS. hang = { path = "../hang" } srt-protocol = "0.4" -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-subscriber = { workspace = true } [lints] workspace = true diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 2d496d0a47b..7f9a35a045f 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -773,35 +773,60 @@ mod tests { /// A video PID that goes silent behind a running mux is logged against its ingest path, /// and neither the same feed intact nor the audio that kept delivering is. #[tokio::test(start_paused = true)] - #[tracing_test::traced_test] async fn publisher_reports_a_silent_pid() { const VIDEO: u16 = 0x100; + // `moq-mux` logs the report, and `traced_test` keeps only this crate's lines. The + // current-thread runtime runs the ingest on this thread, where this subscriber is. + #[derive(Clone, Default)] + struct Logs(std::sync::Arc>>); + impl std::io::Write for Logs { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let logs = Logs::default(); + let writer = logs.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + // With one dispatcher registered, tracing caches each callsite's interest from the + // first thread to hit it, so a parallel test without a subscriber could disable the + // ingest span for good. A second one leaves every thread to ask its own subscriber. + let _second = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default()); + let origin = produce_origin(); ingest(&origin, "intact", &[BBB5S]).await; let stimulus = suppress(BBB5S, VIDEO, Duration::from_millis(1500), Duration::from_millis(3500)); ingest(&origin, "silent", &[&stimulus]).await; - logs_assert(|lines: &[&str]| { - let stopped: Vec<_> = lines + let logs = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + let stopped: Vec<_> = logs + .lines() + .filter(|line| line.contains("stopped delivering access units")) + .collect(); + let reported = |path: &str, pid: u16| { + stopped .iter() - .filter(|line| line.contains("stopped delivering access units")) - .collect(); - let reported = |path: &str, pid: u16| { - stopped - .iter() - .any(|line| line.contains(&format!("path={path}")) && line.contains(&format!("pid={pid} "))) - }; - if !reported("silent", VIDEO) { - return Err(format!("the silent video PID was not reported: {stopped:?}")); - } - if reported("intact", VIDEO) { - return Err(format!("the intact feed's video was reported: {stopped:?}")); - } - if reported("silent", 0x101) || reported("intact", 0x101) { - return Err(format!("the audio kept delivering: {stopped:?}")); - } - Ok(()) - }); + .any(|line| line.contains(&format!("path={path}")) && line.contains(&format!("pid={pid} "))) + }; + assert!( + reported("silent", VIDEO), + "the silent video PID was not reported: {stopped:?}" + ); + assert!( + !reported("intact", VIDEO), + "the intact feed's video was reported: {stopped:?}" + ); + assert!( + !reported("silent", 0x101) && !reported("intact", 0x101), + "the audio kept delivering: {stopped:?}" + ); } /// SRT is a contribution protocol, so SCTE-35 cues survive ingest and egress. diff --git a/rs/moq-tokio/Cargo.toml b/rs/moq-tokio/Cargo.toml index acf8a04d277..56602658250 100644 --- a/rs/moq-tokio/Cargo.toml +++ b/rs/moq-tokio/Cargo.toml @@ -152,7 +152,7 @@ tempfile = "3" tokio = { workspace = true, features = ["test-util"] } tokio-rustls = { workspace = true } toml = "1.1" -tracing-test = { workspace = true, features = ["no-env-filter"] } +tracing-test = { workspace = true } # The examples and integration tests run MoQ sessions, which a driver-only build lacks. [[example]] diff --git a/rs/moq-tokio/tests/integration/backend.rs b/rs/moq-tokio/tests/integration/backend.rs index cc3f9ca6882..e0356b9b453 100644 --- a/rs/moq-tokio/tests/integration/backend.rs +++ b/rs/moq-tokio/tests/integration/backend.rs @@ -337,6 +337,7 @@ async fn cert_sources_test() { /// set follows, without the listener being rebuilt. #[cfg(feature = "quic")] async fn reload_test() { + let logs = crate::Logs::capture(); let dir = tempfile::tempdir().expect("tempdir"); let (cert, key) = write_self_signed(dir.path(), "server", "localhost"); @@ -372,10 +373,7 @@ async fn reload_test() { .await .expect("certificate reload timed out"); - assert!( - tracing_test::internal::logs_with_scope_contain("moq_tokio", "reloading server certificates"), - "no reload log" - ); + assert!(logs.text().contains("reloading server certificates"), "no reload log"); assert_eq!(reloaded.len(), 1); drop(server); } @@ -523,7 +521,6 @@ async fn mtls_test(scheme: &str, reject: bool) { // ── Iroh backend ──────────────────────────────────────────────────── #[cfg(feature = "iroh")] -#[tracing_test::traced_test] #[tokio::test] async fn iroh_connect() { iroh_connect_test(None).await; @@ -532,7 +529,6 @@ async fn iroh_connect() { /// `moq-lite-07-wip` is opt-in, so it is on the wire only when both ends configure it. /// iroh once built its ALPNs from the default constant and ignored that opt-in. #[cfg(feature = "iroh")] -#[tracing_test::traced_test] #[tokio::test] async fn iroh_connect_lite_07_wip() { iroh_connect_test(Some("moq-lite-07-wip")).await; @@ -541,7 +537,6 @@ async fn iroh_connect_lite_07_wip() { /// A dialer and a listener that share no version never connect, instead of the /// listener falling back to the default set. #[cfg(feature = "iroh")] -#[tracing_test::traced_test] #[tokio::test] async fn iroh_versions_disjoint() { use moq_tokio::iroh::Config as IrohConfig; @@ -741,7 +736,6 @@ async fn iroh_connect_test(version: Option<&str>) { /// A client that closes before its runtime stops tells the server at once, instead of /// leaving it to the idle timeout, which is what a process exiting on a signal does. #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_client_close_reaches_server() { // An idle timeout inside TIMEOUT would hide a lost close. @@ -803,7 +797,6 @@ async fn quic_client_close_reaches_server() { /// A client that finishes its track and closes before its runtime stops still delivers /// the queued group and the track's finish, retaining its token-only grant through the drain. #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_client_close_drains_finished_track() { // Several congestion windows, so the token must cover data still in flight @@ -941,7 +934,6 @@ async fn quic_client_close_drains_finished_track() { /// A GOAWAY leaves the old session serving its subscriptions after the replacement /// connects, and a close then drains that predecessor too rather than dropping it. #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_client_close_drains_migrated_predecessor() { // Several congestion windows, so it is still in flight when the close starts. @@ -1089,7 +1081,6 @@ async fn quic_client_close_drains_migrated_predecessor() { /// A close cuts a predecessor that cannot drain off at its handover deadline, /// rather than holding it for the full one second close window. #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_client_close_keeps_predecessor_handover() { let quic = moq_tokio::quic::Config::default(); @@ -1198,70 +1189,60 @@ async fn quic_client_close_keeps_predecessor_handover() { } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_raw_quic() { backend_test("moqt").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_raw_quic_no_sni() { no_sni_test("moqt").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_raw_quic_path() { path_test("moqt").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_raw_quic_moql_path() { path_test("moql").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_webtransport_path() { path_test("https").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_webtransport() { backend_test("https").await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_mtls() { mtls_test("https", false).await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_sni_certificate() { sni_test().await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_cert_sources() { cert_sources_test().await; } #[cfg(feature = "quic")] -#[tracing_test::traced_test] #[tokio::test] async fn quic_cert_reload() { reload_test().await; diff --git a/rs/moq-tokio/tests/integration/broadcast.rs b/rs/moq-tokio/tests/integration/broadcast.rs index a55e77baf86..394bfb2879f 100644 --- a/rs/moq-tokio/tests/integration/broadcast.rs +++ b/rs/moq-tokio/tests/integration/broadcast.rs @@ -243,7 +243,6 @@ async fn lite05_timestamp_roundtrip(scheme: &str) { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_05_timestamps_webtransport() { lite05_timestamp_roundtrip("https").await; @@ -360,7 +359,6 @@ async fn lite05_fetch_roundtrip(scheme: &str) { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_05_fetch_webtransport() { // Exercises the WebTransport path; lite-05 is forced via config on both ends. @@ -470,13 +468,11 @@ async fn transport_fetch_roundtrip(version: &str) { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn fetch_moq_transport_18() { transport_fetch_roundtrip("moq-transport-18").await; } -#[tracing_test::traced_test] #[tokio::test] async fn fetch_moq_transport_22() { transport_fetch_roundtrip("moq-transport-22").await; @@ -602,7 +598,6 @@ async fn lite05_fetch_during_subscribe(scheme: &str) { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_05_fetch_during_subscribe_webtransport() { lite05_fetch_during_subscribe("https").await; @@ -611,7 +606,6 @@ async fn broadcast_moq_lite_05_fetch_during_subscribe_webtransport() { /// Lite05 can't declare an untimed track, so a track that sets no timescale goes out /// in milliseconds, its frames stamped with their send time. The subscriber drops them, /// so the track arrives untimed. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_05_untimed_track() { let pub_origin = moq_tokio::origin::spawn(); @@ -703,7 +697,6 @@ async fn broadcast_moq_lite_05_untimed_track() { /// Both halves have to land in one group, in order and exactly once. Without the fill the /// publisher delivers nothing before the live edge, so the join would start at the next /// group boundary instead. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_transport_22_current_group_join() { let pub_origin = moq_tokio::origin::spawn(); @@ -810,7 +803,6 @@ async fn next_announce(announcements: &mut moq_net::announce::Consumer) -> (moq_ /// re-announce, and replacement. On lite-06 every retraction references the implicit /// announce id rather than repeating the path (the path form doesn't even encode), so /// this exercises the id bookkeeping on both sides of the session. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_06_announce_lifecycle() { announce_lifecycle_test("moq-lite-06").await; @@ -819,7 +811,6 @@ async fn broadcast_moq_lite_06_announce_lifecycle() { /// The same lifecycle on lite-07, where every path shares its head with a live one, /// so the announcements after the first copy it from a base that the retractions /// keep moving. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_07_announce_lifecycle() { announce_lifecycle_test("moq-lite-07-wip").await; @@ -945,7 +936,6 @@ async fn read_payloads(sub: &mut moq_net::track::Subscriber, count: usize) -> Ve /// re-splices through the standby at a group boundary, without the path ever /// being retracted: both routes carry the same epoch over lite-07, so they serve /// the same bytes and the subscription rides the failover. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_route_migration() { use moq_net::Timestamp; @@ -1125,7 +1115,6 @@ async fn broadcast_route_migration() { /// rejoining player seconds behind live. /// /// Pre-06 lite is skipped: nothing on those wires says where a subscription starts. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_rejoin_skips_a_stale_warm_cache() { let pre06 = [ @@ -1263,7 +1252,6 @@ async fn rejoin_skips_a_stale_warm_cache(version: &str) { /// publisher answers at once. Asking past it would wait for a group that a quiet track (a /// catalog) may never send. Covers a finished newest group (a catalog) and one that stays /// open (a JSON log appending frames to group 0), on every version. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_rejoin_replays_a_current_warm_cache() { for version in (0xff0dad00..=0xff0dadff) @@ -1503,14 +1491,12 @@ async fn route_reannounce_test(version: Option<&str>) { } /// Route re-advertisement on the default version (lite-06: ANNOUNCE_RESTART by id). -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_route_reannounce() { route_reannounce_test(None).await; } /// Route re-advertisement on the opt-in lite-07-wip (an explicit ANNOUNCE_RESTART by id). -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_route_reannounce_lite_07() { route_reannounce_test(Some("moq-lite-07-wip")).await; @@ -1518,43 +1504,36 @@ async fn broadcast_route_reannounce_lite_07() { // ── Raw QUIC (moqt://) – same version on both sides ───────────────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_01() { broadcast_test("moqt", Some("moq-lite-01"), Some("moq-lite-01")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_02() { broadcast_test("moqt", Some("moq-lite-02"), Some("moq-lite-02")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_03() { broadcast_test("moqt", Some("moq-lite-03"), Some("moq-lite-03")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_06() { broadcast_test("moqt", Some("moq-lite-06"), Some("moq-lite-06")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_lite_07() { broadcast_test("moqt", Some("moq-lite-07-wip"), Some("moq-lite-07-wip")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_transport_18() { broadcast_test("moqt", Some("moq-transport-18"), Some("moq-transport-18")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_moq_transport_22() { broadcast_test("moqt", Some("moq-transport-22"), Some("moq-transport-22")).await; @@ -1562,25 +1541,21 @@ async fn broadcast_moq_transport_22() { // ── Raw QUIC – server supports all versions, client pins one ───────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_server_all_client_lite_01() { broadcast_test("moqt", Some("moq-lite-01"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_server_all_client_lite_02() { broadcast_test("moqt", Some("moq-lite-02"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_server_all_client_lite_03() { broadcast_test("moqt", Some("moq-lite-03"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_server_all_client_transport_18() { broadcast_test("moqt", Some("moq-transport-18"), None).await; @@ -1588,25 +1563,21 @@ async fn broadcast_negotiate_server_all_client_transport_18() { // ── Raw QUIC – client supports all versions, server pins one ───────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_client_all_server_lite_01() { broadcast_test("moqt", None, Some("moq-lite-01")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_client_all_server_lite_02() { broadcast_test("moqt", None, Some("moq-lite-02")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_client_all_server_lite_03() { broadcast_test("moqt", None, Some("moq-lite-03")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_negotiate_client_all_server_transport_18() { broadcast_test("moqt", None, Some("moq-transport-18")).await; @@ -1716,37 +1687,31 @@ async fn broadcast_max_age_legacy_omits_property() { // ── WebTransport (https://) – same version on both sides ──────────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport() { broadcast_test("https", None, None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_moq_lite_01() { broadcast_test("https", Some("moq-lite-01"), Some("moq-lite-01")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_moq_lite_02() { broadcast_test("https", Some("moq-lite-02"), Some("moq-lite-02")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_moq_lite_03() { broadcast_test("https", Some("moq-lite-03"), Some("moq-lite-03")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_moq_transport_18() { broadcast_test("https", Some("moq-transport-18"), Some("moq-transport-18")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_moq_transport_22() { broadcast_test("https", Some("moq-transport-22"), Some("moq-transport-22")).await; @@ -1754,25 +1719,21 @@ async fn broadcast_webtransport_moq_transport_22() { // ── WebTransport – server supports all, client pins one ───────────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_server_all_client_lite_01() { broadcast_test("https", Some("moq-lite-01"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_server_all_client_lite_02() { broadcast_test("https", Some("moq-lite-02"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_server_all_client_lite_03() { broadcast_test("https", Some("moq-lite-03"), None).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_server_all_client_transport_18() { broadcast_test("https", Some("moq-transport-18"), None).await; @@ -1780,25 +1741,21 @@ async fn broadcast_webtransport_negotiate_server_all_client_transport_18() { // ── WebTransport – client supports all, server pins one ───────────── -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_client_all_server_lite_01() { broadcast_test("https", None, Some("moq-lite-01")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_client_all_server_lite_02() { broadcast_test("https", None, Some("moq-lite-02")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_client_all_server_lite_03() { broadcast_test("https", None, Some("moq-lite-03")).await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_webtransport_negotiate_client_all_server_transport_18() { broadcast_test("https", None, Some("moq-transport-18")).await; @@ -1810,7 +1767,6 @@ async fn broadcast_webtransport_negotiate_client_all_server_transport_18() { /// /// The server binds a WebSocket TCP listener on a separate port. /// The client connects directly via ws://, bypassing QUIC entirely. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_websocket() { // ── publisher (server) ────────────────────────────────────────── @@ -1929,7 +1885,6 @@ async fn broadcast_websocket() { /// The client connects via `http://` to the WebSocket port. QUIC tries to /// reach that port over UDP and fails (no QUIC listener there). The WebSocket /// fallback converts `http://` → `ws://` and connects over TCP, succeeding. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_websocket_fallback() { // ── publisher (server) ────────────────────────────────────────── @@ -2060,7 +2015,6 @@ const NEWEST_LITE: &str = "moq-lite-06"; /// the qmux subprotocol negotiation produced a bare `moql` (or no match) /// instead of the newest lite ALPN, which falls through to legacy SETUP negotiation /// and picks Lite02. This test fails immediately if that happens. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_websocket_uses_newest_version() { let pub_origin = moq_tokio::origin::spawn(); @@ -2134,15 +2088,12 @@ async fn broadcast_websocket_uses_newest_version() { /// (subscribe/track/announce) silently loses that span's fields. Mirrors /// `moq-relay`'s `Connection::run`, whose `#[instrument("conn", fields(addr = /// ...))]` span is exactly what this depends on for a QUIC connection. -/// -/// Needs the crate's `tracing-test/no-env-filter` feature: plain `traced_test` -/// scopes its `EnvFilter` to this test binary's own crate name, which never -/// matches `moq-net`, the crate whose log lines this test inspects. -#[tracing_test::traced_test] #[tokio::test] async fn quic_driver_task_inherits_connection_span() { use tracing::Instrument; + let logs = crate::Logs::capture(); + let pub_origin = moq_tokio::origin::spawn(); let broadcast = pub_origin.create_broadcast("test").expect("failed to create broadcast"); broadcast @@ -2237,19 +2188,15 @@ async fn quic_driver_task_inherits_connection_span() { // not enough for the marker to appear *somewhere* in the logs; it must be on // the exact line moq-net emits from inside the driver task the runtime // spawns, or the assertion would pass even without the fix. - logs_assert(|lines| { - let subscribed = lines - .iter() - .find(|line| line.contains("subscribed started")) - .ok_or("\"subscribed started\" not logged")?; - if subscribed.contains("203.0.113.1:56789") { - Ok(()) - } else { - Err(format!( - "subscribe log lost the connection span's addr field, the runtime isn't instrumenting the driver task\nline: {subscribed}" - )) - } - }); + let logs = logs.text(); + let subscribed = logs + .lines() + .find(|line| line.contains("subscribed started")) + .expect("\"subscribed started\" not logged"); + assert!( + subscribed.contains("203.0.113.1:56789"), + "subscribe log lost the connection span's addr field, the runtime isn't instrumenting the driver task\nline: {subscribed}" + ); drop(session); server_handle @@ -2502,7 +2449,6 @@ async fn idle_subscription_releases_the_viewer_count() { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn websocket_unauthorized_handshake_is_explicit() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -2539,7 +2485,6 @@ async fn websocket_unauthorized_handshake_is_explicit() { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn reconnect_stops_on_websocket_unauthorized() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -2587,7 +2532,6 @@ async fn reconnect_stops_on_websocket_unauthorized() { /// /// The assertion is that `closed()` resolves at all; before the fix the GOAWAY /// arm was not polled without reconnecting, and this hung until the test timeout. -#[tracing_test::traced_test] #[tokio::test] async fn one_shot_goaway_ends_the_connection() { let (mut server, addr) = test_server().await; @@ -2623,7 +2567,6 @@ async fn one_shot_goaway_ends_the_connection() { /// With reconnecting disabled, the session ending ends the connection: its close /// reason surfaces through `closed()` and the loop never dials again. -#[tracing_test::traced_test] #[tokio::test] async fn one_shot_connect_surfaces_the_session_close() { use std::sync::Arc; @@ -2684,7 +2627,6 @@ async fn one_shot_connect_surfaces_the_session_close() { /// retries forever, which used to keep them announced forever). The reconnect /// exists to restore service, not to hide the outage: the application observes /// the loss, and a successful redial re-announces. -#[tracing_test::traced_test] #[tokio::test] async fn a_dead_session_unannounces_while_the_reconnect_retries() { let (mut server, addr) = test_server().await; @@ -2746,7 +2688,6 @@ async fn a_dead_session_unannounces_while_the_reconnect_retries() { /// A peer that expresses announce-interest in a prefix the publisher can't serve (e.g. a /// subscribe-restricted token) must not tear down the whole session. The publisher FINs that /// announce stream cleanly; the connection and other announce streams keep working. -#[tracing_test::traced_test] #[tokio::test] async fn announce_interest_unauthorized_keeps_session_alive() { // ── publisher (server): only allowed to announce under "allowed" ── @@ -2952,27 +2893,23 @@ async fn wildcard_scope_test(version: &str, server_scope: &str) { .expect("server task failed"); } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_wildcard_scopes_lite_05() { wildcard_scope_test("moq-lite-05", "room/alice/**").await; } -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_wildcard_scopes_lite_06() { wildcard_scope_test("moq-lite-06", "room/alice/**").await; } /// A leading-wildcard server grant still filters literal prefix announcements. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_wildcard_server_scope_lite_06() { wildcard_scope_test("moq-lite-06", "*/alice/chat").await; } /// Older wires use the same literal-head interest and local filtering. -#[tracing_test::traced_test] #[tokio::test] async fn broadcast_wildcard_server_scope_lite_05() { wildcard_scope_test("moq-lite-05", "*/alice/chat").await; @@ -2982,7 +2919,6 @@ async fn broadcast_wildcard_server_scope_lite_05() { /// serving a subscribe-only server (`with_subscriber`, no `with_publisher`). The server is also /// interested in a disjoint "denied" prefix the client can't serve, so the server's /// subscriber must survive that FIN and still receive the served broadcast. -#[tracing_test::traced_test] #[tokio::test] async fn publish_only_client_to_subscribe_only_server() { // ── subscriber (server): interested in both "allowed" and "denied" ── @@ -3366,7 +3302,6 @@ async fn connect_once( /// `initial` is both the first delay and the bar a session must clear to count as /// healthy, so at zero every session looked healthy, which reset the give-up /// window on every pass. The loop then redialed forever, timeout or not. -#[tracing_test::traced_test] #[tokio::test] async fn zero_initial_backoff_still_gives_up_on_a_flapping_peer() { let (mut server, addr) = test_server().await; @@ -3402,7 +3337,6 @@ async fn zero_initial_backoff_still_gives_up_on_a_flapping_peer() { /// rides the specified UNAUTHORIZED session code, so it decodes back into a typed /// error rather than an opaque transport string, even in one-shot mode where the /// session ending is what ends the connection. -#[tracing_test::traced_test] #[tokio::test] async fn session_close_surfaces_a_rejection_code() { let (mut server, addr) = test_server().await; @@ -3431,7 +3365,6 @@ async fn session_close_surfaces_a_rejection_code() { /// The same rejection under the reconnect loop: each auth code is specified, so /// the loop treats it as terminal instead of retrying the same credentials with /// backoff until the give-up timeout. -#[tracing_test::traced_test] #[tokio::test] async fn reconnect_stops_on_a_session_level_rejection() { for code in [ @@ -3470,7 +3403,6 @@ async fn reconnect_stops_on_a_session_level_rejection() { /// One-shot mode is the other way a caller observes the rejection: with no /// reconnect loop in the way, the session's close is the terminal error. -#[tracing_test::traced_test] #[tokio::test] async fn one_shot_surfaces_a_session_level_rejection() { let (mut server, addr) = test_server().await; @@ -3499,7 +3431,6 @@ async fn one_shot_surfaces_a_session_level_rejection() { /// This covers the ordinary path, where a session is live when `abort` runs. The /// race it is paired with (a redial landing inside the abort window) is not /// reachable from here, since it turns on when tokio cancels the loop task. -#[tracing_test::traced_test] #[tokio::test] async fn abort_carries_its_code_to_the_peer() { let (mut server, addr) = test_server().await; diff --git a/rs/moq-tokio/tests/integration/main.rs b/rs/moq-tokio/tests/integration/main.rs index 74f904a3bb0..f5b5ff9853e 100644 --- a/rs/moq-tokio/tests/integration/main.rs +++ b/rs/moq-tokio/tests/integration/main.rs @@ -34,3 +34,51 @@ async fn unwanted(mut demand: moq_net::track::Demand) -> moq_net::Result<()> { while demand.wanted_changed().await?.is_some() {} Ok(()) } + +/// Every crate's log lines on this thread, captured until it drops. +/// +/// `traced_test` keeps only this binary's own crate, never the library ones +/// these tests inspect. A current-thread runtime runs spawned tasks on this +/// thread, so their lines are captured too. +struct Logs { + buf: LogBuf, + _guard: tracing::subscriber::DefaultGuard, + _second: tracing::Dispatch, +} + +impl Logs { + fn capture() -> Self { + let buf = LogBuf::default(); + let writer = buf.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + // With one dispatcher registered, tracing caches each callsite's interest from the + // first thread to hit it, so a parallel test without a subscriber could disable a + // span for good. A second one leaves every thread to ask its own subscriber. + let _second = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default()); + Self { buf, _guard, _second } + } + + /// The lines captured so far. + fn text(&self) -> String { + String::from_utf8(self.buf.0.lock().unwrap().clone()).unwrap() + } +} + +/// The shared buffer [`Logs`] formats into. +#[derive(Clone, Default)] +struct LogBuf(std::sync::Arc>>); + +impl std::io::Write for LogBuf { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} From 4c216e827b453ef8c408b4d4cb1b65718639a076 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 6/8] chore: point quest and rust2 references at moq-dev (#7272) * chore: point quest and rust2 references at moq-dev Both repositories moved from the kixelated account to the moq-dev organization. The flake input keeps the same revision and hash. Generated quest skills refresh with the next quest bump. Co-authored-by: Claude Opus 5.5 --- AGENTS.md | 2 +- bench/README.md | 2 +- flake.lock | 4 ++-- flake.nix | 2 +- quest/m0/ci/README.md | 6 +++--- quest/m0/ci/land.md | 6 +++--- quest/m0/ci/nightly-levers.md | 4 ++-- quest/m0/ci/read-only-secrets.md | 6 +++--- quest/m0/ci/slow-tests.md | 4 ++-- quest/m1/api-audit/README.md | 2 +- quest/m1/poll-first/README.md | 4 ++-- quest/m2/README.md | 4 ++-- quest/m2/c/README.md | 2 +- quest/m2/c/async-failure-contract.md | 2 +- quest/m2/ffi-poll-migrate.md | 4 ++-- quest/m2/ffi-size-profile.md | 2 +- quest/m2/nested-options.md | 6 +++--- quest/m2/performance-survey/README.md | 6 +++--- quest/m2/quic-sansio-driver.md | 6 +++--- quest/m2/rs2ts-wasm2js.md | 6 +++--- quest/m2/rs2ts/README.md | 2 +- quest/m2/rs2ts/wasm-bindings.md | 2 +- quest/m2/rust2-uniffi.md | 4 ++-- quest/m2/uniffi-fork.md | 16 ++++++++-------- quest/m3/ci-host.md | 4 ++-- quest/m3/rs2ts-model-ready.md | 2 +- quest/m3/rs2ts-model-subset.md | 4 ++-- test/wasm/generator.md | 6 +++--- 28 files changed, 60 insertions(+), 60 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 26213129986..6271abeeec6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ just fix # Auto-fix lint/formatting, same scope These diff the branch against its base and only run the affected packages. Quests: before starting work, run `quest guide` and follow it. -The `quest` binary comes from the kixelated/quest flake input and serves the quest skills; change them upstream and bump the input. +The `quest` binary comes from the moq-dev/quest flake input and serves the quest skills; change them upstream and bump the input. # Cross-Package Sync diff --git a/bench/README.md b/bench/README.md index 199700c33e3..417e30aa28a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -286,7 +286,7 @@ process CPU (user + system, every thread) per MB into budget (default 1); every received frame is checked byte for byte. The binding benchmark itself lives in -[rust2](https://github.com/kixelated/rust2/blob/main/bench/ffi/README.md), with +[rust2](https://github.com/moq-dev/rust2/blob/main/bench/ffi/README.md), with the UniFFI fork whose changes it measures: its `just bench-ffi` runs these runners nightly at a pinned moq revision, against the fork, and reports each wrapper's ratio to Rust. So that they keep working here, moq's Nightly runs every diff --git a/flake.lock b/flake.lock index 40238e24c89..4087ca2afd4 100644 --- a/flake.lock +++ b/flake.lock @@ -61,13 +61,13 @@ "locked": { "lastModified": 1793286109, "narHash": "sha256-er9ekp0xFZGY2EA/D+01UsbLL3i3H2lMmEYSCUNF7OY=", - "owner": "kixelated", + "owner": "moq-dev", "repo": "quest", "rev": "01d051cf5dca8cd29c664bf8bcbcab63cdc76445", "type": "github" }, "original": { - "owner": "kixelated", + "owner": "moq-dev", "repo": "quest", "rev": "01d051cf5dca8cd29c664bf8bcbcab63cdc76445", "type": "github" diff --git a/flake.nix b/flake.nix index d35670b7d8e..ba2c1356d9e 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,7 @@ # The quest CLI, which also serves the quest guide and skills the stubs in # .claude/skills call. Bump the rev to upgrade them. quest = { - url = "github:kixelated/quest/01d051cf5dca8cd29c664bf8bcbcab63cdc76445"; + url = "github:moq-dev/quest/01d051cf5dca8cd29c664bf8bcbcab63cdc76445"; inputs.nixpkgs.follows = "nixpkgs"; inputs.flake-utils.follows = "flake-utils"; }; diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index e650572c619..bdc4db31900 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -71,7 +71,7 @@ Decided 2026-10-26 (maintainer), from 414 queue merges over 2026-10-23 to (about 1 minute per merge), with a median of 18 entries ahead and 12.7% of runs failing. GitHub's merge queue cannot test several PRs in one run, so a batched gate replaces it on the one self-hosted runner: `quest land`, shared -through kixelated/quest, lands a tested chain of squash commits by +through moq-dev/quest, lands a tested chain of squash commits by fast-forward ([land](/quest/m0/ci/land.md)). This supersedes concurrent queue entries on a second runner. Batching gains most when failures are rare, so the [flake epic](/quest/m0/test-flakes-2/README.md) moved to m0 and @@ -108,7 +108,7 @@ so the work is making builds identical enough to hit it (flags in `.cargo/config.toml`, cargo-hakari), and auditing the package selection `just ci` already does. Nix builds run sandboxed and cannot reach R2, so they stay on crane for packaging only; CI builds with cargo and mbx. Decided 2026-10-26 from -kixelated/rust2's [Nix build benchmark](https://github.com/kixelated/rust2/pull/63), +moq-dev/rust2's [Nix build benchmark](https://github.com/moq-dev/rust2/pull/63), in billions of instructions over a clean build, a second package, the moq-net edit, and a lockfile bump: cargo with mbx totals 9,692 and crane 14,756. crate2nix built to the release profile totals 14,538: it wins only the lockfile bump (1,255 against @@ -118,7 +118,7 @@ Decided 2026-10-25 (maintainer), from a profile of moq-net: monomorphized generics dominate its build (53% of the library's CPU), and its test target costs 3x the library. A "thin generics" profile shrinks them: generic entry points delegate to non-generic bodies, and BTreeMap instantiations are shared. -kixelated/rust2's driver lints the rule later. Making IETF support a cargo +moq-dev/rust2's driver lints the rule later. Making IETF support a cargo feature was considered and not planned: the churn outweighs the build saving. Measure each change against a current baseline and record the result in its PR. diff --git a/quest/m0/ci/land.md b/quest/m0/ci/land.md index f715f313d7b..fc87a3a220c 100644 --- a/quest/m0/ci/land.md +++ b/quest/m0/ci/land.md @@ -48,8 +48,8 @@ Decided 2026-10-26 (maintainer): it can neither guarantee the tested tree nor resolve quest-file conflicts. - One self-hosted runner stays the gate. This replaces the planned second queue runner (`queue-concurrency.md`, deleted). -- The engine is `quest land` in kixelated/quest, shared with kixelated/quest, - moq.pro, and kixelated/rust2. This repository adds a thin workflow, as +- The engine is `quest land` in moq-dev/quest, shared with moq-dev/quest, + moq.pro, and moq-dev/rust2. This repository adds a thin workflow, as moq.pro's `gate.yml` and `queue.yml` do today. - The gate lands by fast-forward. For each PR it writes the squash commit onto the chain, with the engine's default message, and resolves quest-file conflicts @@ -60,7 +60,7 @@ Decided 2026-10-26 (maintainer): exactly the tested one. A conflict outside `quest/` ejects that PR, and so does a fork branch that blocks maintainer edits, whose head the engine cannot move: it is ejected as `no_maintainer_edits` - ([kixelated/quest#229](https://github.com/kixelated/quest/pull/229)). + ([moq-dev/quest#229](https://github.com/moq-dev/quest/pull/229)). - Batches take every queued PR, with no size constant. On failure the gate tests the first half. A passing prefix lands at once and the rest continues. A PR that fails alone is ejected with a comment linking the diff --git a/quest/m0/ci/nightly-levers.md b/quest/m0/ci/nightly-levers.md index adc05fdf608..fbbdc848854 100644 --- a/quest/m0/ci/nightly-levers.md +++ b/quest/m0/ci/nightly-levers.md @@ -18,8 +18,8 @@ total queue time, with Cranelift included. Risks to measure, not assume: - The gate no longer compiles with the toolchain that ships, and nightly mbx keys split from Nightly's stable R2 writers. -Decided 2026-10-27 (maintainer): moq keeps LLVM, accepting kixelated/rust2's -Cranelift verdict ([build benchmark](https://github.com/kixelated/rust2/blob/main/bench/build/README.md), +Decided 2026-10-27 (maintainer): moq keeps LLVM, accepting moq-dev/rust2's +Cranelift verdict ([build benchmark](https://github.com/moq-dev/rust2/blob/main/bench/build/README.md), "Cranelift for moq's CI", 2026-10-25). 42 of moq's 5418 tests fail under Cranelift (panics it cannot unwind, and an x86 SIMD intrinsic it lacks), and the CPU saving is only 4% with a warm cache and 12% cold. This quest is diff --git a/quest/m0/ci/read-only-secrets.md b/quest/m0/ci/read-only-secrets.md index af9e8a767c2..33344da5699 100644 --- a/quest/m0/ci/read-only-secrets.md +++ b/quest/m0/ci/read-only-secrets.md @@ -17,11 +17,11 @@ deleted. ## Plan -Decided 2026-10-28 (maintainer, in kixelated/quest's -[trust audit](https://github.com/kixelated/quest/pull/264)): enrolled +Decided 2026-10-28 (maintainer, in moq-dev/quest's +[trust audit](https://github.com/moq-dev/quest/pull/264)): enrolled repositories keep pull-request CI read-only rather than kix shielding CI behind forks. moq-dev/moq is a kix pilot repository, so it should pass -kix's [read-only CI check](https://github.com/kixelated/quest/blob/main/quest/m0/kix/ci-read-only.md) +kix's [read-only CI check](https://github.com/moq-dev/quest/blob/main/quest/m0/kix/ci-read-only.md) before contributors are invited. Done in [moq-dev/moq#7166](https://github.com/moq-dev/moq/pull/7166): the diff --git a/quest/m0/ci/slow-tests.md b/quest/m0/ci/slow-tests.md index 9a41aa83fbf..27135ad7aec 100644 --- a/quest/m0/ci/slow-tests.md +++ b/quest/m0/ci/slow-tests.md @@ -11,8 +11,8 @@ it is compared against; missing or failed cases are not imputed. ## Plan -Decided with the maintainer on 2026-10-26, from kixelated/rust2's -[test selection report](https://github.com/kixelated/rust2/pull/69): the slowest 1% +Decided with the maintainer on 2026-10-26, from moq-dev/rust2's +[test selection report](https://github.com/moq-dev/rust2/pull/69): the slowest 1% of tests make up 57% of what either package selection or a function-level selector would run, so cutting them beats building a selector. diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index f625a23c2cd..ca8687ccb54 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -36,7 +36,7 @@ Rules every child follows: - Inherited debt is fixed here only when it is cheap or sits on a surface this release breaks anyway. Otherwise it joins the later quest that reshapes it (the ordered track reader joins rust2's - [exporter](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-export.md), + [exporter](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-export.md), which absorbed its foreign poll quest on 2026-10-25). - Each child lists its public API and wire impact in its PR, and its Impact bullet says how to migrate. diff --git a/quest/m1/poll-first/README.md b/quest/m1/poll-first/README.md index a9fd2328bc2..83fcde68492 100644 --- a/quest/m1/poll-first/README.md +++ b/quest/m1/poll-first/README.md @@ -55,8 +55,8 @@ and their alternatives: does not move toward sans-IO. Pull pure wire rules into plain functions where that turns scripted async tests into plain ones, as ordinary refactors. - **Gate the wrappers regardless** of the build measurement, because poll-first is - the style moq wants. kixelated/rust2's - [semantic API hash quest](https://github.com/kixelated/rust2/blob/main/quest/m3/api-cutoff.md) + the style moq wants. moq-dev/rust2's + [semantic API hash quest](https://github.com/moq-dev/rust2/blob/main/quest/m3/api-cutoff.md) replays moq-net's history with and without the wrappers, to report the build effect. - **Adopt `#[kio::machine]` for straight-line code only**, decided with the diff --git a/quest/m2/README.md b/quest/m2/README.md index 7fc17cfabb8..4f4f2452b0a 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -35,8 +35,8 @@ lite-07; bitrate enforcement is not part of this release. - [Byte streams](/quest/m2/stream/README.md) - TCP connections ride MoQ losslessly through relays, proxied per hop rather than as IP packets - [Mesh](/quest/m2/mesh/README.md) - `moq mesh` enrolls a host or robot and reaches any node by endpoint ID or name for SSH, forwards, and agents - [P2P](/quest/m2/p2p/README.md) - application-selected peer sessions use shared priorities, routing, and same-epoch handover -- [rust2 publishes its UniFFI fork](/quest/m2/rust2-uniffi.md) - waiting on kixelated/rust2 to import UniFFI and moq's generators and publish them -- [Switch to rust2's UniFFI](/quest/m2/uniffi-fork.md) - moq-ffi and every wrapper build from kixelated/rust2's fork under one pin, and moq's generator code is deleted +- [rust2 publishes its UniFFI fork](/quest/m2/rust2-uniffi.md) - waiting on moq-dev/rust2 to import UniFFI and moq's generators and publish them +- [Switch to rust2's UniFFI](/quest/m2/uniffi-fork.md) - moq-ffi and every wrapper build from moq-dev/rust2's fork under one pin, and moq's generator code is deleted - [Generated C bindings](/quest/m2/c/README.md) - C generated from moq-ffi ships as `moq-c` 0.8.0 and replaces the hand-written crate - [A runtime-free QUIC driver](/quest/m2/quic-sansio-driver.md) - a caller's own loop moves UDP datagrams and polls moq sessions over moq-quic, for rust2's bindings - [MoQ on rind](/quest/m2/rind/README.md) - every MoQ crate takes its crypto from rind, a pure-Rust graviola fork, so no build compiles C crypto diff --git a/quest/m2/c/README.md b/quest/m2/c/README.md index b8d303197dd..1b1ec4a442f 100644 --- a/quest/m2/c/README.md +++ b/quest/m2/c/README.md @@ -14,7 +14,7 @@ Decided 2026-10-25 (maintainer): this line ships after the cut, once moq builds its bindings from rust2's UniFFI fork. The hand-written crate ships at the cut with [parity](/quest/m1/c/README.md) instead. The generator fix for the C name clash landed in rust2's fork -([kixelated/rust2#66](https://github.com/kixelated/rust2/pull/66)), not in a +([moq-dev/rust2#66](https://github.com/moq-dev/rust2/pull/66)), not in a `kixelated/uniffi-bindgen-cpp` release, and reaches moq with the switch. Decided earlier, still standing: diff --git a/quest/m2/c/async-failure-contract.md b/quest/m2/c/async-failure-contract.md index 91f0cb0ea2c..71414022b28 100644 --- a/quest/m2/c/async-failure-contract.md +++ b/quest/m2/c/async-failure-contract.md @@ -4,7 +4,7 @@ Every async call in the generated C API has a callback contract a C caller can rely on for freeing `user_data`: the call returns a task, never NULL, and its callback runs exactly once, with the result or, when the task was freed first, with `cancelled` set. -[kixelated/rust2#169](https://github.com/kixelated/rust2/pull/169) implements it in rust2's C generator. Waiting on [Switch to rust2's UniFFI](/quest/m2/uniffi-fork.md) to move the C generator pin: `c/moq` adopts the new callback shape then, along with the CamelCase names, and `c/moq/test/probe.c`'s callbacks take `cancelled` and release `user_data` there. Delete this quest with that PR. +[moq-dev/rust2#169](https://github.com/moq-dev/rust2/pull/169) implements it in rust2's C generator. Waiting on [Switch to rust2's UniFFI](/quest/m2/uniffi-fork.md) to move the C generator pin: `c/moq` adopts the new callback shape then, along with the CamelCase names, and `c/moq/test/probe.c`'s callbacks take `cancelled` and release `user_data` there. Delete this quest with that PR. ## Plan diff --git a/quest/m2/ffi-poll-migrate.md b/quest/m2/ffi-poll-migrate.md index 5f242f5701e..c211625940c 100644 --- a/quest/m2/ffi-poll-migrate.md +++ b/quest/m2/ffi-poll-migrate.md @@ -50,7 +50,7 @@ once this lands. - **Still lost on cancel**: when a RustFuture finishes and the foreign side then drops it as cancelled (Python asyncio cancel, Go ctx), the result is gone. The spike measured this in both `Task` and Shape A. Keep the Go - wrapper's terminal cancel. [Foreign poll](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-poll-protocol.md) + wrapper's terminal cancel. [Foreign poll](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-poll-protocol.md) fixes it later for the hot reads. - **What stays**: migrate an export only when it is pure kio polling over handle state. Anything that awaits tokio channels, codecs, or I/O keeps @@ -76,5 +76,5 @@ get items in call order; sequence and arrival order are unchanged. Wire: none. ## Related -- [Foreign poll for hot reads](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-poll-protocol.md) - the generated protocol that follows this, once the fork exists +- [Foreign poll for hot reads](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-poll-protocol.md) - the generated protocol that follows this, once the fork exists - [Libraries spawn nothing](/quest/m1/poll-first/no-spawn.md) - moq-ffi spawns only its root at the binding boundary diff --git a/quest/m2/ffi-size-profile.md b/quest/m2/ffi-size-profile.md index 788b5d8c2c0..553c611eafa 100644 --- a/quest/m2/ffi-size-profile.md +++ b/quest/m2/ffi-size-profile.md @@ -19,7 +19,7 @@ Guidance: opt-level 3 with per-package overrides, and compare against building everything at "s". - Measure what the bindings actually do: publish and subscribe throughput - through moq-ffi on rust2's [binding benchmark](https://github.com/kixelated/rust2/blob/main/bench/ffi/README.md), plus + through moq-ffi on rust2's [binding benchmark](https://github.com/moq-dev/rust2/blob/main/bench/ffi/README.md), plus audio and video encode and decode. - If it's adopted, use a named profile (for example `[profile.ffi]`, inheriting release) that every binding build path selects, so the relay diff --git a/quest/m2/nested-options.md b/quest/m2/nested-options.md index 6382134fc51..ea49139b951 100644 --- a/quest/m2/nested-options.md +++ b/quest/m2/nested-options.md @@ -3,13 +3,13 @@ ## Goal `resolve_fetch_group`, `request_start`, and `Live::base` in moq-net stop building an -`Option>`, so kixelated/rust2's TypeScript translator can translate them. +`Option>`, so moq-dev/rust2's TypeScript translator can translate them. Behavior is unchanged, and the code stays at least as clear. ## Plan -Decided with the maintainer on 2026-10-26, from kixelated/rust2's -[census fixes](https://github.com/kixelated/rust2/pull/78): rewrite these three +Decided with the maintainer on 2026-10-26, from moq-dev/rust2's +[census fixes](https://github.com/moq-dev/rust2/pull/78): rewrite these three functions in moq rather than give the translator a second `Option` representation. Use a named enum or an early return where it reads better, and keep a change only if it does. diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 99a92465d42..708bad5c8f5 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -155,9 +155,9 @@ No duplicate quest or workflow edit belongs here. and fixed QMux copying in [moq-dev/moq#6381](https://github.com/moq-dev/moq/pull/6381). MKV fragmentation below concerns parser replay before those queues. -- [Binding bytes](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-bytes.md), - [foreign polling](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-poll-protocol.md), - [Kotlin calls](https://github.com/kixelated/rust2/blob/main/quest/m1/kt-ffi-calls.md), +- [Binding bytes](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-bytes.md), + [foreign polling](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-poll-protocol.md), + [Kotlin calls](https://github.com/moq-dev/rust2/blob/main/quest/m1/kt-ffi-calls.md), [NVENC pooling](/quest/m2/nvenc-pool.md), and [watch worker planning](/quest/m2/plan-watch-worker.md) retain their scopes. - [Route wakes](/quest/m2/route-wakes.md), diff --git a/quest/m2/quic-sansio-driver.md b/quest/m2/quic-sansio-driver.md index 73aa68c98b8..d862cc45a92 100644 --- a/quest/m2/quic-sansio-driver.md +++ b/quest/m2/quic-sansio-driver.md @@ -9,14 +9,14 @@ on each wake and at the deadline the poll returns. Its connections implement moq `transport::poll::Session`, so moq-net's `Client::connect` and `Server::accept` take them. No tokio or io_uring is in its dependency tree. -The first consumer is [kixelated/rust2](https://github.com/kixelated/rust2)'s exporter, +The first consumer is [moq-dev/rust2](https://github.com/moq-dev/rust2)'s exporter, whose C, C++, and Kotlin bindings drive it from their own loops -([sessions on sans-IO QUIC](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-export/session-quic.md)). +([sessions on sans-IO QUIC](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-export/session-quic.md)). ## Plan Decided with the maintainer on 2026-10-28, from -[kixelated/rust2#215](https://github.com/kixelated/rust2/pull/215): +[moq-dev/rust2#215](https://github.com/moq-dev/rust2/pull/215): - **After [waits](/quest/m1/quic-api/waits.md) and [poll](/quest/m1/quic-api/poll.md).** Once moq-quic owns its waits, a driver is I/O, timers, and a lock. This one is the diff --git a/quest/m2/rs2ts-wasm2js.md b/quest/m2/rs2ts-wasm2js.md index a1c63abc894..127136a1d67 100644 --- a/quest/m2/rs2ts-wasm2js.md +++ b/quest/m2/rs2ts-wasm2js.md @@ -36,7 +36,7 @@ Options to retain in the report: | Charon MIR/LLBC -> TS | Control JS representations and preserve Rust ownership semantics | A custom emitter/runtime, extraction coverage, source maps, and pinned toolchain | | rust-js | Reuse an existing Rust-to-JS compiler and runtime | Audit its supported subset and semantic differences, especially eager Promise execution versus lazy Rust futures | | Component Model + jco | Reusable typed binding and distribution machinery | Additional interface/adapters; default output still uses WASM, while `--js` offers JS-only output | -| Own MIR/THIR backend | Control the extraction and lowering choices; under research in kixelated/rust2 | Highest direct ownership of compiler integration and runtime semantics | +| Own MIR/THIR backend | Control the extraction and lowering choices; under research in moq-dev/rust2 | Highest direct ownership of compiler integration and runtime semantics | Sources: [Charon](https://github.com/AeneasVerif/charon), [rust-js](https://github.com/rust-js-lang/rust-js), and @@ -53,8 +53,8 @@ Split independently landable prerequisites into quests. Decided with the maintainer on 2026-10-23: an own rustc-driver translator is developed as a parallel research track in -[kixelated/rust2](https://github.com/kixelated/rust2), starting with moq-net's -codec ([its quest](https://github.com/kixelated/rust2/blob/main/quest/m2/ts.md)). +[moq-dev/rust2](https://github.com/moq-dev/rust2), starting with moq-net's +codec ([its quest](https://github.com/moq-dev/rust2/blob/main/quest/m2/ts.md)). It does not replace wasm2js as the first route here. Once it translates reachable moq-net code, compare it as the source-translator row below; moq consumes it as a pinned flake input, never by vendoring. diff --git a/quest/m2/rs2ts/README.md b/quest/m2/rs2ts/README.md index 4ae9502b11e..bc78f22d171 100644 --- a/quest/m2/rs2ts/README.md +++ b/quest/m2/rs2ts/README.md @@ -81,5 +81,5 @@ any necessary breaks. Wire: no changes intended. - [WASM-to-JS evaluation](/quest/m2/rs2ts-wasm2js.md) - the first JS-only candidate, with source translators as alternatives - [JS-only @moq/net](/quest/m2/rs2ts-js.md) - the eventual distribution uses no WASM runtime -- [rust2's TS translator](https://github.com/kixelated/rust2/blob/main/quest/m2/ts.md) - a parallel research track translating moq-net's codec through an own rustc driver +- [rust2's TS translator](https://github.com/moq-dev/rust2/blob/main/quest/m2/ts.md) - a parallel research track translating moq-net's codec through an own rustc driver - [Browser media report](/quest/m2/rs2ts-browser-report.md) - broader media measurements after adoption diff --git a/quest/m2/rs2ts/wasm-bindings.md b/quest/m2/rs2ts/wasm-bindings.md index c2eae337ee1..d56a8b766a9 100644 --- a/quest/m2/rs2ts/wasm-bindings.md +++ b/quest/m2/rs2ts/wasm-bindings.md @@ -27,7 +27,7 @@ moves to [adoption](/quest/m2/rs2ts/wasm-adopt.md). Until Gaps the spike found against `@moq/net` and moq-wasm: moq-ffi has no WebSocket fallback on wasm, and payload copies cost about 7 to 8 ns -per byte (see [Payload bytes](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-bytes.md)). Measure boundary calls, Promise allocation, +per byte (see [Payload bytes](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-bytes.md)). Measure boundary calls, Promise allocation, payload copies, and retained memory. Compare a batched poll/event-drain boundary if these dominate, including latency and backpressure effects; do not require a sans-IO rewrite or impose batching latency to win throughput. diff --git a/quest/m2/rust2-uniffi.md b/quest/m2/rust2-uniffi.md index 60ff2a56c19..1925b3376e9 100644 --- a/quest/m2/rust2-uniffi.md +++ b/quest/m2/rust2-uniffi.md @@ -2,8 +2,8 @@ ## Goal -[kixelated/rust2](https://github.com/kixelated/rust2) has merged its -[UniFFI import quest](https://github.com/kixelated/rust2/blob/main/quest/m1/uniffi.md) +[moq-dev/rust2](https://github.com/moq-dev/rust2) has merged its +[UniFFI import quest](https://github.com/moq-dev/rust2/blob/main/quest/m1/uniffi.md) and published `rust2-uniffi` and `rust2-uniffi-bindgen` to crates.io, with a tag moq can pin as a flake input. Check: `cargo search rust2-uniffi` lists a release, and rust2's `quest/m1/uniffi.md` is gone from its `main`. Delete this diff --git a/quest/m2/uniffi-fork.md b/quest/m2/uniffi-fork.md index babe42a4326..cdd3a225911 100644 --- a/quest/m2/uniffi-fork.md +++ b/quest/m2/uniffi-fork.md @@ -4,7 +4,7 @@ moq-ffi depends on the UniFFI fork's renamed crates from crates.io, and every generator moq uses (Kotlin, Swift, Python, Go, Dart, C++, and C) comes from -[kixelated/rust2](https://github.com/kixelated/rust2) through one pinned flake +[moq-dev/rust2](https://github.com/moq-dev/rust2) through one pinned flake input. `rs/uniffi-bindgen`, `nix/uniffi-dart-*`, and the per-generator pins are deleted, and the three `kixelated/uniffi-*` generator forks are archived. The browser's TypeScript, generated through ubrn's `wasm2` flavour, still @@ -17,7 +17,7 @@ undoes. Decided with the maintainer on 2026-10-13, after a review of how moq uses UniFFI: -- **Base**: UniFFI. [The BoltFFI spike](https://github.com/kixelated/rust2/blob/main/bench/ffi/README.md#boltffi-comparison) +- **Base**: UniFFI. [The BoltFFI spike](https://github.com/moq-dev/rust2/blob/main/bench/ffi/README.md#boltffi-comparison) recommended it. Decided later in #5838 (2026-10-17, maintainer): UniFFI-generated moq-ffi serves the browser too, through ubrn's `wasm2` flavour (chosen over BoltFFI and hand-written moq-wasm), so one binding @@ -36,11 +36,11 @@ UniFFI: silently with upstream UniFFI. Decided with the maintainer on 2026-10-23: the hard fork lives in -`kixelated/rust2`, not a new `moq-dev/uniffi`. rust2 groups Rust-to-other-language +`moq-dev/rust2`, not a new `moq-dev/uniffi`. rust2 groups Rust-to-other-language tools (bindings, a TypeScript translator, build-speed work) under one setup. moq's generator code moves with it, including `rs/uniffi-bindgen`. The import and crates.io publishing are rust2's -[UniFFI import quest](https://github.com/kixelated/rust2/blob/main/quest/m1/uniffi.md), +[UniFFI import quest](https://github.com/moq-dev/rust2/blob/main/quest/m1/uniffi.md), which publishes `rust2-uniffi`, `rust2-uniffi-core`, `rust2-uniffi-macros`, and `rust2-uniffi-bindgen`. This quest is the moq side: @@ -53,7 +53,7 @@ which publishes `rust2-uniffi`, `rust2-uniffi-core`, `rust2-uniffi-macros`, and differences. - **C names** (maintainer, 2026-10-24 and 2026-10-25): rust2's C generator names in GLib style - ([kixelated/rust2#66](https://github.com/kixelated/rust2/pull/66)). + ([moq-dev/rust2#66](https://github.com/moq-dev/rust2/pull/66)). Types are CamelCase with the namespace prefix (`MoqError`, `MoqStringList`, enum tags `Tag`, async callbacks `Callback`), functions and enum constants keep their names, and any remaining clash fails generation @@ -98,6 +98,6 @@ none. ## Related - [ubrn generates UniFFI 0.32 TypeScript](/quest/m2/rs2ts/ubrn-032.md) - the browser generator this switch must keep working -- [Payload bytes](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-bytes.md) - the first patch the fork carries -- [Kotlin calls](https://github.com/kixelated/rust2/blob/main/quest/m1/kt-ffi-calls.md) - a Kotlin generator patch the fork carries -- [Foreign poll for hot reads](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-poll-protocol.md) - a generated poll protocol the fork carries; dropping a future at cancel is optional here, since no moq quest waits on it +- [Payload bytes](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-bytes.md) - the first patch the fork carries +- [Kotlin calls](https://github.com/moq-dev/rust2/blob/main/quest/m1/kt-ffi-calls.md) - a Kotlin generator patch the fork carries +- [Foreign poll for hot reads](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-poll-protocol.md) - a generated poll protocol the fork carries; dropping a future at cancel is optional here, since no moq quest waits on it diff --git a/quest/m3/ci-host.md b/quest/m3/ci-host.md index 75db78808bf..e1946405641 100644 --- a/quest/m3/ci-host.md +++ b/quest/m3/ci-host.md @@ -2,7 +2,7 @@ ## Goal -The merge gates for moq-dev/moq, kixelated/quest, and kixelated/rust2 run on +The merge gates for moq-dev/moq, moq-dev/quest, and moq-dev/rust2 run on one rented always-on host with a warm on-disk cache, instead of the maintainer's home `nixos` box. Gate entries are no slower than the home runner's measured times, and the home box is off the merge path. moq.pro's @@ -75,7 +75,7 @@ Remaining work, once [the host is rented](/quest/m3/ci-host-rented.md): generated source trees another job's next build reads. Verify with overlapping jobs under cache pressure. Safe collection need not serialize jobs. -- Give kixelated/quest and kixelated/rust2 a gate that runs on the rented +- Give moq-dev/quest and moq-dev/rust2 a gate that runs on the rented runner. Their Check workflows hardcode hosted runners (`ubuntu-latest`, `ubuntu-24.04-arm`), and quest's has no `merge_group` trigger, so a runner variable alone leaves them off the host. Route only trusted gate runs diff --git a/quest/m3/rs2ts-model-ready.md b/quest/m3/rs2ts-model-ready.md index f8538425637..cc4dad73704 100644 --- a/quest/m3/rs2ts-model-ready.md +++ b/quest/m3/rs2ts-model-ready.md @@ -2,7 +2,7 @@ ## Goal -[rust2's model epic](https://github.com/kixelated/rust2/blob/main/quest/m2/ts-model/README.md) +[rust2's model epic](https://github.com/moq-dev/rust2/blob/main/quest/m2/ts-model/README.md) has landed the translation of safe dependencies and bindings, so moq-net's model is the next thing it translates. Check by reading that epic's Required list. diff --git a/quest/m3/rs2ts-model-subset.md b/quest/m3/rs2ts-model-subset.md index ff28f35378c..33d53ba6ce0 100644 --- a/quest/m3/rs2ts-model-subset.md +++ b/quest/m3/rs2ts-model-subset.md @@ -3,14 +3,14 @@ ## Goal moq-net's model, and the kio, moq-clock, and moq-pattern code it calls, stay inside the -subset [rust2-ts](https://github.com/kixelated/rust2/blob/main/ts/README.md) translates, +subset [rust2-ts](https://github.com/moq-dev/rust2/blob/main/ts/README.md) translates, so the [source-translator route](/quest/m2/rs2ts-wasm2js.md) can be measured on the model with no hand-written stand-ins for moq's own logic. ## Plan Decided with the rust2 maintainer on 2026-10-27, from a census of the model in -[rust2's model epic](https://github.com/kixelated/rust2/blob/main/quest/m2/ts-model/README.md): +[rust2's model epic](https://github.com/moq-dev/rust2/blob/main/quest/m2/ts-model/README.md): rust2-ts stays general, translating safe dependencies from source and refusing `unsafe` and trait objects, so the moq-specific changes are made here. Waits until rust2's epic reaches the model; none of these changes behavior. diff --git a/test/wasm/generator.md b/test/wasm/generator.md index 18b236417b2..2a2e4913a8b 100644 --- a/test/wasm/generator.md +++ b/test/wasm/generator.md @@ -5,13 +5,13 @@ Which Rust binding `@moq/net` wraps in the browser: moq-ffi through UniFFI (ubrn This compares them on the [baseline harness](README.md#backend-baseline). BoltFFI cannot bind moq-ffi as written (see [What blocks BoltFFI](#what-blocks-boltffi)), -so its network numbers come from the [BoltFFI spike](https://github.com/kixelated/rust2/blob/main/bench/ffi/README.md#boltffi-comparison)'s slice with a browser client added. The +so its network numbers come from the [BoltFFI spike](https://github.com/moq-dev/rust2/blob/main/bench/ffi/README.md#boltffi-comparison)'s slice with a browser client added. The same slice also runs through UniFFI, so the two generators compare on identical Rust. **Decision** (maintainer, 2026-10-17, [#5838](https://github.com/moq-dev/moq/pull/5838)): moq-ffi through UniFFI over ubrn. It keeps one binding surface for every language with no API break, and passes every parity case. Its two measured costs have follow-ups: -[size](/quest/m2/rs2ts/net-only.md) and [payload copies](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-bytes.md). +[size](/quest/m2/rs2ts/net-only.md) and [payload copies](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-bytes.md). [Complete WASM bindings](/quest/m2/rs2ts/wasm-bindings.md) builds on this harness. ## Reproduce @@ -151,7 +151,7 @@ The slice, six ABBA rounds: Both generators resolve one JS Promise per `rust_future_poll` wakeup; BoltFFI's async path is 1.4 to 2.1 times cheaper. UniFFI's bytes cost about 7 ns per byte written and 8 ns per byte read, from its per-element `Vec` lift and lower in Rust, against BoltFFI's -bulk copies at 0.5 and 1.3. That is [Payload bytes](https://github.com/kixelated/rust2/blob/main/quest/m1/ffi-bytes.md)'s target. +bulk copies at 0.5 and 1.3. That is [Payload bytes](https://github.com/moq-dev/rust2/blob/main/quest/m1/ffi-bytes.md)'s target. `@ubjs/core` also arms and clears a `setTimeout` on every async call, a Node.js keep-alive that runs in the browser too. From febf8f667f4e6b9f03701e552edda404d82adb11 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:37 +0000 Subject: [PATCH 7/8] Verify the R2 Rust cache (#7030) * chore(quest): claim quest/m0/ci/mbx-r2-verify.md Quest-Claim: 9a556a77ba203b39 * quest(m0/ci): record the first mbx 1.28 Nightly in the R2 verify quest The bump's checks passed: every hosted Nightly job writes, macOS refused only an already oversized manifest, and the merge queue shows no E0463. Reuse is still absent on every platform; record the lockfile-identity evidence and narrow the quest to what remains. * ci(platform): install uniffi-bindgen-cpp after the Rust build The Nightly Windows build loaded the install's 124-prediction mbx manifest under its own identity on 2026-10-27 and 2026-10-28, which blocked inheriting the previous lockfile's manifest. Installing after the build lets the build inherit first. Reported upstream as jdx/mr-boxington#869. * quest(m0/ci): record upstream's CRLF cause for the Windows cache miss jdx/mr-boxington#870 traced the Windows miss to CRLF lockfile digests, not the uniffi-bindgen-cpp install, and the macOS build did inherit. * quest(m0/ci): revert the uniffi-bindgen-cpp step move, record settled decisions #870 showed the Windows miss comes from CRLF lockfile digests, not the install order, so platform.yml keeps main's order. Windows now waits on an mbx release with #870 and a pin bump; Linux full history waits on Windows hitting; lockfile churn past 8 states counts under the rebuild item. Co-authored-by: Claude Opus 5.5 --- quest/m0/ci/README.md | 2 +- quest/m0/ci/mbx-r2-verify.md | 101 +++++++++++++++++++++-------------- 2 files changed, 61 insertions(+), 42 deletions(-) diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index bdc4db31900..9e7ab7f44d1 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -130,7 +130,7 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Land main through quest land](/quest/m0/ci/land.md) - one self-hosted runner tests every queued PR as a squash chain and fast-forwards `main` to the longest passing prefix - [Gate each diff in under five minutes](/quest/m0/ci/five-minutes.md) - `just check` and `just test` each finish within a worker's 5-minute prompt cache, admission wait included - [Measure the parallel frontend](/quest/m0/ci/nightly-levers.md) - a pinned nightly's parallel frontend, adopted only if the numbers beat its risks; Cranelift is ruled out -- [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - measure the hosted Nightly jobs' hits, restore time, and expiry once the bump lands +- [Verify the R2 Rust cache](/quest/m0/ci/mbx-r2-verify.md) - confirm the hosted Nightly jobs reuse the previous Nightly, then measure restore time and expiry - [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - confirm releases use `nix.moq.dev`, check its bucket sizes, and retire Cachix - [Unify dependency features](/quest/m0/ci/hakari.md) - a cargo-hakari workspace-hack so every job compiles each dependency alike and mbx hits across jobs - [Fewer BTreeMap instantiations](/quest/m0/ci/btree-keys.md) - moq-net shares BTreeMap key and value types, kept only if its CPU time drops diff --git a/quest/m0/ci/mbx-r2-verify.md b/quest/m0/ci/mbx-r2-verify.md index 01515718628..2ee49c8852d 100644 --- a/quest/m0/ci/mbx-r2-verify.md +++ b/quest/m0/ci/mbx-r2-verify.md @@ -6,20 +6,21 @@ Confirm the cache earns its keep for the hosted jobs that still use R2 (Nightly on `main`, release pull requests, and the macOS and Windows jobs), and record the numbers in the PR that closes this: -- Each Nightly job's `Check the cache` step reports `effective read-write`, - and a release pull request's reports `effective read-only`. +- A release pull request's `Check the cache` step reports `effective + read-only`. Condition: `release` carries the Rust cache, which its + `platform.yml` still lacks on 2026-10-28; check with + `git show origin/release:.github/workflows/platform.yml | grep rust-cache`. - Nightly's Linux, macOS and Windows jobs hit what the previous Nightly wrote. -- After the mbx 1.28.0 bump, the merge queue reports no new - `error[E0463]: can't find crate`, and the next scheduled Nightly's macOS - job reports no new oversized-manifest digest. - Restore plus build time against the 150-210s Actions restore of the ~6 GB store it replaced, from the same commit before and after. - Each object expires 30 days after upload, so a crate no Nightly builds in that window stays cold for the next hosted job. Count how often a Nightly - job pays that rebuild. -- Bucket size and request counts after a month, against the 30-day expiry in - `infra/mbx/lifecycle.json`. Shorten or lengthen it from that. + job pays that rebuild, including when `Cargo.lock` changed more than 8 times + since the previous Nightly (see Plan). +- Bucket size and request counts after a month on mbx 1.28.0 (from about + 2026-11-26), against the 30-day expiry in `infra/mbx/lifecycle.json`. + Shorten or lengthen it from that. Use scheduled Nightly runs on `main` only (`gh run list --workflow nightly.yml --branch main --event schedule`), never a dispatch on a branch. @@ -31,43 +32,61 @@ Decided 2026-10-22 (maintainer): narrowed to hosted jobs. Since self-hosted `moq-queue` runner with mbx's local store, so it neither reads nor writes R2. -Evidence so far, from the platform jobs of scheduled Nightlies -[39600361153](https://github.com/moq-dev/moq/actions/runs/39600361153) -(2026-10-21, mbx 1.25.0), -[39779729461](https://github.com/moq-dev/moq/actions/runs/39779729461) -(2026-10-22, 1.25.0) and -[39951870847](https://github.com/moq-dev/moq/actions/runs/39951870847) -(2026-10-23, 1.26.0): - -- The bootstrap is done: Windows and macOS report `configured read-write, - effective read-write`. That proves credentials and policy, not reuse. -- macOS refuses task manifests written past the reader's 16 MiB bound - (`a90076a4...` at 17204435 bytes, then `5cb8fb79...` and `b0310ce4...`), so - those tasks get no remote predictions. Two more were refused before the - bump, both under mbx 1.25.0: `2ed9df47...` (17218002 bytes) on 2026-10-25 - ([40192013303](https://github.com/moq-dev/moq/actions/runs/40192013303)) - and 2026-10-26, and `0d4f4e3f...` (17290876 bytes) on 2026-10-26 - ([40332938121](https://github.com/moq-dev/moq/actions/runs/40332938121)). No macOS summary downloads more than - 15 KiB. Reported as - [jdx/mr-boxington#792](https://github.com/jdx/mr-boxington/issues/792) and - fixed by [jdx/mr-boxington#798](https://github.com/jdx/mr-boxington/pull/798), - which keeps the read bound and caps the writer instead. The Rust cache - now pins 1.28.0, the first release carrying the fix. It does not repair - manifests already oversized on R2: if task identities carry over, the - five known digests may still refuse until the 30-day lifecycle deletes - them, about 2026-11-22 for the first three and 2026-11-26 for the last - two. A refusal for any other digest means the fix did not hold. -- Windows logs no refusal but still no reuse. Its first build reports 0 hits - and 1084 of 1091 compilations "not looked up" for want of a matching - prediction on 2026-10-22, after a 1.25.0 Nightly wrote the day before, and - 1091 of 1098 on 1.26.0. The manifest fix does not explain this. If it - persists after the bump, find why before timing builds. -- No release pull request has run with the R2 cache yet. - Decided 2026-10-27 (maintainer): the cross-job miss report moved to [unifying dependency features](/quest/m0/ci/hakari.md), which measures its own before and after baseline and no longer waits on this quest. +The first scheduled Nightly on mbx 1.28.0, +[40506938673](https://github.com/moq-dev/moq/actions/runs/40506938673) +(2026-10-27), settled the bump: every hosted job reports `configured +read-write, effective read-write`, and macOS refused only a manifest already +oversized before it. Six such manifests stay unreadable until the 30-day +lifecycle deletes them, the last about 2026-11-26: `a90076a4...`, +`5cb8fb79...`, `b0310ce4...`, `18d136a6...` (17235558 bytes, first refused in +[40090959108](https://github.com/moq-dev/moq/actions/runs/40090959108)), +`2ed9df47...` and `0d4f4e3f...`. A refusal for any other digest means the +writer cap in mbx 1.28.0 did not hold. + +No platform reuses the previous Nightly yet. mbx names a Cargo build's +prediction manifest by its workspace's `Cargo.lock` digest. When a lockfile +has no manifest yet, mbx inherits one from up to 8 earlier lockfile states, +found through Git history (`previous_build_identities` in `mbx-cache-cargo`, +`inherit_task_manifest` in `mbx-cache-core`). Without a prediction, a +compilation is "not looked up". + +- Windows: 0 hits on 1.28.0 on both 2026-10-27 and + [2026-10-28](https://github.com/moq-dev/moq/actions/runs/40698329026), each + reporting that "a manifest predicting 124 compilations was loaded, but none + matched this build". We reported it upstream as + [jdx/mr-boxington#869](https://github.com/jdx/mr-boxington/issues/869), + blaming the preceding `cargo install uniffi-bindgen-cpp`. The fix, + [jdx/mr-boxington#870](https://github.com/jdx/mr-boxington/pull/870) + (merged 2026-10-29, after v1.29.0), found a different cause. Git for + Windows checks `Cargo.lock` out with CRLF, while history holds LF, so no + earlier lockfile state ever matched. The build then fell back to the newest + local manifest, which was the install's. Since 1.28.0 an install builds + under its own workspace, so the install order does not matter; it stays + where `main` has it. Windows waits on an mbx release that includes #870 and + a bump of the `version` pin in `.github/actions/rust-cache/action.yml`. + Then check that the first scheduled Nightly on it loads a manifest of more + than 124 predictions and hits on Windows. +- macOS: on 2026-10-28 the build did inherit, prefetching 732 predictions, + yet hit only 13, with 1565 compilations not looked up. The install order is + not the cause. Find out why the inherited predictions mostly miss, for + example the lockfile changes or the 30-day expiry. +- Linux: Nightly's Linux jobs check out at depth 1, so mbx finds no earlier + lockfile and reuses only a manifest written for the identical `Cargo.lock`. + The doctest job hit 0 on 2026-10-24 and 2026-10-27, 115 on 2026-10-25, and + 46 on 2026-10-26. Decided 2026-10-29 (maintainer): give those jobs full + history (`fetch-depth: 0` with `filter: blob:none`) only once Windows hits + after the mbx bump, which shows inheritance pays. +- `Cargo.lock` changed 7, 2, 6, 10, 8 and 8 times between consecutive + scheduled Nightlies from 2026-10-21 to 2026-10-27. Nightly is the only + hosted writer for Windows and macOS, so after more than 8 changes the + previous Nightly is out of inheritance reach even with full history. + Decided 2026-10-29 (maintainer): accept that, and count it under the + rebuild item in the Goal before asking upstream for a deeper lookback. + ## Related - [Verify the Nix cache](/quest/m0/ci/nix-cache-verify.md) - the same R2 and Environment bootstrap for Nix From 652ba002e25db18ee7052de329e57716a475045c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 08:32:38 +0000 Subject: [PATCH 8/8] feat(json): a public snapshot group budget, refusing an oversized snapshot before parsing (#7072) * chore(quest): claim quest/m0/ci/stats-oversize.md Quest-Claim: 4a6d2df5874a8c48 * perf(json): refuse an oversized snapshot before parsing or compressing it * chore(quest): finish quest/m0/ci/stats-oversize.md * feat(json): a public snapshot group budget, refusing a snapshot past it before parsing snapshot::Config gains max_group_bytes (JS maxGroupBytes, no longer internal), clamped to moq-net's cache limit. The encoder refuses a snapshot past it before the baseline parse, so moq-stats' oversize tests lower it to 64 KiB under test instead of building 32 MiB of stats. * chore(json): regenerate the API report for maxGroupBytes Co-authored-by: Claude Opus 5.5 --- js/json/api/index.api.md | 1 + js/json/src/snapshot/encoder.ts | 17 +++--- js/json/src/snapshot/snapshot.test.ts | 9 ++++ quest/m0/ci/README.md | 1 - quest/m0/ci/stats-oversize.md | 14 ----- rs/moq-json/src/snapshot/encoder.rs | 76 +++++++++++++++++++++++---- rs/moq-stats/src/produce.rs | 39 +++++++++----- 7 files changed, 113 insertions(+), 44 deletions(-) delete mode 100644 quest/m0/ci/stats-oversize.md diff --git a/js/json/api/index.api.md b/js/json/api/index.api.md index af78c1c76b3..a6ebe608bfd 100644 --- a/js/json/api/index.api.md +++ b/js/json/api/index.api.md @@ -23,6 +23,7 @@ export namespace Snapshot { export interface Config { deltas?: boolean; initial?: T; + maxGroupBytes?: number; schema?: ZodMiniType; } export namespace Consumer { diff --git a/js/json/src/snapshot/encoder.ts b/js/json/src/snapshot/encoder.ts index 06ad29340c3..7ac55193be3 100644 --- a/js/json/src/snapshot/encoder.ts +++ b/js/json/src/snapshot/encoder.ts @@ -1,4 +1,4 @@ -import { Group } from "@moq/net"; +import { Group, Error as NetError } from "@moq/net"; import type * as z from "@zod/mini"; import { deepEqual, diff } from "../diff.ts"; @@ -30,10 +30,12 @@ export interface Config { initial?: T; /** - * Bytes a group may hold before it rolls, defaulting to moq-net's per-group cache limit. Lets a - * test reach the limit without megabytes of JSON. Must fit the snapshot's own frame. + * The most bytes a group may hold, snapshot and deltas together. * - * @internal + * A delta that would pass it rolls a fresh snapshot instead, and a snapshot past it throws + * `FrameTooLarge`. Lowering it lets a test reach the limit without megabytes of JSON; a value + * above moq-net's per-group cache limit is clamped to it, since moq-net refuses a larger group + * anyway. Defaults to that limit. */ maxGroupBytes?: number; } @@ -126,7 +128,7 @@ export class Encoder { constructor(config: Config = {}) { this.#config = config; - this.#maxGroupBytes = config.maxGroupBytes ?? Group.MAX_GROUP_CACHE_BYTES; + this.#maxGroupBytes = Math.min(config.maxGroupBytes ?? Infinity, Group.MAX_GROUP_CACHE_BYTES); } /** @@ -255,8 +257,11 @@ export class Encoder { return new TextEncoder().encode(JSON.stringify(result.patch)); } - // Encode a group's snapshot (frame 0), returning its payload. + // Encode a group's snapshot (frame 0), returning its payload. A snapshot past the budget is + // refused before any state changes, so the previous value stays readable rather than superseded + // by a frame moq-net would refuse on write. #snapshot(frame: Uint8Array): Uint8Array { + if (frame.length > this.#maxGroupBytes) throw new NetError.FrameTooLarge(); this.#groupBytes = frame.length; this.#groupFrames = 1; this.#resync = false; diff --git a/js/json/src/snapshot/snapshot.test.ts b/js/json/src/snapshot/snapshot.test.ts index f8cf2b4c1fe..2a898256989 100644 --- a/js/json/src/snapshot/snapshot.test.ts +++ b/js/json/src/snapshot/snapshot.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "bun:test"; import { Milli } from "@moq/clock"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; import { Consumer } from "./consumer.ts"; +import { Encoder } from "./encoder.ts"; import { Producer } from "./producer.ts"; type Value = Record; @@ -551,6 +552,14 @@ test("a delta that would overflow the snapshot rolls a new one instead", async ( expect(values[values.length - 1]).toEqual({ v: "y".repeat(size) }); }); +test("a snapshot past the budget is refused", () => { + for (const deltas of [false, true]) { + const encoder = new Encoder({ deltas, maxGroupBytes: 100 }); + expect(() => encoder.update({ v: "x".repeat(1000) })).toThrow(NetError.FrameTooLarge); + expect(encoder.value).toBeUndefined(); + } +}); + // A publisher gives up a group a reader is still on once a newer one supersedes it, which every // moq-lite version reports as OLD. The reader skips to the newer snapshot instead of failing. test("snapshot consumer skips a group the publisher dropped mid-read", async () => { diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 9e7ab7f44d1..67d0fd4d449 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -135,7 +135,6 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Unify dependency features](/quest/m0/ci/hakari.md) - a cargo-hakari workspace-hack so every job compiles each dependency alike and mbx hits across jobs - [Fewer BTreeMap instantiations](/quest/m0/ci/btree-keys.md) - moq-net shares BTreeMap key and value types, kept only if its CPU time drops - [Halve the slowest tests' CPU](/quest/m0/ci/slow-tests.md) - the slowest 1% of tests take under half their 289 CPU-seconds, with coverage kept -- [Stats oversize](/quest/m0/ci/stats-oversize.md) - moq-stats refuses an oversized plain snapshot before parsing it back - [Gate send-depth](/quest/m0/ci/send-depth-gate.md) - the merge gate runs send-depth on a pinned nightly when moq-net changes, after #6890 fixes main - [Cargo aliases in nested checkouts](/quest/m0/ci/cargo-alias-strings.md) - `cargo build-ffi` and `cargo moq` work from a worktree nested inside another checkout - [Release credentials off main](/quest/m0/ci/publish-release-only.md) - `publish` admits only `release` and release tags once its `workflow_run` publishers run on their tag diff --git a/quest/m0/ci/stats-oversize.md b/quest/m0/ci/stats-oversize.md deleted file mode 100644 index 10501a0cd4c..00000000000 --- a/quest/m0/ci/stats-oversize.md +++ /dev/null @@ -1,14 +0,0 @@ -# [XS] moq-stats refuses an oversized plain snapshot early - -## Goal - -moq-stats' snapshot encoder refuses an oversized plain snapshot before -parsing it back, as it already does for deflate, so the oversize tests stop -spending about 4.5 s of CPU building and re-reading a snapshot they refuse. - -## Plan - -Found by [moq-dev/moq#6987](https://github.com/moq-dev/moq/pull/6987)'s -profile. Keep the refusal error the same. - -Public API: none. diff --git a/rs/moq-json/src/snapshot/encoder.rs b/rs/moq-json/src/snapshot/encoder.rs index bda433de9e4..a4052b91a2d 100644 --- a/rs/moq-json/src/snapshot/encoder.rs +++ b/rs/moq-json/src/snapshot/encoder.rs @@ -65,6 +65,16 @@ pub struct Config { /// /// Defaults to `true`. pub deltas: bool, + + /// The most bytes a group may hold, snapshot and deltas together. + /// + /// A delta that would pass it rolls a fresh snapshot instead, and a snapshot past it is refused + /// with [`FrameTooLarge`](moq_net::Error::FrameTooLarge). Lowering it lets a test reach the limit + /// without megabytes of JSON; a value above moq-net's cache limit is clamped to it, since moq-net + /// refuses a larger group anyway. + /// + /// Defaults to [`moq_net::group::MAX_CACHE_BYTES`]. + pub max_group_bytes: u64, } impl Config { @@ -73,11 +83,20 @@ impl Config { self.deltas = deltas; self } + + /// Set [`max_group_bytes`](Self::max_group_bytes). + pub fn with_max_group_bytes(mut self, max_group_bytes: u64) -> Self { + self.max_group_bytes = max_group_bytes; + self + } } impl Default for Config { fn default() -> Self { - Self { deltas: true } + Self { + deltas: true, + max_group_bytes: moq_net::group::MAX_CACHE_BYTES, + } } } @@ -206,8 +225,7 @@ pub struct Encoder { /// Frames emitted into the current group, snapshot included. group_frames: usize, - /// The most bytes a group may hold, the cap a delta is admitted under. A field rather than the - /// constant so a test can shrink it to a size it can reach. + /// [`Config::max_group_bytes`], clamped to moq-net's cache limit. max_group_bytes: u64, /// Whether the next frame has to be a full snapshot, because a frame was lost or the caller cut @@ -223,12 +241,12 @@ impl Encoder { /// Create an encoder with a cold baseline, so the first [`update`](Self::update) is a snapshot. pub fn new(config: Config) -> Self { Self { + max_group_bytes: config.max_group_bytes.min(moq_net::group::MAX_CACHE_BYTES), config, last: None, scratch: RefCell::new(crate::diff::Scratch::memoized()), group_bytes: 0, group_frames: 0, - max_group_bytes: moq_net::group::MAX_CACHE_BYTES, resync: false, _marker: PhantomData, } @@ -274,6 +292,8 @@ impl Encoder { Some(Baseline::Encoded { bytes, .. }) => bytes.clone(), Some(Baseline::Parsed(value)) => Bytes::from(serde_json::to_vec(value)?), }; + // A baseline built by patches was never held to the budget as one snapshot. + self.check_snapshot(&bytes)?; let encoded = self.snapshot_encoded(bytes); Ok(Some(Pending { encoder: self, @@ -318,7 +338,9 @@ impl Encoder { // deltas off there is no diff to rebase and no reason to pay it at all. // // Every fallible step runs before any state changes, so a failure leaves the encoder exactly - // as it was rather than half-advanced with no frame to show for it. + // as it was rather than half-advanced with no frame to show for it. The size check runs + // first, so a value no reader could hold costs no parse. + self.check_snapshot(&snapshot)?; let last = if self.config.deltas { Baseline::Parsed(serde_json::from_slice(&snapshot)?) } else { @@ -334,6 +356,15 @@ impl Encoder { Ok((encoded, Edit::Snapshot(last))) } + /// Refuse a snapshot past the group budget before anything is published, so the previous value + /// stays readable rather than superseded by a frame moq-net would refuse on write. + fn check_snapshot(&self, snapshot: &[u8]) -> Result<()> { + match snapshot.len() as u64 > self.max_group_bytes { + true => Err(moq_net::Error::FrameTooLarge.into()), + false => Ok(()), + } + } + /// Encode an already-serialized value as a full snapshot, opening a new group. fn snapshot_encoded(&mut self, snapshot: Bytes) -> Encoded { self.group_bytes = snapshot.len() as u64; @@ -556,7 +587,7 @@ mod test { let sequence = (any::(), property_sequence()); runner .run(&sequence, |(deltas, steps)| { - let mut encoder = Encoder::::new(Config { deltas }); + let mut encoder = Encoder::::new(Config::default().with_deltas(deltas)); let mut decoder = super::super::Decoder::::new(super::super::consumer::Config::default()); for (index, step) in steps.iter().enumerate() { if step.reset { @@ -682,7 +713,7 @@ mod test { #[test] fn snapshot_restates_the_retained_value() { for deltas in [true, false] { - let mut encoder = Encoder::::new(Config { deltas }); + let mut encoder = Encoder::::new(Config::default().with_deltas(deltas)); assert!(encoder.snapshot().unwrap().is_none(), "nothing to restate yet"); commit(&mut encoder, &json!({ "a": 1 })).unwrap(); @@ -864,8 +895,8 @@ mod test { let value = json!({ "v": "x".repeat(1000) }); let patched = json!({ "v": "y".repeat(1000) }); - let mut encoder = Encoder::::new(Config::default()); - encoder.max_group_bytes = (serde_json::to_vec(&value).unwrap().len() + 500) as u64; + let budget = (serde_json::to_vec(&value).unwrap().len() + 500) as u64; + let mut encoder = Encoder::::new(Config::default().with_max_group_bytes(budget)); assert!(commit(&mut encoder, &value).unwrap().keyframe); let rolled = commit(&mut encoder, &patched).expect("a rolled snapshot"); assert!( @@ -883,6 +914,33 @@ mod test { assert!(!next.keyframe); } + /// A snapshot past the budget is refused before its baseline is parsed, with or without deltas. + #[test] + fn a_snapshot_past_the_budget_is_refused() { + for deltas in [false, true] { + let config = Config::default().with_deltas(deltas).with_max_group_bytes(100); + let mut encoder = Encoder::::new(config); + assert!( + matches!( + encoder.update(&json!({ "v": "x".repeat(1000) })), + Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) + ), + "deltas {deltas}" + ); + assert!( + encoder.value().is_none(), + "a refused snapshot leaves the encoder as it was" + ); + } + } + + /// A budget above moq-net's cache limit is clamped to it. + #[test] + fn the_budget_is_clamped_to_the_cache_limit() { + let encoder = Encoder::::new(Config::default().with_max_group_bytes(u64::MAX)); + assert_eq!(encoder.max_group_bytes, moq_net::group::MAX_CACHE_BYTES); + } + /// A value whose serialization changes on every call, standing in for a `Serialize` impl backed by /// a clock, an atomic, or interior mutable state. struct Ticking(std::cell::Cell); diff --git a/rs/moq-stats/src/produce.rs b/rs/moq-stats/src/produce.rs index ab2855283f4..7a9bf3323be 100644 --- a/rs/moq-stats/src/produce.rs +++ b/rs/moq-stats/src/produce.rs @@ -412,10 +412,9 @@ impl Snapshot { /// written; `None` means the value was unchanged, the interval already has a group for a /// snapshot, or the track already ended, and nothing was written. /// - /// A frame over the cache limit aborts the track with + /// A frame over the group budget aborts the track with /// [`moq_net::Error::FrameTooLarge`]: skipping it would leave readers on the - /// previous snapshot with no sign it stopped tracking. A compressed track - /// holds a frame to the same limit by its uncompressed size. + /// previous snapshot with no sign it stopped tracking. fn update(&mut self, value: &T, stamp: Stamp) -> moq_json::Result> { #[cfg(feature = "bench")] { @@ -433,9 +432,10 @@ impl Snapshot { self.encoder.reset(); } let frame = match self.encoder.update(value) { - Ok(Some(frame)) if frame.payload.len() as u64 <= moq_net::group::MAX_CACHE_BYTES => frame, + Ok(Some(frame)) => frame, Ok(None) => return Ok(None), - Ok(Some(_)) | Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) => { + // The encoder refuses a frame past the group budget before anything is written. + Err(moq_json::Error::Net(moq_net::Error::FrameTooLarge)) => { let err = moq_net::Error::FrameTooLarge; if let Some(group) = self.group.take() { let _ = group.abort(err.clone()); @@ -505,6 +505,15 @@ fn info() -> track::Info { .with_refresh(8) } +/// The snapshot codec every stats track encodes with. +fn config() -> moq_json::snapshot::Config { + let config = moq_json::snapshot::Config::default(); + // A test overflows a group with a few KiB rather than moq-net's 32 MiB. + #[cfg(test)] + let config = config.with_max_group_bytes(tests::MAX_GROUP_BYTES); + config +} + impl Output { fn create(broadcast: &broadcast::Producer, name: &str) -> Result { Ok(Self::new(broadcast.create_track(name, info())?)) @@ -524,7 +533,7 @@ impl Output { fn new(track: track::Producer) -> Self { Self { - snapshot: Snapshot::new(track, moq_json::snapshot::Config::default()), + snapshot: Snapshot::new(track, config()), frame: Frame::default(), } } @@ -808,7 +817,7 @@ impl GroupTrack { let track = broadcast.create_track(name.as_str(), info())?; Ok(Self { name, - snapshot: Snapshot::new(track, moq_json::snapshot::Config::default()), + snapshot: Snapshot::new(track, config()), body: GroupBody { // A new instance of the name: readers keep its counters apart from // an earlier instance's. @@ -1299,6 +1308,10 @@ fn advertised_path(prefix: &Path, node: Option<&Path>) -> Result moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) @@ -1524,10 +1537,8 @@ mod tests { } } - /// A frame that outgrows the cache ends the track with `FrameTooLarge` - /// instead of leaving readers on their last snapshot. DEFLATE would shrink - /// the frame under the limit, but a reader inflates it whole, so the track - /// holds it to the same budget by its uncompressed size. + /// A frame that outgrows the group budget ends the track with `FrameTooLarge` + /// instead of leaving readers on their last snapshot. #[tokio::test] async fn oversized_frame_ends_the_track() { use futures::FutureExt; @@ -1566,11 +1577,11 @@ mod tests { )); } - /// Distinct paths under `prefix` whose JSON overflows a group's cache, since one - /// path holds at most [`Path::MAX_BYTES`]. + /// Distinct paths under `prefix` whose JSON overflows [`MAX_GROUP_BYTES`], since + /// one path holds at most [`Path::MAX_BYTES`]. fn oversized_paths(prefix: &str) -> Vec { let filler = "a".repeat(Path::MAX_BYTES - 16); - (0..=moq_net::group::MAX_CACHE_BYTES as usize / filler.len()) + (0..=MAX_GROUP_BYTES as usize / filler.len()) .map(|i| Path::new(&format!("{prefix}{i}/{filler}")).unwrap()) .collect() }