From 63604eb9ac5dc46cf3af5c48cb83bf09eea74e75 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 01:48:05 +0100 Subject: [PATCH 01/14] feat(http2): streamed response bodies - WIP, do not merge yet The response half only, as agreed: an h2 handler can now push body bytes as it produces them, each flush becoming a DATA frame, instead of returning a finished Http2Response. That is what unblocks a streaming IResponseContent, and what makes an endless response expressible at all. The part HTTP/3 did not have is flow control. Every stream here rides one TCP connection and a write needs credit in BOTH the stream's window and the connection's, so FlushAsync waits for a WINDOW_UPDATE rather than resetting the stream the way the buffered path does - a whole body was always in hand there, so running out of credit could only mean the peer had stopped reading. Verified correct: 8192 bytes exact on an 8 x 1KiB response, and 1 MiB delivered whole - sixteen times the default 65535 window, which is what proves the wait works rather than the stream dying at the first window. One leak found and fixed on the way. The streamed dispatch skips the buffered path's tail, and that tail is what retires the stream, so every response was leaking its PendingRequest arena: 20 GB of RSS over eight seconds of load. Retiring the stream in the writer's finally brings that to 97 MB, and doubles throughput as a side effect. NOT READY. Streamed still runs 93990 req/s against the buffered path's 1773623 on the same 8 KiB - a 19x gap I have not diagnosed. Eight flushes per response against one is part of it, but not obviously all of it, and shipping a streaming path that slow would mislead anyone who reached for it. No playground pane, no registry entry, no version bump until that is understood. Unit 29, E2E 46, Http 35 pass. --- .../Playground.Http2.ManagedStreamed.csproj | 19 ++ Playground/Http2/ManagedStreamed/Program.cs | 128 +++++++++++ ioxide.slnx | 1 + .../ioxide.http2/Http2Connection.Frames.cs | 4 + .../ioxide.http2/Http2Connection.Streamed.cs | 204 ++++++++++++++++++ src/protocols/ioxide.http2/Http2Connection.cs | 8 + .../ioxide.http2/Http2ResponseWriter.cs | 201 +++++++++++++++++ 7 files changed, 565 insertions(+) create mode 100644 Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj create mode 100644 Playground/Http2/ManagedStreamed/Program.cs create mode 100644 src/protocols/ioxide.http2/Http2Connection.Streamed.cs create mode 100644 src/protocols/ioxide.http2/Http2ResponseWriter.cs diff --git a/Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj b/Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj new file mode 100644 index 0000000..01e3783 --- /dev/null +++ b/Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj @@ -0,0 +1,19 @@ + + + + Exe + net11.0 + enable + enable + true + Playground.Http2.ManagedStreamed + Playground.Http2.ManagedStreamed + + + + + + + + + diff --git a/Playground/Http2/ManagedStreamed/Program.cs b/Playground/Http2/ManagedStreamed/Program.cs new file mode 100644 index 0000000..f05661b --- /dev/null +++ b/Playground/Http2/ManagedStreamed/Program.cs @@ -0,0 +1,128 @@ +using System.Text; +using ioxide; +using ioxide.http2; +using Playground.Shared; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// http2-managed-streamed - HTTP/2 in pure C# with the RESPONSE BODY STREAMED: the handler pushes +// bytes as it produces them and each flush becomes a DATA frame, instead of returning a finished +// Http2Response. +// +// That is what "/feed" shows - an endless response has no final byte, so a buffered API cannot +// express it at all. It is also what lets a large file be served without ever holding it whole. +// +// The flow-control story is the part HTTP/3 does not have. Every stream here shares one TCP +// connection, so a write needs credit in BOTH the stream's window and the connection's, and +// FlushAsync waits for a WINDOW_UPDATE rather than failing. That wait is the backpressure: a +// peer that stops reading stops the producer instead of growing a queue behind it. +// +// dotnet run -c Release --project Playground/Http2/ManagedStreamed +// curl --http2-prior-knowledge http://127.0.0.1:8080/ # chunked +// curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed # never ends +// +// Needs: ioxide, ioxide.http2 +// ───────────────────────────────────────────────────────────────────────────────────────────── + +// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── +// Edit these. That is the whole mechanism - there is no config file and nothing else to find. +// An Env.Override line means the value can also be set from the environment, which is how +// bench/run.sh drives the sample; the literal is what applies otherwise. Delete those lines when +// you copy this out and the literals above them are the entire configuration. + +ushort port = 8080; +int reactors = Environment.ProcessorCount; +int bodyBytes = 2; // unused here: the body is produced chunk by chunk + +Env.Override(ref port, ref reactors, ref bodyBytes); + +// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The +// handler code is identical either way; this only changes how recv buffers are handed out. +bool incrementalBuffers = false; + +Env.OverrideIncremental(ref incrementalBuffers); + +// Chunks written per response on "/", and the size of each. Their product is never held at once. +int chunkCount = 64; +int chunkBytes = 16 * 1024; + +Env.Override(ref chunkCount, "PLAYGROUND_CHUNKS"); +Env.Override(ref chunkBytes, "PLAYGROUND_CHUNK_BYTES"); +// ───────────────────────────────────────────────────────────────────────────────────────────── + +var config = new ServerConfig +{ + ReactorCount = reactors, // io_uring rings/threads - one per core + RingEntries = 8192, // SQ/CQ depth per ring + DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped + RecvBufferSize = 32 * 1024, // bytes per shared recv buffer + RecvSlots = 4096, // shared recv buffer-ring depth + Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental + Udp = null, // no raw UDP sockets (TCP-only server) + Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn + Tcp = new TcpOptions + { + Port = port, + ExtraPorts = [], // extra listener ports (one handler, several doors) + ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener + WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in + PoolMax = 1024, // pooled connection objects kept per reactor + WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG + ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes + RecvQueueEntries = 64, // per-connection recv completion queue depth + }, +}; + +byte[] body = bodyBytes == 2 + ? "ok"u8.ToArray() + : [.. Enumerable.Repeat((byte)'x', bodyBytes)]; + +var threads = new Thread[config.ReactorCount]; + +byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n"); + +for (int i = 0; i < threads.Length; i++) +{ + var reactor = new Reactor(i, config); + + reactor.TcpHandle = async (r, conn) => + { + try + { + await new Http2Connection(conn).RunAsync(async (request, writer) => + { + bool endless = request.Path.Span.SequenceEqual("/feed"u8); + + // Headers first and once. No content-length: the length is not known yet, and for + // /feed never will be - END_STREAM is what marks the end instead. + var response = new Http2Response { Status = 200 }; + response.Headers.Add("content-type"u8.ToArray(), + endless ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray()); + writer.WriteHeaders(response); + + for (int n = 0; endless || n < chunkCount; n++) + { + chunk.CopyTo(writer.GetSpan(chunk.Length)); + writer.Advance(chunk.Length); + + // Waits when either window is exhausted, and resumes on the WINDOW_UPDATE. + await writer.FlushAsync(); + } + }); + } + finally + { + conn.DecRef(); + } + }; + + threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; + threads[i].Start(); +} + +Console.WriteLine($"[http2-managed-streamed] {config.ReactorCount} reactors on :{config.Tcp!.Port} " + + $"(pure C#), {chunkCount} x {chunkBytes}-byte chunks per response"); + +foreach (Thread thread in threads) +{ + thread.Join(); +} diff --git a/ioxide.slnx b/ioxide.slnx index 8bef36e..9f29cf0 100644 --- a/ioxide.slnx +++ b/ioxide.slnx @@ -57,6 +57,7 @@ + diff --git a/src/protocols/ioxide.http2/Http2Connection.Frames.cs b/src/protocols/ioxide.http2/Http2Connection.Frames.cs index 53726df..456854e 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Frames.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Frames.cs @@ -345,6 +345,10 @@ private void HandleWindowUpdate(in FrameHeader header, ReadOnlySpan payloa { pending.SendWindow += increment; } + + // Credit arrived, so a streamed response parked on it can carry on. Stream 0 credits the + // connection and therefore unblocks every parked writer, not just one. + ReleaseCreditWaiters(header.StreamId); } private void HandlePing(in FrameHeader header, ReadOnlySpan payload) diff --git a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs new file mode 100644 index 0000000..b9143bf --- /dev/null +++ b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs @@ -0,0 +1,204 @@ +using System.Buffers; + +namespace ioxide.http2; + +/// +/// The STREAMED-RESPONSE half: headers first, then the body as DATA frames pushed by the handler +/// through an as it produces them. +/// +/// Owning the framing keeps this a push - a chunk is written the moment it exists rather than +/// when a library asks for it. What HTTP/2 adds over HTTP/3 is that credit is shared: every +/// stream rides one TCP connection and both a per-stream and a connection window must allow a +/// write, so a flush can block on either and is woken by a WINDOW_UPDATE for either. +/// +public sealed partial class Http2Connection +{ + private readonly Stack _writerPool = new(); + private readonly Dictionary> _creditWaiters = new(); + private Func? _streamedHandler; + + /// + /// Serve this connection with each response body produced through a writer rather than + /// returned whole. The handler owns its stream until it completes the writer. + /// + public Task RunAsync(Func handler) + { + _streamedHandler = handler; + return RunBufferedAsync(NoBufferedHandler); + } + + // Never invoked: a streamed connection hands every ready request to a writer before the + // buffered path would reach this. It exists only because RunBufferedAsync needs a handler. + private static Http2Response NoBufferedHandler(Http2Request _) + => throw new InvalidOperationException("A streamed connection dispatches through its writer."); + + /// + /// Hand a ready request to the streamed path, if this connection has one. Called from the + /// dispatch that made it ready, so the writer's first frames ride that same pass. + /// + private bool TryDispatchStreamed(Http2Request request, PendingRequest pending) + { + if (_streamedHandler is null) + { + return false; + } + + Http2ResponseWriter writer = RentWriter(request.StreamId); + _ = ServeStreamedAsync(_streamedHandler, request, writer, pending); + return true; + } + + private async Task ServeStreamedAsync(Func handler, + Http2Request request, Http2ResponseWriter writer, PendingRequest pending) + { + try + { + await handler(request, writer); + await writer.CompleteAsync(); + } + catch (Exception exception) + { + Console.Error.WriteLine($"[ioxide.http2] request handler faulted: {exception.GetBaseException().Message}"); + try + { + await writer.CompleteAsync(); + } + catch + { + // The stream is already unusable; nothing more to say to this peer. + } + } + finally + { + // The buffered dispatch retires the stream in its own finally; the streamed one skips + // that path, so it has to do it here. Without this every response leaks its arena and + // the connection grows without bound - 20 GB over eight seconds of load, measured. + _creditWaiters.Remove(writer.StreamId); + _streams.Remove(writer.StreamId); + pending.Dispose(); + _writerPool.Push(writer); + } + } + + private Http2ResponseWriter RentWriter(int streamId) + { + if (_writerPool.TryPop(out Http2ResponseWriter? pooled)) + { + pooled.Reset(streamId); + return pooled; + } + return new Http2ResponseWriter(this, streamId); + } + + /// Headers of a streamed response - no END_STREAM, the body follows. + internal void SendStreamedHeaders(int streamId, Http2Response response) + => WriteHeaders(streamId, response, endStream: false); + + /// + /// How many body bytes may be sent on this stream right now: the smaller of the connection + /// window, the stream window and the peer's maximum frame size. Zero means wait. + /// + internal int SendCredit(int streamId) + { + if (IsBroken) + { + return 0; + } + + int credit = Math.Min(_peerConnectionWindow, _peerMaxFrameSize); + if (_streams.TryGetValue(streamId, out PendingRequest? pending)) + { + credit = Math.Min(credit, pending.SendWindow); + } + return Math.Max(credit, 0); + } + + /// One DATA frame, already known to fit both windows. + internal void SendStreamedData(int streamId, ReadOnlySpan body, bool endStream) + { + Span header = stackalloc byte[FrameHeader.Size]; + new FrameHeader(body.Length, FrameType.Data, + endStream ? FrameFlags.EndStream : FrameFlags.None, streamId).Write(header); + + Stage(header); + if (!body.IsEmpty) + { + Stage(body); + } + + _peerConnectionWindow -= body.Length; + if (_streams.TryGetValue(streamId, out PendingRequest? pending)) + { + pending.SendWindow -= body.Length; + } + } + + /// Push whatever is staged out to the transport. + internal ValueTask FlushOutboundAsync() => FlushAsync(); + + /// Completes when this stream has credit again, or the connection gives up. + internal Task WaitForSendCreditAsync(int streamId) + { + if (IsBroken) + { + return Task.CompletedTask; + } + + if (!_creditWaiters.TryGetValue(streamId, out List? waiters)) + { + waiters = []; + _creditWaiters[streamId] = waiters; + } + + var waiter = new TaskCompletionSource(); + waiters.Add(waiter); + return waiter.Task; + } + + /// + /// A WINDOW_UPDATE arrived. Stream 0 credits the connection, so it can unblock every parked + /// writer; anything else unblocks only its own. + /// + private void ReleaseCreditWaiters(int streamId) + { + if (_creditWaiters.Count == 0) + { + return; + } + + if (streamId == 0) + { + foreach (List waiters in _creditWaiters.Values.ToArray()) + { + Release(waiters); + } + _creditWaiters.Clear(); + return; + } + + if (_creditWaiters.Remove(streamId, out List? forStream)) + { + Release(forStream); + } + + static void Release(List waiters) + { + foreach (TaskCompletionSource waiter in waiters) + { + waiter.TrySetResult(); + } + } + } + + private void ReleaseAllCreditWaiters() + { + foreach (List waiters in _creditWaiters.Values) + { + foreach (TaskCompletionSource waiter in waiters) + { + waiter.TrySetResult(); + } + } + _creditWaiters.Clear(); + } +} diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 6cef3b5..1a43145 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -141,6 +141,8 @@ public async Task RunBufferedAsync(Func> } finally { + // A writer parked on flow-control credit will never be woken by a dead connection. + ReleaseAllCreditWaiters(); Dispose(); } } @@ -201,6 +203,12 @@ private async ValueTask DispatchReadyAsync(Func +/// The write half of a STREAMED response: the handler pushes body bytes as it produces them and +/// each flush becomes a DATA frame, instead of the whole body being handed over at once. +/// +/// It is an on purpose - that is the shape a serializer, a file +/// copy or a framework's response sink already writes into, so streaming through it needs no +/// adapter. +/// +/// The difference from the HTTP/3 writer is flow control. QUIC gives each stream its own +/// transport-level credit; HTTP/2 multiplexes every stream onto one TCP connection and tracks +/// TWO windows - one per stream, one for the connection - so a flush can be blocked by either. +/// waits for a WINDOW_UPDATE rather than failing, which is what makes a +/// response longer than the peer's window possible at all. The buffered path resets the stream +/// instead, because a whole body was always in hand and running out of credit could only mean the +/// peer had stopped reading entirely. +/// +/// Reactor thread only, like everything else on the connection. +public sealed class Http2ResponseWriter : IBufferWriter +{ + private const int DefaultChunk = 16 * 1024; + + private readonly Http2Connection _connection; + private int _streamId; + + private byte[] _staging = []; + private int _staged; + + private bool _headersSent; + private bool _completed; + + internal Http2ResponseWriter(Http2Connection connection, int streamId) + { + _connection = connection; + _streamId = streamId; + } + + /// The stream this response belongs to. + public int StreamId => _streamId; + + /// True once the body has been finished and END_STREAM sent. + public bool IsCompleted => _completed; + + /// + /// Send the response headers. Exactly once, before any body byte - HTTP/2 puts HEADERS ahead + /// of DATA and there is no correcting it later. + /// + /// No content-length is written: a streamed response does not know its length yet, and an + /// endless one never will. HTTP/2 needs none - each DATA frame carries its own, and + /// END_STREAM marks the end. + /// + public void WriteHeaders(Http2Response response) + { + ArgumentNullException.ThrowIfNull(response); + + if (_headersSent) + { + throw new InvalidOperationException("Response headers have already been written for this stream."); + } + if (!response.Body.IsEmpty) + { + throw new ArgumentException( + "A streamed response carries its body through the writer; leave Response.Body empty.", + nameof(response)); + } + + _headersSent = true; + _connection.SendStreamedHeaders(_streamId, response); + } + + /// + public Span GetSpan(int sizeHint = 0) + { + EnsureStaging(sizeHint <= 0 ? 1 : sizeHint); + return _staging.AsSpan(_staged); + } + + /// + public Memory GetMemory(int sizeHint = 0) + { + EnsureStaging(sizeHint <= 0 ? 1 : sizeHint); + return _staging.AsMemory(_staged); + } + + /// + public void Advance(int count) + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + if (_staged + count > _staging.Length) + { + throw new InvalidOperationException("Advanced past the end of the span handed out by GetSpan."); + } + _staged += count; + } + + /// + /// Send everything staged so far as DATA frames, waiting on flow-control credit as needed. + /// That wait is the backpressure: a peer that stops reading stops the producer rather than + /// growing a queue behind it. + /// + public ValueTask FlushAsync() => FlushCore(endStream: false); + + /// + /// Send what is left and mark END_STREAM. A handler that returns without calling this gets it + /// called for it - the peer is owed an end either way. + /// + public async ValueTask CompleteAsync() + { + if (_completed) + { + return; + } + + if (!_headersSent) + { + WriteHeaders(new Http2Response { Status = 500 }); + } + + await FlushCore(endStream: true); + _completed = true; + + if (_staging.Length > 0) + { + ArrayPool.Shared.Return(_staging); + _staging = []; + } + } + + private async ValueTask FlushCore(bool endStream) + { + if (!_headersSent) + { + throw new InvalidOperationException("Write the response headers before flushing a body chunk."); + } + + int sent = 0; + while (sent < _staged) + { + // Bounded by the frame size AND by both windows: exceeding either is a connection + // error the peer would be right to hang up over. + int credit = _connection.SendCredit(_streamId); + if (credit <= 0) + { + if (_connection.IsBroken) + { + return; + } + await _connection.WaitForSendCreditAsync(_streamId); + continue; + } + + int chunk = Math.Min(credit, _staged - sent); + bool last = endStream && sent + chunk == _staged; + + _connection.SendStreamedData(_streamId, _staging.AsSpan(sent, chunk), last); + sent += chunk; + } + + _staged = 0; + + if (endStream && sent == 0) + { + // Nothing left to send, but the stream still needs its end. + _connection.SendStreamedData(_streamId, ReadOnlySpan.Empty, endStream: true); + } + + await _connection.FlushOutboundAsync(); + } + + private void EnsureStaging(int sizeHint) + { + int needed = _staged + sizeHint; + if (_staging.Length >= needed) + { + return; + } + + byte[] grown = ArrayPool.Shared.Rent(Math.Max(needed, DefaultChunk)); + if (_staged > 0) + { + _staging.AsSpan(0, _staged).CopyTo(grown); + } + if (_staging.Length > 0) + { + ArrayPool.Shared.Return(_staging); + } + _staging = grown; + } + + /// Take this writer for another stream, keeping its buffer. + internal void Reset(int streamId) + { + _streamId = streamId; + _staged = 0; + _headersSent = false; + _completed = false; + } +} From 5465b8f96b2eb6675cf7f67a0dba0e9c352516dd Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 11:26:12 +0100 Subject: [PATCH 02/14] perf(http2): coalesce streamed writes into the pass flush - 6.1x A streamed response measured the same whether it carried 2 bytes or 8 KiB - 408k against 355k req/s - and that flatness is the shape of a syscall, not of work. Each response was forcing its own transport write, where the buffered path spreads ONE write across every stream in flight on the connection. That alone was the whole gap, and at 2 bytes it was 12.8x. So a streamed writer no longer writes for itself while the read loop still owes a flush: it stages, and the pass carries it out alongside every other response. Coalescing is the reason buffered h2 is fast and there is no reason a streamed response cannot share it. It has to stay bounded, though not for memory. A producer that loops without awaiting anything of its own has no yield point except that write, so skipping it unconditionally spins the reactor and the response never moves at all - which is exactly how two earlier attempts at this broke the endless case, both times silently, because a benchmark only measures responses that END. Past 16 KiB staged the write happens for real and hands the thread back. A paced producer never reaches the limit: it parks, the pass flush takes its chunk out immediately, and it resumes outside the pass where the write is unconditional. before after buffered 2 bytes 408,483 3,873,462 5,235,162 1 x 8192 355,209 585,830 1,801,105 8 x 1024 93,006 569,832 1,801,105 Chunk count stopped mattering - 8x1024 and 1x8192 now sit within 3% where they differed 3.8x, because writes follow bytes rather than how often the handler asks for a flush. What remains is per-byte, not per-write: raising the limit to 128 KiB changed nothing, and streamed still falls off with body size faster than buffered. That points at the staging copy - the handler fills an ArrayPool block that is then copied into the pipe, where the buffered path writes the body once. Framing the DATA header directly into the pipe's span would remove it. Verified unchanged: 8192 exact, 1 MiB through a 65535 window, endless /feed still trickling, RSS flat at 98 MB. Unit 29, E2E 46, Http 35 pass. --- .../ioxide.http2/Http2Connection.Streamed.cs | 41 +++++++++++++++++++ .../ioxide.http2/Http2Connection.Write.cs | 2 + src/protocols/ioxide.http2/Http2Connection.cs | 6 +++ .../ioxide.http2/Http2ResponseWriter.cs | 2 +- 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs index b9143bf..f116c01 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs @@ -136,6 +136,47 @@ internal void SendStreamedData(int streamId, ReadOnlySpan body, bool endSt /// Push whatever is staged out to the transport. internal ValueTask FlushOutboundAsync() => FlushAsync(); + /// + /// True while the read loop still owes a flush. A streamed writer checks this to decide whether + /// to write for itself: inside a pass the answer is no, and skipping that write is what lets + /// many responses leave on ONE transport write. + /// + /// It matters because the cost here is per-write, not per-byte - a streamed response measured + /// the same whether it carried 2 bytes or 8 KiB, which is the shape of a syscall rather than of + /// work. Buffered spread one write across every stream in flight; streamed paid one each, and + /// that alone was the 12.8x. + /// + internal bool PassFlushPending => _passFlushPending; + + private bool _passFlushPending; + private int _stagedBytes; + + /// + /// Write only when nothing else will. A handler that parked - on credit, on a timer, on a + /// database - resumes after the pass flush has gone by, so its bytes would otherwise sit staged + /// until the peer happened to send something, which for an endless response is never. + /// + internal ValueTask MaybeFlushAsync() + { + // Coalescing has to stay BOUNDED, and not for memory reasons. A producer that loops without + // awaiting anything of its own - generating as fast as it can - has no yield point except + // this write, so skipping it unconditionally spins the reactor and the response never moves + // at all. Past the limit the write happens for real, which both drains the staging and + // hands the thread back. + // + // A PACED producer (an SSE feed waiting on an event) never reaches the limit: it parks, the + // pass flush carries its chunk out immediately, and it resumes outside the pass where this + // returns a real flush. So latency stays tight where it matters, and only a bandwidth-bound + // producer trades a little of it for throughput. + if (!_passFlushPending || _stagedBytes >= CoalesceLimit) + { + return FlushAsync(); + } + return ValueTask.CompletedTask; + } + + private const int CoalesceLimit = 16 * 1024; + /// Completes when this stream has credit again, or the connection gives up. internal Task WaitForSendCreditAsync(int streamId) { diff --git a/src/protocols/ioxide.http2/Http2Connection.Write.cs b/src/protocols/ioxide.http2/Http2Connection.Write.cs index 9c86a7e..77903f1 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Write.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Write.cs @@ -15,6 +15,7 @@ public sealed partial class Http2Connection private void Stage(ReadOnlySpan bytes) { _pipe.Output.Write(bytes); + _stagedBytes += bytes.Length; _staged = true; } @@ -25,6 +26,7 @@ private async ValueTask FlushAsync() return; } _staged = false; + _stagedBytes = 0; await _pipe.Output.FlushAsync(); } diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 1a43145..2d5f6df 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -122,8 +122,14 @@ public async Task RunBufferedAsync(Func> if (received) { + // A streamed writer that finishes inside this window needs no write of its own: + // the flush below carries it out together with every other response the pass + // produced. That coalescing is the whole reason buffered h2 is fast, and there + // is no reason a streamed response cannot share it. + _passFlushPending = true; ParseAvailable(); await DispatchReadyAsync(handler); + _passFlushPending = false; await FlushAsync(); } diff --git a/src/protocols/ioxide.http2/Http2ResponseWriter.cs b/src/protocols/ioxide.http2/Http2ResponseWriter.cs index 18f05f1..7bf151f 100644 --- a/src/protocols/ioxide.http2/Http2ResponseWriter.cs +++ b/src/protocols/ioxide.http2/Http2ResponseWriter.cs @@ -167,7 +167,7 @@ private async ValueTask FlushCore(bool endStream) _connection.SendStreamedData(_streamId, ReadOnlySpan.Empty, endStream: true); } - await _connection.FlushOutboundAsync(); + await _connection.MaybeFlushAsync(); } private void EnsureStaging(int sizeHint) From 61efa487bcde1cfa2e88f0beb005d4be4de10ac5 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 12:52:01 +0100 Subject: [PATCH 03/14] fix(http2): a slow handler no longer blocks every other stream DispatchReadyAsync awaited each handler in turn, so one request that parked - a database, an upstream, a disk - held up every other stream on that TCP connection, including responses already produced and staged with nowhere to go. Two requests on one connection, one sleeping a second: /slow /fast before 1.02s 1.02s <- waited for /slow after 1.02s 20.99ms That is the whole point of multiplexing, and h2 is the worst place to lose it: QUIC streams are independent, so an h3 handler that parks inconveniences only itself, but on h2 everything shares one connection and one dispatch loop. Both h3 modules already dispatch this way, buffered and streamed alike. h2 was the outlier. A handler that answers synchronously - nearly all of them - stays inline, so its response is still staged in time for the pass flush and still leaves with every other one in a single write, and there is no Task to allocate. Only a handler that actually parks is detached, and it writes its own bytes when it finishes, since the pass flush has gone by. Detaching means nothing is awaiting the tail, so the tail has to do the work the loop used to do in its finally: retire the request, and flush. Skipping exactly that is what leaked 20 GB in the streamed path. An escaping exception would also vanish silently and leave the peer waiting on a stream that never comes, so it is caught, logged and answered with a 500 - the buffered path previously let it kill the connection. No cost to the fast path: 2 bytes 5353304 req/s (was 5235162), 8 KiB 1783354 (was 1801105) - both inside noise. Unit 29, E2E 46, Http 35 pass. nghttp2 has the same defect at RunBuffered.cs:94 and is deliberately left alone; the managed stack is where the effort goes. --- src/protocols/ioxide.http2/Http2Connection.cs | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 2d5f6df..958d581 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -206,6 +206,8 @@ private async ValueTask DispatchReadyAsync(Func inFlight; + try { Http2Request request = pending.Freeze(); @@ -215,13 +217,66 @@ private async ValueTask DispatchReadyAsync(Func + /// The tail of a handler that parked. Nothing is awaiting this, so everything the dispatch loop + /// would have done afterwards has to happen here instead - retiring the request, and writing, + /// since the pass flush has long gone by. Forgetting that tail in the streamed path is what + /// leaked 20 GB. + /// + private async Task CompleteBufferedAsync(ValueTask inFlight, PendingRequest pending) + { + try + { + Http2Response response = await inFlight; + WriteResponse(pending.StreamId, response); + } + catch (Exception exception) + { + // Nobody can observe this task, so an escaping exception would vanish silently and the + // peer would wait on a stream that is never coming. + Console.Error.WriteLine($"[ioxide.http2] request handler faulted: {exception.GetBaseException().Message}"); + + if (!IsBroken) + { + WriteResponse(pending.StreamId, new Http2Response { Status = 500 }); } } + finally + { + pending.Dispose(); + await MaybeFlushAsync(); + } } } From b771050a0f4e8fb386b4f3d357e8f768840d8706 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 14:17:45 +0100 Subject: [PATCH 04/14] fix(http2): async handlers can write again - staging and flushing take turns Non-blocking dispatch made handlers complete outside the pass, and the write path could not accept them: a PipeWriter permits no Write while a flush is outstanding, so every asynchronous handler faulted with "Cannot write while flush is in progress" and served nothing at all. Frames produced during a flush now land in a queue; when the flush completes the whole queue moves into the pipe and leaves as the next one. So responses that completed during one transport write share the single write after it, which extends the pass coalescing to handlers that finish outside the pass - which is every handler that touches a database or an upstream. Callers who queued await that turn, keeping backpressure and the yield a real flush gave them. Kestrel's Http2FrameWriter and Go's net/http2 writer take this shape. h2load -t4 -c32 -m16 -D8, 2 reactors, 3 reps: async handler, Task.Yield 0 bytes (faulted) -> 1928617 req/s async handler, Task.Delay(1) 0 bytes (faulted) -> 352000 req/s streamed 8x1KiB 551000 -> 1346000 (2.44x) buffered 2 B 5180000 -> 5110000 (-1%) buffered 8 KiB 1754280 -> 1768103 The coalesce limit is now per writer rather than per pass: many responses coalescing into one large write is the point, and the per-pass version split that write and cost a third of streamed throughput. First tests for any of this - the fake transport enforces the real contract, that a write during a flush throws and so does a second flush, so the case that used to break is the case under test. --- .../ioxide.http2/Http2Connection.Streamed.cs | 66 ++-- .../ioxide.http2/Http2Connection.Write.cs | 134 ++++++- src/protocols/ioxide.http2/Http2Connection.cs | 12 + .../ioxide.http2/Http2ResponseWriter.cs | 20 +- .../Http2OutputQueueTests.cs | 329 ++++++++++++++++++ .../Ioxide.Tests.Unit.csproj | 1 + tests/Ioxide.Tests.Unit/Program.cs | 1 + 7 files changed, 520 insertions(+), 43 deletions(-) create mode 100644 tests/Ioxide.Tests.Unit/Http2OutputQueueTests.cs diff --git a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs index f116c01..1cf9b5c 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs @@ -133,49 +133,45 @@ internal void SendStreamedData(int streamId, ReadOnlySpan body, bool endSt } } - /// Push whatever is staged out to the transport. - internal ValueTask FlushOutboundAsync() => FlushAsync(); - - /// - /// True while the read loop still owes a flush. A streamed writer checks this to decide whether - /// to write for itself: inside a pass the answer is no, and skipping that write is what lets - /// many responses leave on ONE transport write. - /// - /// It matters because the cost here is per-write, not per-byte - a streamed response measured - /// the same whether it carried 2 bytes or 8 KiB, which is the shape of a syscall rather than of - /// work. Buffered spread one write across every stream in flight; streamed paid one each, and - /// that alone was the 12.8x. - /// - internal bool PassFlushPending => _passFlushPending; - + // True while the read loop still owes a flush. Inside a pass a writer skips its own write and + // rides that one, which is what lets many responses leave on ONE transport write. The cost here + // is per-write, not per-byte - a streamed response measured the same whether it carried 2 bytes + // or 8 KiB - and buffered spreading one write across every stream in flight was the 12.8x. private bool _passFlushPending; - private int _stagedBytes; + + /// True while the dispatch that owes the pass flush is still running. + internal bool InDispatchPass => _passFlushPending; /// /// Write only when nothing else will. A handler that parked - on credit, on a timer, on a /// database - resumes after the pass flush has gone by, so its bytes would otherwise sit staged - /// until the peer happened to send something, which for an endless response is never. + /// until the peer happened to send something, which for an endless response is never. When it + /// resumes mid-flush instead, the bytes join the write queue and this await is the turn of the + /// pump that carries them (Http2Connection.Write.cs) - so completions sharing a reactor turn + /// still share a write. /// internal ValueTask MaybeFlushAsync() - { - // Coalescing has to stay BOUNDED, and not for memory reasons. A producer that loops without - // awaiting anything of its own - generating as fast as it can - has no yield point except - // this write, so skipping it unconditionally spins the reactor and the response never moves - // at all. Past the limit the write happens for real, which both drains the staging and - // hands the thread back. - // - // A PACED producer (an SSE feed waiting on an event) never reaches the limit: it parks, the - // pass flush carries its chunk out immediately, and it resumes outside the pass where this - // returns a real flush. So latency stays tight where it matters, and only a bandwidth-bound - // producer trades a little of it for throughput. - if (!_passFlushPending || _stagedBytes >= CoalesceLimit) - { - return FlushAsync(); - } - return ValueTask.CompletedTask; - } + => _passFlushPending ? ValueTask.CompletedTask : FlushAsync(); + + /// A streamed writer's flush: out to the transport, or the write queue's next turn. + internal ValueTask FlushOutboundAsync() => FlushAsync(); - private const int CoalesceLimit = 16 * 1024; + /// + /// How much ONE response may stage before its writer flushes for real even inside a pass. The + /// bound is per writer, never per pass - many responses coalescing into one large pass write + /// is the point of the design, and an earlier per-pass version of this limit split those + /// writes and cost a third of streamed throughput. What it exists for is the single producer + /// that loops without awaiting anything of its own: that loop has no yield but this write, so + /// skipping it unconditionally would spin the reactor with the response never moving at all. + /// Past the limit the write happens for real, which both drains the staging and hands the + /// thread back. + /// + /// A PACED producer (an SSE feed waiting on an event) never reaches the limit: it parks, the + /// pass flush carries its chunk out immediately, and it resumes outside the pass where the + /// flush is real. So latency stays tight where it matters, and only a bandwidth-bound producer + /// trades a little of it for throughput. + /// + internal const int CoalesceLimit = 16 * 1024; /// Completes when this stream has credit again, or the connection gives up. internal Task WaitForSendCreditAsync(int streamId) diff --git a/src/protocols/ioxide.http2/Http2Connection.Write.cs b/src/protocols/ioxide.http2/Http2Connection.Write.cs index 77903f1..e4dac95 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Write.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Write.cs @@ -7,27 +7,147 @@ namespace ioxide.http2; /// /// The write side: everything is staged into the connection's write slab and flushed once per pass, /// so a batch of multiplexed responses leaves in one send rather than one each. +/// +/// Staging and flushing take TURNS, because the transport permits neither a write during a flush +/// nor a second flush - and with non-blocking dispatch a handler finishes whenever its database +/// answers, which is as likely as not to be mid-flush. So while a flush is in flight, frames land +/// in a queue instead of the pipe; when it completes, the whole queue moves into the pipe and goes +/// out as the next flush. Every response that completed during one transport write leaves on the +/// single write after it, which is what extends the pass coalescing to handlers that finish +/// outside the pass. Kestrel's Http2FrameWriter and Go's net/http2 writer land on this same shape. /// public sealed partial class Http2Connection { - private bool _staged; + private bool _staged; // bytes sit in the pipe writer, awaiting a flush + private bool _flushing; // a transport flush is in flight; the pipe writer is untouchable + private bool _writeDead; // a flush faulted; the transport takes no more bytes, ever + + // Frames produced while a flush was in flight, in stage order. Drained into the pipe the + // moment the flush completes, so wire order is exactly stage order. + private byte[] _queued = []; + private int _queuedUsed; + + // Completed when the flush AFTER the current one finishes - the one that carries the queue. + // Callers who staged into the queue await this, which keeps the two things a real flush + // provided: backpressure, and the yield that hands the reactor back. + private TaskCompletionSource? _turnWaiter; private void Stage(ReadOnlySpan bytes) { - _pipe.Output.Write(bytes); - _stagedBytes += bytes.Length; - _staged = true; + if (_disposed || _writeDead) + { + return; // a detached tail outlived the connection; there is nobody to send to + } + + if (_flushing) + { + Enqueue(bytes); + } + else + { + _pipe.Output.Write(bytes); + _staged = true; + } } private async ValueTask FlushAsync() { + if (_disposed || _writeDead) + { + return; + } + + if (_flushing) + { + // The pump owns the pipe. If this caller queued bytes they leave on the pump's next + // turn - await it, so the caller keeps real backpressure. With nothing queued there + // is nothing to wait for. + if (_queuedUsed > 0) + { + await TurnAsync(); + } + return; + } + if (!_staged) { return; } - _staged = false; - _stagedBytes = 0; - await _pipe.Output.FlushAsync(); + + // This caller becomes the pump: it drives turn after turn until a flush completes with + // the queue empty. Everyone else who staged meanwhile is parked on TurnAsync. + _flushing = true; + try + { + while (true) + { + _staged = false; + + // Waiters captured BEFORE the flush are the ones whose bytes are in it; anyone + // arriving during the await parks a fresh waiter for the turn after. + TaskCompletionSource? turn = _turnWaiter; + _turnWaiter = null; + + await _pipe.Output.FlushAsync(); + + // May resume writers inline; _flushing is still true, so anything they stage + // lands in the queue and is picked up by the check just below. + turn?.TrySetResult(); + + if (_queuedUsed == 0 || _disposed) + { + return; + } + + _pipe.Output.Write(_queued.AsSpan(0, _queuedUsed)); + _queuedUsed = 0; + } + } + catch + { + // The transport refused a write. Nothing staged can ever leave now, so later stages + // must drop rather than throw from detached tails nobody observes. + _writeDead = true; + _failed = true; + throw; + } + finally + { + _flushing = false; + // Never strand a waiter: on a fault the turn it is waiting for will not come, and it + // has to wake to see the connection is broken rather than hang forever. + _turnWaiter?.TrySetResult(); + _turnWaiter = null; + } + } + + private Task TurnAsync() + { + _turnWaiter ??= new TaskCompletionSource(); + return _turnWaiter.Task; + } + + private void Enqueue(ReadOnlySpan bytes) + { + if (_queued.Length - _queuedUsed < bytes.Length) + { + long size = Math.Max(16 * 1024, (long)_queued.Length * 2); + while (size < (long)_queuedUsed + bytes.Length) + { + size *= 2; + } + + byte[] grown = ArrayPool.Shared.Rent((int)Math.Min(size, Array.MaxLength)); + _queued.AsSpan(0, _queuedUsed).CopyTo(grown); + if (_queued.Length > 0) + { + ArrayPool.Shared.Return(_queued); + } + _queued = grown; + } + + bytes.CopyTo(_queued.AsSpan(_queuedUsed)); + _queuedUsed += bytes.Length; } private void WriteSettings() diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 958d581..19ce0c3 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -97,6 +97,18 @@ public void Dispose() ArrayPool.Shared.Return(_inbound); _inbound = []; } + + if (_queued.Length > 0) + { + ArrayPool.Shared.Return(_queued); + _queued = []; + } + _queuedUsed = 0; + + // A writer parked on its turn of the write pump has to wake into IsBroken, not hang: the + // flush that would have completed its turn is never coming. + _turnWaiter?.TrySetResult(); + _turnWaiter = null; } /// Serve until the peer goes away, answering each request with . diff --git a/src/protocols/ioxide.http2/Http2ResponseWriter.cs b/src/protocols/ioxide.http2/Http2ResponseWriter.cs index 7bf151f..6fb4589 100644 --- a/src/protocols/ioxide.http2/Http2ResponseWriter.cs +++ b/src/protocols/ioxide.http2/Http2ResponseWriter.cs @@ -29,6 +29,13 @@ public sealed class Http2ResponseWriter : IBufferWriter private byte[] _staging = []; private int _staged; + // Bytes THIS response has staged since its last real flush. Inside a pass a writer normally + // rides the pass flush, but past Http2Connection.CoalesceLimit it flushes for real anyway - + // the yield that keeps a producer looping without any await of its own from spinning the + // reactor. Per writer on purpose: a per-pass version of this bound split the pass write and + // cost a third of streamed throughput. + private int _sinceRealFlush; + private bool _headersSent; private bool _completed; @@ -160,6 +167,7 @@ private async ValueTask FlushCore(bool endStream) } _staged = 0; + _sinceRealFlush += sent; if (endStream && sent == 0) { @@ -167,7 +175,16 @@ private async ValueTask FlushCore(bool endStream) _connection.SendStreamedData(_streamId, ReadOnlySpan.Empty, endStream: true); } - await _connection.MaybeFlushAsync(); + // Inside a pass the pass flush carries these frames with every other response's, so the + // writer stays out of the way - unless this one response has already staged past the + // limit, where it must write for real to yield. Outside a pass the flush is always real. + if (_connection.InDispatchPass && _sinceRealFlush < Http2Connection.CoalesceLimit) + { + return; + } + + _sinceRealFlush = 0; + await _connection.FlushOutboundAsync(); } private void EnsureStaging(int sizeHint) @@ -195,6 +212,7 @@ internal void Reset(int streamId) { _streamId = streamId; _staged = 0; + _sinceRealFlush = 0; _headersSent = false; _completed = false; } diff --git a/tests/Ioxide.Tests.Unit/Http2OutputQueueTests.cs b/tests/Ioxide.Tests.Unit/Http2OutputQueueTests.cs new file mode 100644 index 0000000..92c6891 --- /dev/null +++ b/tests/Ioxide.Tests.Unit/Http2OutputQueueTests.cs @@ -0,0 +1,329 @@ +using System.Buffers.Binary; +using System.IO.Pipelines; +using ioxide.http2; + +namespace Ioxide.Tests; + +/// +/// The HTTP/2 write queue: staging and flushing take turns on the connection's pipe, so a handler +/// that completes while a transport flush is in flight queues its frames and they leave - together +/// with every other completion from that window - on the single flush after it. +/// +/// The fake transport here enforces the same contract as the real TcpConnection: any write while a +/// flush is in flight throws, and so does a second flush. That is the constraint the queue exists +/// to solve, so a test that passed against a lenient fake would prove nothing. +/// +internal static class Http2OutputQueueTests +{ + public static void Register(Runner runner) + { + runner.Test("h2 queue: a pass's responses still leave on one flush", () => + { + using var client = new StrictClient(); + Task run = client.Connection.RunBufferedAsync(_ => Http2Response.Text("hi")); + + client.ReleaseFlush(); // server SETTINGS + client.SendRequests(1, 3); + + // One pass: SETTINGS ack plus both responses, staged together, flushed once. + Assert.Equal(1, client.PendingFlushes); + byte[] pass = client.ReleaseFlush(); + + List frames = Frame.Walk(pass); + Assert.True(frames.Any(f => f.Type == 0x4 && (f.Flags & 0x1) != 0), "settings ack in pass flush"); + AssertResponse(frames, streamId: 1); + AssertResponse(frames, streamId: 3); + + client.Close(run); + }); + + runner.Test("h2 queue: completions during a flush share the next one", () => + { + using var client = new StrictClient(); + + var parked = new Dictionary>(); + Task run = client.Connection.RunBufferedAsync(request => + { + var tcs = new TaskCompletionSource(); + parked[request.StreamId] = tcs; + return new ValueTask(tcs.Task); + }); + + client.ReleaseFlush(); // server SETTINGS + client.SendRequests(1, 3); + + // The pass flushed only the SETTINGS ack; both handlers are parked. Complete them + // while that flush is still in flight - exactly where the old code threw + // "Cannot write while flush is in progress" and killed the connection. + Assert.Equal(1, client.PendingFlushes); + parked[1].SetResult(Http2Response.Text("first")); + parked[3].SetResult(Http2Response.Text("second")); + + Assert.Equal(1, client.PendingFlushes); // both queued; nothing new in flight + client.ReleaseFlush(); // the ack + + // ONE flush now carries both responses - the write the queue exists to coalesce. + Assert.Equal(1, client.PendingFlushes); + List frames = Frame.Walk(client.ReleaseFlush()); + AssertResponse(frames, streamId: 1); + AssertResponse(frames, streamId: 3); + + client.Close(run); + }); + + runner.Test("h2 queue: a streamed body resumed outside the pass keeps frame order", () => + { + using var client = new StrictClient(); + + var gate = new TaskCompletionSource(); + Task run = client.Connection.RunAsync(async (_, writer) => + { + writer.WriteHeaders(new Http2Response { Status = 200 }); + Push(writer, "live"); + await writer.FlushAsync(); // inside the pass: rides the pass flush + await gate.Task; // park past the pass, like a slow upstream + Push(writer, "wire"); + await writer.FlushAsync(); // outside: a real transport write + }); + + client.ReleaseFlush(); // server SETTINGS + client.SendRequests(1); + + List pass = Frame.Walk(client.ReleaseFlush()); + Assert.True(pass.Any(f => f.Type == 0x1 && f.StreamId == 1), "HEADERS rode the pass flush"); + Assert.True(pass.Any(f => f is { Type: 0x0, StreamId: 1, Length: 4, EndStream: false }), + "first chunk rode the pass flush, stream open"); + + gate.SetResult(); // resume: chunk, then END_STREAM on complete + List chunk = Frame.Walk(client.ReleaseFlush()); + Assert.True(chunk.Any(f => f is { Type: 0x0, StreamId: 1, Length: 4, EndStream: false }), + "second chunk flushed on resume"); + + List fin = Frame.Walk(client.ReleaseFlush()); + Assert.True(fin.Any(f => f is { Type: 0x0, StreamId: 1, Length: 0, EndStream: true }), + "completion sent END_STREAM"); + + client.Close(run); + }); + + runner.Test("h2 queue: a faulted flush wakes parked writers instead of stranding them", () => + { + using var client = new StrictClient(); + + var parked = new Dictionary>(); + Task run = client.Connection.RunBufferedAsync(request => + { + var tcs = new TaskCompletionSource(); + parked[request.StreamId] = tcs; + return new ValueTask(tcs.Task); + }); + + client.ReleaseFlush(); // server SETTINGS + client.SendRequests(1); + + // The completion queues behind the in-flight ack flush and parks on its turn. Then the + // transport dies. The waiter must wake and the connection must finish, not hang. + parked[1].SetResult(Http2Response.Text("never sent")); + client.FaultFlush(); + + Assert.True(run.Wait(5_000), "connection wound down after the transport fault"); + }); + } + + private static void Push(Http2ResponseWriter writer, string text) + { + Span span = writer.GetSpan(text.Length); + for (int i = 0; i < text.Length; i++) + { + span[i] = (byte)text[i]; + } + writer.Advance(text.Length); + } + + private static void AssertResponse(List frames, int streamId) + { + Assert.True(frames.Any(f => f.Type == 0x1 && f.StreamId == streamId), + $"HEADERS for stream {streamId}"); + Assert.True(frames.Any(f => f is { Type: 0x0, EndStream: true } && f.StreamId == streamId), + $"DATA with END_STREAM for stream {streamId}"); + } + + private readonly record struct Frame(int Length, byte Type, byte Flags, int StreamId) + { + public bool EndStream => (Flags & 0x1) != 0; + + public static List Walk(ReadOnlySpan wire) + { + var frames = new List(); + int at = 0; + while (at + 9 <= wire.Length) + { + int length = (wire[at] << 16) | (wire[at + 1] << 8) | wire[at + 2]; + byte type = wire[at + 3]; + byte flags = wire[at + 4]; + int stream = (int)(BinaryPrimitives.ReadUInt32BigEndian(wire[(at + 5)..]) & 0x7FFFFFFF); + frames.Add(new Frame(length, type, flags, stream)); + at += 9 + length; + } + Assert.Equal(wire.Length, at); // no torn frame: every byte accounted for + return frames; + } + } + + /// + /// An HTTP/2 client the test steps by hand: an inline input pipe to feed frames, and a strict + /// output writer whose flushes complete only when the test releases them. + /// + private sealed class StrictClient : IDuplexPipe, IDisposable + { + private readonly Pipe _input = new(new PipeOptions( + readerScheduler: PipeScheduler.Inline, + writerScheduler: PipeScheduler.Inline, + useSynchronizationContext: false)); + + private readonly StrictWriter _writer = new(); + + public StrictClient() => Connection = new Http2Connection(this); + + public Http2Connection Connection { get; } + + public PipeReader Input => _input.Reader; + public PipeWriter Output => _writer; + + public int PendingFlushes => _writer.PendingFlushes; + public byte[] ReleaseFlush() => _writer.ReleaseFlush(); + public void FaultFlush() => _writer.FaultFlush(); + + /// Preface, an empty SETTINGS, then one indexed-HPACK GET per stream id. + public void SendRequests(params int[] streamIds) + { + var bytes = new List("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"u8.ToArray()); + bytes.AddRange(FrameHeaderBytes(0, 0x4, 0, 0)); + foreach (int streamId in streamIds) + { + // 0x82 :method GET, 0x86 :scheme http, 0x84 :path / - static table only, so the + // request needs no HPACK encoder of its own. + bytes.AddRange(FrameHeaderBytes(3, 0x1, 0x5, streamId)); + bytes.AddRange([0x82, 0x86, 0x84]); + } + Feed(bytes.ToArray()); + } + + public void Feed(byte[] bytes) + => _input.Writer.WriteAsync(bytes).GetAwaiter().GetResult(); + + /// End the input and wait out the connection's run task. + public void Close(Task run) + { + _input.Writer.Complete(); + Assert.True(run.Wait(5_000), "connection wound down"); + } + + public void Dispose() => Connection.Dispose(); + + private static byte[] FrameHeaderBytes(int length, byte type, byte flags, int streamId) => + [ + (byte)(length >> 16), (byte)(length >> 8), (byte)length, + type, flags, + (byte)(streamId >> 24), (byte)(streamId >> 16), (byte)(streamId >> 8), (byte)streamId, + ]; + } + + /// + /// The real transport's write contract, distilled: writes throw while a flush is in flight, a + /// second flush throws, and a flush completes only when the reactor - here, the test - says so. + /// + private sealed class StrictWriter : PipeWriter + { + private readonly List _staged = []; + private readonly Queue<(TaskCompletionSource Signal, byte[] Payload)> _inFlight = new(); + private byte[] _scratch = new byte[4096]; + private bool _flushing; + + public int PendingFlushes => _inFlight.Count; + + public override Memory GetMemory(int sizeHint = 0) + { + ThrowIfFlushing(); + EnsureScratch(sizeHint); + return _scratch; + } + + public override Span GetSpan(int sizeHint = 0) + { + ThrowIfFlushing(); + EnsureScratch(sizeHint); + return _scratch; + } + + public override void Advance(int bytes) + { + ThrowIfFlushing(); + _staged.AddRange(_scratch.AsSpan(0, bytes)); + } + + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) + { + if (_flushing) + { + throw new InvalidOperationException("FlushAsync already in progress."); + } + if (_staged.Count == 0) + { + return new ValueTask(new FlushResult(isCanceled: false, isCompleted: false)); + } + + _flushing = true; + var signal = new TaskCompletionSource(); + _inFlight.Enqueue((signal, _staged.ToArray())); + _staged.Clear(); + return new ValueTask(signal.Task); + } + + /// Complete the oldest in-flight flush and hand back the bytes it carried. + public byte[] ReleaseFlush() + { + Assert.True(_inFlight.Count > 0, "a flush was expected to be in flight"); + (TaskCompletionSource signal, byte[] payload) = _inFlight.Dequeue(); + + // Writable again BEFORE the signal: the queue drain runs inline on SetResult and its + // first act is to write. + _flushing = false; + signal.SetResult(new FlushResult(isCanceled: false, isCompleted: false)); + return payload; + } + + /// Fail the oldest in-flight flush, as a torn-down transport would. + public void FaultFlush() + { + Assert.True(_inFlight.Count > 0, "a flush was expected to be in flight"); + (TaskCompletionSource signal, _) = _inFlight.Dequeue(); + _flushing = false; + signal.SetException(new IOException("transport torn down")); + } + + public override void Complete(Exception? exception = null) + { + } + + public override void CancelPendingFlush() + { + } + + private void ThrowIfFlushing() + { + if (_flushing) + { + throw new InvalidOperationException("Cannot write while flush is in progress."); + } + } + + private void EnsureScratch(int sizeHint) + { + if (_scratch.Length < sizeHint) + { + _scratch = new byte[sizeHint]; + } + } + } +} diff --git a/tests/Ioxide.Tests.Unit/Ioxide.Tests.Unit.csproj b/tests/Ioxide.Tests.Unit/Ioxide.Tests.Unit.csproj index 4cb4715..55a63eb 100644 --- a/tests/Ioxide.Tests.Unit/Ioxide.Tests.Unit.csproj +++ b/tests/Ioxide.Tests.Unit/Ioxide.Tests.Unit.csproj @@ -13,6 +13,7 @@ + diff --git a/tests/Ioxide.Tests.Unit/Program.cs b/tests/Ioxide.Tests.Unit/Program.cs index ae158df..caa9c1b 100644 --- a/tests/Ioxide.Tests.Unit/Program.cs +++ b/tests/Ioxide.Tests.Unit/Program.cs @@ -16,6 +16,7 @@ private static int Main() MessageTests.Register(runner); ResponseAssemblyTests.Register(runner); ResponseCapTests.Register(runner); + Http2OutputQueueTests.Register(runner); return runner.Summary(); } From 778fb628940256972547000472db1a6b26ac4506 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 14:51:21 +0100 Subject: [PATCH 05/14] feat(httpclient): the HTTP/2 client is pure C# now The h2 client was the last thing holding the nghttp2 binding in the tree. It is the same framing, HPACK and flow control ioxide.http2 already runs for the server, pointed the other way round: the preface, odd stream ids, requests instead of responses, and the retry rules that decide whether a failed exchange may be sent again. Those types stay internal to ioxide.http2 and reach the client through InternalsVisibleTo. Duplicating them was the alternative, and two copies of an HPACK encoder is how the two drift; promoting them to public API would have made the package's surface bigger than its job. Kept from the binding, because the hazards are the client's own rather than nghttp2's: completions are recorded during a parse and resumed after it unwinds, so a resumed caller that submits again - or retries through the pool, which may dispose this connection - never re-enters the parser from inside itself. Verified against nginx over h2c and over TLS with ALPN, not just against our own server: GET, a POST body that reaches the origin, 25 requests multiplexed onto one connection, MaxResponseBytes, and a trailered response. Those five were skipping for want of a sidecar; they run now. Two more cover what a 1 KiB GET cannot reach - a 1 MiB body that has to park on the flow-control window and resume on each WINDOW_UPDATE, and a header block that has to leave as HEADERS plus CONTINUATION. Http 37 pass, 0 skipped. --- .../Http2/Tls/Playground.Http2.Tls.csproj | 19 - Playground/Http2/Tls/Program.cs | 230 ---- .../Playground.Http2.Nghttp2.csproj | 0 .../Playground.Http2.Nghttp2}/Program.cs | 0 {scripts => dropped}/build-nghttp2-native.sh | 0 .../Connection/Nghttp2Connection.Callbacks.cs | 0 .../Connection/Nghttp2Connection.Egress.cs | 0 .../Nghttp2Connection.RunBuffered.cs | 0 .../Connection/Nghttp2Connection.cs | 0 .../Connection/Nghttp2Options.cs | 0 .../ioxide.nghttp2/Http/CookieEnumerator.cs | 0 .../ioxide.nghttp2/Http/KeyValueList.cs | 0 .../ioxide.nghttp2/Http/Nghttp2Request.cs | 0 .../ioxide.nghttp2/Http/Nghttp2Response.cs | 0 .../ioxide.nghttp2/Interop/Nghttp2.cs | 0 .../ioxide.nghttp2/ioxide.nghttp2.csproj | 0 .../native/ioxide_nghttp2_shim.c | 0 .../linux-x64/native/libioxide_nghttp2.so | Bin .../Http2/Http2ClientConnection.cs | 994 +++++++++++++----- .../Http2/Http2ClientPool.cs | 3 +- .../ioxide.httpclient.csproj | 4 +- .../ioxide.http2/ioxide.http2.csproj | 9 +- tests/Ioxide.Tests.Http/Http2ClientTests.cs | 146 ++- .../Ioxide.Tests.Http.csproj | 1 + 24 files changed, 878 insertions(+), 528 deletions(-) delete mode 100644 Playground/Http2/Tls/Playground.Http2.Tls.csproj delete mode 100644 Playground/Http2/Tls/Program.cs rename {Playground/Http2/Nghttp2 => dropped/Playground.Http2.Nghttp2}/Playground.Http2.Nghttp2.csproj (100%) rename {Playground/Http2/Nghttp2 => dropped/Playground.Http2.Nghttp2}/Program.cs (100%) rename {scripts => dropped}/build-nghttp2-native.sh (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Connection/Nghttp2Connection.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Connection/Nghttp2Options.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Http/CookieEnumerator.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Http/KeyValueList.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Http/Nghttp2Request.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Http/Nghttp2Response.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/Interop/Nghttp2.cs (100%) rename {src/protocols => dropped}/ioxide.nghttp2/ioxide.nghttp2.csproj (100%) rename {src/protocols => dropped}/ioxide.nghttp2/native/ioxide_nghttp2_shim.c (100%) rename {src/protocols => dropped}/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so (100%) diff --git a/Playground/Http2/Tls/Playground.Http2.Tls.csproj b/Playground/Http2/Tls/Playground.Http2.Tls.csproj deleted file mode 100644 index 7517c6f..0000000 --- a/Playground/Http2/Tls/Playground.Http2.Tls.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - Exe - net11.0 - enable - enable - true - Playground.Http2.Tls - Playground.Http2.Tls - - - - - - - - - diff --git a/Playground/Http2/Tls/Program.cs b/Playground/Http2/Tls/Program.cs deleted file mode 100644 index aa991d7..0000000 --- a/Playground/Http2/Tls/Program.cs +++ /dev/null @@ -1,230 +0,0 @@ -using System.IO.Pipelines; -using System.Text; -using ioxide; -using ioxide.nghttp2; -using ioxide.tls; -using Playground.Shared; - -// ───────────────────────────────────────────────────────────────────────────────────────────── -// http2-tls - HTTP/2 over TLS, negotiated by ALPN, alongside HTTP/1.1 on the SAME port. This is -// what a browser expects: it offers "h2,http/1.1" and the server chooses. -// -// dotnet run -c Release --project Playground/Http2/Tls -// curl -k --http2 https://127.0.0.1:8443/ # negotiates h2 -// curl -k --http1.1 https://127.0.0.1:8443/ # same port, gets http/1.1 -// -// Two things make this work. TlsOptions.Alpn is an ORDERED list, most preferred first, and the -// server walks it to pick the first entry the client also offered - so the order below is the -// policy. And TlsSession.NegotiatedAlpn reports what was chosen, which is what lets one handler -// run two different protocol loops. -// -// Note what Nghttp2Connection is handed: a TlsConnectionDualPipe. It never learns that TLS is -// involved - the pipe decrypts on the way in and encrypts on the way out, so the protocol -// code is byte-for-byte the same as the h2c sample. Needs: ioxide, ioxide.nghttp2 -// ───────────────────────────────────────────────────────────────────────────────────────────── - -// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── -// Edit these. That is the whole mechanism - there is no config file and nothing else to find. -// Env.Override exists only so bench/run.sh can drive the sample from outside; delete that line -// when you copy this out and the literals above it are the entire configuration. - -ushort port = 8443; // https://127.0.0.1:8443/ -int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core -int bodyBytes = 2; // "ok" - this sample is about ALPN, not throughput - -Env.Override(ref port, ref reactors, ref bodyBytes); - -// A real PEM pair, or null to generate a self-signed localhost cert on first run. -string? certOverride = null; -string? keyOverride = null; - -// Hand OUTBOUND encryption to the kernel: the handler writes plaintext and the kernel makes the -// records. Off by default - OpenSSL both ways is the portable path, and on loopback the kernel -// is not faster. Its real payoff is sendfile and NIC offload, which a benchmark here cannot see. -bool kernelTx = false; - -// Hand INBOUND decryption to the kernel as well. Requires kernelTx - the RX handoff happens at -// the same moment as the TX one - and is experimental for a reason: a TLS 1.3 KeyUpdate cannot be -// read through IORING_OP_RECV, and roughly one first connection in twelve fails outright. -bool kernelRx = false; - -Env.OverrideKtls(ref kernelTx, ref kernelRx); - -// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The -// handler code is identical either way; this only changes how recv buffers are handed out. -bool incrementalBuffers = false; - -Env.OverrideIncremental(ref incrementalBuffers); -// ───────────────────────────────────────────────────────────────────────────────────────────── - -(string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); - -var config = new ServerConfig -{ - ReactorCount = reactors, // io_uring rings/threads - one per core - RingEntries = 8192, // SQ/CQ depth per ring - DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped - RecvBufferSize = 32 * 1024, // bytes per shared recv buffer - RecvSlots = 4096, // shared recv buffer-ring depth - Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null, // per-connection recv rings (6.12+) - see Tcp/Incremental - Udp = null, // no raw UDP sockets (TCP-only server) - Quic = null, // no QUIC transport - see Http3/* and Quic/Alpn - Tcp = new TcpOptions - { - Port = port, - ExtraPorts = [], // extra listener ports (one handler, several doors) - ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener - WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in - PoolMax = 1024, // pooled connection objects kept per reactor - WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG - ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes - RecvQueueEntries = 64, // per-connection recv completion queue depth - }, -}; - -byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)]; - -byte[] http11Response = -[ - .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\n\r\n"), - .. body, -]; - -var threads = new Thread[config.ReactorCount]; - -for (int i = 0; i < threads.Length; i++) -{ - var reactor = new Reactor(i, config); - - reactor.OnStart = r => TlsService.Start(r, new TlsOptions - { - CertificatePath = certPath, // PEM certificate chain file (leaf first) - CertificatePem = null, // in-memory PEM alternative - set one, not both - KeyPath = keyPath, // PEM private key file - KeyPem = null, // in-memory PEM alternative to KeyPath - Alpn = ["h2", "http/1.1"], // ORDERED, most preferred first - a client offering both gets h2 - KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = OpenSSL both ways) - KernelRx = kernelRx, // kTLS decrypt: needs KernelTx; experimental (see the knob above) - }); - - reactor.TcpHandle = async (r, conn) => - { - TlsSession? tls = null; - try - { - tls = await r.GetService()!.AcceptAsync(conn); - - if (tls.NegotiatedAlpn == "h2") - { - // The decrypt lives in the pipe, so the HTTP/2 code below is identical to the - // cleartext sample. Which halves the pipe uses is decided by what the handshake - // achieved, not by anything chosen here. - await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false); - - await new Nghttp2Connection(pipe).RunBufferedAsync(_ => new Nghttp2Response - { - Status = 200, - Body = body, - }); - return; - } - - // Anything else: HTTP/1.1 on the same port. - // - // The carry is not incidental. TLS hands back RECORDS, not requests, so a request - // split across two records decrypts twice - and answering on "plaintext arrived" - // would answer twice to one request. Framing is ours; ioxide does not parse HTTP. - var carry = new Carry(); - - // The client's first request usually rides in with its Finished flight, so the - // handshake already decrypted it and it is sitting in the session, not in any recv - // buffer. Miss this and that request is dropped and the loop parks on bytes that - // already arrived - which is exactly what happened here before. - carry.Append(tls.DrainPlaintext()); - - while (true) - { - bool wrote = false; - int end; - while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0) - { - carry.Consume(end + 4); - // Correct whichever backend the session ended up with. - tls.Write(conn, http11Response); - wrote = true; - } - - if (wrote) - { - await conn.FlushAsync(); - } - - RecvSnapshot snapshot = await conn.ReadAsync(); - - unsafe - { - while (conn.TryGetItem(snapshot, out ioxide.utils.SpscRecvRing.Item item)) - { - if (item.HasBuffer) - { - carry.Append(tls.Decrypt(item.Ptr, item.Len)); - conn.ReturnBuffer(in item); - } - } - } - - if (snapshot.IsClosed || tls.Closed) return; - conn.ResetRead(); - } - } - catch (Exception e) - { - Console.Error.WriteLine($"[http2-tls] connection failed: {e.Message}"); - } - finally - { - tls?.Dispose(); - conn.DecRef(); - } - }; - - threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; - threads[i].Start(); -} - -Console.WriteLine($"[http2-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " - + $"ALPN h2 then http/1.1, cert {certPath}, " - + $"rx={(kernelTx && kernelRx ? "kernel" : "openssl")}, " - + $"tx={(kernelTx ? "kernel" : "openssl")}"); - -foreach (Thread thread in threads) -{ - thread.Join(); -} - -// Decrypted-but-unframed bytes: append at the end, consume from the front. A List pressed -// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain -// array does both directly. -sealed class Carry -{ - private byte[] _buf = new byte[8 * 1024]; - private int _len; - - public ReadOnlySpan Span => _buf.AsSpan(0, _len); - - public void Append(ReadOnlySpan bytes) - { - if (_buf.Length - _len < bytes.Length) - { - Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length)); - } - bytes.CopyTo(_buf.AsSpan(_len)); - _len += bytes.Length; - } - - public void Consume(int count) - { - _buf.AsSpan(count, _len - count).CopyTo(_buf); - _len -= count; - } -} diff --git a/Playground/Http2/Nghttp2/Playground.Http2.Nghttp2.csproj b/dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj similarity index 100% rename from Playground/Http2/Nghttp2/Playground.Http2.Nghttp2.csproj rename to dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj diff --git a/Playground/Http2/Nghttp2/Program.cs b/dropped/Playground.Http2.Nghttp2/Program.cs similarity index 100% rename from Playground/Http2/Nghttp2/Program.cs rename to dropped/Playground.Http2.Nghttp2/Program.cs diff --git a/scripts/build-nghttp2-native.sh b/dropped/build-nghttp2-native.sh similarity index 100% rename from scripts/build-nghttp2-native.sh rename to dropped/build-nghttp2-native.sh diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs b/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs rename to dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs b/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs rename to dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs b/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs rename to dropped/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.cs b/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.cs rename to dropped/ioxide.nghttp2/Connection/Nghttp2Connection.cs diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Options.cs b/dropped/ioxide.nghttp2/Connection/Nghttp2Options.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Connection/Nghttp2Options.cs rename to dropped/ioxide.nghttp2/Connection/Nghttp2Options.cs diff --git a/src/protocols/ioxide.nghttp2/Http/CookieEnumerator.cs b/dropped/ioxide.nghttp2/Http/CookieEnumerator.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Http/CookieEnumerator.cs rename to dropped/ioxide.nghttp2/Http/CookieEnumerator.cs diff --git a/src/protocols/ioxide.nghttp2/Http/KeyValueList.cs b/dropped/ioxide.nghttp2/Http/KeyValueList.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Http/KeyValueList.cs rename to dropped/ioxide.nghttp2/Http/KeyValueList.cs diff --git a/src/protocols/ioxide.nghttp2/Http/Nghttp2Request.cs b/dropped/ioxide.nghttp2/Http/Nghttp2Request.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Http/Nghttp2Request.cs rename to dropped/ioxide.nghttp2/Http/Nghttp2Request.cs diff --git a/src/protocols/ioxide.nghttp2/Http/Nghttp2Response.cs b/dropped/ioxide.nghttp2/Http/Nghttp2Response.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Http/Nghttp2Response.cs rename to dropped/ioxide.nghttp2/Http/Nghttp2Response.cs diff --git a/src/protocols/ioxide.nghttp2/Interop/Nghttp2.cs b/dropped/ioxide.nghttp2/Interop/Nghttp2.cs similarity index 100% rename from src/protocols/ioxide.nghttp2/Interop/Nghttp2.cs rename to dropped/ioxide.nghttp2/Interop/Nghttp2.cs diff --git a/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj b/dropped/ioxide.nghttp2/ioxide.nghttp2.csproj similarity index 100% rename from src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj rename to dropped/ioxide.nghttp2/ioxide.nghttp2.csproj diff --git a/src/protocols/ioxide.nghttp2/native/ioxide_nghttp2_shim.c b/dropped/ioxide.nghttp2/native/ioxide_nghttp2_shim.c similarity index 100% rename from src/protocols/ioxide.nghttp2/native/ioxide_nghttp2_shim.c rename to dropped/ioxide.nghttp2/native/ioxide_nghttp2_shim.c diff --git a/src/protocols/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so b/dropped/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so similarity index 100% rename from src/protocols/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so rename to dropped/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so diff --git a/src/clients/ioxide.httpclient/Http2/Http2ClientConnection.cs b/src/clients/ioxide.httpclient/Http2/Http2ClientConnection.cs index 76ca6c4..229b8b3 100644 --- a/src/clients/ioxide.httpclient/Http2/Http2ClientConnection.cs +++ b/src/clients/ioxide.httpclient/Http2/Http2ClientConnection.cs @@ -1,50 +1,71 @@ using System.Buffers; -using System.Runtime.InteropServices; +using System.Buffers.Binary; using System.Threading.Tasks.Sources; -using ioxide.httpclient; -using ioxide.nghttp2; +using ioxide.http2; namespace ioxide.httpclient; /// -/// One HTTP/2 connection over a ring socket: many requests in flight at once, multiplexed by -/// nghttp2 onto a single TCP byte stream. +/// One HTTP/2 connection over a ring socket: many requests in flight at once, multiplexed onto a +/// single TCP byte stream. /// -/// nghttp2 is sans-I/O, so the split is clean - it owns framing, HPACK and flow control, and this -/// class owns the socket, the loop and the waiters. A pump task reads bytes, feeds them to -/// ih2_read, and drains whatever that produced back to the socket. +/// Framing, HPACK and flow control are ioxide.http2's - the same managed code the server side runs, +/// pointed the other way round. What is here is the client's half of the protocol: the preface, odd +/// stream ids, requests instead of responses, and the retry rules that decide whether a failed +/// exchange may be sent again. /// -/// This is h2c - cleartext HTTP/2 with prior knowledge (RFC 9113 §3.3). There is no TLS and -/// therefore no ALPN, so the peer must already be expecting HTTP/2 on this port; there is no -/// Upgrade dance and no h2-over-TLS here. +/// This speaks h2c with prior knowledge (RFC 9113 section 3.3) on cleartext and h2 by +/// ALPN over TLS. There is no Upgrade dance in either direction: on a plaintext port the origin +/// must already be expecting HTTP/2. /// /// -/// Reactor thread only. Response completions are recorded by the native callbacks and resumed -/// AFTER ih2_read returns, so a caller that immediately submits another request can never -/// re-enter nghttp2 while it is still on the stack. +/// Reactor thread only. Completions are recorded while frames are being parsed and resumed after +/// the parse unwinds, so a caller that immediately submits another request - or retries through the +/// pool, which may dispose this connection - never re-enters the parser from inside itself. /// public sealed class Http2ClientConnection : IDisposable { - // nghttp2's largest frame is 16 MiB, but it only emits SETTINGS_MAX_FRAME_SIZE-sized DATA - // frames (16 KiB by default) plus a 9-byte header. 64 KiB leaves ample room for a drain pass. - private const int EgressBufferSize = 64 * 1024; private const int IngressBufferSize = 64 * 1024; + /// What we advertise per stream. Large, because the origin is the one streaming to us. + private const int SelfInitialWindow = 1 << 20; + private readonly IClientTransport _transport; private readonly byte[] _authority; - private readonly byte[] _egress = new byte[EgressBufferSize]; + private readonly int _maxResponseBytes; + + private readonly Dictionary _pending = new(); + private readonly HpackDecoder _decoder = new(); + private byte[] _headerScratch = []; + + // Client-initiated streams are odd and strictly increasing (RFC 9113 section 5.1.1). + private int _nextStreamId = 1; + + // Ingress accumulator. A frame boundary has nothing to do with a recv boundary, so a partial + // frame has to survive to the next read. + private byte[] _inbound = []; + private int _inboundUsed; + + // Egress staging, double-buffered: a drain swaps the buffers and sends the full one, so bytes + // staged while that send is awaited land in the empty one and keep their order. + private byte[] _egress = new byte[16 * 1024]; + private byte[] _sending = new byte[16 * 1024]; + private int _egressUsed; + + // Peer flow-control state. The connection window and per-stream windows both have to allow a + // DATA frame before it may go out. + private int _peerConnectionWindow = 65535; + private int _peerInitialStreamWindow = 65535; + private int _peerMaxFrameSize = 16384; - private nint _nghttp2Handle; - private GCHandle _self; private bool _failed; private bool _disposed; - private readonly Dictionary _pending = new(); + // Set when the peer stops accepting new streams (GOAWAY, or a max-requests limit). What is in + // flight can still finish; the pool must stop handing this connection new work. + private bool _retiring; - // Completions and stream failures recorded during the current ih2_read/ih2_write call, - // resumed once it unwinds. Failures ride the same list because a resumed waiter can retry - // through the pool - and the pool may dispose this connection, which must never happen while - // nghttp2 is still on the stack. + // Completions and stream failures recorded during the current parse, resumed once it unwinds. private readonly List<(PendingRequest Pending, HttpClientResponse? Response, Exception? Error)> _completedThisPass = []; // Set while the pump owns the drain, so a request submitted by a resumed caller rides the @@ -52,18 +73,12 @@ public sealed class Http2ClientConnection : IDisposable private bool _inPumpPass; // One drain at a time. Both the pump and SendAsync can start one, and a drain awaits socket - // sends - so without this two of them interleave writes out of the single _egress buffer and - // put a corrupt frame stream on the wire (the peer then closes on a protocol error). + // sends - so without this two of them interleave and put a corrupt frame stream on the wire. private bool _draining; private bool _drainAgain; private readonly TaskCompletionSource _ready = new(TaskCreationOptions.RunContinuationsAsynchronously); - // Ceiling on headers + body per response. nghttp2 streams a response in through callbacks with - // no length known up front, so without this a single origin can grow the arena until the - // process dies. - private readonly int _maxResponseBytes; - private Http2ClientConnection(IClientTransport transport, string authority, int maxResponseBytes) { _transport = transport; @@ -73,11 +88,6 @@ private Http2ClientConnection(IClientTransport transport, string authority, int public bool IsBroken => _failed || _disposed || _retiring; - // Set when the peer refuses a stream (GOAWAY / max-requests reached). The connection can still - // finish what is in flight, but the pool must stop handing it new requests and open a - // replacement. - private bool _retiring; - public int InFlight => _pending.Count; public static async Task ConnectAsync(IRingHost host, string ip, ushort port, @@ -97,36 +107,15 @@ public static async Task ConnectAsync(IRingHost host, str } var connection = new Http2ClientConnection(transport, authority, maxResponseBytes); - connection.Setup(); + connection.WritePrefaceAndSettings(); _ = connection.PumpLoopAsync(); await connection._ready.Task; // preface + SETTINGS on the wire before the first request return connection; } - private unsafe void Setup() - { - _self = GCHandle.Alloc(this); - var callbacks = new Nghttp2.Callbacks - { - OnBeginHeaders = &CallbackBeginHeaders, - OnHeader = &CallbackHeader, - OnEndHeaders = &CallbackEndHeaders, - OnData = &CallbackData, - OnEndStream = &CallbackEndStream, - OnStreamError = &CallbackStreamError, - }; - - _nghttp2Handle = Nghttp2.ih2_client_new(callbacks, (void*)GCHandle.ToIntPtr(_self)); - if (_nghttp2Handle == 0) - { - _failed = true; - _ready.TrySetException(new Http2ClientException("nghttp2 client init failed")); - } - } - // --- requests ------------------------------------------------------------------------------- - public unsafe ValueTask SendAsync(HttpClientRequest request) + public ValueTask SendAsync(HttpClientRequest request) { if (IsBroken) { @@ -135,24 +124,20 @@ public unsafe ValueTask SendAsync(HttpClientRequest request) throw new Http2StreamRefusedException("connection is retiring or closed; request was not submitted"); } - byte[] headers = PackRequestHeaders(request, out int headersLength); - int streamId; - fixed (byte* headersPointer = headers) - fixed (byte* bodyPointer = request.Body.Span) - { - streamId = Nghttp2.ih2_submit_request(_nghttp2Handle, headersPointer, (nuint)headersLength, - bodyPointer, (nuint)request.Body.Length); - } - ArrayPool.Shared.Return(headers); + int streamId = _nextStreamId; + _nextStreamId += 2; - if (streamId < 0) + var pending = new PendingRequest { - throw new Http2ClientException($"submit_request failed: {Nghttp2.StrError(streamId)}"); - } - - var pending = new PendingRequest(); + StreamId = streamId, + SendWindow = _peerInitialStreamWindow, + BodyRemaining = request.Body, + }; _pending[streamId] = pending; + WriteRequestHeaders(request, streamId, endStream: request.Body.Length == 0); + PumpBody(pending); + if (!_inPumpPass) { _ = DrainDetachedAsync(); // issued from outside a pass: put it on the wire now @@ -177,60 +162,112 @@ private async Task DrainDetachedAsync() } } - // [u16 namelen][name][u16 valuelen][value]... - pseudo-headers first, in the order h2 requires. - private byte[] PackRequestHeaders(HttpClientRequest request, out int written) + /// + /// The request's field section, HPACK-encoded into one HEADERS frame. + /// + /// + /// The encoder never touches the dynamic table, so it holds no per-connection compression state + /// and cannot desynchronise from the origin's decoder. On a client that costs more than it does + /// on a server - a client repeats its own header set on every request, which is exactly what + /// indexing is for - but the pseudo-headers that dominate a small request are static-table hits + /// either way, and a desynchronised HPACK table poisons every later stream on the connection. + /// + private void WriteRequestHeaders(HttpClientRequest request, int streamId, bool endStream) { - int capacity = 64 + request.Method.Length + request.Path.Length + _authority.Length + 32; + int capacity = HpackEncoder.MaxEncodedLength(":authority".Length, _authority.Length) + + HpackEncoder.MaxEncodedLength(":method".Length, request.Method.Length) + + HpackEncoder.MaxEncodedLength(":path".Length, request.Path.Length) + + HpackEncoder.MaxEncodedLength(":scheme".Length, 5); foreach (KeyValuePair, ReadOnlyMemory> field in request.Headers.AsSpan()) { - capacity += 4 + field.Key.Length + field.Value.Length; + capacity += HpackEncoder.MaxEncodedLength(field.Key.Length, field.Value.Length); } - byte[] buffer = ArrayPool.Shared.Rent(capacity); - int cursor = 0; + byte[] block = ArrayPool.Shared.Rent(capacity); + int used = 0; + + // Pseudo-headers first and in this order, per RFC 9113 section 8.3.1. + used += HpackEncoder.Encode(block.AsSpan(used), ":method"u8, request.Method.Span); + // :scheme must match the transport. An origin reached over TLS sees ":scheme http" as a + // mismatch, and the strict ones reject the stream for it. + used += HpackEncoder.Encode(block.AsSpan(used), ":scheme"u8, _transport.IsSecure ? "https"u8 : "http"u8); + used += HpackEncoder.Encode(block.AsSpan(used), ":authority"u8, _authority); + used += HpackEncoder.Encode(block.AsSpan(used), ":path"u8, request.Path.Span); - static void Write(byte[] destination, ref int offset, ReadOnlySpan name, ReadOnlySpan value) + foreach (KeyValuePair, ReadOnlyMemory> field in request.Headers.AsSpan()) { - // Lengths are framed as u16. Truncating one while still copying the bytes would - // desync the whole packed block, so refuse instead. - if (name.Length > ushort.MaxValue || value.Length > ushort.MaxValue) - { - throw new Http2ClientException("header name or value exceeds 65535 bytes"); - } + used += HpackEncoder.Encode(block.AsSpan(used), field.Key.Span, field.Value.Span); + } + + WriteHeaderBlock(streamId, block.AsSpan(0, used), endStream); + ArrayPool.Shared.Return(block); + } - destination[offset++] = (byte)name.Length; - destination[offset++] = (byte)(name.Length >> 8); - name.CopyTo(destination.AsSpan(offset)); - offset += name.Length; + // One HEADERS, then CONTINUATION for whatever did not fit the peer's frame size. The block + // cannot be split anywhere else: HPACK is a stream, so a decoder needs the pieces contiguous + // and uninterrupted by any other stream's frames. + private void WriteHeaderBlock(int streamId, ReadOnlySpan block, bool endStream) + { + int first = Math.Min(block.Length, _peerMaxFrameSize); + bool complete = first == block.Length; - destination[offset++] = (byte)value.Length; - destination[offset++] = (byte)(value.Length >> 8); - value.CopyTo(destination.AsSpan(offset)); - offset += value.Length; - } + FrameFlags flags = FrameFlags.None; + if (complete) flags |= FrameFlags.EndHeaders; + if (endStream) flags |= FrameFlags.EndStream; - Write(buffer, ref cursor, ":method"u8, request.Method.Span); - // Must match the transport. An origin reached over TLS sees ":scheme http" as a - // mismatch, and the strict ones reject the stream for it - harmless while this client was - // h2c-only, reachable the moment h2-over-TLS works. - Write(buffer, ref cursor, ":scheme"u8, _transport.IsSecure ? "https"u8 : "http"u8); - Write(buffer, ref cursor, ":authority"u8, _authority); - Write(buffer, ref cursor, ":path"u8, request.Path.Span); + Span header = stackalloc byte[FrameHeader.Size]; + new FrameHeader(first, FrameType.Headers, flags, streamId).Write(header); + Stage(header); + Stage(block[..first]); - foreach (KeyValuePair, ReadOnlyMemory> field in request.Headers.AsSpan()) + int offset = first; + while (offset < block.Length) { - Write(buffer, ref cursor, field.Key.Span, field.Value.Span); + int chunk = Math.Min(block.Length - offset, _peerMaxFrameSize); + bool last = offset + chunk == block.Length; + new FrameHeader(chunk, FrameType.Continuation, + last ? FrameFlags.EndHeaders : FrameFlags.None, streamId).Write(header); + Stage(header); + Stage(block.Slice(offset, chunk)); + offset += chunk; } + } + + /// + /// Push as much of the request body as the peer's windows allow. Called at submit and again on + /// every WINDOW_UPDATE, because credit is what decides when the rest may go - a body larger + /// than the initial 65535-byte window leaves in pieces as the origin reads it. + /// + private void PumpBody(PendingRequest pending) + { + while (pending.BodyRemaining.Length > 0) + { + int credit = Math.Min(pending.SendWindow, _peerConnectionWindow); + if (credit <= 0) + { + return; // parked; a WINDOW_UPDATE brings us back + } + + int chunk = Math.Min(Math.Min(credit, _peerMaxFrameSize), pending.BodyRemaining.Length); + bool last = chunk == pending.BodyRemaining.Length; - written = cursor; - return buffer; + Span header = stackalloc byte[FrameHeader.Size]; + new FrameHeader(chunk, FrameType.Data, last ? FrameFlags.EndStream : FrameFlags.None, + pending.StreamId).Write(header); + Stage(header); + Stage(pending.BodyRemaining.Span[..chunk]); + + pending.BodyRemaining = pending.BodyRemaining[chunk..]; + pending.SendWindow -= chunk; + _peerConnectionWindow -= chunk; + } } // --- the pump ------------------------------------------------------------------------------- private async Task PumpLoopAsync() { - nint receive = Marshal.AllocHGlobal(IngressBufferSize); + nint receive = System.Runtime.InteropServices.Marshal.AllocHGlobal(IngressBufferSize); try { await PumpEgressAsync(); // connection preface + SETTINGS @@ -244,16 +281,14 @@ private async Task PumpLoopAsync() throw new Http2ClientException(n == 0 ? "peer closed the connection" : $"recv failed: errno {-n}"); } + Accumulate(receive, n); + _inPumpPass = true; try { - nint consumed = Feed(receive, n); - if (consumed < 0) - { - throw new Http2ClientException($"read failed: {Nghttp2.StrError((int)consumed)}"); - } + ParseAvailable(); - // nghttp2 has unwound: safe to resume the waiters. Their follow-up requests + // The parser has unwound: safe to resume the waiters. Their follow-up requests // submit while _inPumpPass is set, so they all ride the drain below. CompleteFinishedRequests(); } @@ -264,10 +299,9 @@ private async Task PumpLoopAsync() await PumpEgressAsync(); // ACKs, WINDOW_UPDATEs and any newly submitted requests - if (_nghttp2Handle != 0 && Nghttp2.ih2_is_dead(_nghttp2Handle) != 0) + if (_retiring && _pending.Count == 0) { - _retiring = true; // GOAWAY drained; the pool opens a replacement - break; + break; // GOAWAY drained and nothing left in flight; the pool opens a replacement } } } @@ -277,7 +311,7 @@ private async Task PumpLoopAsync() } finally { - Marshal.FreeHGlobal(receive); + System.Runtime.InteropServices.Marshal.FreeHGlobal(receive); // A clean retirement (GOAWAY drained, nothing left to read) leaves only streams the // peer never processed: ones past last_stream_id or never sent. RFC 9113 8.7 makes @@ -290,11 +324,33 @@ private async Task PumpLoopAsync() } } + private unsafe void Accumulate(nint receive, int length) + { + if (_inbound.Length - _inboundUsed < length) + { + long size = Math.Max(IngressBufferSize, (long)_inbound.Length * 2); + while (size < (long)_inboundUsed + length) + { + size *= 2; + } + byte[] grown = ArrayPool.Shared.Rent((int)Math.Min(size, Array.MaxLength)); + _inbound.AsSpan(0, _inboundUsed).CopyTo(grown); + if (_inbound.Length > 0) + { + ArrayPool.Shared.Return(_inbound); + } + _inbound = grown; + } + + new ReadOnlySpan((void*)receive, length).CopyTo(_inbound.AsSpan(_inboundUsed)); + _inboundUsed += length; + } + private async ValueTask PumpEgressAsync() { if (_draining) { - _drainAgain = true; // a drain is mid-flight; it will pick up what we just queued + _drainAgain = true; // a drain is mid-flight; it will pick up what we just staged return; } @@ -312,36 +368,20 @@ private async ValueTask PumpEgressAsync() finally { _draining = false; - - // Stream failures can be deposited DURING a drain: after a GOAWAY, nghttp2 refuses - // queued streams inside mem_send. Resolve them here, outside the native call - on the - // error path too, or they would be lost (their pendings left _pending already). - CompleteFinishedRequests(); } } private async ValueTask DrainLoopAsync() { - while (true) + while (_egressUsed > 0 && !_disposed) { - if (_nghttp2Handle == 0 || _disposed) - { - return; - } - - nint produced = DrainEgress(); + // Swap rather than copy: whatever is staged during the await below lands in the buffer + // this one just vacated, so ordering holds without a second pass over the bytes. + (_egress, _sending) = (_sending, _egress); + int length = _egressUsed; + _egressUsed = 0; - if (produced < 0) - { - _failed = true; - throw new Http2ClientException($"write failed: {Nghttp2.StrError((int)produced)}"); - } - if (produced == 0) - { - return; - } - - await SendAllAsync(_egress, (int)produced); + await SendAllAsync(_sending, length); } } @@ -363,128 +403,300 @@ private async ValueTask SendAllAsync(byte[] buffer, int length) } } - // Pointer-touching helpers, kept out of the async methods above (no await may appear in an - // unsafe context). - private unsafe nint Feed(nint receive, int length) - => Nghttp2.ih2_read(_nghttp2Handle, (byte*)receive, (nuint)length); + private static unsafe nint PointerOf(MemoryHandle pinned) => (nint)pinned.Pointer; - private unsafe nint DrainEgress() + private void Stage(ReadOnlySpan bytes) { - fixed (byte* egressPointer = _egress) + if (_disposed) + { + return; + } + + if (_egress.Length - _egressUsed < bytes.Length) { - return Nghttp2.ih2_write(_nghttp2Handle, egressPointer, (nuint)_egress.Length); + long size = Math.Max(16 * 1024, (long)_egress.Length * 2); + while (size < (long)_egressUsed + bytes.Length) + { + size *= 2; + } + byte[] grown = new byte[(int)Math.Min(size, Array.MaxLength)]; + _egress.AsSpan(0, _egressUsed).CopyTo(grown); + _egress = grown; } + + bytes.CopyTo(_egress.AsSpan(_egressUsed)); + _egressUsed += bytes.Length; } - private static unsafe nint PointerOf(MemoryHandle pinned) => (nint)pinned.Pointer; + // --- frames out ----------------------------------------------------------------------------- - // Runs OUTSIDE ih2_read/ih2_write, so a resumed caller may safely submit again - or retry - // through the pool, which may dispose this very connection. - private void CompleteFinishedRequests() + private void WritePrefaceAndSettings() { - if (_completedThisPass.Count == 0) + Stage(FrameHeader.ClientPreface); + + // ENABLE_PUSH off: this client has no cache to push into, and refusing it here is simpler + // than answering PUSH_PROMISE frames. INITIAL_WINDOW_SIZE is ours to set - the origin is + // the side sending bodies, so a small window here would throttle every download. + Span frame = stackalloc byte[FrameHeader.Size + 12]; + new FrameHeader(12, FrameType.Settings, FrameFlags.None, 0).Write(frame); + BinaryPrimitives.WriteUInt16BigEndian(frame[9..], Http2Setting.EnablePush); + BinaryPrimitives.WriteUInt32BigEndian(frame[11..], 0); + BinaryPrimitives.WriteUInt16BigEndian(frame[15..], Http2Setting.InitialWindowSize); + BinaryPrimitives.WriteUInt32BigEndian(frame[17..], SelfInitialWindow); + Stage(frame); + + // Lift the connection window to match what we just advertised per stream. The connection + // window is NOT covered by SETTINGS_INITIAL_WINDOW_SIZE and stays at 65535 otherwise, which + // would cap every download on the connection no matter what the streams allow. + WriteWindowUpdate(0, SelfInitialWindow - 65535); + } + + private void WriteSettingsAck() + { + Span frame = stackalloc byte[FrameHeader.Size]; + new FrameHeader(0, FrameType.Settings, FrameFlags.Ack, 0).Write(frame); + Stage(frame); + } + + private void WriteWindowUpdate(int streamId, int increment) + { + if (increment <= 0) { return; } - // Snapshot-and-clear first: a resumed caller can land new completions here, and the list - // must not be mutated while it is being walked. - var finished = _completedThisPass.ToArray(); - _completedThisPass.Clear(); + Span frame = stackalloc byte[FrameHeader.Size + 4]; + new FrameHeader(4, FrameType.WindowUpdate, FrameFlags.None, streamId).Write(frame); + BinaryPrimitives.WriteUInt32BigEndian(frame[FrameHeader.Size..], (uint)increment); + Stage(frame); + } - foreach ((PendingRequest pending, HttpClientResponse? response, Exception? error) in finished) + private void WritePingAck(ReadOnlySpan opaque) + { + Span frame = stackalloc byte[FrameHeader.Size + 8]; + new FrameHeader(8, FrameType.Ping, FrameFlags.Ack, 0).Write(frame); + opaque.CopyTo(frame[FrameHeader.Size..]); + Stage(frame); + } + + private void GoAway(uint error) + { + Span frame = stackalloc byte[FrameHeader.Size + 8]; + new FrameHeader(8, FrameType.GoAway, FrameFlags.None, 0).Write(frame); + BinaryPrimitives.WriteUInt32BigEndian(frame[FrameHeader.Size..], 0); + BinaryPrimitives.WriteUInt32BigEndian(frame[(FrameHeader.Size + 4)..], error); + Stage(frame); + _failed = true; + } + + // --- frames in ------------------------------------------------------------------------------ + + private void ParseAvailable() + { + int position = 0; + + while (!_failed) { - if (error is not null) + if (!FrameHeader.TryRead(_inbound.AsSpan(position, _inboundUsed - position), out FrameHeader header)) { - pending.Fail(error); - continue; + break; // not even a header yet } - // The arena refused bytes for exceeding MaxResponseBytes. The callbacks could only - // record that; failing the caller happens here, where a throw no longer crosses - // nghttp2's frames. - if (response!.Overflowed) + int total = FrameHeader.Size + header.Length; + if (_inboundUsed - position < total) { - response.Dispose(); - pending.Fail(new Http2ClientException( - $"response exceeds MaxResponseBytes ({_maxResponseBytes})")); - continue; + break; // header is in, payload is not } - pending.Complete(response); + Handle(header, _inbound.AsSpan(position + FrameHeader.Size, header.Length)); + position += total; } + + Compact(position); } - private void FailAll(Exception error) + private void Compact(int consumed) { - // Broken before anything resumes: an inline-resumed caller that immediately retries must - // see IsBroken and go to the pool, not submit onto this connection. - _failed = true; - _ready.TrySetException(error); - - // Deposited but not yet flushed: those pendings already left _pending, so they would be - // missed below. A recorded response is a real completed exchange - deliver it. - CompleteFinishedRequests(); - - if (_pending.Count == 0) + if (consumed == 0) { return; } - // Snapshot-and-clear: resumes run inline and may re-enter (a retry submitting elsewhere - // still touches pool state), and _pending must not change under the enumeration. - PendingRequest[] pendings = [.. _pending.Values]; - _pending.Clear(); - foreach (PendingRequest pending in pendings) + int remaining = _inboundUsed - consumed; + if (remaining > 0) { - pending.Fail(error); + _inbound.AsSpan(consumed, remaining).CopyTo(_inbound); + } + _inboundUsed = remaining; + } + + private void Handle(in FrameHeader header, ReadOnlySpan payload) + { + switch (header.Type) + { + case FrameType.Headers: HandleHeaders(header, payload); break; + case FrameType.Continuation: HandleContinuation(header, payload); break; + case FrameType.Data: HandleData(header, payload); break; + case FrameType.Settings: HandleSettings(header, payload); break; + case FrameType.WindowUpdate: HandleWindowUpdate(header, payload); break; + case FrameType.Ping: HandlePing(header, payload); break; + case FrameType.RstStream: HandleRstStream(header, payload); break; + case FrameType.GoAway: HandleGoAway(payload); break; + + // We advertised ENABLE_PUSH=0, so a PUSH_PROMISE is the origin breaking the setting we + // gave it. PRIORITY carries no obligation and is ignored. + case FrameType.PushPromise: + GoAway(Http2Error.ProtocolError); + break; + + case FrameType.Priority: + default: + break; } } - // --- native callbacks: deposit only --------------------------------------------------------- + private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload) + { + ReadOnlySpan block = payload; - private static unsafe Http2ClientConnection FromUser(void* user) - => (Http2ClientConnection)GCHandle.FromIntPtr((nint)user).Target!; + // Padding and priority sit in front of the header block and are not part of it. + if ((header.Flags & FrameFlags.Padded) != 0) + { + if (block.Length < 1) + { + GoAway(Http2Error.FrameSizeError); + return; + } + int padding = block[0]; + block = block[1..]; + if (padding > block.Length) + { + GoAway(Http2Error.ProtocolError); + return; + } + block = block[..^padding]; + } + + if ((header.Flags & FrameFlags.Priority) != 0) + { + if (block.Length < 5) + { + GoAway(Http2Error.FrameSizeError); + return; + } + block = block[5..]; + } + + // A response on a stream we no longer have (cancelled, or already complete) still has to be + // decoded: HPACK is stateful across the whole connection, so skipping the block would + // desynchronise the table for every stream after it. + if (_pending.TryGetValue(header.StreamId, out PendingRequest? pending)) + { + pending.AppendHeaderBlock(block); + } + else + { + _orphanBlock.AppendHeaderBlock(block); + } - [UnmanagedCallersOnly] - private static unsafe void CallbackBeginHeaders(void* user, int streamId) + // Recorded rather than acted on: END_STREAM is carried by the HEADERS that OPENS the block, + // but the block may run on into CONTINUATION frames. Completing here would hand back a + // response whose header fields have not been decoded yet. + if ((header.Flags & FrameFlags.EndStream) != 0 && pending is not null) + { + pending.HeadersEndedStream = true; + } + + if ((header.Flags & FrameFlags.EndHeaders) != 0) + { + if (!DecodeHeaderBlock(pending)) + { + return; + } + + if (pending is { HeadersEndedStream: true }) + { + Complete(pending); + } + } + } + + private void HandleContinuation(in FrameHeader header, ReadOnlySpan payload) { - Http2ClientConnection connection = FromUser(user); + _pending.TryGetValue(header.StreamId, out PendingRequest? pending); + (pending ?? _orphanBlock).AppendHeaderBlock(payload); - // One response per request, created on the FIRST field section and kept. nghttp2 reports - // trailers as HCAT_HEADERS, the same category as the real response after a 1xx, so this - // callback fires again at the end of a trailered stream. Replacing the response there - // would throw away the assembled one while BodyStart/BodyLength still describe its arena, - // and CallbackEndStream would then slice those offsets out of the fresh, near-empty one. - // Interim (1xx) heads are cleared by ResetForInterim instead, which reuses this object. - if (connection._pending.TryGetValue(streamId, out PendingRequest? pending) && - pending.Response is null) + if ((header.Flags & FrameFlags.EndHeaders) != 0) { - var response = new HttpClientResponse(); - response.SetMaxArenaBytes(connection._maxResponseBytes); - pending.Response = response; + if (!DecodeHeaderBlock(pending)) + { + return; + } + + // CONTINUATION carries no END_STREAM of its own; it is on the HEADERS that opened the + // block, which we recorded there. + if (pending is { HeadersEndedStream: true }) + { + Complete(pending); + } } } - [UnmanagedCallersOnly] - private static unsafe void CallbackHeader(void* user, int streamId, byte* name, nuint nameLen, - byte* value, nuint valueLen) + /// Header blocks for streams we no longer track, decoded only to keep HPACK in step. + private readonly PendingRequest _orphanBlock = new(); + + private bool DecodeHeaderBlock(PendingRequest? pending) { - Http2ClientConnection connection = FromUser(user); - if (!connection._pending.TryGetValue(streamId, out PendingRequest? pending) || - pending.Response is not { } response) + PendingRequest target = pending ?? _orphanBlock; + try { - return; + ReadOnlySpan block = target.HeaderBlock; + + // Huffman can expand a literal well past its encoded length, so the scratch has to be + // able to hold the worst case for the whole block. + int needed = Huffman.MaxDecodedLength(block.Length) + block.Length; + if (_headerScratch.Length < needed) + { + _headerScratch = new byte[needed]; + } + + if (pending is null) + { + _decoder.Decode(block, _headerScratch, static (_, _) => { }); + } + else + { + HttpClientResponse response = pending.Response ??= NewResponse(); + _decoder.Decode(block, _headerScratch, (name, value) => AddHeader(response, name, value)); + pending.Assembly.EndFieldSection(response); + } + + target.ClearHeaderBlock(); + return true; } + catch (HpackDecoder.HpackException) + { + // The dynamic tables have diverged; every later block on this connection would decode + // to nonsense, so the connection - not the stream - is what has to end. + GoAway(Http2Error.CompressionError); + return false; + } + } - var nameSpan = new ReadOnlySpan(name, (int)nameLen); - var valueSpan = new ReadOnlySpan(value, (int)valueLen); + private HttpClientResponse NewResponse() + { + var response = new HttpClientResponse(); + response.SetMaxArenaBytes(_maxResponseBytes); + return response; + } + private static void AddHeader(HttpClientResponse response, ReadOnlySpan name, ReadOnlySpan value) + { // ":status" is the status line's stand-in; everything else is an ordinary field. - if (nameSpan.SequenceEqual(":status"u8)) + if (name.SequenceEqual(":status"u8)) { int status = 0; - foreach (byte digit in valueSpan) + foreach (byte digit in value) { status = (status * 10) + (digit - (byte)'0'); } @@ -492,43 +704,62 @@ private static unsafe void CallbackHeader(void* user, int streamId, byte* name, return; } - (int Offset, int Length) nameRange = response.Append(nameSpan); - (int Offset, int Length) valueRange = response.Append(valueSpan); + (int Offset, int Length) nameRange = response.Append(name); + (int Offset, int Length) valueRange = response.Append(value); response.AddHeaderRange(nameRange, valueRange); } - [UnmanagedCallersOnly] - private static unsafe void CallbackEndHeaders(void* user, int streamId) + private void HandleData(in FrameHeader header, ReadOnlySpan payload) { - Http2ClientConnection connection = FromUser(user); - if (!connection._pending.TryGetValue(streamId, out PendingRequest? pending) || - pending.Response is not { } response) + ReadOnlySpan body = payload; + + if ((header.Flags & FrameFlags.Padded) != 0) { - return; + if (body.Length < 1) + { + GoAway(Http2Error.FrameSizeError); + return; + } + int padding = body[0]; + body = body[1..]; + if (padding > body.Length) + { + GoAway(Http2Error.ProtocolError); + return; + } + body = body[..^padding]; } - pending.Assembly.EndFieldSection(response); - } - - [UnmanagedCallersOnly] - private static unsafe void CallbackData(void* user, int streamId, byte* data, nuint dataLen) - { - Http2ClientConnection connection = FromUser(user); - if (connection._pending.TryGetValue(streamId, out PendingRequest? pending) && + if (_pending.TryGetValue(header.StreamId, out PendingRequest? pending) && pending.Response is { } response) { // Count what the arena ACCEPTED, not what arrived: past MaxResponseBytes the append is // refused and returns a zero-length range, and a body length that outran the arena - // would make Freeze slice out of bounds - inside this callback, which is fatal. - pending.Assembly.BodyLength += response.Append(new ReadOnlySpan(data, (int)dataLen)).Length; + // would make Freeze slice out of bounds. + pending.Assembly.BodyLength += response.Append(body).Length; + } + + // The whole payload counts against the window, padding included, so the peer's accounting + // and ours agree. Replenished immediately: this client buffers the response anyway, so + // holding the window back would only stall the origin. + if (payload.Length > 0) + { + WriteWindowUpdate(0, payload.Length); + if (header.StreamId != 0 && pending is not null) + { + WriteWindowUpdate(header.StreamId, payload.Length); + } + } + + if ((header.Flags & FrameFlags.EndStream) != 0 && pending is not null) + { + Complete(pending); } } - [UnmanagedCallersOnly] - private static unsafe void CallbackEndStream(void* user, int streamId) + private void Complete(PendingRequest pending) { - Http2ClientConnection connection = FromUser(user); - if (!connection._pending.Remove(streamId, out PendingRequest? pending)) + if (!_pending.Remove(pending.StreamId)) { return; } @@ -537,75 +768,306 @@ private static unsafe void CallbackEndStream(void* user, int streamId) response.SetBodyRange(pending.Assembly.BodyRange); response.Freeze(); - // Record only - resumed by CompleteFinishedRequests once nghttp2 unwinds. - connection._completedThisPass.Add((pending, response, null)); + // Record only - resumed by CompleteFinishedRequests once the parse unwinds. + _completedThisPass.Add((pending, response, null)); + } + + private void HandleSettings(in FrameHeader header, ReadOnlySpan payload) + { + if ((header.Flags & FrameFlags.Ack) != 0) + { + return; // our own SETTINGS acknowledged + } + + if (payload.Length % 6 != 0) + { + GoAway(Http2Error.FrameSizeError); + return; + } + + for (int offset = 0; offset + 6 <= payload.Length; offset += 6) + { + ushort id = BinaryPrimitives.ReadUInt16BigEndian(payload[offset..]); + uint value = BinaryPrimitives.ReadUInt32BigEndian(payload[(offset + 2)..]); + + switch (id) + { + case Http2Setting.InitialWindowSize: + if (value > int.MaxValue) + { + GoAway(Http2Error.FlowControlError); + return; + } + // Applies retroactively to every open stream, per RFC 9113 section 6.9.2. + int delta = (int)value - _peerInitialStreamWindow; + _peerInitialStreamWindow = (int)value; + foreach (PendingRequest stream in _pending.Values) + { + stream.SendWindow += delta; + } + break; + + case Http2Setting.MaxFrameSize: + if (value is < 16384 or > 16777215) + { + GoAway(Http2Error.ProtocolError); + return; + } + _peerMaxFrameSize = (int)value; + break; + } + } + + WriteSettingsAck(); + + // A raised window may have unparked a body mid-send. + PumpParkedBodies(); } - [UnmanagedCallersOnly] - private static unsafe void CallbackStreamError(void* user, int streamId, uint errorCode) + private void HandleWindowUpdate(in FrameHeader header, ReadOnlySpan payload) { - Http2ClientConnection connection = FromUser(user); + if (payload.Length != 4) + { + GoAway(Http2Error.FrameSizeError); + return; + } + + int increment = (int)(BinaryPrimitives.ReadUInt32BigEndian(payload) & 0x7FFFFFFF); + if (increment == 0) + { + GoAway(Http2Error.ProtocolError); + return; + } + + if (header.StreamId == 0) + { + _peerConnectionWindow += increment; + PumpParkedBodies(); // connection credit unparks every stream, not one + } + else if (_pending.TryGetValue(header.StreamId, out PendingRequest? pending)) + { + pending.SendWindow += increment; + PumpBody(pending); + } + } + + private void PumpParkedBodies() + { + foreach (PendingRequest pending in _pending.Values) + { + if (pending.BodyRemaining.Length > 0) + { + PumpBody(pending); + } + } + } + + private void HandlePing(in FrameHeader header, ReadOnlySpan payload) + { + if (payload.Length != 8) + { + GoAway(Http2Error.FrameSizeError); + return; + } + if ((header.Flags & FrameFlags.Ack) != 0) + { + return; + } + + WritePingAck(payload); + } + + private void HandleRstStream(in FrameHeader header, ReadOnlySpan payload) + { + uint error = payload.Length >= 4 ? BinaryPrimitives.ReadUInt32BigEndian(payload) : Http2Error.NoError; // REFUSED_STREAM means the peer never processed the request - it is retiring the connection // (GOAWAY, or a max-requests limit like nginx's keepalive_requests). RFC 9113 8.7 makes // retrying it explicitly safe, so it is surfaced as retryable and the connection retires. - bool refused = errorCode == RefusedStream; + bool refused = error == Http2Error.RefusedStream; if (refused) { - connection._retiring = true; + _retiring = true; + } + + if (_pending.Remove(header.StreamId, out PendingRequest? pending)) + { + // Record only, exactly like a completion: failing here would resume the waiter mid-parse, + // and a resumed caller retries through the pool, which prunes retiring connections - + // disposing this one while its own parser is still on the stack. + _completedThisPass.Add((pending, null, refused + ? new Http2StreamRefusedException($"stream {header.StreamId} refused; connection is retiring") + : new Http2ClientException($"stream {header.StreamId} reset (error {error})"))); } + } - if (connection._pending.Remove(streamId, out PendingRequest? pending)) + private void HandleGoAway(ReadOnlySpan payload) + { + _retiring = true; + + // Everything above last_stream_id was never processed, so RFC 9113 8.7 makes resending it + // safe whatever the method. Below it the origin may have acted on the request, so those + // streams are left in flight to finish or fail on their own. + int lastStreamId = payload.Length >= 4 + ? (int)(BinaryPrimitives.ReadUInt32BigEndian(payload) & 0x7FFFFFFF) + : 0; + + int[] refused = [.. _pending.Keys.Where(id => id > lastStreamId)]; + foreach (int streamId in refused) { - // Record only, exactly like EndStream. Failing here would resume the waiter INSIDE - // ih2_read/ih2_write (this callback fires during both - GOAWAY closes live streams in - // mem_recv and refuses queued ones in mem_send), and a resumed caller retries through - // the pool, which prunes retiring connections - Dispose would then run ih2_free while - // nghttp2 is still on this very stack. - connection._completedThisPass.Add((pending, null, refused - ? new Http2StreamRefusedException($"stream {streamId} refused; connection is retiring") - : new Http2ClientException($"stream {streamId} reset (error {errorCode})"))); + if (_pending.Remove(streamId, out PendingRequest? pending)) + { + _completedThisPass.Add((pending, null, new Http2StreamRefusedException( + $"stream {streamId} is past the GOAWAY last-stream-id; request was not processed"))); + } } } - /// HTTP/2 REFUSED_STREAM (RFC 9113 section 7). - private const uint RefusedStream = 7; + // --- completion ----------------------------------------------------------------------------- - public unsafe void Dispose() + // Runs OUTSIDE the parse, so a resumed caller may safely submit again - or retry through the + // pool, which may dispose this very connection. + private void CompleteFinishedRequests() { - if (_disposed) + if (_completedThisPass.Count == 0) { return; } - _disposed = true; + + // Snapshot-and-clear first: a resumed caller can land new completions here, and the list + // must not be mutated while it is being walked. + var finished = _completedThisPass.ToArray(); + _completedThisPass.Clear(); + + foreach ((PendingRequest pending, HttpClientResponse? response, Exception? error) in finished) + { + if (error is not null) + { + pending.Dispose(); + pending.Fail(error); + continue; + } + + // The arena refused bytes for exceeding MaxResponseBytes. Parsing could only record + // that; failing the caller happens here, where a throw no longer unwinds the parser. + if (response!.Overflowed) + { + response.Dispose(); + pending.Dispose(); + pending.Fail(new Http2ClientException( + $"response exceeds MaxResponseBytes ({_maxResponseBytes})")); + continue; + } + + pending.Dispose(); + pending.Complete(response); + } + } + + private void FailAll(Exception error) + { + // Broken before anything resumes: an inline-resumed caller that immediately retries must + // see IsBroken and go to the pool, not submit onto this connection. _failed = true; + _ready.TrySetException(error); + + // Recorded but not yet flushed: those pendings already left _pending, so they would be + // missed below. A recorded response is a real completed exchange - deliver it. + CompleteFinishedRequests(); + + if (_pending.Count == 0) + { + return; + } + + // Snapshot-and-clear: resumes run inline and may re-enter (a retry submitting elsewhere + // still touches pool state), and _pending must not change under the enumeration. + PendingRequest[] pendings = [.. _pending.Values]; + _pending.Clear(); + foreach (PendingRequest pending in pendings) + { + pending.Dispose(); + pending.Fail(error); + } + } - if (_nghttp2Handle != 0) + public void Dispose() + { + if (_disposed) { - Nghttp2.ih2_free(_nghttp2Handle); - _nghttp2Handle = 0; + return; } - if (_self.IsAllocated) + _disposed = true; + _failed = true; + + _orphanBlock.Dispose(); + if (_inbound.Length > 0) { - _self.Free(); + ArrayPool.Shared.Return(_inbound); + _inbound = []; } + _inboundUsed = 0; + _transport.Dispose(); } /// One in-flight request. IValueTaskSource with asynchronous continuations OFF, so the /// caller resumes inline on the reactor thread. - private sealed class PendingRequest : IValueTaskSource + private sealed class PendingRequest : IValueTaskSource, IDisposable { private ManualResetValueTaskSourceCore _core = new() { RunContinuationsAsynchronously = false, }; + private byte[] _block = []; + private int _blockUsed; + + public int StreamId; public HttpClientResponse? Response; public ResponseAssembly Assembly; + /// What the peer will still accept on this stream. Starts at its advertised default. + public int SendWindow = 65535; + + /// Request body not yet on the wire, held back by flow control. + public ReadOnlyMemory BodyRemaining; + + /// END_STREAM arrived on a HEADERS whose block is still being continued. + public bool HeadersEndedStream; + public ValueTask Task => new(this, _core.Version); + public ReadOnlySpan HeaderBlock => _block.AsSpan(0, _blockUsed); + + // A header block can span HEADERS + CONTINUATION frames, and HPACK cannot be decoded + // piecewise - the whole block has to be in hand first. + public void AppendHeaderBlock(ReadOnlySpan data) + { + if (_block.Length - _blockUsed < data.Length) + { + byte[] grown = ArrayPool.Shared.Rent(Math.Max(4096, (_blockUsed + data.Length) * 2)); + _block.AsSpan(0, _blockUsed).CopyTo(grown); + if (_block.Length > 0) + { + ArrayPool.Shared.Return(_block); + } + _block = grown; + } + data.CopyTo(_block.AsSpan(_blockUsed)); + _blockUsed += data.Length; + } + + public void ClearHeaderBlock() + { + if (_block.Length > 0) + { + ArrayPool.Shared.Return(_block); + _block = []; + } + _blockUsed = 0; + } + public void Complete(HttpClientResponse response) => _core.SetResult(response); public void Fail(Exception error) => _core.SetException(error); @@ -617,6 +1079,8 @@ private sealed class PendingRequest : IValueTaskSource public void OnCompleted(Action continuation, object? state, short token, ValueTaskSourceOnCompletedFlags flags) => _core.OnCompleted(continuation, state, token, flags & ~ValueTaskSourceOnCompletedFlags.UseSchedulingContext); + + public void Dispose() => ClearHeaderBlock(); } } diff --git a/src/clients/ioxide.httpclient/Http2/Http2ClientPool.cs b/src/clients/ioxide.httpclient/Http2/Http2ClientPool.cs index f0e04f3..b1b2934 100644 --- a/src/clients/ioxide.httpclient/Http2/Http2ClientPool.cs +++ b/src/clients/ioxide.httpclient/Http2/Http2ClientPool.cs @@ -1,5 +1,4 @@ using ioxide.httpclient; -using ioxide.nghttp2; namespace ioxide.httpclient; @@ -216,7 +215,7 @@ private async Task OpenOneAsync() catch (Exception e) { _lastOpenFailure = e.Message; - Console.Error.WriteLine($"[nghttp2] connect to {_options.Host}:{_options.Port} failed: {e.Message}"); + Console.Error.WriteLine($"[http2] connect to {_options.Host}:{_options.Port} failed: {e.Message}"); WakeWaiters(); // fail fast: waiters re-check and time out rather than hang } finally diff --git a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj index 60b4d78..07d1ce7 100644 --- a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj +++ b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj @@ -10,7 +10,7 @@ ioxide.httpclient 0.4.169 MDA2AV - The ring-native HTTP client for the ioxide io_uring runtime: HTTP/1.1, HTTP/2 and HTTP/3 behind one API, with the protocol chosen per origin via Alt-Svc. One package - the h1 parser, the nghttp2 and nghttp3 bridges, client-side TLS (SNI, ALPN and certificate verification) for https:// origins, and the negotiating client - sharing one set of message types. Every response resumes the awaiting handler inline on its own reactor thread. + The ring-native HTTP client for the ioxide io_uring runtime: HTTP/1.1, HTTP/2 and HTTP/3 behind one API, with the protocol chosen per origin via Alt-Svc. One package - the h1 parser, a pure-C# HTTP/2 client on ioxide.http2's framing, the nghttp3 bridge, client-side TLS (SNI, ALPN and certificate verification) for https:// origins, and the negotiating client - sharing one set of message types. Every response resumes the awaiting handler inline on its own reactor thread. MIT https://mda2av.github.io/ioxide/ https://github.com/MDA2AV/ioxide @@ -23,7 +23,7 @@ - + diff --git a/src/protocols/ioxide.http2/ioxide.http2.csproj b/src/protocols/ioxide.http2/ioxide.http2.csproj index 1e01c2d..eb318c3 100644 --- a/src/protocols/ioxide.http2/ioxide.http2.csproj +++ b/src/protocols/ioxide.http2/ioxide.http2.csproj @@ -10,7 +10,7 @@ ioxide.http2 0.4.169 MDA2AV - Pure-C# HTTP/2 for the ioxide io_uring runtime: framing, HPACK (static and dynamic tables, Huffman) and flow control, with zero native code. A drop-in alternative to ioxide.nghttp2 for deployments that would rather not ship a native library - same connection shape, same request and response types. + Pure-C# HTTP/2 for the ioxide io_uring runtime: framing, HPACK (static and dynamic tables, Huffman) and flow control, with zero native code. Serves h2c with prior knowledge and h2 over TLS by ALPN, buffered or streamed in either direction, and the same framing drives ioxide.httpclient's HTTP/2 client. MIT https://mda2av.github.io/ioxide/ https://github.com/MDA2AV/ioxide @@ -24,4 +24,11 @@ + + + + + diff --git a/tests/Ioxide.Tests.Http/Http2ClientTests.cs b/tests/Ioxide.Tests.Http/Http2ClientTests.cs index 3e58313..215b1e5 100644 --- a/tests/Ioxide.Tests.Http/Http2ClientTests.cs +++ b/tests/Ioxide.Tests.Http/Http2ClientTests.cs @@ -39,9 +39,9 @@ public static void Register(Runner runner) Assert.Equal("200|1024", body); // the sidecar's 1 KiB object }, skip: noSidecar); - // Regression guard: request bodies were silently dropped by the shim (NGHTTP2_DATA_FLAG_NO_COPY - // with a no-op send_data callback made nghttp2 account each DATA frame as sent without ever - // emitting it). Every earlier test was a GET, so nothing caught it. + // Regression guard: request bodies were once silently dropped - accounted for as sent + // without a DATA frame ever leaving. Every other test here is a GET, so nothing else on + // this connection would notice. runner.Test("httpclient h2: POST body actually reaches the origin", () => { int driver = TestServer.Start(PostDriverHandler, onStart: reactor => @@ -88,12 +88,11 @@ public static void Register(Runner runner) runner.Test("httpclient h2: a trailered response survives and keeps its body", () => { - // nghttp2 reports TRAILERS as HCAT_HEADERS - the same category as the real response - // after a 1xx - so begin_headers fires a SECOND time at the end of a trailered stream. - // While that callback replaced the response, the assembled one was discarded with - // BodyStart/BodyLength still describing its arena, and end_stream then sliced those - // offsets out of the fresh, near-empty one: a large body threw - // ArgumentOutOfRangeException from inside [UnmanagedCallersOnly] and killed the + // Trailers are a SECOND field section on a stream that already has one, which is the + // same shape as the real response arriving after a 1xx. While the second section + // replaced the response, the assembled one was discarded with BodyStart/BodyLength + // still describing its arena, and end-of-stream then sliced those offsets out of the + // fresh, near-empty one: a large body threw ArgumentOutOfRangeException and killed the // process, a small one came back as silent garbage with status 0. // // The 20 KB object is deliberate. It cannot fit the trailer section's arena, so a @@ -109,6 +108,135 @@ public static void Register(Runner runner) (_, string second) = Client.Get(driver, "/big.html", timeoutMs: 20_000); Assert.Equal("200|20000", second); }, skip: noTrailerSidecar); + + // The two paths below need an origin that reports back what it RECEIVED, which nginx has no + // way to do - so they run against ioxide's own HTTP/2 server. Both exercise client code the + // sidecar tests never reach, because a 1 KiB GET fits in one window and one frame. + + runner.Test("httpclient h2: a body past the flow-control window arrives whole", () => + { + // 1 MiB against a 65535-byte connection window: the body cannot go out in one pass, so + // the client has to send what it has credit for, park, and resume on each WINDOW_UPDATE + // the origin sends back. Getting this wrong either truncates the body or blows the + // window and earns a FLOW_CONTROL_ERROR - the origin echoes the length, so both show. + const int BodyBytes = 1024 * 1024; + + int origin = StartEchoOrigin(); + int driver = TestServer.Start(PostSizeDriver(BodyBytes), onStart: reactor => + Http2ClientPool.Start(reactor, OriginOptions(origin))); + + (int status, string body) = Client.Get(driver, "/echo", timeoutMs: 30_000); + Assert.Equal(200, status); + Assert.Equal($"200|{BodyBytes}", body); + }); + + runner.Test("httpclient h2: a header block past the frame size continues", () => + { + // 40 headers of ~512 bytes overflows the 16 KiB maximum frame size, so the field + // section has to leave as HEADERS + CONTINUATION. The block cannot be split anywhere + // else on the connection either - HPACK is one stream, and a decoder needs the pieces + // contiguous - so a mistake here desynchronises the table rather than failing cleanly. + int origin = StartEchoOrigin(); + int driver = TestServer.Start(ManyHeadersDriver(count: 40, valueBytes: 512), onStart: reactor => + Http2ClientPool.Start(reactor, OriginOptions(origin))); + + (int status, string body) = Client.Get(driver, "/headers", timeoutMs: 30_000); + Assert.Equal(200, status); + Assert.Equal("200|40", body); + }); + } + + private static Http2ClientOptions OriginOptions(int port) => new() + { + Host = "127.0.0.1", + Port = (ushort)port, + PoolSize = 1, + }; + + /// + /// An ioxide HTTP/2 origin that answers with what it received: the body length for a request + /// that carried one, otherwise the number of ordinary header fields. + /// + private static int StartEchoOrigin() => TestServer.Start(async (_, connection) => + { + try + { + await new ioxide.http2.Http2Connection(connection).RunBufferedAsync(request => + new ioxide.http2.Http2Response + { + Status = 200, + Body = Encoding.ASCII.GetBytes( + (request.Body.Length > 0 ? request.Body.Length : request.Headers.Count).ToString()), + }); + } + finally + { + connection.DecRef(); + } + }); + + private static Func PostSizeDriver(int bodyBytes) + => (reactor, connection) => DriveOnce(reactor, connection, upstream => + { + byte[] payload = new byte[bodyBytes]; + payload.AsSpan().Fill((byte)'z'); + return upstream.PostAsync("/echo"u8.ToArray(), payload); + }); + + private static Func ManyHeadersDriver(int count, int valueBytes) + => (reactor, connection) => DriveOnce(reactor, connection, upstream => + { + var request = new HttpClientRequest(HttpMethods.Get, "/headers"); + for (int i = 0; i < count; i++) + { + request.Headers.Add( + Encoding.ASCII.GetBytes($"x-filler-{i:D3}"), + Encoding.ASCII.GetBytes(new string('v', valueBytes))); + } + return upstream.SendAsync(request); + }); + + // One request per inbound connection, answering with "status|body" so the assertion reads the + // origin's own account of what arrived. + private static async Task DriveOnce(Reactor reactor, TcpConnection connection, + Func> exchange) + { + try + { + Http2ClientPool upstream = reactor.GetService()!; + + while (true) + { + RecvSnapshot snapshot = await connection.ReadAsync(); + if (snapshot.IsClosed) + { + return; + } + Wire.ReadPath(connection, snapshot); + + string detail; + int status; + try + { + using HttpClientResponse response = await exchange(upstream); + status = response.Status; + detail = Encoding.ASCII.GetString(response.Body.Span); + } + catch (Exception e) + { + status = 599; + detail = e.Message; + } + + Wire.Write(connection, 200, $"{status}|{detail}"); + await connection.FlushAsync(); + connection.ResetRead(); + } + } + finally + { + connection.DecRef(); + } } // Sends a POST with a 4 KiB body and reports the status, so a dropped body shows up as a diff --git a/tests/Ioxide.Tests.Http/Ioxide.Tests.Http.csproj b/tests/Ioxide.Tests.Http/Ioxide.Tests.Http.csproj index 0233ecf..7a09723 100644 --- a/tests/Ioxide.Tests.Http/Ioxide.Tests.Http.csproj +++ b/tests/Ioxide.Tests.Http/Ioxide.Tests.Http.csproj @@ -13,6 +13,7 @@ + From b7fafdf3dd8b91c7d68777083b45e0d441d4dd93 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 14:51:35 +0100 Subject: [PATCH 06/14] chore(nghttp2): retire the binding into dropped/ ioxide.http2 started as the drop-in that needed no native library and ended as the only HTTP/2 here. It measured level with the binding (0.98x-1.09x on a small body, the ordering depending on the connection-to-reactor ratio rather than the codec), then grew past it: streamed responses, streamed request bodies and non-blocking dispatch all landed on the managed side, while the binding kept the blocking dispatch loop where one slow handler held up every other stream on the connection. Two implementations of one protocol is a tax paid in samples, docs, tests and benchmark fixtures, and the second one had stopped buying coverage of the protocol's darker corners - it was buying a native build step. dropped/ is where retired code goes: out of ioxide.slnx, out of CI, off NuGet, kept readable because the reasoning is easier to follow with the thing itself still there. Its README says so. The five samples that used it move to ioxide.http2. Four are the three-name swap the packages were designed for - Proxy/H2ToH1, H2ToH2, H2ToH3 and Http2/SslStream. The fifth, Http2/Tls, was already ported: Http2/ManagedTls is that same server on the managed stack, so keeping both would have been one sample twice. Also corrects comments the drop made false - the pure-C# module described itself as an alternative to nghttp2, its response type claimed bytes were copied into nghttp2 at submit, and the h2c sample said its read loop fed it. Unit 33, Http 37, E2E 46 pass. Solution builds with no reference to the binding left outside dropped/. --- .github/workflows/build.yml | 7 ++-- Playground/Http2/Managed/Program.cs | 4 +-- .../Playground.Http2.SslStream.csproj | 2 +- Playground/Http2/SslStream/Program.cs | 10 +++--- .../H2ToH1/Playground.Proxy.H2ToH1.csproj | 2 +- Playground/Proxy/H2ToH1/Program.cs | 16 ++++----- .../H2ToH2/Playground.Proxy.H2ToH2.csproj | 2 +- Playground/Proxy/H2ToH2/Program.cs | 16 ++++----- .../H2ToH3/Playground.Proxy.H2ToH3.csproj | 2 +- Playground/Proxy/H2ToH3/Program.cs | 14 ++++---- dropped/README.md | 33 +++++++++++++++++++ ioxide.slnx | 4 --- src/protocols/ioxide.http2/HpackEncoder.cs | 12 +++---- src/protocols/ioxide.http2/Http2Connection.cs | 13 ++++---- src/protocols/ioxide.http2/Http2Response.cs | 4 +-- 15 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 dropped/README.md diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c624037..3147d61 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,8 +85,8 @@ jobs: # Paths mirror the src/ grouping in ioxide.slnx (core, protocols/, clients/, serving/). # Every project carrying a PackageId is packed. ioxide.httpclient is the whole client - - # h1, h2c (nghttp2 bundled inside it) and h3 - and project-references ngtcp2/nghttp3, - # so those ship as its NuGet dependencies. + # h1, h2 and h3 - and project-references http2/ngtcp2/nghttp3, so those ship as its NuGet + # dependencies. - name: Pack ioxide run: dotnet pack src/ioxide/ioxide.csproj --configuration Release --no-build --output ./artifacts @@ -96,9 +96,6 @@ jobs: - name: Pack ioxide.nghttp3 run: dotnet pack src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj --configuration Release --no-build --output ./artifacts - - name: Pack ioxide.nghttp2 - run: dotnet pack src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj --configuration Release --no-build --output ./artifacts - - name: Pack ioxide.http2 run: dotnet pack src/protocols/ioxide.http2/ioxide.http2.csproj --configuration Release --no-build --output ./artifacts diff --git a/Playground/Http2/Managed/Program.cs b/Playground/Http2/Managed/Program.cs index c0ebf23..df3f464 100644 --- a/Playground/Http2/Managed/Program.cs +++ b/Playground/Http2/Managed/Program.cs @@ -71,8 +71,8 @@ { try { - // The connection owns the read loop from here: it feeds nghttp2, dispatches each - // request once its stream ends, and drains the egress once per batch. + // The connection owns the read loop from here: it parses frames, dispatches each + // request once its stream ends, and flushes the batch in one write. await new Http2Connection(conn).RunBufferedAsync(_ => new Http2Response { Status = 200, diff --git a/Playground/Http2/SslStream/Playground.Http2.SslStream.csproj b/Playground/Http2/SslStream/Playground.Http2.SslStream.csproj index e4ca597..c4ea47b 100644 --- a/Playground/Http2/SslStream/Playground.Http2.SslStream.csproj +++ b/Playground/Http2/SslStream/Playground.Http2.SslStream.csproj @@ -13,7 +13,7 @@ - + diff --git a/Playground/Http2/SslStream/Program.cs b/Playground/Http2/SslStream/Program.cs index f006302..770a690 100644 --- a/Playground/Http2/SslStream/Program.cs +++ b/Playground/Http2/SslStream/Program.cs @@ -3,7 +3,7 @@ using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using ioxide; -using ioxide.nghttp2; +using ioxide.http2; using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── @@ -13,12 +13,12 @@ // dotnet run -c Release --project Playground/Http2/SslStream // curl -k --http2 https://127.0.0.1:8443/ // -// The point of this sample is what it demonstrates about the shape: Nghttp2Connection takes an +// The point of this sample is what it demonstrates about the shape: Http2Connection takes an // IDuplexPipe, and a Stream can be one in about ten lines (below). So the same HTTP/2 code runs // over the ring directly, over kTLS, or over SslStream, without knowing which - the transport is // a constructor argument, not a branch inside the protocol. // -// Compare with Playground/Http2/Tls for the ioxide.tls version. Needs: ioxide, ioxide.nghttp2 +// Compare with Playground/Http2/Tls for the ioxide.tls version. Needs: ioxide, ioxide.http2 // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -90,8 +90,8 @@ await ssl.AuthenticateAsServerAsync(new SslServerAuthenticationOptions return; // this sample only serves h2; see Playground/Http2/Tls for the fallback } - await new Nghttp2Connection(new StreamDuplexPipe(ssl)).RunBufferedAsync( - _ => new Nghttp2Response { Status = 200, Body = body }); + await new Http2Connection(new StreamDuplexPipe(ssl)).RunBufferedAsync( + _ => new Http2Response { Status = 200, Body = body }); } catch (Exception e) { diff --git a/Playground/Proxy/H2ToH1/Playground.Proxy.H2ToH1.csproj b/Playground/Proxy/H2ToH1/Playground.Proxy.H2ToH1.csproj index 832ba35..9a63cc7 100644 --- a/Playground/Proxy/H2ToH1/Playground.Proxy.H2ToH1.csproj +++ b/Playground/Proxy/H2ToH1/Playground.Proxy.H2ToH1.csproj @@ -13,7 +13,7 @@ - + diff --git a/Playground/Proxy/H2ToH1/Program.cs b/Playground/Proxy/H2ToH1/Program.cs index 633e636..347bb59 100644 --- a/Playground/Proxy/H2ToH1/Program.cs +++ b/Playground/Proxy/H2ToH1/Program.cs @@ -1,7 +1,7 @@ using System.Text; using ioxide; using ioxide.httpclient; -using ioxide.nghttp2; +using ioxide.http2; using ioxide.tls; using Playground.Shared; @@ -14,7 +14,7 @@ // from one. ALPN is what makes it work: the server offers "h2" and the client picks it during // the handshake, before a single byte of HTTP exists. // -// Note what Nghttp2Connection is handed - a TlsConnectionDualPipe. It never learns that TLS is +// Note what Http2Connection is handed - a TlsConnectionDualPipe. It never learns that TLS is // involved: the pipe decrypts on the way in and encrypts on the way out, so the HTTP/2 code // is byte-for-byte the h2c version. // @@ -30,7 +30,7 @@ // unbounded, with the whole acquire bounded by HttpClientOptions.AcquireTimeoutMs - a saturated // origin surfaces as a 502 on that stream, not as an fd leak. // -// Needs: ioxide, ioxide.nghttp2, ioxide.httpclient +// Needs: ioxide, ioxide.http2, ioxide.httpclient // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -154,7 +154,7 @@ // Buffered + async: each stream dispatches with its body assembled, and the handler // may await - the upstream round trip resumes inline on this reactor. Concurrent // streams interleave here, which is exactly why the h1 pool has to be deep. - await new Nghttp2Connection(pipe).RunBufferedAsync(async request => + await new Http2Connection(pipe).RunBufferedAsync(async request => { try { @@ -163,11 +163,11 @@ using HttpClientResponse response = await client.SendAsync(new HttpClientRequest( request.Method, request.Path) { Body = request.Body }); - // Copy before Dispose: the response arena is freed then, and nghttp2 copies - // the h2 response only AFTER this handler returns. A real proxy would also + // Copy before Dispose: the response arena is freed then, and the h2 response + // is framed only AFTER this handler returns. A real proxy would also // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all // illegal in h2 and would be a protocol error to forward. - var proxied = new Nghttp2Response + var proxied = new Http2Response { Status = response.Status, Body = response.Body.ToArray(), @@ -183,7 +183,7 @@ // Upstream down is a gateway error on this stream, not a dead h2 connection: // every other stream on it keeps working. A refused certificate arrives the // same way - the handshake is part of opening the upstream connection. - return new Nghttp2Response + return new Http2Response { Status = 502, Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"), diff --git a/Playground/Proxy/H2ToH2/Playground.Proxy.H2ToH2.csproj b/Playground/Proxy/H2ToH2/Playground.Proxy.H2ToH2.csproj index 77d7535..9c5e145 100644 --- a/Playground/Proxy/H2ToH2/Playground.Proxy.H2ToH2.csproj +++ b/Playground/Proxy/H2ToH2/Playground.Proxy.H2ToH2.csproj @@ -13,7 +13,7 @@ - + diff --git a/Playground/Proxy/H2ToH2/Program.cs b/Playground/Proxy/H2ToH2/Program.cs index e4daf83..4644311 100644 --- a/Playground/Proxy/H2ToH2/Program.cs +++ b/Playground/Proxy/H2ToH2/Program.cs @@ -1,7 +1,7 @@ using System.Text; using ioxide; using ioxide.httpclient; -using ioxide.nghttp2; +using ioxide.http2; using ioxide.tls; using Playground.Shared; @@ -13,7 +13,7 @@ // ALPN carries h2 in both directions - offered by the server on the way in, offered by the // client on the way out. Neither side ever assumes it. // -// Two different nghttp2 sessions are involved and they share nothing. The inbound one decodes +// Two different HTTP/2 sessions are involved and they share nothing. The inbound one decodes // HPACK against the client's dynamic table; the outbound one re-encodes against the origin's. // Header state is per-connection in HTTP/2 and cannot be forwarded - a proxy always re-encodes, // which is why "just splice the frames" is not a shortcut that exists. TLS makes that doubly @@ -24,7 +24,7 @@ // dotnet run -c Release --project Playground/Proxy/H2ToH2 // curl -k --http2 https://127.0.0.1:8443/ // -// Needs: ioxide, ioxide.nghttp2, ioxide.httpclient +// Needs: ioxide, ioxide.http2, ioxide.httpclient // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -145,7 +145,7 @@ // Buffered + async: each stream dispatches with its body assembled, and the handler // may await - the upstream round trip resumes inline on this reactor. - await new Nghttp2Connection(pipe).RunBufferedAsync(async request => + await new Http2Connection(pipe).RunBufferedAsync(async request => { try { @@ -154,11 +154,11 @@ using HttpClientResponse response = await client.SendAsync(new HttpClientRequest( request.Method, request.Path) { Body = request.Body }); - // Copy before Dispose: the response arena is freed then, and nghttp2 copies - // the h2 response only AFTER this handler returns. A real proxy would also + // Copy before Dispose: the response arena is freed then, and the h2 response + // is framed only AFTER this handler returns. A real proxy would also // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all // illegal in h2 and would be a protocol error to forward. - var proxied = new Nghttp2Response + var proxied = new Http2Response { Status = response.Status, Body = response.Body.ToArray(), @@ -174,7 +174,7 @@ // Upstream down is a gateway error on this stream, not a dead h2 connection: // every other stream on it keeps working. A refused certificate arrives the // same way - the handshake is part of opening the upstream connection. - return new Nghttp2Response + return new Http2Response { Status = 502, Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"), diff --git a/Playground/Proxy/H2ToH3/Playground.Proxy.H2ToH3.csproj b/Playground/Proxy/H2ToH3/Playground.Proxy.H2ToH3.csproj index 81a5f5d..087924d 100644 --- a/Playground/Proxy/H2ToH3/Playground.Proxy.H2ToH3.csproj +++ b/Playground/Proxy/H2ToH3/Playground.Proxy.H2ToH3.csproj @@ -13,7 +13,7 @@ - + diff --git a/Playground/Proxy/H2ToH3/Program.cs b/Playground/Proxy/H2ToH3/Program.cs index 97b6e5c..6108205 100644 --- a/Playground/Proxy/H2ToH3/Program.cs +++ b/Playground/Proxy/H2ToH3/Program.cs @@ -1,7 +1,7 @@ using System.Text; using ioxide; using ioxide.httpclient; -using ioxide.nghttp2; +using ioxide.http2; using ioxide.tls; using Playground.Shared; @@ -23,7 +23,7 @@ // PLAYGROUND_UPSTREAM_PORT=8443 dotnet run -c Release --project Playground/Proxy/H2ToH3 // curl -k --http2 https://127.0.0.1:8443/ // -// Needs: ioxide, ioxide.nghttp2, ioxide.httpclient +// Needs: ioxide, ioxide.http2, ioxide.httpclient // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -127,7 +127,7 @@ // Buffered + async: each stream dispatches with its body assembled, and the handler // may await - the upstream round trip resumes inline on this reactor. - await new Nghttp2Connection(pipe).RunBufferedAsync(async request => + await new Http2Connection(pipe).RunBufferedAsync(async request => { try { @@ -136,11 +136,11 @@ using HttpClientResponse response = await client.SendAsync(new HttpClientRequest( request.Method, request.Path) { Body = request.Body }); - // Copy before Dispose: the response arena is freed then, and nghttp2 copies - // the h2 response only AFTER this handler returns. A real proxy would also + // Copy before Dispose: the response arena is freed then, and the h2 response + // is framed only AFTER this handler returns. A real proxy would also // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all // illegal in h2 and would be a protocol error to forward. - var proxied = new Nghttp2Response + var proxied = new Http2Response { Status = response.Status, Body = response.Body.ToArray(), @@ -156,7 +156,7 @@ // Upstream down is a gateway error on this stream, not a dead h2 connection: // every other stream on it keeps working. A refused certificate arrives the // same way - the handshake is part of opening the upstream connection. - return new Nghttp2Response + return new Http2Response { Status = 502, Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"), diff --git a/dropped/README.md b/dropped/README.md new file mode 100644 index 0000000..824adda --- /dev/null +++ b/dropped/README.md @@ -0,0 +1,33 @@ +# dropped + +Code that used to ship and no longer does. It is kept because it was real, it was measured, and the +reasoning behind retiring it is easier to follow with the thing itself still readable. + +Nothing here is in `ioxide.slnx`, nothing here is built by CI, and nothing here is published to +NuGet. It will not compile against the current tree forever, and that is expected - if you need it, +take it from the last release tag that shipped it rather than from here. + +## ioxide.nghttp2 + +The nghttp2 binding: HTTP/2 framing, HPACK and flow control from the reference C implementation, +driven sans-I/O over an `IDuplexPipe`. + +It was replaced by `ioxide.http2`, which does the same job in pure C#. That started as the +drop-in-without-a-native-library option and ended as the only one: + +- **It measured at least as well.** Interleaved warm runs put the two within `0.98x`-`1.09x` of each + other on a small body; where they diverged, the ordering depended on the connection-to-reactor + ratio rather than on the codec. +- **It grew past what the binding could reach.** Streamed responses, streamed request bodies and + non-blocking dispatch all landed on the managed side. The binding kept the blocking dispatch loop + it was written with, where one slow handler held up every other stream on the connection. +- **Two implementations of one protocol is a tax on every change**, paid in samples, docs, tests and + benchmark fixtures, and the second one was no longer buying coverage of the protocol's darker + corners - it was buying a native build step. + +The last thing holding it in the tree was `ioxide.httpclient`, whose HTTP/2 *client* was built on +it. That client is pure C# now too, on the same framing and HPACK the server uses, so the binding +had no callers left. + +`build-nghttp2-native.sh` built the native library it bound to. `Playground.Http2.Nghttp2` was its +h2c sample; `Playground/Http2/Managed` is the same server on the managed stack. diff --git a/ioxide.slnx b/ioxide.slnx index 9f29cf0..fcd586e 100644 --- a/ioxide.slnx +++ b/ioxide.slnx @@ -4,7 +4,6 @@ - @@ -55,10 +54,8 @@ - - @@ -124,7 +121,6 @@ - diff --git a/src/protocols/ioxide.http2/HpackEncoder.cs b/src/protocols/ioxide.http2/HpackEncoder.cs index f6e69c9..aa8d8d9 100644 --- a/src/protocols/ioxide.http2/HpackEncoder.cs +++ b/src/protocols/ioxide.http2/HpackEncoder.cs @@ -4,12 +4,12 @@ namespace ioxide.http2; /// The encoding half of HPACK. Deliberately simple: it uses the static table where a name or a /// name+value pair matches exactly, and sends everything else as a literal WITHOUT indexing. /// -/// That choice is worth stating plainly, because it is the one place this differs from nghttp2 in -/// output rather than only in implementation. Never adding to the dynamic table means the encoder -/// holds no per-connection compression state, so it cannot desynchronise from the peer's decoder - -/// the failure mode that makes HPACK bugs so unpleasant. It costs bytes on repeated custom headers, -/// which a server sends far fewer of than a client; the pseudo-headers and common response fields -/// that dominate a small response are all static-table hits either way. +/// That choice is worth stating plainly, because it is where this produces different bytes from a +/// fully general encoder rather than merely reaching them differently. Never adding to the dynamic +/// table means the encoder holds no per-connection compression state, so it cannot desynchronise +/// from the peer's decoder - the failure mode that makes HPACK bugs so unpleasant. It costs bytes +/// on repeated custom headers, which a server sends far fewer of than a client; the pseudo-headers +/// and common response fields that dominate a small response are all static-table hits either way. /// /// Literals are sent unencoded rather than Huffman-coded. Huffman saves roughly 20% on header /// octets and costs CPU per response; for a server whose headers are mostly static-table indices diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 19ce0c3..366bc49 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -12,14 +12,13 @@ namespace ioxide.http2; /// new Http2Connection(conn).RunBufferedAsync(request => Http2Response.Text("hello")); /// /// -/// A drop-in alternative to Nghttp2Connection - same shape, same request and response -/// surface - the way ioxide.http3 is for ioxide.nghttp3. Take this one when shipping -/// a native library is inconvenient; take nghttp2 when you want the reference implementation's -/// coverage of the protocol's darker corners. +/// This is the only HTTP/2 in ioxide: the nghttp2 binding it began as an alternative to was +/// retired once this measured level with it and then grew past it - see dropped/. The same +/// framing and HPACK drive ioxide.httpclient's HTTP/2 client, pointed the other way round. /// -/// Like its nghttp2 counterpart it speaks to an and knows nothing about -/// TLS: hand it a TcpConnectionDualPipe for h2c or a TlsConnectionDualPipe for h2 -/// over TLS, and the protocol code is identical either way. +/// It speaks to an and knows nothing about TLS: hand it a +/// TcpConnectionDualPipe for h2c or a TlsConnectionDualPipe for h2 over TLS, and the +/// protocol code is identical either way. /// /// Reactor thread only. public sealed partial class Http2Connection : IDisposable diff --git a/src/protocols/ioxide.http2/Http2Response.cs b/src/protocols/ioxide.http2/Http2Response.cs index 55cf7a7..b0287e8 100644 --- a/src/protocols/ioxide.http2/Http2Response.cs +++ b/src/protocols/ioxide.http2/Http2Response.cs @@ -3,8 +3,8 @@ namespace ioxide.http2; /// /// One HTTP/2 response: status, headers, and an in-memory body - bytes throughout, mirroring /// . Header names must be ASCII (they're lowercased as they're packed); -/// values are raw octets. Everything is copied into nghttp2 synchronously at submit, so the -/// memories can be pooled, stackallocated behind, or static. +/// values are raw octets. Everything is framed into the connection's write buffer synchronously +/// when the handler returns, so the memories can be pooled, stackallocated behind, or static. /// public sealed class Http2Response { From 0964f98d9a9e255ed7dd01d4c1f0498520473f52 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 14:59:13 +0100 Subject: [PATCH 07/14] feat(http2): request bodies can be streamed, not only responses h2 could stream a response but never a request: Http2Request carried only Body, so the whole upload was assembled before the handler saw any of it, and MaxRequestBytes was the only thing standing between a hostile peer and the arena. h3 has had Http3Request.BodyReader all along; this is its counterpart. Http2Options.StreamRequestBodies dispatches at the HEADERS and hands the handler an Http2BodyReader. The stream then stays in _streams while it runs - DATA frames still have somewhere to go - and is retired when the handler is done rather than when the request ends. What makes it worth having is where the credit goes. A chunk opens the peer's window only as the handler READS it, so a slow consumer stops replenishing and the peer stops sending: memory is bound by one window instead of by the body. Crediting on arrival, which is what the buffered path does and should, would leave the bound off. Unlike h3 the credit is shared - every stream is on one TCP connection - so a handler that never reads holds down the connection window for every other stream too, and the comment says so. Wakes are deferred to after the parser unwinds, the same discipline the h3 reader and the write queue already use, so a resumed handler cannot re-enter the parser mid-frame. Tests: 1 MiB uploaded through the pure-C# client into a streaming origin, and three unit tests on the part end-to-end cannot see - that no WINDOW_UPDATE is emitted while a handler holds the body unread, that a bodyless request reads empty instead of parking forever, and that buffered dispatch still assembles and still credits on arrival. Unit 36, Http 38, both 0 failed. --- src/protocols/ioxide.http2/Http2BodyReader.cs | 161 +++++++++++ .../ioxide.http2/Http2Connection.Frames.cs | 68 ++++- src/protocols/ioxide.http2/Http2Connection.cs | 65 ++++- src/protocols/ioxide.http2/Http2Options.cs | 12 + src/protocols/ioxide.http2/Http2Request.cs | 9 +- tests/Ioxide.Tests.Http/Http2ClientTests.cs | 55 ++++ .../Http2StreamedRequestTests.cs | 254 ++++++++++++++++++ tests/Ioxide.Tests.Unit/Program.cs | 1 + 8 files changed, 612 insertions(+), 13 deletions(-) create mode 100644 src/protocols/ioxide.http2/Http2BodyReader.cs create mode 100644 tests/Ioxide.Tests.Unit/Http2StreamedRequestTests.cs diff --git a/src/protocols/ioxide.http2/Http2BodyReader.cs b/src/protocols/ioxide.http2/Http2BodyReader.cs new file mode 100644 index 0000000..b92d656 --- /dev/null +++ b/src/protocols/ioxide.http2/Http2BodyReader.cs @@ -0,0 +1,161 @@ +using System.Buffers; +using System.Threading.Tasks.Sources; + +namespace ioxide.http2; + +/// +/// Pull surface for a streaming request body, handed to the handler as +/// when is on. +/// The connection loop pushes chunks in as DATA frames arrive; the handler pulls with +/// . An empty chunk means end of body (END_STREAM or a reset stream). +/// +/// Backpressure is real rather than buffered-and-hoped-for: a chunk is credited back to the peer's +/// window only as it is handed over, so a slow consumer stops replenishing, the peer runs out of +/// window and stops sending. Memory is bound by one window instead of by the whole body, which is +/// the difference that makes a hostile upload harmless. +/// +/// Unlike HTTP/3, credit here is shared. Every stream rides one TCP connection, so a chunk opens +/// both the stream's window and the CONNECTION's - and a handler that never reads holds down the +/// connection window for every other stream on it, not only its own. +/// +/// +/// Single consumer, reactor thread only. Each invalidates the previous +/// chunk's memory - its pooled buffer goes back - so a handler that needs to keep bytes past the +/// next read must copy them. Wakes are deferred by the connection loop until the parser has +/// unwound, so a resumed handler cannot re-enter it mid-frame. +/// +public sealed class Http2BodyReader : IValueTaskSource> +{ + private readonly Http2Connection _owner; + private readonly int _streamId; + + private readonly Queue<(byte[] Buffer, int Length)> _chunks = new(); + private (byte[]? Buffer, int Length) _handedOut; + private bool _ended; + private bool _armed; + + private ManualResetValueTaskSourceCore> _core = new() + { + RunContinuationsAsynchronously = false, + }; + + internal Http2BodyReader(Http2Connection owner, int streamId, bool ended) + { + _owner = owner; + _streamId = streamId; + _ended = ended; + } + + /// + /// The next body chunk; empty means end of body. The memory is valid until the next + /// call, or until the handler returns, whichever comes first. + /// + public ValueTask> ReadAsync() + { + ReleaseHandedOut(); + + if (_chunks.TryDequeue(out (byte[] Buffer, int Length) chunk)) + { + _handedOut = chunk; + _owner.CreditBody(_streamId, chunk.Length); // consumption is what opens the window + return new ValueTask>(chunk.Buffer.AsMemory(0, chunk.Length)); + } + + if (_ended) + { + return default; + } + + _core.Reset(); + _armed = true; + return new ValueTask>(this, _core.Version); + } + + // Body bytes straight off a DATA frame. The span points into the connection's accumulator and + // dies when the parser moves on, so it has to be copied. Never completes a parked reader + // inline: the wake is deferred to FireIfReady, after the parser unwinds. + internal void Push(ReadOnlySpan data) + { + if (data.IsEmpty) + { + return; + } + + byte[] buffer = ArrayPool.Shared.Rent(data.Length); + data.CopyTo(buffer); + _chunks.Enqueue((buffer, data.Length)); + + if (_armed) + { + _owner.NoteBodyWake(this); + } + } + + /// End of body: END_STREAM, a reset stream, or the connection going away. Idempotent. + internal void End() + { + if (_ended) + { + return; + } + _ended = true; + + if (_armed) + { + _owner.NoteBodyWake(this); + } + } + + // The deferred wake: complete a parked ReadAsync now that the parser is off the stack. + internal void FireIfReady() + { + if (!_armed) + { + return; + } + + if (_chunks.TryDequeue(out (byte[] Buffer, int Length) chunk)) + { + _armed = false; + _handedOut = chunk; + _owner.CreditBody(_streamId, chunk.Length); + _core.SetResult(chunk.Buffer.AsMemory(0, chunk.Length)); + } + else if (_ended) + { + _armed = false; + _core.SetResult(default); + } + } + + // Teardown while chunks may still be queued: recycle everything and wake anyone parked. + internal void Drop() + { + End(); + ReleaseHandedOut(); + + while (_chunks.TryDequeue(out (byte[] Buffer, int Length) chunk)) + { + ArrayPool.Shared.Return(chunk.Buffer); + } + } + + private void ReleaseHandedOut() + { + if (_handedOut.Buffer is not null) + { + ArrayPool.Shared.Return(_handedOut.Buffer); + _handedOut = (null, 0); + } + } + + // Completes on the reactor thread only; the context-post is stripped so resumes stay inline. + ReadOnlyMemory IValueTaskSource>.GetResult(short token) => _core.GetResult(token); + + ValueTaskSourceStatus IValueTaskSource>.GetStatus(short token) => _core.GetStatus(token); + + void IValueTaskSource>.OnCompleted(Action continuation, object? state, + short token, ValueTaskSourceOnCompletedFlags flags) + => _core.OnCompleted(continuation, state, token, + flags & ~ValueTaskSourceOnCompletedFlags.UseSchedulingContext); +} diff --git a/src/protocols/ioxide.http2/Http2Connection.Frames.cs b/src/protocols/ioxide.http2/Http2Connection.Frames.cs index 456854e..5685cc9 100644 --- a/src/protocols/ioxide.http2/Http2Connection.Frames.cs +++ b/src/protocols/ioxide.http2/Http2Connection.Frames.cs @@ -154,15 +154,33 @@ private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload) { return; } + BeginStreamedBody(pending, ended: (header.Flags & FrameFlags.EndStream) != 0); } if ((header.Flags & FrameFlags.EndStream) != 0) { pending.RequestEnded = true; + pending.BodyReader?.End(); TryComplete(pending); } } + /// + /// Hand this request over the moment its headers are in, with the body still to come. The + /// stream stays in _streams - unlike the buffered path, later DATA frames still have + /// somewhere to go - and is retired when the handler finishes instead. + /// + private void BeginStreamedBody(PendingRequest pending, bool ended) + { + if (!_options.StreamRequestBodies || pending.BodyReader is not null) + { + return; + } + + pending.BodyReader = new Http2BodyReader(this, pending.StreamId, ended); + _ready.Add(pending); + } + private void HandleContinuation(in FrameHeader header, ReadOnlySpan payload) { if (!_streams.TryGetValue(header.StreamId, out PendingRequest? pending)) @@ -179,6 +197,8 @@ private void HandleContinuation(in FrameHeader header, ReadOnlySpan payloa { return; } + // END_STREAM rode the HEADERS that opened this block, not the CONTINUATION closing it. + BeginStreamedBody(pending, ended: pending.RequestEnded); TryComplete(pending); } } @@ -233,7 +253,13 @@ private void HandleData(in FrameHeader header, ReadOnlySpan payload) body = body[..^padding]; } - if (_streams.TryGetValue(header.StreamId, out PendingRequest? pending)) + _streams.TryGetValue(header.StreamId, out PendingRequest? pending); + + if (pending?.BodyReader is { } reader) + { + reader.Push(body); + } + else if (pending is not null) { if (pending.BodyLength + body.Length > _options.MaxRequestBytes) { @@ -244,9 +270,14 @@ private void HandleData(in FrameHeader header, ReadOnlySpan payload) } // The whole payload counts against the window, padding included, so the peer's accounting - // and ours agree. Replenished immediately: this server buffers the request anyway, so - // holding the window back would only stall the peer. - if (payload.Length > 0) + // and ours agree. Replenished immediately here because this server buffers the request + // anyway, so holding the window back would only stall the peer for nothing. + // + // A STREAMED body is the exception, and the reason the option exists: its credit is + // returned as the handler reads (Http2BodyReader.ReadAsync), so a consumer that falls + // behind stops replenishing and the peer stops sending. Crediting here as well would hand + // the window back before the bytes were consumed and put the bound back on memory. + if (payload.Length > 0 && pending?.BodyReader is null) { WriteWindowUpdate(0, payload.Length); if (header.StreamId != 0) @@ -258,12 +289,20 @@ private void HandleData(in FrameHeader header, ReadOnlySpan payload) if ((header.Flags & FrameFlags.EndStream) != 0 && pending is not null) { pending.RequestEnded = true; + pending.BodyReader?.End(); TryComplete(pending); } } private void TryComplete(PendingRequest pending) { + // A streamed request was handed over at its headers and is being served right now; its + // stream is retired by whoever is serving it, not here. + if (pending.BodyReader is not null) + { + return; + } + if (!pending.HeadersDone || !pending.RequestEnded) { return; @@ -395,6 +434,9 @@ private sealed class PendingRequest : IDisposable public bool HeadersDone; public bool RequestEnded; + /// Set only when the body is being streamed; the arena stays empty then. + public Http2BodyReader? BodyReader; + /// What the peer will still accept on this stream. Starts at its advertised default. public int SendWindow = 65535; @@ -491,12 +533,13 @@ public Http2Request Freeze() { var request = new Http2Request { - StreamId = StreamId, - Method = Slice(Method), - Path = Slice(Path), - Scheme = Slice(Scheme), - Authority = Slice(Authority), - Body = Slice(_body), + StreamId = StreamId, + Method = Slice(Method), + Path = Slice(Path), + Scheme = Slice(Scheme), + Authority = Slice(Authority), + Body = Slice(_body), + BodyReader = BodyReader, }; foreach ((int nameOffset, int nameLength, int valueOffset, int valueLength) in _fields) @@ -513,6 +556,11 @@ private ReadOnlyMemory Slice((int Offset, int Length) range) public void Dispose() { + // Recycles any chunk still queued and wakes a handler parked on a body that will + // never finish arriving. + BodyReader?.Drop(); + BodyReader = null; + ClearHeaderBlock(); _fields.Clear(); if (_arena.Length > 0) diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs index 366bc49..4d9550c 100644 --- a/src/protocols/ioxide.http2/Http2Connection.cs +++ b/src/protocols/ioxide.http2/Http2Connection.cs @@ -39,6 +39,10 @@ public sealed partial class Http2Connection : IDisposable private readonly Dictionary _streams = new(); private readonly List _ready = []; + // Readers whose parked ReadAsync has something to hand over. Collected during parsing and + // fired once it has unwound, so a resumed handler cannot re-enter the parser mid-frame. + private readonly List _bodyWakes = []; + private bool _prefaceSeen; private bool _disposed; private bool _failed; @@ -90,6 +94,7 @@ public void Dispose() pending.Dispose(); } _ready.Clear(); + _bodyWakes.Clear(); if (_inbound.Length > 0) { @@ -140,6 +145,12 @@ public async Task RunBufferedAsync(Func> _passFlushPending = true; ParseAvailable(); await DispatchReadyAsync(handler); + + // Body chunks reach their handlers here, inside the pass: the WINDOW_UPDATEs a + // read stages then ride the same flush as everything else, so credit gets back + // to the peer without a write of its own. + FireBodyWakes(); + _passFlushPending = false; await FlushAsync(); } @@ -247,7 +258,7 @@ private async ValueTask DispatchReadyAsync(Func inFlight, Pend } finally { - pending.Dispose(); + RetireStream(pending); await MaybeFlushAsync(); } } + + /// + /// Done with a stream. The buffered path already took it out of _streams when it became + /// ready; a streamed request is still in there, because DATA frames were arriving the whole + /// time the handler ran. + /// + private void RetireStream(PendingRequest pending) + { + _streams.Remove(pending.StreamId); + pending.Dispose(); + } + + /// Return a consumed chunk's credit to the peer, on both windows it was charged to. + internal void CreditBody(int streamId, int length) + { + if (length <= 0 || IsBroken) + { + return; + } + + WriteWindowUpdate(0, length); + WriteWindowUpdate(streamId, length); + } + + /// A reader has something for a parked ReadAsync; wake it once the parser is done. + internal void NoteBodyWake(Http2BodyReader reader) + { + if (!_bodyWakes.Contains(reader)) + { + _bodyWakes.Add(reader); + } + } + + private void FireBodyWakes() + { + if (_bodyWakes.Count == 0) + { + return; + } + + // Snapshot-and-clear: a resumed handler reads again, which can land another wake here, and + // the list must not be mutated while it is being walked. + Http2BodyReader[] wakes = _bodyWakes.ToArray(); + _bodyWakes.Clear(); + + foreach (Http2BodyReader reader in wakes) + { + reader.FireIfReady(); + } + } } diff --git a/src/protocols/ioxide.http2/Http2Options.cs b/src/protocols/ioxide.http2/Http2Options.cs index 2547894..aca78b0 100644 --- a/src/protocols/ioxide.http2/Http2Options.cs +++ b/src/protocols/ioxide.http2/Http2Options.cs @@ -17,4 +17,16 @@ public sealed record Http2Options /// Streams the peer may have open at once. public int MaxConcurrentStreams { get; init; } = 1000; + + /// + /// Dispatch each request as soon as its HEADERS are in, with the body arriving through + /// instead of assembled into + /// . + /// + /// The trade is what memory is bound by. Buffered holds the whole body, which suits ordinary + /// requests and not hostile uploads; streamed holds one flow-control window, because credit is + /// only returned to the peer as the handler reads. stops applying + /// to the body when this is on - there is no arena for it to bound. + /// + public bool StreamRequestBodies { get; init; } } diff --git a/src/protocols/ioxide.http2/Http2Request.cs b/src/protocols/ioxide.http2/Http2Request.cs index 549c3a2..1bbe0fa 100644 --- a/src/protocols/ioxide.http2/Http2Request.cs +++ b/src/protocols/ioxide.http2/Http2Request.cs @@ -46,6 +46,13 @@ public bool TryGetHeader(ReadOnlySpan name, out ReadOnlyMemory value return false; } - /// Request body, empty when there was none. + /// Request body, empty when there was none - or when it is being streamed. public ReadOnlyMemory Body { get; internal set; } + + /// + /// The body as it arrives, set only when is on; + /// null otherwise, when already holds it whole. The handler runs while the + /// body is still coming in, so reading it is what lets the peer send more. + /// + public Http2BodyReader? BodyReader { get; internal set; } } diff --git a/tests/Ioxide.Tests.Http/Http2ClientTests.cs b/tests/Ioxide.Tests.Http/Http2ClientTests.cs index 215b1e5..44834d6 100644 --- a/tests/Ioxide.Tests.Http/Http2ClientTests.cs +++ b/tests/Ioxide.Tests.Http/Http2ClientTests.cs @@ -130,6 +130,23 @@ public static void Register(Runner runner) Assert.Equal($"200|{BodyBytes}", body); }); + runner.Test("h2 server: a streamed request body is read as it arrives", () => + { + // Same 1 MiB upload, but the origin never holds it: StreamRequestBodies dispatches at + // the headers and hands the handler a reader. Window credit goes back only as chunks + // are read, so if that crediting were wrong the upload would stall at the first window + // and this would time out rather than come back short. + const int BodyBytes = 1024 * 1024; + + int origin = StartStreamedOrigin(); + int driver = TestServer.Start(PostSizeDriver(BodyBytes), onStart: reactor => + Http2ClientPool.Start(reactor, OriginOptions(origin))); + + (int status, string body) = Client.Get(driver, "/echo", timeoutMs: 30_000); + Assert.Equal(200, status); + Assert.Equal($"200|{BodyBytes}", body); + }); + runner.Test("httpclient h2: a header block past the frame size continues", () => { // 40 headers of ~512 bytes overflows the 16 KiB maximum frame size, so the field @@ -175,6 +192,44 @@ private static int StartEchoOrigin() => TestServer.Start(async (_, connection) = } }); + /// + /// The same origin with the body STREAMED: it counts the bytes it is handed and never keeps + /// them, so the answer proves the whole body arrived without any of it being held. + /// + private static int StartStreamedOrigin() => TestServer.Start(async (_, connection) => + { + try + { + var options = new ioxide.http2.Http2Options { StreamRequestBodies = true }; + await new ioxide.http2.Http2Connection(connection, options).RunBufferedAsync(async request => + { + int total = 0; + if (request.BodyReader is { } reader) + { + while (true) + { + ReadOnlyMemory chunk = await reader.ReadAsync(); + if (chunk.IsEmpty) + { + break; + } + total += chunk.Length; + } + } + + return new ioxide.http2.Http2Response + { + Status = 200, + Body = Encoding.ASCII.GetBytes(total.ToString()), + }; + }); + } + finally + { + connection.DecRef(); + } + }); + private static Func PostSizeDriver(int bodyBytes) => (reactor, connection) => DriveOnce(reactor, connection, upstream => { diff --git a/tests/Ioxide.Tests.Unit/Http2StreamedRequestTests.cs b/tests/Ioxide.Tests.Unit/Http2StreamedRequestTests.cs new file mode 100644 index 0000000..f9a5e72 --- /dev/null +++ b/tests/Ioxide.Tests.Unit/Http2StreamedRequestTests.cs @@ -0,0 +1,254 @@ +using System.Buffers; +using System.Buffers.Binary; +using System.IO.Pipelines; +using ioxide.http2; + +namespace Ioxide.Tests; + +/// +/// Streamed request bodies: the handler runs while the body is still arriving, and flow-control +/// credit goes back to the peer only as it READS. +/// +/// That last part is the entire feature, and it is the part an end-to-end test cannot see - an +/// upload succeeds either way. What differs is what bounds memory: crediting on arrival lets a peer +/// send as fast as it likes, and the bytes pile up behind a slow handler. So these tests watch the +/// WINDOW_UPDATE frames rather than the body. +/// +internal static class Http2StreamedRequestTests +{ + public static void Register(Runner runner) + { + runner.Test("h2 streamed request: credit is returned on read, not on arrival", () => + { + var gate = new TaskCompletionSource(); + var chunks = new List(); + + using var peer = new Peer(new Http2Options { StreamRequestBodies = true }); + Task run = peer.Connection.RunBufferedAsync(async request => + { + await gate.Task; // hold the body unread, like a slow consumer + + while (true) + { + ReadOnlyMemory chunk = await request.BodyReader!.ReadAsync(); + if (chunk.IsEmpty) + { + break; + } + chunks.Add(chunk.Length); + } + + return Http2Response.Text("done"); + }); + + peer.OpenRequest(streamId: 1, endStream: false); + peer.SendData(streamId: 1, bytes: 400, endStream: false); + peer.SendData(streamId: 1, bytes: 600, endStream: true); + + // The handler is parked before its first read. The bytes are in - and the peer has been + // told nothing, so it may not send more. + Assert.Equal(0, peer.CreditFor(streamId: 1)); + Assert.Equal(0, peer.CreditFor(streamId: 0)); + + gate.SetResult(); + peer.Pump(); + + // Read, and only now does the window open - on the stream AND on the connection, which + // is the part HTTP/3 does not have to do. + Assert.Equal(1000, peer.CreditFor(streamId: 1)); + Assert.Equal(1000, peer.CreditFor(streamId: 0)); + Assert.Equal(2, chunks.Count); + Assert.Equal(400, chunks[0]); + Assert.Equal(600, chunks[1]); + + peer.Close(run); + }); + + runner.Test("h2 streamed request: a request with no body reads empty at once", () => + { + bool sawEmpty = false; + + using var peer = new Peer(new Http2Options { StreamRequestBodies = true }); + Task run = peer.Connection.RunBufferedAsync(async request => + { + sawEmpty = (await request.BodyReader!.ReadAsync()).IsEmpty; + return Http2Response.Text("done"); + }); + + // END_STREAM on the HEADERS: there is no body coming, so the reader has to end rather + // than park forever waiting for a DATA frame that cannot arrive. + peer.OpenRequest(streamId: 1, endStream: true); + peer.Pump(); + + Assert.True(sawEmpty, "a bodyless request should read empty immediately"); + peer.Close(run); + }); + + runner.Test("h2 streamed request: buffered dispatch still assembles the body", () => + { + int seen = -1; + + using var peer = new Peer(new Http2Options()); // streaming OFF - the default + Task run = peer.Connection.RunBufferedAsync(request => + { + seen = request.Body.Length; + Assert.True(request.BodyReader is null, "buffered dispatch hands over no reader"); + return Http2Response.Text("done"); + }); + + peer.OpenRequest(streamId: 1, endStream: false); + peer.SendData(streamId: 1, bytes: 400, endStream: false); + peer.SendData(streamId: 1, bytes: 600, endStream: true); + peer.Pump(); + + // The other half of the trade: the whole body is in hand before the handler runs, and + // the window was credited as it arrived rather than as it was read. + Assert.Equal(1000, seen); + Assert.Equal(1000, peer.CreditFor(streamId: 1)); + + peer.Close(run); + }); + } + + /// + /// A peer driven by hand: frames in through an inline pipe, everything the server wrote back + /// captured for inspection. + /// + private sealed class Peer : IDuplexPipe, IDisposable + { + private readonly Pipe _input = new(new PipeOptions( + readerScheduler: PipeScheduler.Inline, + writerScheduler: PipeScheduler.Inline, + useSynchronizationContext: false)); + + private readonly CaptureWriter _output = new(); + + public Peer(Http2Options options) => Connection = new Http2Connection(this, options); + + public Http2Connection Connection { get; } + + public PipeReader Input => _input.Reader; + public PipeWriter Output => _output; + + /// Total WINDOW_UPDATE credit the server has handed back for a stream (0 = connection). + public int CreditFor(int streamId) + { + int total = 0; + ReadOnlySpan wire = _output.Written; + int at = 0; + + while (at + 9 <= wire.Length) + { + int length = (wire[at] << 16) | (wire[at + 1] << 8) | wire[at + 2]; + byte type = wire[at + 3]; + int stream = (int)(BinaryPrimitives.ReadUInt32BigEndian(wire[(at + 5)..]) & 0x7FFFFFFF); + + if (type == 0x8 && stream == streamId) // WINDOW_UPDATE + { + total += (int)(BinaryPrimitives.ReadUInt32BigEndian(wire[(at + 9)..]) & 0x7FFFFFFF); + } + at += 9 + length; + } + + return total; + } + + /// Preface, an empty SETTINGS, then one indexed-HPACK POST that opens a stream. + public void OpenRequest(int streamId, bool endStream) + { + var bytes = new List("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"u8.ToArray()); + bytes.AddRange(Header(0, 0x4, 0, 0)); + + // 0x83 :method POST, 0x86 :scheme http, 0x84 :path / - static table only, so this + // needs no HPACK encoder of its own. + byte flags = (byte)(0x4 | (endStream ? 0x1 : 0)); + bytes.AddRange(Header(3, 0x1, flags, streamId)); + bytes.AddRange([0x83, 0x86, 0x84]); + Feed(bytes.ToArray()); + } + + public void SendData(int streamId, int bytes, bool endStream) + { + var frame = new List(Header(bytes, 0x0, (byte)(endStream ? 0x1 : 0), streamId)); + frame.AddRange(Enumerable.Repeat((byte)'z', bytes)); + Feed(frame.ToArray()); + } + + /// Let the connection loop run whatever the last feed made possible. + public void Pump() => Feed([]); + + public void Close(Task run) + { + _input.Writer.Complete(); + Assert.True(run.Wait(5_000), "connection wound down"); + } + + public void Dispose() => Connection.Dispose(); + + private void Feed(byte[] bytes) + { + if (bytes.Length > 0) + { + _input.Writer.WriteAsync(bytes).GetAwaiter().GetResult(); + } + else + { + _input.Writer.FlushAsync().GetAwaiter().GetResult(); + } + } + + private static byte[] Header(int length, byte type, byte flags, int streamId) => + [ + (byte)(length >> 16), (byte)(length >> 8), (byte)length, + type, flags, + (byte)(streamId >> 24), (byte)(streamId >> 16), (byte)(streamId >> 8), (byte)streamId, + ]; + } + + /// Keeps every byte the server wrote, so the test can walk the frames afterwards. + private sealed class CaptureWriter : PipeWriter + { + private readonly List _written = []; + private byte[] _scratch = new byte[4096]; + private int _pending; + + public ReadOnlySpan Written => System.Runtime.InteropServices.CollectionsMarshal.AsSpan(_written); + + public override Memory GetMemory(int sizeHint = 0) + { + Grow(sizeHint); + return _scratch.AsMemory(_pending); + } + + public override Span GetSpan(int sizeHint = 0) + { + Grow(sizeHint); + return _scratch.AsSpan(_pending); + } + + public override void Advance(int bytes) => _pending += bytes; + + public override ValueTask FlushAsync(CancellationToken cancellationToken = default) + { + _written.AddRange(_scratch.AsSpan(0, _pending)); + _pending = 0; + return new ValueTask(new FlushResult(isCanceled: false, isCompleted: false)); + } + + public override void Complete(Exception? exception = null) + { + } + + public override void CancelPendingFlush() + { + } + + private void Grow(int sizeHint) + { + if (_scratch.Length - _pending < Math.Max(sizeHint, 1)) + { + Array.Resize(ref _scratch, Math.Max(_scratch.Length * 2, _pending + Math.Max(sizeHint, 4096))); + } + } + } +} diff --git a/tests/Ioxide.Tests.Unit/Program.cs b/tests/Ioxide.Tests.Unit/Program.cs index caa9c1b..c80a935 100644 --- a/tests/Ioxide.Tests.Unit/Program.cs +++ b/tests/Ioxide.Tests.Unit/Program.cs @@ -17,6 +17,7 @@ private static int Main() ResponseAssemblyTests.Register(runner); ResponseCapTests.Register(runner); Http2OutputQueueTests.Register(runner); + Http2StreamedRequestTests.Register(runner); return runner.Summary(); } From 86c44babfc16f4750a9f13ba5eea4537b4c4b8cb Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 15:11:01 +0100 Subject: [PATCH 08/14] feat(playground): name the samples by what streams, and cover h2 both ways "Managed" stopped meaning anything when nghttp2 left - all HTTP/2 here is pure C# now - and "streamed" never said WHICH direction, which is the thing a reader actually needs to know. Samples are named for what they do, and the library appears only where two of them still exist: Http2/Managed -> Http2/Buffered Http2/ManagedStreamed-> Http2/StreamedResponse Http2/ManagedTls -> Http2/Tls Http3/Managed -> Http3/Buffered Http3/ManagedStreamed-> Http3/StreamedBoth Http3/Nghttp3 -> Http3/Nghttp3Request Http3/Buffered -> Http3/Nghttp3Buffered Http3/Streamed -> Http3/Nghttp3Response Two new h2 samples close the ladder against h3's: StreamedRequest, where the body arrives a chunk at a time and reading it is what credits the peer, and StreamedBoth, whose /echo reads and writes at once - the shape a proxy needs, and the reason the two directions are separate switches rather than one. Writing them found two real bugs, which is the argument for samples that run: ReleaseAllCreditWaiters enumerated _creditWaiters while waking writers, and those wake INLINE and re-register at once - "collection was modified", thrown from the teardown finally, so it escaped the catch that exists to stop a malformed peer looking like a server fault. The stream-0 path had the mirror of it, clearing after releasing and so discarding the waiter a resumed writer had just added, parking it forever. Both take the waiters out before waking any. And PendingRequest.SendWindow opened at the RFC default of 65535 instead of what the peer's SETTINGS advertised. Streams are created long after those SETTINGS arrive, so a response longer than 65535 bytes stalled waiting for a WINDOW_UPDATE the peer had no reason to send - it believed we still held its whole window. Streamed responses had been dodging it by dropping the stream from _streams, which also dropped the per-stream window from the credit calculation; keeping the stream for a streamed request is what exposed it. /feed went from 64 KB in three seconds to 926 MB. Verified by running them: 8 MiB through /echo exactly, 50 MiB uploaded to /upload, /feed endless. Unit 36, Http 38, E2E 46, Chaos 37. --- Playground/Clients/Http/Program.cs | 2 +- Playground/Dockerfile | 4 +- .../Playground.Http2.Buffered.csproj} | 4 +- .../Http2/{Managed => Buffered}/Program.cs | 4 +- .../Playground.Http2.StreamedBoth.csproj | 19 ++ Playground/Http2/StreamedBoth/Program.cs | 136 ++++++++++++ .../Playground.Http2.StreamedRequest.csproj} | 4 +- Playground/Http2/StreamedRequest/Program.cs | 121 +++++++++++ .../Playground.Http2.StreamedResponse.csproj | 19 ++ .../Program.cs | 6 +- .../Playground.Http2.Tls.csproj} | 4 +- .../Http2/{ManagedTls => Tls}/Program.cs | 14 +- .../Buffered/Playground.Http3.Buffered.csproj | 2 +- Playground/Http3/Buffered/Program.cs | 177 +++------------ Playground/Http3/Managed/Program.cs | 88 -------- .../Playground.Http3.Nghttp3Buffered.csproj} | 4 +- Playground/Http3/Nghttp3Buffered/Program.cs | 205 ++++++++++++++++++ .../Playground.Http3.Nghttp3Request.csproj} | 4 +- .../{Nghttp3 => Nghttp3Request}/Program.cs | 10 +- .../Playground.Http3.Nghttp3Response.csproj} | 6 +- .../{Streamed => Nghttp3Response}/Program.cs | 8 +- .../Playground.Http3.StreamedBoth.csproj} | 4 +- .../Program.cs | 6 +- Playground/Proxy/H1ToH3/Program.cs | 2 +- Playground/Proxy/H2ToH3/Program.cs | 2 +- Playground/Proxy/H3ToH3/Program.cs | 2 +- Playground/Quic/Alpn/Program.cs | 2 +- Playground/README.md | 16 +- bench/h2-matrix.sh | 4 +- bench/results/20260809T181559Z.json | 2 +- bench/results/20260810T001150Z.json | 2 +- bench/results/20260810T001948Z.json | 2 +- bench/results/latest.json | 2 +- bench/run.sh | 2 +- bench/samples.tsv | 20 +- docs/index.html | 12 +- docs/learn/quic-h3.html | 2 +- dropped/README.md | 2 +- ioxide.slnx | 16 +- scripts/gen-docs-panes.py | 16 +- scripts/gen-docs-proxy-panes.py | 6 +- src/protocols/ioxide.http2/Http2BodyReader.cs | 8 +- .../ioxide.http2/Http2Connection.Frames.cs | 10 +- .../ioxide.http2/Http2Connection.Streamed.cs | 24 +- 44 files changed, 669 insertions(+), 336 deletions(-) rename Playground/Http2/{ManagedTls/Playground.Http2.ManagedTls.csproj => Buffered/Playground.Http2.Buffered.csproj} (81%) rename Playground/Http2/{Managed => Buffered}/Program.cs (97%) create mode 100644 Playground/Http2/StreamedBoth/Playground.Http2.StreamedBoth.csproj create mode 100644 Playground/Http2/StreamedBoth/Program.cs rename Playground/Http2/{ManagedStreamed/Playground.Http2.ManagedStreamed.csproj => StreamedRequest/Playground.Http2.StreamedRequest.csproj} (81%) create mode 100644 Playground/Http2/StreamedRequest/Program.cs create mode 100644 Playground/Http2/StreamedResponse/Playground.Http2.StreamedResponse.csproj rename Playground/Http2/{ManagedStreamed => StreamedResponse}/Program.cs (96%) rename Playground/Http2/{Managed/Playground.Http2.Managed.csproj => Tls/Playground.Http2.Tls.csproj} (82%) rename Playground/Http2/{ManagedTls => Tls}/Program.cs (94%) delete mode 100644 Playground/Http3/Managed/Program.cs rename Playground/Http3/{Streamed/Playground.Http3.Streamed.csproj => Nghttp3Buffered/Playground.Http3.Nghttp3Buffered.csproj} (82%) create mode 100644 Playground/Http3/Nghttp3Buffered/Program.cs rename Playground/Http3/{Nghttp3/Playground.Http3.Nghttp3.csproj => Nghttp3Request/Playground.Http3.Nghttp3Request.csproj} (83%) rename Playground/Http3/{Nghttp3 => Nghttp3Request}/Program.cs (95%) rename Playground/Http3/{ManagedStreamed/Playground.Http3.ManagedStreamed.csproj => Nghttp3Response/Playground.Http3.Nghttp3Response.csproj} (71%) rename Playground/Http3/{Streamed => Nghttp3Response}/Program.cs (94%) rename Playground/Http3/{Managed/Playground.Http3.Managed.csproj => StreamedBoth/Playground.Http3.StreamedBoth.csproj} (83%) rename Playground/Http3/{ManagedStreamed => StreamedBoth}/Program.cs (97%) diff --git a/Playground/Clients/Http/Program.cs b/Playground/Clients/Http/Program.cs index c673654..8d1ed8b 100644 --- a/Playground/Clients/Http/Program.cs +++ b/Playground/Clients/Http/Program.cs @@ -14,7 +14,7 @@ // the same GetAsync is h1 on the first request and h3 later. Http1Only / Http2Only (h2c) / // Http3Only pin it instead, which is what the nine Proxy/* samples do. // -// dotnet run -c Release --project Playground/Http3/Nghttp3 # an origin that advertises h3 +// dotnet run -c Release --project Playground/Http3/Nghttp3Request # an origin that advertises h3 // PLAYGROUND_UPSTREAM_PORT=8080 dotnet run -c Release --project Playground/Clients/Http // curl http://127.0.0.1:8090/ // diff --git a/Playground/Dockerfile b/Playground/Dockerfile index 32739c0..753a16a 100644 --- a/Playground/Dockerfile +++ b/Playground/Dockerfile @@ -3,11 +3,11 @@ # docker build -f Playground/Dockerfile --build-arg SAMPLE=Tcp/Raw -t playground-raw . # docker run --rm -p 8080:8080 playground-raw # -# docker build -f Playground/Dockerfile --build-arg SAMPLE=Http3/Nghttp3 -t playground-h3 . +# docker build -f Playground/Dockerfile --build-arg SAMPLE=Http3/Nghttp3Request -t playground-h3 . # docker run --rm -p 8080:8080 -p 8443:8443/udp playground-h3 # # SAMPLE is the directory path under Playground/ - any directory that carries a csproj: -# Tcp/Raw, Tcp/Pipe, Tls/OpenSsl, Http2/Nghttp2, Http3/Nghttp3, Clients/Pg, Proxy/H1ToH1, ... +# Tcp/Raw, Tcp/Pipe, Tls/OpenSsl, Http2/Nghttp2, Http3/Nghttp3Request, Clients/Pg, Proxy/H1ToH1, ... ARG SAMPLE=Tcp/Raw FROM mcr.microsoft.com/dotnet/sdk:11.0-preview AS build diff --git a/Playground/Http2/ManagedTls/Playground.Http2.ManagedTls.csproj b/Playground/Http2/Buffered/Playground.Http2.Buffered.csproj similarity index 81% rename from Playground/Http2/ManagedTls/Playground.Http2.ManagedTls.csproj rename to Playground/Http2/Buffered/Playground.Http2.Buffered.csproj index aadcb98..dcee586 100644 --- a/Playground/Http2/ManagedTls/Playground.Http2.ManagedTls.csproj +++ b/Playground/Http2/Buffered/Playground.Http2.Buffered.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http2.ManagedTls - Playground.Http2.ManagedTls + Playground.Http2.Buffered + Playground.Http2.Buffered diff --git a/Playground/Http2/Managed/Program.cs b/Playground/Http2/Buffered/Program.cs similarity index 97% rename from Playground/Http2/Managed/Program.cs rename to Playground/Http2/Buffered/Program.cs index df3f464..0c01544 100644 --- a/Playground/Http2/Managed/Program.cs +++ b/Playground/Http2/Buffered/Program.cs @@ -7,7 +7,7 @@ // http2 - an HTTP/2 server in pure C#: framing, HPACK and flow control; ioxide owns the // ring, the loop and the connection, so a response is written straight into the write slab. // -// dotnet run -c Release --project Playground/Http2/Managed +// dotnet run -c Release --project Playground/Http2/Buffered // curl --http2-prior-knowledge http://127.0.0.1:8080/hello // // This is h2c with PRIOR KNOWLEDGE: the peer opens with the HTTP/2 connection preface and no @@ -89,7 +89,7 @@ threads[i].Start(); } -Console.WriteLine($"[http2] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " +Console.WriteLine($"[http2-buffered] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " + $"{body.Length}-byte body (h2c prior knowledge)"); foreach (Thread thread in threads) diff --git a/Playground/Http2/StreamedBoth/Playground.Http2.StreamedBoth.csproj b/Playground/Http2/StreamedBoth/Playground.Http2.StreamedBoth.csproj new file mode 100644 index 0000000..e86955c --- /dev/null +++ b/Playground/Http2/StreamedBoth/Playground.Http2.StreamedBoth.csproj @@ -0,0 +1,19 @@ + + + + Exe + net11.0 + enable + enable + true + Playground.Http2.StreamedBoth + Playground.Http2.StreamedBoth + + + + + + + + + diff --git a/Playground/Http2/StreamedBoth/Program.cs b/Playground/Http2/StreamedBoth/Program.cs new file mode 100644 index 0000000..b21b14b --- /dev/null +++ b/Playground/Http2/StreamedBoth/Program.cs @@ -0,0 +1,136 @@ +using ioxide; +using ioxide.http2; +using Playground.Shared; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// http2-streamed-both - HTTP/2 with BOTH directions streamed: the request body pulled a chunk at +// a time, the response body pushed a chunk at a time, in the same handler. +// +// dotnet run -c Release --project Playground/Http2/StreamedBoth +// curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed # never ends +// head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \ +// http://127.0.0.1:8080/echo # neither side held +// +// /echo is the shape that needs both: read a chunk, write a chunk, and neither the upload nor +// the download is ever held whole. That is what a proxy does, and it is why the two directions +// are separate features rather than one "streaming" switch - they solve different problems. +// +// REQUEST streamed (Http2Request.BodyReader, StreamRequestBodies on) +// bounds what an upload can make the server hold. A read returns the peer's credit, so a +// slow handler makes the PEER slow down. +// +// RESPONSE streamed (Http2ResponseWriter, RunAsync) +// lets a body exist that has no length and no end - /feed has no final byte, so a +// buffered API cannot express it at all. A flush waits when either window is exhausted +// and resumes on the WINDOW_UPDATE. +// +// The windows are what HTTP/2 adds over HTTP/3 here: every stream shares one TCP connection, so +// both a per-stream and a connection window must allow a write, and a handler that stops reading +// holds down the connection window for every other stream on it. +// Needs: ioxide, ioxide.http2 +// ───────────────────────────────────────────────────────────────────────────────────────────── + +// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── +// Edit these. That is the whole mechanism - there is no config file and nothing else to find. + +ushort port = 8080; +int reactors = Environment.ProcessorCount; +int chunkBytes = 1024; // one DATA frame per flush on /feed + +Env.Override(ref port, ref reactors); +// ───────────────────────────────────────────────────────────────────────────────────────────── + +var config = new ServerConfig +{ + ReactorCount = reactors, // io_uring rings/threads - one per core + RingEntries = 8192, // SQ/CQ depth per ring + DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped + RecvBufferSize = 32 * 1024, // bytes per shared recv buffer + RecvSlots = 4096, // shared recv buffer-ring depth + Udp = null, // no raw UDP sockets (TCP-only server) + Quic = null, // no QUIC transport - see Http3/* + Tcp = new TcpOptions + { + Port = port, + ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener + WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in + PoolMax = 1024, // pooled connection objects kept per reactor + WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG + ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes + RecvQueueEntries = 64, // per-connection recv completion queue depth + }, +}; + +// Both halves are opt-in and independent: this one turns the REQUEST direction on, RunAsync below +// is what turns the RESPONSE direction on. +var http2 = new Http2Options { StreamRequestBodies = true }; + +byte[] chunk = [.. Enumerable.Repeat((byte)'x', chunkBytes)]; + +var threads = new Thread[config.ReactorCount]; + +for (int i = 0; i < threads.Length; i++) +{ + var reactor = new Reactor(i, config); + + reactor.TcpHandle = async (r, conn) => + { + try + { + await new Http2Connection(conn, http2).RunAsync(async (request, writer) => + { + bool echo = request.Path.Span.SequenceEqual("/echo"u8); + + // No content-length: on /feed the length will never be known, and on /echo it is + // not known yet. END_STREAM is what marks the end instead. + var response = new Http2Response { Status = 200 }; + response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray()); + writer.WriteHeaders(response); + + if (echo) + { + // Both directions at once. Nothing here holds more than one chunk: the read + // credits the peer for what it hands back, and the flush waits for room on the + // way out - so a fast uploader is paced by the slower of the two, not buffered. + while (true) + { + ReadOnlyMemory incoming = await request.BodyReader!.ReadAsync(); + if (incoming.IsEmpty) + { + break; + } + + incoming.Span.CopyTo(writer.GetSpan(incoming.Length)); + writer.Advance(incoming.Length); + await writer.FlushAsync(); + } + return; + } + + // /feed: a response with no end at all. There is no final byte to wait for, which + // is the case a buffered API has no way to express. + while (true) + { + chunk.CopyTo(writer.GetSpan(chunk.Length)); + writer.Advance(chunk.Length); + await writer.FlushAsync(); + } + }); + } + finally + { + conn.DecRef(); + } + }; + + threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; + threads[i].Start(); +} + +Console.WriteLine($"[http2-streamed-both] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " + + $"request pulled and response pushed ({chunkBytes}-byte chunks on /feed)"); + +foreach (Thread thread in threads) +{ + thread.Join(); +} diff --git a/Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj b/Playground/Http2/StreamedRequest/Playground.Http2.StreamedRequest.csproj similarity index 81% rename from Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj rename to Playground/Http2/StreamedRequest/Playground.Http2.StreamedRequest.csproj index 01e3783..45275af 100644 --- a/Playground/Http2/ManagedStreamed/Playground.Http2.ManagedStreamed.csproj +++ b/Playground/Http2/StreamedRequest/Playground.Http2.StreamedRequest.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http2.ManagedStreamed - Playground.Http2.ManagedStreamed + Playground.Http2.StreamedRequest + Playground.Http2.StreamedRequest diff --git a/Playground/Http2/StreamedRequest/Program.cs b/Playground/Http2/StreamedRequest/Program.cs new file mode 100644 index 0000000..c04083c --- /dev/null +++ b/Playground/Http2/StreamedRequest/Program.cs @@ -0,0 +1,121 @@ +using System.Text; +using ioxide; +using ioxide.http2; +using Playground.Shared; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// http2-streamed-request - HTTP/2 where the REQUEST body arrives a chunk at a time instead of +// assembled. The handler runs as soon as the headers are in, while the upload is still coming. +// +// dotnet run -c Release --project Playground/Http2/StreamedRequest +// head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \ +// http://127.0.0.1:8080/upload +// +// The knob is one line - StreamRequestBodies - and what it changes is what bounds memory. +// BUFFERED holds the whole body before your handler starts, so MaxRequestBytes is the only thing +// between a hostile peer and the arena. STREAMED holds one flow-control window, because a chunk +// credits the peer's window only as this handler READS it: fall behind and the peer runs out of +// credit and stops sending. That is backpressure with the peer actually participating, rather +// than a buffer you hope is big enough. +// +// Note the response is still returned whole here - one direction at a time, so the difference is +// the only thing on screen. Playground/Http2/StreamedBoth does both. +// Needs: ioxide, ioxide.http2 +// ───────────────────────────────────────────────────────────────────────────────────────────── + +// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── +// Edit these. That is the whole mechanism - there is no config file and nothing else to find. + +ushort port = 8080; +int reactors = Environment.ProcessorCount; + +Env.Override(ref port, ref reactors); + +// Advertised per stream. This is the ceiling on how far ahead of the handler a peer may get, so +// on a streamed request it is the memory bound - not a throughput knob. +int streamWindow = 256 * 1024; +// ───────────────────────────────────────────────────────────────────────────────────────────── + +var config = new ServerConfig +{ + ReactorCount = reactors, // io_uring rings/threads - one per core + RingEntries = 8192, // SQ/CQ depth per ring + DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped + RecvBufferSize = 32 * 1024, // bytes per shared recv buffer + RecvSlots = 4096, // shared recv buffer-ring depth + Udp = null, // no raw UDP sockets (TCP-only server) + Quic = null, // no QUIC transport - see Http3/* + Tcp = new TcpOptions + { + Port = port, + ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener + WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in + PoolMax = 1024, // pooled connection objects kept per reactor + WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG + ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes + RecvQueueEntries = 64, // per-connection recv completion queue depth + }, +}; + +var http2 = new Http2Options +{ + StreamRequestBodies = true, // the whole point: dispatch at the headers, body follows + InitialWindowSize = streamWindow, // how far ahead of the handler the peer may run +}; + +var threads = new Thread[config.ReactorCount]; + +for (int i = 0; i < threads.Length; i++) +{ + var reactor = new Reactor(i, config); + + reactor.TcpHandle = async (r, conn) => + { + try + { + await new Http2Connection(conn, http2).RunBufferedAsync(async request => + { + // BodyReader is set because StreamRequestBodies is on; with it off the body would + // be in request.Body instead and this would be null. + long total = 0; + if (request.BodyReader is { } body) + { + while (true) + { + // Empty means end of body. Each read hands back the peer's credit for the + // chunk it returns, which is what lets the next one arrive - and the + // memory it points at is recycled by the NEXT read, so anything worth + // keeping has to be copied out here. + ReadOnlyMemory chunk = await body.ReadAsync(); + if (chunk.IsEmpty) + { + break; + } + total += chunk.Length; + } + } + + return new Http2Response + { + Status = 200, + Body = Encoding.ASCII.GetBytes($"{total} bytes\n"), + }; + }); + } + finally + { + conn.DecRef(); + } + }; + + threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; + threads[i].Start(); +} + +Console.WriteLine($"[http2-streamed-request] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " + + $"request bodies streamed, {streamWindow / 1024} KiB window per stream"); + +foreach (Thread thread in threads) +{ + thread.Join(); +} diff --git a/Playground/Http2/StreamedResponse/Playground.Http2.StreamedResponse.csproj b/Playground/Http2/StreamedResponse/Playground.Http2.StreamedResponse.csproj new file mode 100644 index 0000000..98a63af --- /dev/null +++ b/Playground/Http2/StreamedResponse/Playground.Http2.StreamedResponse.csproj @@ -0,0 +1,19 @@ + + + + Exe + net11.0 + enable + enable + true + Playground.Http2.StreamedResponse + Playground.Http2.StreamedResponse + + + + + + + + + diff --git a/Playground/Http2/ManagedStreamed/Program.cs b/Playground/Http2/StreamedResponse/Program.cs similarity index 96% rename from Playground/Http2/ManagedStreamed/Program.cs rename to Playground/Http2/StreamedResponse/Program.cs index f05661b..1886cde 100644 --- a/Playground/Http2/ManagedStreamed/Program.cs +++ b/Playground/Http2/StreamedResponse/Program.cs @@ -4,7 +4,7 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// http2-managed-streamed - HTTP/2 in pure C# with the RESPONSE BODY STREAMED: the handler pushes +// http2-streamed-response - HTTP/2 with the RESPONSE BODY STREAMED: the handler pushes // bytes as it produces them and each flush becomes a DATA frame, instead of returning a finished // Http2Response. // @@ -16,7 +16,7 @@ // FlushAsync waits for a WINDOW_UPDATE rather than failing. That wait is the backpressure: a // peer that stops reading stops the producer instead of growing a queue behind it. // -// dotnet run -c Release --project Playground/Http2/ManagedStreamed +// dotnet run -c Release --project Playground/Http2/StreamedResponse // curl --http2-prior-knowledge http://127.0.0.1:8080/ # chunked // curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed # never ends // @@ -119,7 +119,7 @@ threads[i].Start(); } -Console.WriteLine($"[http2-managed-streamed] {config.ReactorCount} reactors on :{config.Tcp!.Port} " +Console.WriteLine($"[http2-streamed-response] {config.ReactorCount} reactors on :{config.Tcp!.Port} " + $"(pure C#), {chunkCount} x {chunkBytes}-byte chunks per response"); foreach (Thread thread in threads) diff --git a/Playground/Http2/Managed/Playground.Http2.Managed.csproj b/Playground/Http2/Tls/Playground.Http2.Tls.csproj similarity index 82% rename from Playground/Http2/Managed/Playground.Http2.Managed.csproj rename to Playground/Http2/Tls/Playground.Http2.Tls.csproj index a9824dc..3f5ab87 100644 --- a/Playground/Http2/Managed/Playground.Http2.Managed.csproj +++ b/Playground/Http2/Tls/Playground.Http2.Tls.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http2.Managed - Playground.Http2.Managed + Playground.Http2.Tls + Playground.Http2.Tls diff --git a/Playground/Http2/ManagedTls/Program.cs b/Playground/Http2/Tls/Program.cs similarity index 94% rename from Playground/Http2/ManagedTls/Program.cs rename to Playground/Http2/Tls/Program.cs index 03dcf4d..ae486da 100644 --- a/Playground/Http2/ManagedTls/Program.cs +++ b/Playground/Http2/Tls/Program.cs @@ -6,10 +6,12 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// http2-managed-tls - HTTP/2 over TLS in PURE C#: the same server as Playground/Http2/Tls with -// ioxide.http2 in place of ioxide.nghttp2. The diff is three type names and a package - no -// native library anywhere - because both take an IDuplexPipe and neither learns what is -// under it. +// http2-tls - HTTP/2 over TLS, negotiated by ALPN, alongside HTTP/1.1 on the SAME port. This +// is what a browser expects: it offers "h2,http/1.1" and the server chooses. +// +// Note what Http2Connection is handed: a TlsConnectionDualPipe. It never learns that TLS is +// involved - the pipe decrypts on the way in and encrypts on the way out - so the protocol +// code here is byte-for-byte the h2c sample's. // // dotnet run -c Release --project Playground/Http2/Tls // curl -k --http2 https://127.0.0.1:8443/ # negotiates h2 @@ -181,7 +183,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\ } catch (Exception e) { - Console.Error.WriteLine($"[http2-managed-tls] connection failed: {e.Message}"); + Console.Error.WriteLine($"[http2-tls] connection failed: {e.Message}"); } finally { @@ -194,7 +196,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\ threads[i].Start(); } -Console.WriteLine($"[http2-managed-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " +Console.WriteLine($"[http2-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, " + $"ALPN h2 then http/1.1, cert {certPath}, " + $"rx={(kernelTx && kernelRx ? "kernel" : "openssl")}, " + $"tx={(kernelTx ? "kernel" : "openssl")}"); diff --git a/Playground/Http3/Buffered/Playground.Http3.Buffered.csproj b/Playground/Http3/Buffered/Playground.Http3.Buffered.csproj index fcbc9f5..2619d1c 100644 --- a/Playground/Http3/Buffered/Playground.Http3.Buffered.csproj +++ b/Playground/Http3/Buffered/Playground.Http3.Buffered.csproj @@ -14,7 +14,7 @@ - + diff --git a/Playground/Http3/Buffered/Program.cs b/Playground/Http3/Buffered/Program.cs index 7803df3..123fa42 100644 --- a/Playground/Http3/Buffered/Program.cs +++ b/Playground/Http3/Buffered/Program.cs @@ -1,121 +1,72 @@ -using System.Runtime.InteropServices; using ioxide; -using ioxide.nghttp3; +using ioxide.http3; using ioxide.ngtcp2; -using ioxide.utils; using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// nghttp3-buffered - the same HTTP/3 server as Playground/Http3/Nghttp3, with the OTHER dispatch mode. +// http3-buffered - HTTP/3 in PURE C#, dispatched BUFFERED: the handler runs once the request +// has fully arrived, so request.Body already holds the whole body. Frames, QPACK and Huffman +// are all managed code, so nothing native ships but the QUIC transport underneath. // -// BUFFERED: dispatch waits for end-of-stream, so the whole body is already in request.Body when -// your handler runs - no BodyReader, no pacing - and the handler may still await (a PgPool query, -// Redis, anything ioxide-native resumes inline on the reactor). -// -// The trade: memory holds the entire body, so this suits normal-sized requests. Use Playground/Http3/Nghttp3 -// when uploads can be large or hostile. -// -// It doubles as the QUIC/HTTP3 tuning reference: the Knobs block below shows every h3-path option -// (engine, listener, UDP, QPACK) as a literal, including maxSendRetentionBytes - the knob that -// bounds memory when serving large responses. +// The same server as Playground/Http3/Nghttp3Request otherwise - the diff is the package and the type +// names - which is what makes the two directly comparable: // // dotnet run -c Release --project Playground/Http3/Buffered // curl --http3-only -k https://127.0.0.1:8443/ // -// Needs: ioxide, ioxide.ngtcp2, ioxide.nghttp3 +// Needs: ioxide, ioxide.ngtcp2, ioxide.http3 // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── // Edit these. That is the whole mechanism - there is no config file and nothing else to find. -// Env.Override exists only so bench/run.sh can drive the sample from outside; delete those lines -// when you copy this out and the literals above them are the entire configuration. +// Each Env.Override names the variable that can drive it instead, which is how the bench scripts +// run this sample; the literal is what applies otherwise. -// This sample is the QUIC/HTTP3 tuning reference: every knob the h3 path exposes is here as a -// literal, grouped by the type it feeds. The defaults are the shipping defaults - shown, not -// changed - so you can see the whole surface and edit one line. - -ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP - h3 lives here -ushort tcpPort = 8080; // the TCP listener, so the process serves both +ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core -Env.Override(ref tcpPort, ref reactors); Env.OverrideQuic(ref quicPort, ref reactors); -// ── QuicEngine: the per-endpoint QUIC/TLS state, shared by every connection ─────────────────── -uint cidLength = 8; // connection-id length this endpoint mints (1..20) - -// Per-connection send-retention high-water. A response larger than this is streamed out paced by -// the peer's acks instead of buffered whole, so memory stays ~this-per-connection whatever the -// response size - the knob that lets HTTP/3 serve large files. Raise for more in-flight throughput -// on fat links; lower to cap memory under many connections. Default 16 MiB. -long maxSendRetentionBytes = 16L << 20; +// Response body size. 13 is "Hello, World!"; anything else is that many 'x'. +int bodyBytes = 13; -// ── QuicOptions: the listener ───────────────────────────────────────────────────────────────── -int idleTimeoutMs = 60_000; // close a connection idle this long (no packets) +Env.Override(ref bodyBytes, "PLAYGROUND_BODY"); -// ── UdpOptions: how datagrams are received ──────────────────────────────────────────────────── -int udpRecvSlots = 16; // multishot recv slots per reactor - datagrams the ring can hold at once -bool gro = true; // UDP_GRO: coalesce received datagrams into one recv (fewer syscalls) - -// ── Nghttp3Options: the HTTP/3 layer ────────────────────────────────────────────────────────── -// QPACK dynamic table. 0 keeps every header literal, which costs bytes but never blocks a stream -// on a table update; raise it and set QpackBlockedStreams to trade one for the other. -long qpackCapacity = 0; -long qpackBlockedStreams = qpackCapacity > 0 ? 100 : 0; +// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once. +int udpRecvSlots = 16; // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; + +Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); -using var engine = new QuicEngine(certPath, keyPath, cidLength, alpn: ["h3"], maxSendRetentionBytes); +using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]); var config = new ServerConfig { - ReactorCount = reactors, - RingEntries = 8192, // SQ/CQ depth per ring - DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped - RecvBufferSize = 32 * 1024, // bytes per shared recv buffer - RecvSlots = 4096, // shared recv buffer-ring depth - Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental - Tcp = new TcpOptions - { - Port = tcpPort, - ExtraPorts = [], // extra listener ports (one handler, several doors) - ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener - WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in - PoolMax = 1024, // pooled connection objects kept per reactor - WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG - ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes - RecvQueueEntries = 64, // per-connection recv completion queue depth - }, - Udp = new UdpOptions { RecvSlots = udpRecvSlots, Gro = gro }, + ReactorCount = reactors, + Tcp = null, // QUIC only + Udp = new UdpOptions { RecvSlots = udpRecvSlots }, Quic = new QuicOptions { Port = quicPort, - LocalCidLength = (int)cidLength, // must match the engine's cidLength - IdleTimeoutMs = idleTimeoutMs, + LocalCidLength = 8, ConnectionFactory = engine.CreateFactory(), }, }; -var h3Options = new Nghttp3Options +// Built once and reused: the h3 layer copies status, headers and body at submit and never retains +// the object, so a hot path should not rebuild it per request. +var response = new Http3Response { - QpackDynamicTableCapacity = qpackCapacity, - QpackBlockedStreams = qpackBlockedStreams, + Body = bodyBytes == 13 ? "Hello, World!"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)], }; - -// Built once and reused for every request - the h3 layer copies it into nghttp3 at submit and never -// retains it, so this costs zero allocations per request. -var response = new Nghttp3Response { Body = "Hello, World!"u8.ToArray() }; -response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray()); -response.Headers.Add("server"u8.ToArray(), "ioxide"u8.ToArray()); - -byte[] tcpResponse = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok"u8.ToArray(); - -List<(Reactor Reactor, Nghttp3Connection Connection)> live = []; +response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray())); +response.Headers.Add(("server"u8.ToArray(), "ioxide"u8.ToArray())); var threads = new Thread[config.ReactorCount]; @@ -123,80 +74,14 @@ { var reactor = new Reactor(i, config); - reactor.QuicHandle = (r, quicConn) => - { - var h3 = new Nghttp3Connection(quicConn, h3Options); - lock (live) - { - live.Add((r, h3)); - } - - // RunBufferedAsync, not RunStreamingAsync - that one call is the whole difference. - return h3.RunBufferedAsync(request => - { - // Dispatch waited for end-of-stream, so the body is ALREADY here: request.Body is - // complete and request.Body.Length is just a property read. No BodyReader, no pacing. - // This overload is synchronous, but the awaiting one exists too - a PgPool query or a - // Redis command resumes inline on this reactor, so you can await it right here. - _ = request.Body.Length; - - // One response object, reused for every request: zero allocations on this path. To - // route, compare request.Path.Span - it is post-QPACK bytes, so SequenceEqual against a - // u8 literal beats decoding it to a string. - return response; - }); - }; - - reactor.TcpHandle = async (r, conn) => - { - try - { - while (true) - { - RecvSnapshot snapshot = await conn.ReadAsync(); - while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item)) - { - if (item.HasBuffer) conn.ReturnBuffer(in item); - } - - conn.Write(tcpResponse); - await conn.FlushAsync(); - - if (snapshot.IsClosed) return; - conn.ResetRead(); - } - } - finally - { - conn.DecRef(); - } - }; + reactor.QuicHandle = (r, conn) => new Http3Connection(conn).RunAsync(_ => response); threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; threads[i].Start(); } -using var drain = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context => -{ - context.Cancel = true; - Console.WriteLine("[nghttp3-buffered] SIGTERM: draining connections (GOAWAY)..."); - - lock (live) - { - foreach ((Reactor r, Nghttp3Connection h3) in live) - { - r.ScheduleOnReactor(static state => ((Nghttp3Connection)state!).Shutdown(), h3); - } - live.Clear(); - } - - Thread.Sleep(2000); - Console.WriteLine("[nghttp3-buffered] drain complete, exiting"); - Environment.Exit(0); -}); - -Console.WriteLine($"[nghttp3-buffered] {config.ReactorCount} reactors - tcp :{config.Tcp.Port}, " - + $"udp :{quicPort} (ngtcp2 {QuicEngine.NativeVersion()})"); +Console.WriteLine($"[h3-managed] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port} " + + $"(pure C#), {bodyBytes}-byte body, cert {certPath}"); foreach (Thread thread in threads) { diff --git a/Playground/Http3/Managed/Program.cs b/Playground/Http3/Managed/Program.cs deleted file mode 100644 index ff4c55d..0000000 --- a/Playground/Http3/Managed/Program.cs +++ /dev/null @@ -1,88 +0,0 @@ -using ioxide; -using ioxide.http3; -using ioxide.ngtcp2; -using Playground.Shared; - -// ───────────────────────────────────────────────────────────────────────────────────────────── -// http3-managed - HTTP/3 in PURE C#: ioxide.http3 in place of ioxide.nghttp3. Frames, QPACK and -// Huffman are all managed code, so nothing native ships but the QUIC transport underneath. -// -// The same server as Playground/Http3/Nghttp3 otherwise - the diff is the package and the type -// names - which is what makes the two directly comparable: -// -// dotnet run -c Release --project Playground/Http3/Managed -// curl --http3-only -k https://127.0.0.1:8443/ -// -// Needs: ioxide, ioxide.ngtcp2, ioxide.http3 -// ───────────────────────────────────────────────────────────────────────────────────────────── - -// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── -// Edit these. That is the whole mechanism - there is no config file and nothing else to find. -// Each Env.Override names the variable that can drive it instead, which is how the bench scripts -// run this sample; the literal is what applies otherwise. - -ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP -int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core - -Env.OverrideQuic(ref quicPort, ref reactors); - -// Response body size. 13 is "Hello, World!"; anything else is that many 'x'. -int bodyBytes = 13; - -Env.Override(ref bodyBytes, "PLAYGROUND_BODY"); - -// UDP receive slots per reactor: how many datagrams the ring can have outstanding at once. -int udpRecvSlots = 16; - -// A real PEM pair, or null to generate a self-signed localhost cert on first run. -string? certOverride = null; -string? keyOverride = null; - -Env.OverrideCert(ref certOverride, ref keyOverride); -// ───────────────────────────────────────────────────────────────────────────────────────────── - -(string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); - -using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"]); - -var config = new ServerConfig -{ - ReactorCount = reactors, - Tcp = null, // QUIC only - Udp = new UdpOptions { RecvSlots = udpRecvSlots }, - Quic = new QuicOptions - { - Port = quicPort, - LocalCidLength = 8, - ConnectionFactory = engine.CreateFactory(), - }, -}; - -// Built once and reused: the h3 layer copies status, headers and body at submit and never retains -// the object, so a hot path should not rebuild it per request. -var response = new Http3Response -{ - Body = bodyBytes == 13 ? "Hello, World!"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)], -}; -response.Headers.Add(("content-type"u8.ToArray(), "text/plain"u8.ToArray())); -response.Headers.Add(("server"u8.ToArray(), "ioxide"u8.ToArray())); - -var threads = new Thread[config.ReactorCount]; - -for (int i = 0; i < threads.Length; i++) -{ - var reactor = new Reactor(i, config); - - reactor.QuicHandle = (r, conn) => new Http3Connection(conn).RunAsync(_ => response); - - threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; - threads[i].Start(); -} - -Console.WriteLine($"[h3-managed] {config.ReactorCount} reactors, h3 on udp :{config.Quic!.Port} " - + $"(pure C#), {bodyBytes}-byte body, cert {certPath}"); - -foreach (Thread thread in threads) -{ - thread.Join(); -} diff --git a/Playground/Http3/Streamed/Playground.Http3.Streamed.csproj b/Playground/Http3/Nghttp3Buffered/Playground.Http3.Nghttp3Buffered.csproj similarity index 82% rename from Playground/Http3/Streamed/Playground.Http3.Streamed.csproj rename to Playground/Http3/Nghttp3Buffered/Playground.Http3.Nghttp3Buffered.csproj index 792a82c..e03f05f 100644 --- a/Playground/Http3/Streamed/Playground.Http3.Streamed.csproj +++ b/Playground/Http3/Nghttp3Buffered/Playground.Http3.Nghttp3Buffered.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http3.Streamed - Playground.Http3.Streamed + Playground.Http3.Nghttp3Buffered + Playground.Http3.Nghttp3Buffered diff --git a/Playground/Http3/Nghttp3Buffered/Program.cs b/Playground/Http3/Nghttp3Buffered/Program.cs new file mode 100644 index 0000000..446428c --- /dev/null +++ b/Playground/Http3/Nghttp3Buffered/Program.cs @@ -0,0 +1,205 @@ +using System.Runtime.InteropServices; +using ioxide; +using ioxide.nghttp3; +using ioxide.ngtcp2; +using ioxide.utils; +using Playground.Shared; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// nghttp3-buffered - the same HTTP/3 server as Playground/Http3/Nghttp3Request, with the +// OTHER request dispatch mode. +// +// BUFFERED: dispatch waits for end-of-stream, so the whole body is already in request.Body when +// your handler runs - no BodyReader, no pacing - and the handler may still await (a PgPool query, +// Redis, anything ioxide-native resumes inline on the reactor). +// +// The trade: memory holds the entire body, so this suits normal-sized requests. Use Playground/Http3/Nghttp3Request +// when uploads can be large or hostile. +// +// It doubles as the QUIC/HTTP3 tuning reference: the Knobs block below shows every h3-path option +// (engine, listener, UDP, QPACK) as a literal, including maxSendRetentionBytes - the knob that +// bounds memory when serving large responses. +// +// dotnet run -c Release --project Playground/Http3/Nghttp3Buffered +// curl --http3-only -k https://127.0.0.1:8443/ +// +// Needs: ioxide, ioxide.ngtcp2, ioxide.nghttp3 +// ───────────────────────────────────────────────────────────────────────────────────────────── + +// ── Knobs ──────────────────────────────────────────────────────────────────────────────────── +// Edit these. That is the whole mechanism - there is no config file and nothing else to find. +// Env.Override exists only so bench/run.sh can drive the sample from outside; delete those lines +// when you copy this out and the literals above them are the entire configuration. + +// This sample is the QUIC/HTTP3 tuning reference: every knob the h3 path exposes is here as a +// literal, grouped by the type it feeds. The defaults are the shipping defaults - shown, not +// changed - so you can see the whole surface and edit one line. + +ushort quicPort = 8443; // https://127.0.0.1:8443/ over UDP - h3 lives here +ushort tcpPort = 8080; // the TCP listener, so the process serves both +int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core + +Env.Override(ref tcpPort, ref reactors); +Env.OverrideQuic(ref quicPort, ref reactors); + +// ── QuicEngine: the per-endpoint QUIC/TLS state, shared by every connection ─────────────────── +uint cidLength = 8; // connection-id length this endpoint mints (1..20) + +// Per-connection send-retention high-water. A response larger than this is streamed out paced by +// the peer's acks instead of buffered whole, so memory stays ~this-per-connection whatever the +// response size - the knob that lets HTTP/3 serve large files. Raise for more in-flight throughput +// on fat links; lower to cap memory under many connections. Default 16 MiB. +long maxSendRetentionBytes = 16L << 20; + +// ── QuicOptions: the listener ───────────────────────────────────────────────────────────────── +int idleTimeoutMs = 60_000; // close a connection idle this long (no packets) + +// ── UdpOptions: how datagrams are received ──────────────────────────────────────────────────── +int udpRecvSlots = 16; // multishot recv slots per reactor - datagrams the ring can hold at once +bool gro = true; // UDP_GRO: coalesce received datagrams into one recv (fewer syscalls) + +// ── Nghttp3Options: the HTTP/3 layer ────────────────────────────────────────────────────────── +// QPACK dynamic table. 0 keeps every header literal, which costs bytes but never blocks a stream +// on a table update; raise it and set QpackBlockedStreams to trade one for the other. +long qpackCapacity = 0; +long qpackBlockedStreams = qpackCapacity > 0 ? 100 : 0; + +// A real PEM pair, or null to generate a self-signed localhost cert on first run. +string? certOverride = null; +string? keyOverride = null; +// ───────────────────────────────────────────────────────────────────────────────────────────── + +(string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); + +using var engine = new QuicEngine(certPath, keyPath, cidLength, alpn: ["h3"], maxSendRetentionBytes); + +var config = new ServerConfig +{ + ReactorCount = reactors, + RingEntries = 8192, // SQ/CQ depth per ring + DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped + RecvBufferSize = 32 * 1024, // bytes per shared recv buffer + RecvSlots = 4096, // shared recv buffer-ring depth + Incremental = null, // per-connection recv rings (6.12+) - see Tcp/Incremental + Tcp = new TcpOptions + { + Port = tcpPort, + ExtraPorts = [], // extra listener ports (one handler, several doors) + ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener + WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in + PoolMax = 1024, // pooled connection objects kept per reactor + WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG + ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes + RecvQueueEntries = 64, // per-connection recv completion queue depth + }, + Udp = new UdpOptions { RecvSlots = udpRecvSlots, Gro = gro }, + Quic = new QuicOptions + { + Port = quicPort, + LocalCidLength = (int)cidLength, // must match the engine's cidLength + IdleTimeoutMs = idleTimeoutMs, + ConnectionFactory = engine.CreateFactory(), + }, +}; + +var h3Options = new Nghttp3Options +{ + QpackDynamicTableCapacity = qpackCapacity, + QpackBlockedStreams = qpackBlockedStreams, +}; + +// Built once and reused for every request - the h3 layer copies it into nghttp3 at submit and never +// retains it, so this costs zero allocations per request. +var response = new Nghttp3Response { Body = "Hello, World!"u8.ToArray() }; +response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray()); +response.Headers.Add("server"u8.ToArray(), "ioxide"u8.ToArray()); + +byte[] tcpResponse = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nok"u8.ToArray(); + +List<(Reactor Reactor, Nghttp3Connection Connection)> live = []; + +var threads = new Thread[config.ReactorCount]; + +for (int i = 0; i < threads.Length; i++) +{ + var reactor = new Reactor(i, config); + + reactor.QuicHandle = (r, quicConn) => + { + var h3 = new Nghttp3Connection(quicConn, h3Options); + lock (live) + { + live.Add((r, h3)); + } + + // RunBufferedAsync, not RunStreamingAsync - that one call is the whole difference. + return h3.RunBufferedAsync(request => + { + // Dispatch waited for end-of-stream, so the body is ALREADY here: request.Body is + // complete and request.Body.Length is just a property read. No BodyReader, no pacing. + // This overload is synchronous, but the awaiting one exists too - a PgPool query or a + // Redis command resumes inline on this reactor, so you can await it right here. + _ = request.Body.Length; + + // One response object, reused for every request: zero allocations on this path. To + // route, compare request.Path.Span - it is post-QPACK bytes, so SequenceEqual against a + // u8 literal beats decoding it to a string. + return response; + }); + }; + + reactor.TcpHandle = async (r, conn) => + { + try + { + while (true) + { + RecvSnapshot snapshot = await conn.ReadAsync(); + while (conn.TryGetItem(snapshot, out SpscRecvRing.Item item)) + { + if (item.HasBuffer) conn.ReturnBuffer(in item); + } + + conn.Write(tcpResponse); + await conn.FlushAsync(); + + if (snapshot.IsClosed) return; + conn.ResetRead(); + } + } + finally + { + conn.DecRef(); + } + }; + + threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" }; + threads[i].Start(); +} + +using var drain = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context => +{ + context.Cancel = true; + Console.WriteLine("[nghttp3-buffered] SIGTERM: draining connections (GOAWAY)..."); + + lock (live) + { + foreach ((Reactor r, Nghttp3Connection h3) in live) + { + r.ScheduleOnReactor(static state => ((Nghttp3Connection)state!).Shutdown(), h3); + } + live.Clear(); + } + + Thread.Sleep(2000); + Console.WriteLine("[nghttp3-buffered] drain complete, exiting"); + Environment.Exit(0); +}); + +Console.WriteLine($"[nghttp3-buffered] {config.ReactorCount} reactors - tcp :{config.Tcp.Port}, " + + $"udp :{quicPort} (ngtcp2 {QuicEngine.NativeVersion()})"); + +foreach (Thread thread in threads) +{ + thread.Join(); +} diff --git a/Playground/Http3/Nghttp3/Playground.Http3.Nghttp3.csproj b/Playground/Http3/Nghttp3Request/Playground.Http3.Nghttp3Request.csproj similarity index 83% rename from Playground/Http3/Nghttp3/Playground.Http3.Nghttp3.csproj rename to Playground/Http3/Nghttp3Request/Playground.Http3.Nghttp3Request.csproj index cf0c8ca..4a660b9 100644 --- a/Playground/Http3/Nghttp3/Playground.Http3.Nghttp3.csproj +++ b/Playground/Http3/Nghttp3Request/Playground.Http3.Nghttp3Request.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http3.Nghttp3 - Playground.Http3.Nghttp3 + Playground.Http3.Nghttp3Request + Playground.Http3.Nghttp3Request diff --git a/Playground/Http3/Nghttp3/Program.cs b/Playground/Http3/Nghttp3Request/Program.cs similarity index 95% rename from Playground/Http3/Nghttp3/Program.cs rename to Playground/Http3/Nghttp3Request/Program.cs index 1b04063..007b908 100644 --- a/Playground/Http3/Nghttp3/Program.cs +++ b/Playground/Http3/Nghttp3Request/Program.cs @@ -6,15 +6,17 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// nghttp3 - a real HTTP/3 server, whole. ngtcp2 + picotls are bundled as one native library +// nghttp3-streamed-request - a real HTTP/3 server, whole, dispatched as the REQUEST body +// streams in: the handler runs while the upload is still arriving, so memory is bound by one +// flow-control window rather than by the body. ngtcp2 + picotls are bundled as one native library // (TLS 1.3 lives inside the transport), and nghttp3 puts HTTP/3 on top. Every reactor binds the // UDP port via SO_REUSEPORT and demuxes its own flows. // // STREAMED dispatch: your handler runs at end-of-headers, while the body is still arriving. Each // chunk you read credits the peer's flow-control window, so memory is bound by one window rather -// than by the size of the upload. See Playground/Http3/Buffered for the other mode. +// than by the size of the upload. See Playground/Http3/Nghttp3Buffered for the other mode. // -// dotnet run -c Release --project Playground/Http3/Nghttp3 +// dotnet run -c Release --project Playground/Http3/Nghttp3Request // curl --http3-only -k https://127.0.0.1:8443/ // h2load --alpn-list=h3 -n 1 -c 1 -d bigfile.bin https://127.0.0.1:8443/ // @@ -58,7 +60,7 @@ // One engine for the whole server. ALPN pinned to h3, so nothing else negotiates. The last arg is // the per-connection send-retention high-water (default 16 MiB): a response larger than it streams // out paced by acks instead of buffering whole, so h3 serves large files in bounded memory. See -// Playground/Http3/Buffered for the full QUIC/h3 knob set. +// Playground/Http3/Nghttp3Buffered for the full QUIC/h3 knob set. using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"], maxSendRetentionBytes: 16L << 20); var config = new ServerConfig diff --git a/Playground/Http3/ManagedStreamed/Playground.Http3.ManagedStreamed.csproj b/Playground/Http3/Nghttp3Response/Playground.Http3.Nghttp3Response.csproj similarity index 71% rename from Playground/Http3/ManagedStreamed/Playground.Http3.ManagedStreamed.csproj rename to Playground/Http3/Nghttp3Response/Playground.Http3.Nghttp3Response.csproj index c444b26..e0d8f75 100644 --- a/Playground/Http3/ManagedStreamed/Playground.Http3.ManagedStreamed.csproj +++ b/Playground/Http3/Nghttp3Response/Playground.Http3.Nghttp3Response.csproj @@ -6,15 +6,15 @@ enable enable true - Playground.Http3.ManagedStreamed - Playground.Http3.ManagedStreamed + Playground.Http3.Nghttp3Response + Playground.Http3.Nghttp3Response - + diff --git a/Playground/Http3/Streamed/Program.cs b/Playground/Http3/Nghttp3Response/Program.cs similarity index 94% rename from Playground/Http3/Streamed/Program.cs rename to Playground/Http3/Nghttp3Response/Program.cs index fcf55d7..869e918 100644 --- a/Playground/Http3/Streamed/Program.cs +++ b/Playground/Http3/Nghttp3Response/Program.cs @@ -5,16 +5,16 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// nghttp3-streamed - HTTP/3 where the RESPONSE body is produced over time instead of handed -// over whole. The other two h3 samples differ in how the REQUEST arrives; this one is about the -// other direction. +// nghttp3-streamed-response - HTTP/3 where the RESPONSE body is produced over time rather +// than handed over whole. The other two nghttp3 samples differ in how the REQUEST arrives; +// this one is about the other direction. // // Buffered and streaming both end with `return new Nghttp3Response { Body = ... }` - the whole // body has to exist before anything can be sent. That is fine for a page and impossible for a // feed: an endpoint that never stops has no final byte to return. Here the handler gets a // WRITER, and each flush becomes DATA on the wire. // -// dotnet run -c Release --project Playground/Http3/Streamed +// dotnet run -c Release --project Playground/Http3/Nghttp3Response // curl --http3-only -k https://127.0.0.1:8443/ # 64 chunks, one per flush // curl --http3-only -kN https://127.0.0.1:8443/feed # never ends; ctrl-c to stop // diff --git a/Playground/Http3/Managed/Playground.Http3.Managed.csproj b/Playground/Http3/StreamedBoth/Playground.Http3.StreamedBoth.csproj similarity index 83% rename from Playground/Http3/Managed/Playground.Http3.Managed.csproj rename to Playground/Http3/StreamedBoth/Playground.Http3.StreamedBoth.csproj index 5e913bb..43ca032 100644 --- a/Playground/Http3/Managed/Playground.Http3.Managed.csproj +++ b/Playground/Http3/StreamedBoth/Playground.Http3.StreamedBoth.csproj @@ -6,8 +6,8 @@ enable enable true - Playground.Http3.Managed - Playground.Http3.Managed + Playground.Http3.StreamedBoth + Playground.Http3.StreamedBoth diff --git a/Playground/Http3/ManagedStreamed/Program.cs b/Playground/Http3/StreamedBoth/Program.cs similarity index 97% rename from Playground/Http3/ManagedStreamed/Program.cs rename to Playground/Http3/StreamedBoth/Program.cs index fc9a565..88bdfcc 100644 --- a/Playground/Http3/ManagedStreamed/Program.cs +++ b/Playground/Http3/StreamedBoth/Program.cs @@ -5,7 +5,7 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// http3-managed-streamed - HTTP/3 in pure C# with BOTH directions streamed. +// http3-streamed-both - HTTP/3 in pure C# with BOTH directions streamed. // // The request body arrives through Http3Request.BodyReader, pulled a chunk at a time under // flow control, so a large upload is never held whole. The response body goes out through an @@ -17,9 +17,9 @@ // // Because ioxide.http3 owns the framing, sending is a push: build [0x00][varint len][payload] // and hand it to the QUIC stream. There is no data-reader callback to answer and nothing to -// defer - which is the difference from the nghttp3 version of this in Playground/Http3/Streamed. +// defer - which is the difference from the nghttp3 version of this in Playground/Http3/Nghttp3Response. // -// dotnet run -c Release --project Playground/Http3/ManagedStreamed +// dotnet run -c Release --project Playground/Http3/StreamedBoth // curl --http3-only -k https://127.0.0.1:8443/ # chunked download // curl --http3-only -kN https://127.0.0.1:8443/feed # endless; ctrl-c to stop // curl --http3-only -k --data-binary @big.bin https://127.0.0.1:8443/upload diff --git a/Playground/Proxy/H1ToH3/Program.cs b/Playground/Proxy/H1ToH3/Program.cs index 540b096..952b82d 100644 --- a/Playground/Proxy/H1ToH3/Program.cs +++ b/Playground/Proxy/H1ToH3/Program.cs @@ -15,7 +15,7 @@ // for the upstream. Being an HTTP/3 client requires no HTTP/3 server - the first connect opens an // ephemeral UDP socket on this reactor's ring and replies route back by connection ID. // -// dotnet run -c Release --project Playground/Http3/Nghttp3 # h3 origin on udp :8443 +// dotnet run -c Release --project Playground/Http3/Nghttp3Request # h3 origin on udp :8443 // PLAYGROUND_UPSTREAM_PORT=8443 dotnet run -c Release --project Playground/Proxy/H1ToH3 // curl -k https://127.0.0.1:8443/ // diff --git a/Playground/Proxy/H2ToH3/Program.cs b/Playground/Proxy/H2ToH3/Program.cs index 6108205..3654eee 100644 --- a/Playground/Proxy/H2ToH3/Program.cs +++ b/Playground/Proxy/H2ToH3/Program.cs @@ -19,7 +19,7 @@ // HTTP/3 client requires no HTTP/3 server - the first connect opens an ephemeral UDP socket on // this reactor's ring and replies route back by connection ID. // -// dotnet run -c Release --project Playground/Http3/Nghttp3 # h3 origin on udp :8443 +// dotnet run -c Release --project Playground/Http3/Nghttp3Request # h3 origin on udp :8443 // PLAYGROUND_UPSTREAM_PORT=8443 dotnet run -c Release --project Playground/Proxy/H2ToH3 // curl -k --http2 https://127.0.0.1:8443/ // diff --git a/Playground/Proxy/H3ToH3/Program.cs b/Playground/Proxy/H3ToH3/Program.cs index 42a5fac..ce73c49 100644 --- a/Playground/Proxy/H3ToH3/Program.cs +++ b/Playground/Proxy/H3ToH3/Program.cs @@ -18,7 +18,7 @@ // the certificate the QuicEngine serves, and the ServerName the pool verifies, are the whole // configuration. Compare the h1 and h2 frontends, where TLS is a layer you add. // -// PLAYGROUND_QUIC_PORT=8444 PLAYGROUND_PORT=8090 dotnet run -c Release --project Playground/Http3/Nghttp3 +// PLAYGROUND_QUIC_PORT=8444 PLAYGROUND_PORT=8090 dotnet run -c Release --project Playground/Http3/Nghttp3Request // dotnet run -c Release --project Playground/Proxy/H3ToH3 // curl --http3-only -ks https://127.0.0.1:8443/anything // diff --git a/Playground/Quic/Alpn/Program.cs b/Playground/Quic/Alpn/Program.cs index 129df53..33133e1 100644 --- a/Playground/Quic/Alpn/Program.cs +++ b/Playground/Quic/Alpn/Program.cs @@ -36,7 +36,7 @@ int udpRecvSlots = 16; // Per-connection send-retention high-water (default 16 MiB): a response larger than it streams out -// paced by acks instead of buffering whole. See Playground/Http3/Buffered for the full knob set. +// paced by acks instead of buffering whole. See Playground/Http3/Nghttp3Buffered for the full knob set. long maxSendRetentionBytes = 16L << 20; // A real PEM pair, or null to generate a self-signed localhost cert on first run. diff --git a/Playground/README.md b/Playground/README.md index 5fa6ae8..40c9b0a 100644 --- a/Playground/README.md +++ b/Playground/README.md @@ -25,11 +25,11 @@ Run any of them with `dotnet run -c Release --project Playground//` | [`Tls.MultiPort`](Tls/MultiPort/Program.cs) | 160 | Plaintext on `:8080`, TLS on `:8081`, ONE pipe serve loop for both doors - multi-port made concrete. | `ioxide` | | [`Tls.SslStream`](Tls/SslStream/Program.cs) | 100 | The BCL `SslStream` over `TcpConnectionStream` - portable userspace TLS, the comparison point. | `ioxide` | | [`Http2.Nghttp2`](Http2/Nghttp2/Program.cs) | 66 | HTTP/2 (h2c, prior knowledge) with nghttp2 doing framing, HPACK and flow control. | `ioxide.nghttp2` | -| [`Http2.Managed`](Http2/Managed/Program.cs) | 66 | The same server with **zero native code** - framing, HPACK and flow control in C#. Drop-in for the above. | `ioxide.http2` | +| [`Http2.Buffered`](Http2/Buffered/Program.cs) | 66 | The same server with **zero native code** - framing, HPACK and flow control in C#. Drop-in for the above. | `ioxide.http2` | | [`Http2.Tls`](Http2/Tls/Program.cs) | 132 | h2 **and** http/1.1 on one port, chosen by ALPN. The HTTP/2 code is unchanged - only the pipe differs. | `ioxide.nghttp2` | | [`Http2.SslStream`](Http2/SslStream/Program.cs) | 100 | The same HTTP/2 over the BCL `SslStream`, via a ten-line `Stream`-to-`IDuplexPipe` adapter. | `ioxide.nghttp2` | -| [`Http3.Nghttp3`](Http3/Nghttp3/Program.cs) | 169 | HTTP/3 with **streamed** dispatch, and a `SIGTERM` GOAWAY drain. | `ioxide.ngtcp2`, `ioxide.nghttp3` | -| [`Http3.Buffered`](Http3/Buffered/Program.cs) | 146 | The same server with **buffered** dispatch - one method call is the whole difference. | `ioxide.ngtcp2`, `ioxide.nghttp3` | +| [`Http3.Nghttp3Request`](Http3/Nghttp3Request/Program.cs) | 169 | HTTP/3 with **streamed** dispatch, and a `SIGTERM` GOAWAY drain. | `ioxide.ngtcp2`, `ioxide.nghttp3` | +| [`Http3.Nghttp3Buffered`](Http3/Nghttp3Buffered/Program.cs) | 146 | The same server with **buffered** dispatch - one method call is the whole difference. | `ioxide.ngtcp2`, `ioxide.nghttp3` | | [`Quic.Alpn`](Quic/Alpn/Program.cs) | 111 | One QUIC listener, two protocols by ALPN: h3, or raw stream echo over the dual pipe. QUIC-only - `Tcp = null`. | `ioxide.ngtcp2`, `ioxide.nghttp3` | | [`Proxy.H1ToH1`](Proxy/H1ToH1/Program.cs) | 207 | TLS both hops, and the one to read first - ioxide TLS in, `TlsClientContext` out. Everything else here is this with one type changed. | `ioxide.httpclient` | | [`Proxy.H1ToH2`](Proxy/H1ToH2/Program.cs) | 208 | The same frontend, h2 upstream chosen by ALPN: `PoolSize` 1 carries every concurrent request. | `ioxide.httpclient` | @@ -48,7 +48,7 @@ Run any of them with `dotnet run -c Release --project Playground//` Read `Tcp.Raw` first - every other sample is that same skeleton with one thing changed. Each project references only the packages it demonstrates: `Tcp.Raw` publishes three assemblies and -no native libraries at all, while `Http3.Nghttp3` pulls in the ngtcp2 and nghttp3 bundles. So the build +no native libraries at all, while `Http3.Nghttp3Request` pulls in the ngtcp2 and nghttp3 bundles. So the build graph, not a comment, decides what each sample is allowed to touch. ## What is shared, and why so little @@ -68,8 +68,8 @@ shared `ServeAsync` would make the Playground shorter and make it useless. ## HTTP/3 Both samples answer every request with a single reused response object, so the handler stays small -enough to read at a glance. `Http3.Nghttp3` reads the request body through `BodyReader` while it is still -arriving; `Http3.Buffered` gets it complete in `request.Body`. That one difference is the reason +enough to read at a glance. `Http3.Nghttp3Request` reads the request body through `BodyReader` while it is still +arriving; `Http3.Nghttp3Buffered` gets it complete in `request.Body`. That one difference is the reason both exist. They also listen on TCP `:8080` alongside UDP `:8443`, and answer @@ -150,7 +150,7 @@ upstream, a `TlsClientContext` for h1 and h2 and again QUIC for h3. The h2 front `h2` in ALPN, which is what makes them reachable from a browser at all. Each needs a **TLS** origin to forward to on `PLAYGROUND_UPSTREAM_PORT` - `Tls/OpenSsl` for an h1 -upstream, `Http2/Tls` for h2, `Http3/Nghttp3` for h3. They verify its certificate against the same +upstream, `Http2/Tls` for h2, `Http3/Nghttp3Request` for h3. They verify its certificate against the same self-signed cert they serve, so they work against each other out of the box. With nothing listening they answer `502` once the acquire timeout elapses, and a certificate that does not verify arrives the same way. @@ -158,7 +158,7 @@ the same way. ## Docker `Playground/Dockerfile` builds one image per sample, selected with `--build-arg SAMPLE=` from -the repo root - `Tcp/Raw`, `Clients/Pg`, `Http3/Nghttp3` and so on, matching the directory layout. Publish +the repo root - `Tcp/Raw`, `Clients/Pg`, `Http3/Nghttp3Request` and so on, matching the directory layout. Publish `8080/tcp`, and `8443/udp` as well for the HTTP/3 samples. io_uring pins memory, so a container running many reactors may need `--ulimit memlock=-1:-1`. diff --git a/bench/h2-matrix.sh b/bench/h2-matrix.sh index 295b9fc..3d583b4 100755 --- a/bench/h2-matrix.sh +++ b/bench/h2-matrix.sh @@ -87,9 +87,9 @@ echo "== h2: nghttp2 vs pure C# ($REACTORS reactors, $CONNS conns, $STREAMS st for pass in $(seq "$PASSES"); do echo "-- pass $pass/$PASSES" cell nghttp2-h2c Http2/Nghttp2 8080 0 - cell managed-h2c Http2/Managed 8080 0 + cell managed-h2c Http2/Buffered 8080 0 cell nghttp2-tls Http2/Tls 8443 1 - cell managed-tls Http2/ManagedTls 8443 1 + cell managed-tls Http2/Tls 8443 1 done echo diff --git a/bench/results/20260809T181559Z.json b/bench/results/20260809T181559Z.json index 864df01..b4c3a20 100644 --- a/bench/results/20260809T181559Z.json +++ b/bench/results/20260809T181559Z.json @@ -5,6 +5,6 @@ "reactors": 2, "conns": 64, "threads": 8, "seconds": 10, "commit": "1f2e52c", "samples": [ - {"sample":"Tcp/Raw","proto":"h1","rps":876837.11,"cpu_us_per_req":2.30,"util_pct":101,"note":""},{"sample":"Tcp/Pipe","proto":"h1","rps":855179.93,"cpu_us_per_req":2.36,"util_pct":101,"note":""},{"sample":"Tcp/Incremental","proto":"h1","rps":911145.06,"cpu_us_per_req":2.22,"util_pct":101,"note":""},{"sample":"Tcp/Big","proto":"h1","rps":234893.95,"cpu_us_per_req":8.60,"util_pct":101,"note":""},{"sample":"Tcp/Hop","proto":"h1","rps":894299.67,"cpu_us_per_req":2.26,"util_pct":101,"note":""},{"sample":"Tcp/TaskRun","proto":"h1","rps":789809.51,"cpu_us_per_req":4.73,"util_pct":187,"note":""},{"sample":"Tls/OpenSsl","proto":"h1s","rps":417981.42,"cpu_us_per_req":4.84,"util_pct":101,"note":""},{"sample":"Tls/Hybrid","proto":"h1s","rps":413799.29,"cpu_us_per_req":4.89,"util_pct":101,"note":""},{"sample":"Tls/Ktls","proto":"h1s","rps":397877.79,"cpu_us_per_req":5.07,"util_pct":101,"note":""},{"sample":"Tls/OpenSslPipes","proto":"h1s","rps":368372.57,"cpu_us_per_req":5.48,"util_pct":101,"note":""},{"sample":"Tls/KtlsPipes","proto":"h1s","rps":389719.00,"cpu_us_per_req":5.19,"util_pct":101,"note":""},{"sample":"Tls/SslStream","proto":"h1s","rps":331645.59,"cpu_us_per_req":9.27,"util_pct":154,"note":""},{"sample":"Tls/MultiPort","proto":"h1","rps":843886.27,"cpu_us_per_req":2.39,"util_pct":101,"note":""},{"sample":"Http2/Nghttp2","proto":"h2c","rps":2359817.60,"cpu_us_per_req":0.84,"util_pct":99,"note":""},{"sample":"Http2/Managed","proto":"h2c","rps":6215459.20,"cpu_us_per_req":0.32,"util_pct":98,"note":""},{"sample":"Http2/Tls","proto":"h2","rps":2194921.60,"cpu_us_per_req":0.91,"util_pct":100,"note":""},{"sample":"Http2/SslStream","proto":"h2","rps":1967763.20,"cpu_us_per_req":1.30,"util_pct":128,"note":""},{"sample":"Http3/Nghttp3","proto":"h3","rps":476256,"cpu_us_per_req":4.20,"util_pct":100,"note":""},{"sample":"Http3/Buffered","proto":"h3","rps":500625,"cpu_us_per_req":4.00,"util_pct":100,"note":""},{"sample":"Quic/Alpn","proto":"echo","rps":161245.22,"cpu_us_per_req":6.83,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Quic/Pipe","proto":"echo","rps":158225.33,"cpu_us_per_req":6.97,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Quic/Raw","proto":"echo","rps":168769.68,"cpu_us_per_req":6.52,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Clients/File","proto":"h1","rps":669613.41,"cpu_us_per_req":3.01,"util_pct":101,"note":""},{"sample":"Clients/Pg","proto":"h1","rps":336951.12,"cpu_us_per_req":5.97,"util_pct":101,"note":""},{"sample":"Clients/Redis","proto":"h1","rps":503128.77,"cpu_us_per_req":3.97,"util_pct":100,"note":""},{"sample":"Clients/Http","proto":"h1","rps":290428.68,"cpu_us_per_req":15.23,"util_pct":221,"note":""},{"sample":"Clients/Https","proto":"h1","rps":144262.73,"cpu_us_per_req":34.48,"util_pct":249,"note":""},{"sample":"Clients/Quic","skipped":"a load driver, not a server workload"},{"sample":"Proxy/H1ToH1","proto":"h1s","rps":134752.48,"cpu_us_per_req":28.93,"util_pct":195,"note":""},{"sample":"Proxy/H1ToH2","proto":"h1s","rps":346024.81,"cpu_us_per_req":5.70,"util_pct":99,"note":""},{"sample":"Proxy/H1ToH3","proto":"h1s","rps":109719.47,"cpu_us_per_req":15.59,"util_pct":86,"note":"unsaturated (86%) - raise CONNS/THREADS"},{"sample":"Proxy/H2ToH1","proto":"h2","rps":140126.20,"cpu_us_per_req":16.36,"util_pct":115,"note":""},{"sample":"Proxy/H2ToH2","proto":"h2","rps":656163.20,"cpu_us_per_req":2.73,"util_pct":90,"note":""},{"sample":"Proxy/H2ToH3","proto":"h2","rps":109491.20,"cpu_us_per_req":11.92,"util_pct":65,"note":"unsaturated (65%) - raise CONNS/THREADS"},{"sample":"Proxy/H3ToH1","proto":"h3","rps":51528,"cpu_us_per_req":51.04,"util_pct":132,"note":""},{"sample":"Proxy/H3ToH2","proto":"h3","rps":270077,"cpu_us_per_req":7.34,"util_pct":99,"note":""},{"sample":"Proxy/H3ToH3","proto":"h3","rps":50649,"cpu_us_per_req":23.63,"util_pct":60,"note":"unsaturated (60%) - raise CONNS/THREADS"} + {"sample":"Tcp/Raw","proto":"h1","rps":876837.11,"cpu_us_per_req":2.30,"util_pct":101,"note":""},{"sample":"Tcp/Pipe","proto":"h1","rps":855179.93,"cpu_us_per_req":2.36,"util_pct":101,"note":""},{"sample":"Tcp/Incremental","proto":"h1","rps":911145.06,"cpu_us_per_req":2.22,"util_pct":101,"note":""},{"sample":"Tcp/Big","proto":"h1","rps":234893.95,"cpu_us_per_req":8.60,"util_pct":101,"note":""},{"sample":"Tcp/Hop","proto":"h1","rps":894299.67,"cpu_us_per_req":2.26,"util_pct":101,"note":""},{"sample":"Tcp/TaskRun","proto":"h1","rps":789809.51,"cpu_us_per_req":4.73,"util_pct":187,"note":""},{"sample":"Tls/OpenSsl","proto":"h1s","rps":417981.42,"cpu_us_per_req":4.84,"util_pct":101,"note":""},{"sample":"Tls/Hybrid","proto":"h1s","rps":413799.29,"cpu_us_per_req":4.89,"util_pct":101,"note":""},{"sample":"Tls/Ktls","proto":"h1s","rps":397877.79,"cpu_us_per_req":5.07,"util_pct":101,"note":""},{"sample":"Tls/OpenSslPipes","proto":"h1s","rps":368372.57,"cpu_us_per_req":5.48,"util_pct":101,"note":""},{"sample":"Tls/KtlsPipes","proto":"h1s","rps":389719.00,"cpu_us_per_req":5.19,"util_pct":101,"note":""},{"sample":"Tls/SslStream","proto":"h1s","rps":331645.59,"cpu_us_per_req":9.27,"util_pct":154,"note":""},{"sample":"Tls/MultiPort","proto":"h1","rps":843886.27,"cpu_us_per_req":2.39,"util_pct":101,"note":""},{"sample":"Http2/Nghttp2","proto":"h2c","rps":2359817.60,"cpu_us_per_req":0.84,"util_pct":99,"note":""},{"sample":"Http2/Buffered","proto":"h2c","rps":6215459.20,"cpu_us_per_req":0.32,"util_pct":98,"note":""},{"sample":"Http2/Tls","proto":"h2","rps":2194921.60,"cpu_us_per_req":0.91,"util_pct":100,"note":""},{"sample":"Http2/SslStream","proto":"h2","rps":1967763.20,"cpu_us_per_req":1.30,"util_pct":128,"note":""},{"sample":"Http3/Nghttp3Request","proto":"h3","rps":476256,"cpu_us_per_req":4.20,"util_pct":100,"note":""},{"sample":"Http3/Nghttp3Buffered","proto":"h3","rps":500625,"cpu_us_per_req":4.00,"util_pct":100,"note":""},{"sample":"Quic/Alpn","proto":"echo","rps":161245.22,"cpu_us_per_req":6.83,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Quic/Pipe","proto":"echo","rps":158225.33,"cpu_us_per_req":6.97,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Quic/Raw","proto":"echo","rps":168769.68,"cpu_us_per_req":6.52,"util_pct":55,"note":"unsaturated (55%) - raise CONNS/THREADS"},{"sample":"Clients/File","proto":"h1","rps":669613.41,"cpu_us_per_req":3.01,"util_pct":101,"note":""},{"sample":"Clients/Pg","proto":"h1","rps":336951.12,"cpu_us_per_req":5.97,"util_pct":101,"note":""},{"sample":"Clients/Redis","proto":"h1","rps":503128.77,"cpu_us_per_req":3.97,"util_pct":100,"note":""},{"sample":"Clients/Http","proto":"h1","rps":290428.68,"cpu_us_per_req":15.23,"util_pct":221,"note":""},{"sample":"Clients/Https","proto":"h1","rps":144262.73,"cpu_us_per_req":34.48,"util_pct":249,"note":""},{"sample":"Clients/Quic","skipped":"a load driver, not a server workload"},{"sample":"Proxy/H1ToH1","proto":"h1s","rps":134752.48,"cpu_us_per_req":28.93,"util_pct":195,"note":""},{"sample":"Proxy/H1ToH2","proto":"h1s","rps":346024.81,"cpu_us_per_req":5.70,"util_pct":99,"note":""},{"sample":"Proxy/H1ToH3","proto":"h1s","rps":109719.47,"cpu_us_per_req":15.59,"util_pct":86,"note":"unsaturated (86%) - raise CONNS/THREADS"},{"sample":"Proxy/H2ToH1","proto":"h2","rps":140126.20,"cpu_us_per_req":16.36,"util_pct":115,"note":""},{"sample":"Proxy/H2ToH2","proto":"h2","rps":656163.20,"cpu_us_per_req":2.73,"util_pct":90,"note":""},{"sample":"Proxy/H2ToH3","proto":"h2","rps":109491.20,"cpu_us_per_req":11.92,"util_pct":65,"note":"unsaturated (65%) - raise CONNS/THREADS"},{"sample":"Proxy/H3ToH1","proto":"h3","rps":51528,"cpu_us_per_req":51.04,"util_pct":132,"note":""},{"sample":"Proxy/H3ToH2","proto":"h3","rps":270077,"cpu_us_per_req":7.34,"util_pct":99,"note":""},{"sample":"Proxy/H3ToH3","proto":"h3","rps":50649,"cpu_us_per_req":23.63,"util_pct":60,"note":"unsaturated (60%) - raise CONNS/THREADS"} ] } diff --git a/bench/results/20260810T001150Z.json b/bench/results/20260810T001150Z.json index a45037c..13193c6 100644 --- a/bench/results/20260810T001150Z.json +++ b/bench/results/20260810T001150Z.json @@ -5,6 +5,6 @@ "reactors": 2, "conns": 64, "threads": 8, "seconds": 10, "commit": "96137b9", "samples": [ - {"sample":"Http3/Nghttp3","proto":"h3","rps":480371,"cpu_us_per_req":4.17,"util_pct":100,"note":""},{"sample":"Http3/Buffered","proto":"h3","rps":516707,"cpu_us_per_req":3.87,"util_pct":100,"note":""},{"sample":"Http3/Managed","proto":"h3","rps":665936,"cpu_us_per_req":2.96,"util_pct":98,"note":""},{"sample":"Http3/Streamed","proto":"h3","rps":89906,"cpu_us_per_req":22.03,"util_pct":99,"note":""} + {"sample":"Http3/Nghttp3Request","proto":"h3","rps":480371,"cpu_us_per_req":4.17,"util_pct":100,"note":""},{"sample":"Http3/Nghttp3Buffered","proto":"h3","rps":516707,"cpu_us_per_req":3.87,"util_pct":100,"note":""},{"sample":"Http3/Buffered","proto":"h3","rps":665936,"cpu_us_per_req":2.96,"util_pct":98,"note":""},{"sample":"Http3/Nghttp3Response","proto":"h3","rps":89906,"cpu_us_per_req":22.03,"util_pct":99,"note":""} ] } diff --git a/bench/results/20260810T001948Z.json b/bench/results/20260810T001948Z.json index 5b61c4d..0ba22d3 100644 --- a/bench/results/20260810T001948Z.json +++ b/bench/results/20260810T001948Z.json @@ -5,6 +5,6 @@ "reactors": 2, "conns": 64, "threads": 8, "seconds": 10, "commit": "3c9a208", "samples": [ - {"sample":"Http3/ManagedStreamed","proto":"h3","rps":28906,"cpu_us_per_req":68.95,"util_pct":100,"note":""} + {"sample":"Http3/StreamedBoth","proto":"h3","rps":28906,"cpu_us_per_req":68.95,"util_pct":100,"note":""} ] } diff --git a/bench/results/latest.json b/bench/results/latest.json index 5b61c4d..0ba22d3 100644 --- a/bench/results/latest.json +++ b/bench/results/latest.json @@ -5,6 +5,6 @@ "reactors": 2, "conns": 64, "threads": 8, "seconds": 10, "commit": "3c9a208", "samples": [ - {"sample":"Http3/ManagedStreamed","proto":"h3","rps":28906,"cpu_us_per_req":68.95,"util_pct":100,"note":""} + {"sample":"Http3/StreamedBoth","proto":"h3","rps":28906,"cpu_us_per_req":68.95,"util_pct":100,"note":""} ] } diff --git a/bench/run.sh b/bench/run.sh index 85bf850..db23ed6 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -85,7 +85,7 @@ else fi # ── h3 server ─────────────────────────────────────────────────────────────────────────────── -play Http3/Nghttp3 PLAYGROUND_REACTORS=$R PLAYGROUND_QUIC_PORT=18444 PLAYGROUND_PORT=18090 +play Http3/Nghttp3Request PLAYGROUND_REACTORS=$R PLAYGROUND_QUIC_PORT=18444 PLAYGROUND_PORT=18090 if [ -x "$H3X" ]; then N=$("$H3X" -k -t 4 --connections 64 -m 8 -d $DUR --send-batch 8 https://127.0.0.1:18444/ 2>&1 \ | grep -oE 'throughput: [0-9]+' | grep -oE '[0-9]+') diff --git a/bench/samples.tsv b/bench/samples.tsv index dfa9843..3a49752 100644 --- a/bench/samples.tsv +++ b/bench/samples.tsv @@ -34,16 +34,16 @@ Tls/SslStream h1s 8443 / - - Tls/MultiPort h1 8080 / - - Http2/Nghttp2 h2c 8080 / - - -Http2/Managed h2c 8080 / - - +Http2/Buffered h2c 8080 / - - Http2/Tls h2 8443 / - - Http2/SslStream h2 8443 / - - -Http2/ManagedTls h2 8443 / - - +Http2/Tls h2 8443 / - - -Http3/Nghttp3 h3 8443 / - - -Http3/Buffered h3 8443 / - - -Http3/Managed h3 8443 / - - -Http3/Streamed h3 8443 / - PLAYGROUND_CHUNKS=8 PLAYGROUND_CHUNK_BYTES=1024 -Http3/ManagedStreamed h3 8443 / - PLAYGROUND_CHUNKS=8 PLAYGROUND_CHUNK_BYTES=1024 +Http3/Nghttp3Request h3 8443 / - - +Http3/Nghttp3Buffered h3 8443 / - - +Http3/Buffered h3 8443 / - - +Http3/Nghttp3Response h3 8443 / - PLAYGROUND_CHUNKS=8 PLAYGROUND_CHUNK_BYTES=1024 +Http3/StreamedBoth h3 8443 / - PLAYGROUND_CHUNKS=8 PLAYGROUND_CHUNK_BYTES=1024 Quic/Alpn echo 8443 / - - Quic/Pipe echo 8443 / - - @@ -58,10 +58,10 @@ Clients/Quic driver - - - - Proxy/H1ToH1 h1s 8443 / Tls/OpenSsl - Proxy/H1ToH2 h1s 8443 / Http2/Tls - -Proxy/H1ToH3 h1s 8443 / Http3/Nghttp3 - +Proxy/H1ToH3 h1s 8443 / Http3/Nghttp3Request - Proxy/H2ToH1 h2 8443 / Tls/OpenSsl - Proxy/H2ToH2 h2 8443 / Http2/Tls - -Proxy/H2ToH3 h2 8443 / Http3/Nghttp3 - +Proxy/H2ToH3 h2 8443 / Http3/Nghttp3Request - Proxy/H3ToH1 h3 8443 / Tls/OpenSsl - Proxy/H3ToH2 h3 8443 / Http2/Tls - -Proxy/H3ToH3 h3 8443 / Http3/Nghttp3 - +Proxy/H3ToH3 h3 8443 / Http3/Nghttp3Request - diff --git a/docs/index.html b/docs/index.html index 3556e8b..b4bc4ae 100644 --- a/docs/index.html +++ b/docs/index.html @@ -1640,7 +1640,7 @@

QUIC · two protocols by ALPN

int udpRecvSlots = 16; // Per-connection send-retention high-water (default 16 MiB): a response larger than it streams out -// paced by acks instead of buffering whole. See Playground/Http3/Buffered for the full knob set. +// paced by acks instead of buffering whole. See Playground/Http3/Nghttp3Buffered for the full knob set. long maxSendRetentionBytes = 16L << 20; // ───────────────────────────────────────────────────────────────────────────────────────────── @@ -4023,7 +4023,7 @@

HTTP/3 · nghttp3

// One engine for the whole server. ALPN pinned to h3, so nothing else negotiates. The last arg is // the per-connection send-retention high-water (default 16 MiB): a response larger than it streams // out paced by acks instead of buffering whole, so h3 serves large files in bounded memory. See -// Playground/Http3/Buffered for the full QUIC/h3 knob set. +// Playground/Http3/Nghttp3Buffered for the full QUIC/h3 knob set. using var engine = new QuicEngine(certPath, keyPath, cidLength: 8, alpn: ["h3"], maxSendRetentionBytes: 16L << 20); var config = new ServerConfig @@ -4697,7 +4697,7 @@

HTTP/1.1 in · HTTP/3 out

ioxide + ioxide.httpclient 
// dotnet add package ioxide ioxide.httpclient
-//   dotnet run --project Playground/Http3/Nghttp3          # h3 origin on udp :8443
+//   dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443
 //   PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H1ToH3
 //   curl -k https://127.0.0.1:8443/
 
@@ -5275,7 +5275,7 @@ 
HTTP/2 in · HTTP/3 out

       ioxide + ioxide.nghttp2 + ioxide.httpclient
     
 
// dotnet add package ioxide ioxide.nghttp2 ioxide.httpclient
-//   dotnet run --project Playground/Http3/Nghttp3          # h3 origin on udp :8443
+//   dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443
 //   PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H2ToH3
 //   curl -k --http2 https://127.0.0.1:8443/
 
@@ -5744,7 +5744,7 @@ 
HTTP/3 in · HTTP/3 out

       ioxide + ioxide.ngtcp2 + ioxide.nghttp3 + ioxide.httpclient
     
 
// dotnet add package ioxide ioxide.ngtcp2 ioxide.nghttp3 ioxide.httpclient
-//   PLAYGROUND_QUIC_PORT=8444 dotnet run --project Playground/Http3/Nghttp3
+//   PLAYGROUND_QUIC_PORT=8444 dotnet run --project Playground/Http3/Nghttp3Request
 //   curl --http3-only -k https://127.0.0.1:8443/
 
 using ioxide;
@@ -5872,7 +5872,7 @@ 
HTTP client · alt-svc

     
 
// dotnet add package ioxide
 // dotnet add package ioxide.httpclient
-//   dotnet run -c Release --project Playground/Http3/Nghttp3   # an origin advertising h3
+//   dotnet run -c Release --project Playground/Http3/Nghttp3Request   # an origin advertising h3
 //   PLAYGROUND_UPSTREAM_PORT=8080 dotnet run -c Release --project Playground/Clients/Http
 
 using System.Text;
diff --git a/docs/learn/quic-h3.html b/docs/learn/quic-h3.html
index 66526dd..1369ab8 100644
--- a/docs/learn/quic-h3.html
+++ b/docs/learn/quic-h3.html
@@ -125,7 +125,7 @@ 
The layer map

     
 
     
Wiring it up, shown with every QUIC/h3 knob at its default
-    (Playground/Http3/Buffered is the same as an editable reference):

+    (Playground/Http3/Nghttp3Buffered is the same as an editable reference):

 
var engine = new QuicEngine(
     certPath, keyPath,
     cidLength: 8,                      // connection-id length this endpoint mints
diff --git a/dropped/README.md b/dropped/README.md
index 824adda..e4f7c5b 100644
--- a/dropped/README.md
+++ b/dropped/README.md
@@ -30,4 +30,4 @@ it. That client is pure C# now too, on the same framing and HPACK the server use
 had no callers left.
 
 `build-nghttp2-native.sh` built the native library it bound to. `Playground.Http2.Nghttp2` was its
-h2c sample; `Playground/Http2/Managed` is the same server on the managed stack.
+h2c sample; `Playground/Http2/Buffered` is the same server on the managed stack.
diff --git a/ioxide.slnx b/ioxide.slnx
index fcd586e..ca623ab 100644
--- a/ioxide.slnx
+++ b/ioxide.slnx
@@ -54,18 +54,20 @@
   
 
   
-    
-    
-    
+    
+    
+    
+    
+    
     
   
 
   
-    
+    
+    
     
-    
-    
-    
+    
+    
   
 
   
diff --git a/scripts/gen-docs-panes.py b/scripts/gen-docs-panes.py
index 8dc50b1..7c9ec1f 100644
--- a/scripts/gen-docs-panes.py
+++ b/scripts/gen-docs-panes.py
@@ -119,7 +119,7 @@
         "OpenSSL both ways by default; the kernelTx/kernelRx "
         "knobs at the top are what move either direction into the kernel."),
     "h2cstls": (
-        "Http2/ManagedTls", "HTTP/2 · pure C# over TLS", "ioxide + ioxide.http2",
+        "Http2/Tls", "HTTP/2 · pure C# over TLS", "ioxide + ioxide.http2",
         ["curl -k --http2 https://127.0.0.1:8443/"],
         "The pure-C# HTTP/2 server behind TLS, and the diff against "
         " is three "
@@ -139,7 +139,7 @@
         "directly, over ioxide's TLS, or over SslStream - the transport is a "
         "constructor argument, not a branch inside the protocol."),
     "h3cs": (
-        "Http3/Managed", "HTTP/3 · pure C#", "ioxide + ioxide.ngtcp2 + ioxide.http3",
+        "Http3/Buffered", "HTTP/3 · pure C#", "ioxide + ioxide.ngtcp2 + ioxide.http3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "HTTP/3 with no native library above the transport - frames, QPACK and Huffman are "
         "all managed code, and only QUIC itself stays native. Drop-in for "
@@ -150,7 +150,7 @@
         "from sending the header frame and a short body in ONE call rather than two; the large-body "
         "lead is the native shim copying every response body at submit, which this never does."),
     "h3stream": (
-        "Http3/Streamed", "HTTP/3 · streamed response", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Response", "HTTP/3 · streamed response", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/",
          "curl --http3-only -kN https://127.0.0.1:8443/feed   # never ends"],
         "The response body produced OVER TIME instead of handed over whole - each flush becomes a "
@@ -162,7 +162,7 @@
         "PULLS body bytes rather than accepting pushes, which is why this carries a resume and a "
         "drain the pure-C# writer does not need."),
     "h3csstream": (
-        "Http3/ManagedStreamed", "HTTP/3 · streamed both ways", "ioxide + ioxide.ngtcp2 + ioxide.http3",
+        "Http3/StreamedBoth", "HTTP/3 · streamed both ways", "ioxide + ioxide.ngtcp2 + ioxide.http3",
         ["curl --http3-only -k https://127.0.0.1:8443/",
          "curl --http3-only -kN https://127.0.0.1:8443/feed        # never ends",
          "curl --http3-only -k --data-binary @big.bin https://127.0.0.1:8443/echo"],
@@ -178,7 +178,7 @@
         "resume and a drain because nghttp3 pulls instead; this measures 1.32× its "
         "throughput on the same 8×1 KiB response."),
     "h3buf": (
-        "Http3/Buffered", "HTTP/3 · buffered dispatch", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Buffered", "HTTP/3 · buffered dispatch", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "The same server as  with the other "
         "dispatch mode - one method call is the whole difference. Buffered waits for "
@@ -237,11 +237,11 @@
         ["curl --http2-prior-knowledge http://127.0.0.1:8080/"],
         'This is h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and there is no upgrade dance. For h2 over TLS see the  tab - the protocol code there is byte-for-byte identical, because Nghttp2Connection takes an IDuplexPipe and never learns what is under it.'),
     "h2cs": (
-        "Http2/Managed", "HTTP/2 · pure C#", "ioxide + ioxide.http2",
+        "Http2/Buffered", "HTTP/2 · pure C#", "ioxide + ioxide.http2",
         ["curl --http2-prior-knowledge http://127.0.0.1:8080/"],
         "Which to take? They measure the same. Interleaved warm runs on one rig - h2load -n 300000 -c 32 -m 32, 4 reactors, 2-byte body - put the ratio between 0.98× and 1.09×, and a 1 KiB body holds the same. On a small response the cost is the loop and the syscalls, not the header codec. So take ioxide.http2 when shipping a native library is inconvenient, and ioxide.nghttp2 when you want the reference implementation's coverage of the protocol's darker corners. The same choice exists one version up: ioxide.http3 is the pure-C# drop-in for ioxide.nghttp3."),
     "h3": (
-        "Http3/Nghttp3", "HTTP/3 · nghttp3", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Request", "HTTP/3 · nghttp3", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "HTTP/3 over QUIC, dispatched as the body streams. Compare "
         ", which waits for end-of-stream "
@@ -263,7 +263,7 @@
         "ioxide.nghttp3 sits on."),
     "http": (
         "Clients/Http", "HTTP client · alt-svc", "ioxide + ioxide.httpclient",
-        ["dotnet run -c Release --project Playground/Http3/Nghttp3   # an origin advertising h3",
+        ["dotnet run -c Release --project Playground/Http3/Nghttp3Request   # an origin advertising h3",
          "PLAYGROUND_UPSTREAM_PORT=8080 dotnet run -c Release --project Playground/Clients/Http"],
         "Both hops - the inbound connection and the outbound call - ride this reactor's ring and "
         "resume inline, so a request never leaves the thread it arrived on. The knob is "
diff --git a/scripts/gen-docs-proxy-panes.py b/scripts/gen-docs-proxy-panes.py
index d2af797..d16f173 100644
--- a/scripts/gen-docs-proxy-panes.py
+++ b/scripts/gen-docs-proxy-panes.py
@@ -28,7 +28,7 @@
     "H1ToH3": ("h1 → h3", "HTTP/1.1 in · HTTP/3 out",
                "ioxide + ioxide.httpclient",
                [
-                'dotnet run --project Playground/Http3/Nghttp3          # h3 origin on udp :8443',
+                'dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443',
                 'PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H1ToH3',
                 'curl -k https://127.0.0.1:8443/']),
     "H2ToH1": ("h2 → h1", "HTTP/2 in · HTTP/1.1 out",
@@ -44,7 +44,7 @@
     "H2ToH3": ("h2 → h3", "HTTP/2 in · HTTP/3 out",
                "ioxide + ioxide.nghttp2 + ioxide.httpclient",
                [
-                'dotnet run --project Playground/Http3/Nghttp3          # h3 origin on udp :8443',
+                'dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443',
                 'PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H2ToH3',
                 'curl -k --http2 https://127.0.0.1:8443/']),
     "H3ToH1": ("h3 → h1", "HTTP/3 in · HTTP/1.1 out",
@@ -60,7 +60,7 @@
     "H3ToH3": ("h3 → h3", "HTTP/3 in · HTTP/3 out",
                "ioxide + ioxide.ngtcp2 + ioxide.nghttp3 + ioxide.httpclient",
                [
-                'PLAYGROUND_QUIC_PORT=8444 dotnet run --project Playground/Http3/Nghttp3',
+                'PLAYGROUND_QUIC_PORT=8444 dotnet run --project Playground/Http3/Nghttp3Request',
                 'curl --http3-only -k https://127.0.0.1:8443/']),
 }
 
diff --git a/src/protocols/ioxide.http2/Http2BodyReader.cs b/src/protocols/ioxide.http2/Http2BodyReader.cs
index b92d656..3f574d3 100644
--- a/src/protocols/ioxide.http2/Http2BodyReader.cs
+++ b/src/protocols/ioxide.http2/Http2BodyReader.cs
@@ -131,13 +131,19 @@ internal void FireIfReady()
     // Teardown while chunks may still be queued: recycle everything and wake anyone parked.
     internal void Drop()
     {
-        End();
         ReleaseHandedOut();
 
         while (_chunks.TryDequeue(out (byte[] Buffer, int Length) chunk))
         {
             ArrayPool.Shared.Return(chunk.Buffer);
         }
+
+        // Drained first, so the wake below reports end-of-body rather than handing out a chunk
+        // whose stream is already gone. Woken directly rather than through the connection's
+        // deferred list: teardown is the last thing that happens, so nothing would fire it, and a
+        // handler parked mid-body would wait forever on a body that stopped arriving.
+        _ended = true;
+        FireIfReady();
     }
 
     private void ReleaseHandedOut()
diff --git a/src/protocols/ioxide.http2/Http2Connection.Frames.cs b/src/protocols/ioxide.http2/Http2Connection.Frames.cs
index 5685cc9..421883d 100644
--- a/src/protocols/ioxide.http2/Http2Connection.Frames.cs
+++ b/src/protocols/ioxide.http2/Http2Connection.Frames.cs
@@ -140,7 +140,15 @@ private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload)
 
         if (!_streams.TryGetValue(header.StreamId, out PendingRequest? pending))
         {
-            pending = new PendingRequest { StreamId = header.StreamId };
+            // Opens at what the peer's SETTINGS advertised, not at the RFC default. Streams are
+            // created long after those SETTINGS arrive, so starting at 65535 and waiting for a
+            // WINDOW_UPDATE means waiting for one the peer has no reason to send - it believes we
+            // still have its whole window. That stalls any response longer than 65535 bytes.
+            pending = new PendingRequest
+            {
+                StreamId = header.StreamId,
+                SendWindow = _peerInitialStreamWindow,
+            };
             _streams[header.StreamId] = pending;
         }
 
diff --git a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs
index 1cf9b5c..00e23b7 100644
--- a/src/protocols/ioxide.http2/Http2Connection.Streamed.cs
+++ b/src/protocols/ioxide.http2/Http2Connection.Streamed.cs
@@ -205,11 +205,17 @@ private void ReleaseCreditWaiters(int streamId)
 
         if (streamId == 0)
         {
-            foreach (List waiters in _creditWaiters.Values.ToArray())
+            // Take the waiters OUT before waking any of them. These complete inline, so a resumed
+            // writer that still has no credit re-registers immediately - and clearing afterwards
+            // threw that new waiter away, leaving the writer parked on a wake that never comes,
+            // while the enumeration it was added during threw "collection was modified".
+            List[] all = [.. _creditWaiters.Values];
+            _creditWaiters.Clear();
+
+            foreach (List waiters in all)
             {
                 Release(waiters);
             }
-            _creditWaiters.Clear();
             return;
         }
 
@@ -229,13 +235,23 @@ static void Release(List waiters)
 
     private void ReleaseAllCreditWaiters()
     {
-        foreach (List waiters in _creditWaiters.Values)
+        if (_creditWaiters.Count == 0)
+        {
+            return;
+        }
+
+        // Same discipline as above, and it matters more here: this runs in the teardown finally, so
+        // an exception escaping it bypasses the catch that exists to keep a malformed peer from
+        // looking like a server fault.
+        List[] all = [.. _creditWaiters.Values];
+        _creditWaiters.Clear();
+
+        foreach (List waiters in all)
         {
             foreach (TaskCompletionSource waiter in waiters)
             {
                 waiter.TrySetResult();
             }
         }
-        _creditWaiters.Clear();
     }
 }

From d7e21205b4f34bfc2a996d59b24891796685ed82 Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 15:16:39 +0100
Subject: [PATCH 09/14] docs(site): the example tabs say which direction
 streams
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The http/3 menu had TWO tabs reading "nghttp3 · streamed" - one was request
streaming, one was response streaming, and nothing on the page distinguished
them. The rest were named after a library ("pure c#"), which says what a sample
is built from and not what it does.

Every tab now names the direction, and the library appears only in http/3 where
two implementations still exist:

  h2c · buffered            h3 · buffered
  h2c · response streamed   h3 · request + response streamed
  h2c · request streamed    h3 · buffered (nghttp3)
  h2c · both streamed       h3 · request streamed (nghttp3)
  h2 · tls & alpn           h3 · response streamed (nghttp3)
  h2 · over sslstream

Three of those h2 tabs are new: response streaming was never on the site at all,
and request streaming and both-directions did not exist until this branch. Their
notes are about the trade rather than the API - buffered bounds nothing but
MaxRequestBytes, streamed bounds one flow-control window because a chunk credits
the peer only as the handler reads it.

The two nghttp2 panes are gone with the package. The proxy panes said they
needed ioxide.nghttp2, which would now fail to restore; they take ioxide.http2.
The h2-over-TLS pane no longer explains itself as a diff against a tab that does
not exist, and the learn pages stop offering a choice between two h2 packages.

Panes regenerate idempotently from the samples, no reference to a removed tab is
left in the page or the stylesheet, and no mention of nghttp2 survives anywhere
under docs/.
---
 docs/assets/style.css           |  19 +-
 docs/index.html                 | 465 +++++++++++++++++---------------
 docs/learn/overview.html        |   2 +-
 docs/learn/tls.html             |   2 +-
 scripts/gen-docs-panes.py       |  91 ++++---
 scripts/gen-docs-proxy-panes.py |   8 +-
 6 files changed, 329 insertions(+), 258 deletions(-)

diff --git a/docs/assets/style.css b/docs/assets/style.css
index 7b2667e..ccb11b6 100644
--- a/docs/assets/style.css
+++ b/docs/assets/style.css
@@ -226,9 +226,10 @@ nav.top .links a.gh svg { display: block; }
 #tab-quicalpn:checked ~ .ex-menu label[for="tab-quicalpn"],
 #tab-qclient:checked ~ .ex-menu label[for="tab-qclient"],
 #tab-https:checked ~ .ex-menu label[for="tab-https"],
-#tab-h2:checked ~ .ex-menu label[for="tab-h2"],
 #tab-h2cs:checked ~ .ex-menu label[for="tab-h2cs"],
-#tab-h2tls:checked ~ .ex-menu label[for="tab-h2tls"],
+#tab-h2sresp:checked ~ .ex-menu label[for="tab-h2sresp"],
+#tab-h2sreq:checked ~ .ex-menu label[for="tab-h2sreq"],
+#tab-h2sboth:checked ~ .ex-menu label[for="tab-h2sboth"],
 #tab-pxmatrix:checked ~ .ex-menu label[for="tab-pxmatrix"],
 #tab-pxh1toh1:checked ~ .ex-menu label[for="tab-pxh1toh1"],
 #tab-pxh1toh2:checked ~ .ex-menu label[for="tab-pxh1toh2"],
@@ -356,9 +357,10 @@ nav.top .links a.gh svg { display: block; }
 #tab-quicalpn:checked ~ .pane-quicalpn { display: block; }
 #tab-qclient:checked ~ .pane-qclient { display: block; }
 #tab-https:checked ~ .pane-https { display: block; }
-#tab-h2:checked ~ .pane-h2 { display: block; }
 #tab-h2cs:checked ~ .pane-h2cs { display: block; }
-#tab-h2tls:checked ~ .pane-h2tls { display: block; }
+#tab-h2sresp:checked ~ .pane-h2sresp { display: block; }
+#tab-h2sreq:checked ~ .pane-h2sreq { display: block; }
+#tab-h2sboth:checked ~ .pane-h2sboth { display: block; }
 #tab-pxmatrix:checked ~ .pane-pxmatrix { display: block; }
 #tab-pxh1toh1:checked ~ .pane-pxh1toh1 { display: block; }
 #tab-pxh1toh2:checked ~ .pane-pxh1toh2 { display: block; }
@@ -491,10 +493,11 @@ nav.top .links a.gh svg { display: block; }
   #tab-quicalpn:checked ~ .ex-menu label[for="tab-quicalpn"],
   #tab-qclient:checked ~ .ex-menu label[for="tab-qclient"],
 #tab-https:checked ~ .ex-menu label[for="tab-https"],
-  #tab-h2:checked ~ .ex-menu label[for="tab-h2"],
-  #tab-h2cs:checked ~ .ex-menu label[for="tab-h2cs"],
-  #tab-h2tls:checked ~ .ex-menu label[for="tab-h2tls"],
-  #tab-pxmatrix:checked ~ .ex-menu label[for="tab-pxmatrix"],
+    #tab-h2cs:checked ~ .ex-menu label[for="tab-h2cs"],
+#tab-h2sresp:checked ~ .ex-menu label[for="tab-h2sresp"],
+#tab-h2sreq:checked ~ .ex-menu label[for="tab-h2sreq"],
+#tab-h2sboth:checked ~ .ex-menu label[for="tab-h2sboth"],
+    #tab-pxmatrix:checked ~ .ex-menu label[for="tab-pxmatrix"],
   #tab-pxh1toh1:checked ~ .ex-menu label[for="tab-pxh1toh1"],
   #tab-pxh1toh2:checked ~ .ex-menu label[for="tab-pxh1toh2"],
   #tab-pxh1toh3:checked ~ .ex-menu label[for="tab-pxh1toh3"],
diff --git a/docs/index.html b/docs/index.html
index b4bc4ae..eb2575f 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -39,9 +39,10 @@
   
   
   
-  
   
-  
+  
+  
+  
   
   
   
@@ -103,20 +104,21 @@
 
     

       http/2
-      
-      
-      
-      
+      
+      
+      
+      
+      
       
     

 
     

       http/3
-      
-      
-      
-      
-      
+      
+      
+      
+      
+      
     

 
     

@@ -772,7 +774,7 @@ 
TCP · ordinary async

   
   

     

-      
HTTP/2 · pure C# over TLS

+      
HTTP/2 · TLS & ALPN

       ioxide + ioxide.http2
     

 
// dotnet add package ioxide
@@ -932,7 +934,7 @@ 
HTTP/2 · pure C# over TLS

         }
         catch (Exception e)
         {
-            Console.Error.WriteLine($"[http2-managed-tls] connection failed: {e.Message}");
+            Console.Error.WriteLine($"[http2-tls] connection failed: {e.Message}");
         }
         finally
         {
@@ -945,7 +947,7 @@ 
HTTP/2 · pure C# over TLS

     threads[i].Start();
 }
 
-Console.WriteLine($"[http2-managed-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+Console.WriteLine($"[http2-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
                 + $"ALPN h2 then http/1.1, cert {certPath}, "
                 + $"rx={(kernelTx && kernelRx ? "kernel" : "openssl")}, "
                 + $"tx={(kernelTx ? "kernel" : "openssl")}");
@@ -981,15 +983,15 @@ 
HTTP/2 · pure C# over TLS

         _len -= count;
     }
 }

-    
The pure-C# HTTP/2 server behind TLS, and the diff against  is three type names and a package - no native library anywhere. Both take an IDuplexPipe, so neither learns what is under it. Measured on this rig at 2 reactors, 64 connections, 32 streams, a 2-byte body: pure C# runs 2.46× nghttp2 over TLS and 2.73× cleartext, at about a third of the CPU per request. That ordering does not generalize - at 32 connections over 4 reactors nghttp2 is ahead instead - and a 2-byte body measures framing and HPACK rather than moving data. TLS itself costs both the same 0.05µs per request.

+    
How a browser actually reaches h2: over TLS, with the protocol chosen during the handshake. Alpn = ["h2", "http/1.1"] is an ordered preference, not a weighting - the server takes the first entry the client also offered, and this sample then branches on what was agreed, so ONE port serves both. That is why the h2c samples are the exception rather than the rule. What Http2Connection is handed is a TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code is byte-for-byte the h2c sample's. TLS is OpenSSL both ways by default; the kernelTx/kernelRx knobs at the top move either direction into the kernel, and cost the same 0.05µs per request either way.

   

   

     

       
HTTP/2 · over SslStream

-      ioxide + ioxide.nghttp2
+      ioxide + ioxide.http2
     

 
// dotnet add package ioxide
-// dotnet add package ioxide.nghttp2
+// dotnet add package ioxide.http2
 //   curl -k --http2 https://127.0.0.1:8443/
 
 using System.IO.Pipelines;
@@ -997,7 +999,7 @@ 
HTTP/2 · over SslStream

 using System.Security.Authentication;
 using System.Security.Cryptography.X509Certificates;
 using ioxide;
-using ioxide.nghttp2;
+using ioxide.http2;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
 
@@ -1062,8 +1064,8 @@ 
HTTP/2 · over SslStream

                 return;   // this sample only serves h2; see Playground/Http2/Tls for the fallback
             }
 
-            await new Nghttp2Connection(new StreamDuplexPipe(ssl)).RunBufferedAsync(
-                _ => new Nghttp2Response { Status = 200, Body = body });
+            await new Http2Connection(new StreamDuplexPipe(ssl)).RunBufferedAsync(
+                _ => new Http2Response { Status = 200, Body = body });
         }
         catch (Exception e)
         {
@@ -1101,7 +1103,7 @@ 
HTTP/2 · over SslStream

   

   

     

-      
HTTP/3 · pure C#

+      
HTTP/3 · buffered

       ioxide + ioxide.ngtcp2 + ioxide.http3
     

 
// dotnet add package ioxide
@@ -1178,7 +1180,7 @@ 
HTTP/3 · pure C#

   

   

     

-      
HTTP/3 · streamed response

+      
HTTP/3 · response streamed (nghttp3)

       ioxide + ioxide.ngtcp2 + ioxide.nghttp3
     

 
// dotnet add package ioxide
@@ -1278,7 +1280,7 @@ 
HTTP/3 · streamed response

   

   

     

-      
HTTP/3 · streamed both ways

+      
HTTP/3 · request + response streamed

       ioxide + ioxide.ngtcp2 + ioxide.http3
     

 
// dotnet add package ioxide
@@ -1428,7 +1430,7 @@ 
HTTP/3 · streamed both ways

   

   

     

-      
HTTP/3 · buffered dispatch

+      
HTTP/3 · buffered (nghttp3)

       ioxide + ioxide.ngtcp2 + ioxide.nghttp3
     

 
// dotnet add package ioxide
@@ -3387,18 +3389,19 @@ 
OpenSSL · pipes

 }

     
Diff this against  and the only functional difference is KernelTx; the rest is the banner and the log tag. That is the point of the pipe seam - TlsConnectionDualPipe pairs TcpConnectionPipeReader or TlsDecryptingPipeReader with TcpConnectionPipeWriter or TlsEncryptingPipeWriter, chosen from the session. It has to be the session and not the config, because a handshake that left a partial record keeps the userspace reader whatever was asked for.

   

-  

+
+  

     

-      
HTTP/2 · nghttp2

-      ioxide + ioxide.nghttp2
+      
HTTP/2 · buffered

+      ioxide + ioxide.http2
     

 
// dotnet add package ioxide
-// dotnet add package ioxide.nghttp2
+// dotnet add package ioxide.http2
 //   curl --http2-prior-knowledge http://127.0.0.1:8080/
 
 using System.Text;
 using ioxide;
-using ioxide.nghttp2;
+using ioxide.http2;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
 
@@ -3449,9 +3452,9 @@ 
HTTP/2 · nghttp2

     {
         try
         {
-            // The connection owns the read loop from here: it feeds nghttp2, dispatches each
-            // request once its stream ends, and drains the egress once per batch.
-            await new Nghttp2Connection(conn).RunBufferedAsync(_ => new Nghttp2Response
+            // The connection owns the read loop from here: it parses frames, dispatches each
+            // request once its stream ends, and flushes the batch in one write.
+            await new Http2Connection(conn).RunBufferedAsync(_ => new Http2Response
             {
                 Status = 200,
                 Body = body,
@@ -3467,24 +3470,24 @@ 
HTTP/2 · nghttp2

     threads[i].Start();
 }
 
-Console.WriteLine($"[nghttp2] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+Console.WriteLine($"[http2-buffered] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
                 + $"{body.Length}-byte body (h2c prior knowledge)");
 
 foreach (Thread thread in threads)
 {
     thread.Join();
 }

-    
This is h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and there is no upgrade dance. For h2 over TLS see the  tab - the protocol code there is byte-for-byte identical, because Nghttp2Connection takes an IDuplexPipe and never learns what is under it.

+    
h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and there is no upgrade dance. For h2 over TLS see  - the protocol code there is byte-for-byte this one, because Http2Connection takes an IDuplexPipe and never learns what is under it. BUFFERED is the dispatch mode: the handler runs once the request has fully arrived, so request.Body holds the whole body and the answer is one Http2Response. That is the right default, and the wrong one for a large upload or an endless response - the three tabs after it are those cases.

   

-
-  

+  

     

-      
HTTP/2 · pure C#

+      
HTTP/2 · response streamed

       ioxide + ioxide.http2
     

 
// dotnet add package ioxide
 // dotnet add package ioxide.http2
 //   curl --http2-prior-knowledge http://127.0.0.1:8080/
+//   curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed   # never ends
 
 using System.Text;
 using ioxide;
@@ -3494,12 +3497,17 @@ 
HTTP/2 · pure C#

 
 ushort port      = 8080;
 int    reactors  = Environment.ProcessorCount;
-int    bodyBytes = 2;
+int    bodyBytes = 2;   // unused here: the body is produced chunk by chunk
 
 
 // Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
 // handler code is identical either way; this only changes how recv buffers are handed out.
 bool incrementalBuffers = false;
+
+
+// Chunks written per response on "/", and the size of each. Their product is never held at once.
+int chunkCount = 64;
+int chunkBytes = 16 * 1024;
 // ─────────────────────────────────────────────────────────────────────────────────────────────
 
 var config = new ServerConfig
@@ -3531,6 +3539,8 @@ 
HTTP/2 · pure C#

 
 var threads = new Thread[config.ReactorCount];
 
+byte[] chunk = Encoding.ASCII.GetBytes(new string('x', chunkBytes - 1) + "\n");
+
 for (int i = 0; i < threads.Length; i++)
 {
     var reactor = new Reactor(i, config);
@@ -3539,12 +3549,25 @@ 
HTTP/2 · pure C#

     {
         try
         {
-            // The connection owns the read loop from here: it feeds nghttp2, dispatches each
-            // request once its stream ends, and drains the egress once per batch.
-            await new Http2Connection(conn).RunBufferedAsync(_ => new Http2Response
+            await new Http2Connection(conn).RunAsync(async (request, writer) =>
             {
-                Status = 200,
-                Body = body,
+                bool endless = request.Path.Span.SequenceEqual("/feed"u8);
+
+                // Headers first and once. No content-length: the length is not known yet, and for
+                // /feed never will be - END_STREAM is what marks the end instead.
+                var response = new Http2Response { Status = 200 };
+                response.Headers.Add("content-type"u8.ToArray(),
+                    endless ? "text/event-stream"u8.ToArray() : "text/plain"u8.ToArray());
+                writer.WriteHeaders(response);
+
+                for (int n = 0; endless || n < chunkCount; n++)
+                {
+                    chunk.CopyTo(writer.GetSpan(chunk.Length));
+                    writer.Advance(chunk.Length);
+
+                    // Waits when either window is exhausted, and resumes on the WINDOW_UPDATE.
+                    await writer.FlushAsync();
+                }
             });
         }
         finally
@@ -3557,72 +3580,52 @@ 
HTTP/2 · pure C#

     threads[i].Start();
 }
 
-Console.WriteLine($"[http2] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
-                + $"{body.Length}-byte body (h2c prior knowledge)");
+Console.WriteLine($"[http2-streamed-response] {config.ReactorCount} reactors on :{config.Tcp!.Port} "
+                + $"(pure C#), {chunkCount} x {chunkBytes}-byte chunks per response");
 
 foreach (Thread thread in threads)
 {
     thread.Join();
 }

-    
Which to take? They measure the same. Interleaved warm runs on one rig - h2load -n 300000 -c 32 -m 32, 4 reactors, 2-byte body - put the ratio between 0.98× and 1.09×, and a 1 KiB body holds the same. On a small response the cost is the loop and the syscalls, not the header codec. So take ioxide.http2 when shipping a native library is inconvenient, and ioxide.nghttp2 when you want the reference implementation's coverage of the protocol's darker corners. The same choice exists one version up: ioxide.http3 is the pure-C# drop-in for ioxide.nghttp3.

+    
The RESPONSE body produced over time instead of returned whole - each flush becomes a DATA frame. /feed is why the mode exists: an endless response has no final byte, so a buffered API cannot express it at all. Http2ResponseWriter is an IBufferWriter<byte>, so a serializer or a framework's response sink writes into it unchanged. What HTTP/2 adds over  is that credit is SHARED: every stream rides one TCP connection, so a flush waits on whichever of the stream and connection windows runs out first, and a WINDOW_UPDATE for either wakes it.

   

-
-  

+  

     

-      
HTTP/2 · TLS & ALPN

-      ioxide + ioxide.nghttp2
+      
HTTP/2 · request streamed

+      ioxide + ioxide.http2
     

 
// dotnet add package ioxide
-// dotnet add package ioxide.nghttp2
-//   curl -k --http2 https://127.0.0.1:8443/
+// dotnet add package ioxide.http2
+//   head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \
+//     http://127.0.0.1:8080/upload
 
-using System.IO.Pipelines;
 using System.Text;
 using ioxide;
-using ioxide.nghttp2;
-using ioxide.tls;
+using ioxide.http2;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
 
-ushort port      = 8443;                        // https://127.0.0.1:8443/
-int    reactors  = Environment.ProcessorCount;  // one ring per reactor, one reactor per core
-int    bodyBytes = 2;                           // "ok" - this sample is about ALPN, not throughput
-
-
-
-// Hand OUTBOUND encryption to the kernel: the handler writes plaintext and the kernel makes the
-// records. Off by default - OpenSSL both ways is the portable path, and on loopback the kernel
-// is not faster. Its real payoff is sendfile and NIC offload, which a benchmark here cannot see.
-bool kernelTx = false;
-
-// Hand INBOUND decryption to the kernel as well. Requires kernelTx - the RX handoff happens at
-// the same moment as the TX one - and is experimental for a reason: a TLS 1.3 KeyUpdate cannot be
-// read through IORING_OP_RECV, and roughly one first connection in twelve fails outright.
-bool kernelRx = false;
+ushort port     = 8080;
+int    reactors = Environment.ProcessorCount;
 
 
-// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
-// handler code is identical either way; this only changes how recv buffers are handed out.
-bool incrementalBuffers = false;
+// Advertised per stream. This is the ceiling on how far ahead of the handler a peer may get, so
+// on a streamed request it is the memory bound - not a throughput knob.
+int streamWindow = 256 * 1024;
 // ─────────────────────────────────────────────────────────────────────────────────────────────
 
-const string certPath = "cert.pem";   // any PEM pair
-const string keyPath  = "key.pem";
-
 var config = new ServerConfig
 {
-    ReactorCount   = reactors,                                              // io_uring rings/threads - one per core
-    RingEntries    = 8192,                                                  // SQ/CQ depth per ring
-    DualStack      = false,                                                 // true = one IPv6 socket also accepts IPv4-mapped
-    RecvBufferSize = 32 * 1024,                                             // bytes per shared recv buffer
-    RecvSlots      = 4096,                                                  // shared recv buffer-ring depth
-    Incremental    = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null,  // per-connection recv rings (6.12+) - see Tcp/Incremental
-    Udp            = null,                                                  // no raw UDP sockets (TCP-only server)
-    Quic           = null,                                                  // no QUIC transport - see Http3/* and Quic/Alpn
+    ReactorCount   = reactors,   // io_uring rings/threads - one per core
+    RingEntries    = 8192,                                                        // SQ/CQ depth per ring
+    DualStack      = false,                                                       // true = one IPv6 socket also accepts IPv4-mapped
+    RecvBufferSize = 32 * 1024,                                                   // bytes per shared recv buffer
+    RecvSlots      = 4096,                                                        // shared recv buffer-ring depth
+    Udp            = null,                                                        // no raw UDP sockets (TCP-only server)
+    Quic           = null,                                                        // no QUIC transport - see Http3/*
     Tcp = new TcpOptions
     {
         Port             = port,
-        ExtraPorts       = [],                             // extra listener ports (one handler, several doors)
         ListenBacklog    = 1024,                           // accept-queue depth per SO_REUSEPORT listener
         WriteSlabSize    = 16 * 1024,                      // per-connection write buffer before overflow kicks in
         PoolMax          = 1024,                           // pooled connection objects kept per reactor
@@ -3632,13 +3635,11 @@ 
HTTP/2 · TLS & ALPN

     },
 };
 
-byte[] body = bodyBytes == 2 ? "ok"u8.ToArray() : [.. Enumerable.Repeat((byte)'x', bodyBytes)];
-
-byte[] http11Response =
-[
-    .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\n\r\n"),
-    .. body,
-];
+var http2 = new Http2Options
+{
+    StreamRequestBodies = true,          // the whole point: dispatch at the headers, body follows
+    InitialWindowSize   = streamWindow,  // how far ahead of the handler the peer may run
+};
 
 var threads = new Thread[config.ReactorCount];
 
@@ -3646,94 +3647,158 @@ 
HTTP/2 · TLS & ALPN

 {
     var reactor = new Reactor(i, config);
 
-    reactor.OnStart = r => TlsService.Start(r, new TlsOptions
-    {
-        CertificatePath = certPath,                        // PEM certificate chain file (leaf first)
-        CertificatePem  = null,                            // in-memory PEM alternative - set one, not both
-        KeyPath         = keyPath,                         // PEM private key file
-        KeyPem          = null,                            // in-memory PEM alternative to KeyPath
-        Alpn            = ["h2", "http/1.1"],              // ORDERED, most preferred first - a client offering both gets h2
-        KernelTx        = kernelTx,                        // kTLS encrypt: kernel makes the records (off = OpenSSL both ways)
-        KernelRx        = kernelRx,                        // kTLS decrypt: needs KernelTx; experimental (see the knob above)
-    });
-
     reactor.TcpHandle = async (r, conn) =>
     {
-        TlsSession? tls = null;
         try
         {
-            tls = await r.GetService<TlsService>()!.AcceptAsync(conn);
-
-            if (tls.NegotiatedAlpn == "h2")
+            await new Http2Connection(conn, http2).RunBufferedAsync(async request =>
             {
-                // The decrypt lives in the pipe, so the HTTP/2 code below is identical to the
-                // cleartext sample. Which halves the pipe uses is decided by what the handshake
-                // achieved, not by anything chosen here.
-                await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);
+                // BodyReader is set because StreamRequestBodies is on; with it off the body would
+                // be in request.Body instead and this would be null.
+                long total = 0;
+                if (request.BodyReader is { } body)
+                {
+                    while (true)
+                    {
+                        // Empty means end of body. Each read hands back the peer's credit for the
+                        // chunk it returns, which is what lets the next one arrive - and the
+                        // memory it points at is recycled by the NEXT read, so anything worth
+                        // keeping has to be copied out here.
+                        ReadOnlyMemory<byte> chunk = await body.ReadAsync();
+                        if (chunk.IsEmpty)
+                        {
+                            break;
+                        }
+                        total += chunk.Length;
+                    }
+                }
 
-                await new Nghttp2Connection(pipe).RunBufferedAsync(_ => new Nghttp2Response
+                return new Http2Response
                 {
                     Status = 200,
-                    Body = body,
-                });
-                return;
-            }
+                    Body = Encoding.ASCII.GetBytes($"{total} bytes\n"),
+                };
+            });
+        }
+        finally
+        {
+            conn.DecRef();
+        }
+    };
 
-            // Anything else: HTTP/1.1 on the same port.
-            //
-            // The carry is not incidental. TLS hands back RECORDS, not requests, so a request
-            // split across two records decrypts twice - and answering on "plaintext arrived"
-            // would answer twice to one request. Framing is ours; ioxide does not parse HTTP.
-            var carry = new Carry();
+    threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
+    threads[i].Start();
+}
 
-            // The client's first request usually rides in with its Finished flight, so the
-            // handshake already decrypted it and it is sitting in the session, not in any recv
-            // buffer. Miss this and that request is dropped and the loop parks on bytes that
-            // already arrived - which is exactly what happened here before.
-            carry.Append(tls.DrainPlaintext());
+Console.WriteLine($"[http2-streamed-request] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+                + $"request bodies streamed, {streamWindow / 1024} KiB window per stream");
 
-            while (true)
-            {
-                bool wrote = false;
-                int end;
-                while ((end = carry.Span.IndexOf("\r\n\r\n"u8)) >= 0)
-                {
-                    carry.Consume(end + 4);
-                    // Correct whichever backend the session ended up with.
-                    tls.Write(conn, http11Response);
-                    wrote = true;
-                }
+foreach (Thread thread in threads)
+{
+    thread.Join();
+}

+    
The other direction, and a different problem. StreamRequestBodies dispatches at the HEADERS and hands the handler an Http2BodyReader, so it runs while the upload is still arriving. What changes is what bounds memory: buffered holds the whole body, so MaxRequestBytes is all that stands between a hostile peer and the arena; streamed holds ONE flow-control window, because a chunk credits the peer's window only as the handler reads it. Fall behind and the peer runs out of credit and stops sending - backpressure the peer takes part in, rather than a buffer you hope is big enough.

+  

+  

+    

+      
HTTP/2 · both directions streamed

+      ioxide + ioxide.http2
+    

+
// dotnet add package ioxide
+// dotnet add package ioxide.http2
+//   curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed
+//   head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \
+//     http://127.0.0.1:8080/echo
 
-                if (wrote)
-                {
-                    await conn.FlushAsync();
-                }
+using ioxide;
+using ioxide.http2;
 
-                RecvSnapshot snapshot = await conn.ReadAsync();
+// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
 
-                unsafe
+ushort port       = 8080;
+int    reactors   = Environment.ProcessorCount;
+int    chunkBytes = 1024;   // one DATA frame per flush on /feed
+// ─────────────────────────────────────────────────────────────────────────────────────────────
+
+var config = new ServerConfig
+{
+    ReactorCount   = reactors,   // io_uring rings/threads - one per core
+    RingEntries    = 8192,                                                        // SQ/CQ depth per ring
+    DualStack      = false,                                                       // true = one IPv6 socket also accepts IPv4-mapped
+    RecvBufferSize = 32 * 1024,                                                   // bytes per shared recv buffer
+    RecvSlots      = 4096,                                                        // shared recv buffer-ring depth
+    Udp            = null,                                                        // no raw UDP sockets (TCP-only server)
+    Quic           = null,                                                        // no QUIC transport - see Http3/*
+    Tcp = new TcpOptions
+    {
+        Port             = port,
+        ListenBacklog    = 1024,                           // accept-queue depth per SO_REUSEPORT listener
+        WriteSlabSize    = 16 * 1024,                      // per-connection write buffer before overflow kicks in
+        PoolMax          = 1024,                           // pooled connection objects kept per reactor
+        WriteOverflow    = WriteOverflowStrategy.Grow,     // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
+        ZeroCopySend     = false,                          // SEND_ZC: kernel copies less, wins on large writes
+        RecvQueueEntries = 64,                             // per-connection recv completion queue depth
+    },
+};
+
+// Both halves are opt-in and independent: this one turns the REQUEST direction on, RunAsync below
+// is what turns the RESPONSE direction on.
+var http2 = new Http2Options { StreamRequestBodies = true };
+
+byte[] chunk = [.. Enumerable.Repeat((byte)'x', chunkBytes)];
+
+var threads = new Thread[config.ReactorCount];
+
+for (int i = 0; i < threads.Length; i++)
+{
+    var reactor = new Reactor(i, config);
+
+    reactor.TcpHandle = async (r, conn) =>
+    {
+        try
+        {
+            await new Http2Connection(conn, http2).RunAsync(async (request, writer) =>
+            {
+                bool echo = request.Path.Span.SequenceEqual("/echo"u8);
+
+                // No content-length: on /feed the length will never be known, and on /echo it is
+                // not known yet. END_STREAM is what marks the end instead.
+                var response = new Http2Response { Status = 200 };
+                response.Headers.Add("content-type"u8.ToArray(), "text/plain"u8.ToArray());
+                writer.WriteHeaders(response);
+
+                if (echo)
                 {
-                    while (conn.TryGetItem(snapshot, out ioxide.utils.SpscRecvRing.Item item))
+                    // Both directions at once. Nothing here holds more than one chunk: the read
+                    // credits the peer for what it hands back, and the flush waits for room on the
+                    // way out - so a fast uploader is paced by the slower of the two, not buffered.
+                    while (true)
                     {
-                        if (item.HasBuffer)
+                        ReadOnlyMemory<byte> incoming = await request.BodyReader!.ReadAsync();
+                        if (incoming.IsEmpty)
                         {
-                            carry.Append(tls.Decrypt(item.Ptr, item.Len));
-                            conn.ReturnBuffer(in item);
+                            break;
                         }
+
+                        incoming.Span.CopyTo(writer.GetSpan(incoming.Length));
+                        writer.Advance(incoming.Length);
+                        await writer.FlushAsync();
                     }
+                    return;
                 }
 
-                if (snapshot.IsClosed || tls.Closed) return;
-                conn.ResetRead();
-            }
-        }
-        catch (Exception e)
-        {
-            Console.Error.WriteLine($"[http2-tls] connection failed: {e.Message}");
+                // /feed: a response with no end at all. There is no final byte to wait for, which
+                // is the case a buffered API has no way to express.
+                while (true)
+                {
+                    chunk.CopyTo(writer.GetSpan(chunk.Length));
+                    writer.Advance(chunk.Length);
+                    await writer.FlushAsync();
+                }
+            });
         }
         finally
         {
-            tls?.Dispose();
             conn.DecRef();
         }
     };
@@ -3742,45 +3807,17 @@ 
HTTP/2 · TLS & ALPN

     threads[i].Start();
 }
 
-Console.WriteLine($"[http2-tls] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
-                + $"ALPN h2 then http/1.1, cert {certPath}, "
-                + $"rx={(kernelTx && kernelRx ? "kernel" : "openssl")}, "
-                + $"tx={(kernelTx ? "kernel" : "openssl")}");
+Console.WriteLine($"[http2-streamed-both] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+                + $"request pulled and response pushed ({chunkBytes}-byte chunks on /feed)");
 
 foreach (Thread thread in threads)
 {
     thread.Join();
-}
-
-// Decrypted-but-unframed bytes: append at the end, consume from the front. A List<byte> pressed
-// into this job needs CollectionsMarshal to be searched and RemoveRange to be consumed; a plain
-// array does both directly.
-sealed class Carry
-{
-    private byte[] _buf = new byte[8 * 1024];
-    private int _len;
-
-    public ReadOnlySpan<byte> Span => _buf.AsSpan(0, _len);
-
-    public void Append(ReadOnlySpan<byte> bytes)
-    {
-        if (_buf.Length - _len < bytes.Length)
-        {
-            Array.Resize(ref _buf, Math.Max(_buf.Length * 2, _len + bytes.Length));
-        }
-        bytes.CopyTo(_buf.AsSpan(_len));
-        _len += bytes.Length;
-    }
-
-    public void Consume(int count)
-    {
-        _buf.AsSpan(count, _len - count).CopyTo(_buf);
-        _len -= count;
-    }
 }

-    
How a browser actually reaches h2: over TLS, with the protocol chosen during the handshake. Alpn = ["h2", "http/1.1"] is an ordered preference, not a weighting - the server takes the first entry the client also offered, and this sample then branches on what was agreed, so one port serves both. That is why the h2c samples are the exception rather than the rule. TLS here is OpenSSL both ways by default; the kernelTx/kernelRx knobs at the top are what move either direction into the kernel.

+    
Both at once, which is the shape a proxy needs: /echo reads a chunk and writes a chunk, so neither the upload nor the download is ever held whole. The two directions are separate switches - StreamRequestBodies for the read side, RunAsync with a writer for the write side - because they solve different problems and most servers want exactly one of them. Mirrors ; the difference is HTTP/2's shared connection window, which a handler that stops reading holds down for every other stream on the connection.

   

 
+
   

     

       
QUIC · pipes

@@ -3984,7 +4021,7 @@ 
QUIC · raw streams

   

   

     

-      
HTTP/3 · nghttp3

+      
HTTP/3 · request streamed (nghttp3)

       ioxide + ioxide.ngtcp2 + ioxide.nghttp3
     

 
// dotnet add package ioxide
@@ -4905,16 +4942,16 @@ 
HTTP/1.1 in · HTTP/3 out

   

     

       
HTTP/2 in · HTTP/1.1 out

-      ioxide + ioxide.nghttp2 + ioxide.httpclient
+      ioxide + ioxide.http2 + ioxide.httpclient
     

-
// dotnet add package ioxide ioxide.nghttp2 ioxide.httpclient
+
// dotnet add package ioxide ioxide.http2 ioxide.httpclient
 //   PLAYGROUND_PORT=8444 dotnet run --project Playground/Tls/Ktls   # a TLS origin
 //   curl -k --http2 https://127.0.0.1:8443/
 
 using System.Text;
 using ioxide;
 using ioxide.httpclient;
-using ioxide.nghttp2;
+using ioxide.http2;
 using ioxide.tls;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
@@ -5024,7 +5061,7 @@ 
HTTP/2 in · HTTP/1.1 out

             // Buffered + async: each stream dispatches with its body assembled, and the handler
             // may await - the upstream round trip resumes inline on this reactor. Concurrent
             // streams interleave here, which is exactly why the h1 pool has to be deep.
-            await new Nghttp2Connection(pipe).RunBufferedAsync(async request =>
+            await new Http2Connection(pipe).RunBufferedAsync(async request =>
             {
                 try
                 {
@@ -5033,11 +5070,11 @@ 
HTTP/2 in · HTTP/1.1 out

                     using HttpClientResponse response = await client.SendAsync(new HttpClientRequest(
                         request.Method, request.Path) { Body = request.Body });
 
-                    // Copy before Dispose: the response arena is freed then, and nghttp2 copies
-                    // the h2 response only AFTER this handler returns. A real proxy would also
+                    // Copy before Dispose: the response arena is freed then, and the h2 response
+                    // is framed only AFTER this handler returns. A real proxy would also
                     // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all
                     // illegal in h2 and would be a protocol error to forward.
-                    var proxied = new Nghttp2Response
+                    var proxied = new Http2Response
                     {
                         Status = response.Status,
                         Body = response.Body.ToArray(),
@@ -5053,7 +5090,7 @@ 
HTTP/2 in · HTTP/1.1 out

                     // Upstream down is a gateway error on this stream, not a dead h2 connection:
                     // every other stream on it keeps working. A refused certificate arrives the
                     // same way - the handshake is part of opening the upstream connection.
-                    return new Nghttp2Response
+                    return new Http2Response
                     {
                         Status = 502,
                         Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"),
@@ -5090,16 +5127,16 @@ 
HTTP/2 in · HTTP/1.1 out

   

     

       
HTTP/2 in · HTTP/2 out

-      ioxide + ioxide.nghttp2 + ioxide.httpclient
+      ioxide + ioxide.http2 + ioxide.httpclient
     

-
// dotnet add package ioxide ioxide.nghttp2 ioxide.httpclient
+
// dotnet add package ioxide ioxide.http2 ioxide.httpclient
 //   PLAYGROUND_PORT=8444 dotnet run --project Playground/Http2/Tls  # an h2-over-TLS origin
 //   curl -k --http2 https://127.0.0.1:8443/
 
 using System.Text;
 using ioxide;
 using ioxide.httpclient;
-using ioxide.nghttp2;
+using ioxide.http2;
 using ioxide.tls;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
@@ -5206,7 +5243,7 @@ 
HTTP/2 in · HTTP/2 out

 
             // Buffered + async: each stream dispatches with its body assembled, and the handler
             // may await - the upstream round trip resumes inline on this reactor.
-            await new Nghttp2Connection(pipe).RunBufferedAsync(async request =>
+            await new Http2Connection(pipe).RunBufferedAsync(async request =>
             {
                 try
                 {
@@ -5215,11 +5252,11 @@ 
HTTP/2 in · HTTP/2 out

                     using HttpClientResponse response = await client.SendAsync(new HttpClientRequest(
                         request.Method, request.Path) { Body = request.Body });
 
-                    // Copy before Dispose: the response arena is freed then, and nghttp2 copies
-                    // the h2 response only AFTER this handler returns. A real proxy would also
+                    // Copy before Dispose: the response arena is freed then, and the h2 response
+                    // is framed only AFTER this handler returns. A real proxy would also
                     // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all
                     // illegal in h2 and would be a protocol error to forward.
-                    var proxied = new Nghttp2Response
+                    var proxied = new Http2Response
                     {
                         Status = response.Status,
                         Body = response.Body.ToArray(),
@@ -5235,7 +5272,7 @@ 
HTTP/2 in · HTTP/2 out

                     // Upstream down is a gateway error on this stream, not a dead h2 connection:
                     // every other stream on it keeps working. A refused certificate arrives the
                     // same way - the handshake is part of opening the upstream connection.
-                    return new Nghttp2Response
+                    return new Http2Response
                     {
                         Status = 502,
                         Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"),
@@ -5267,14 +5304,14 @@ 
HTTP/2 in · HTTP/2 out

 {
     thread.Join();
 }

-    
The narrowest proxy in this set: two sockets per reactor no matter how many requests are in flight. Two independent nghttp2 sessions are involved and they share nothing - HPACK state is per-connection, so a proxy always re-encodes. “Just splice the frames” is not a shortcut that exists.

+    
The narrowest proxy in this set: two sockets per reactor no matter how many requests are in flight. Two independent HTTP/2 sessions are involved and they share nothing - HPACK state is per-connection, so a proxy always re-encodes. “Just splice the frames” is not a shortcut that exists.

   

   

     

       
HTTP/2 in · HTTP/3 out

-      ioxide + ioxide.nghttp2 + ioxide.httpclient
+      ioxide + ioxide.http2 + ioxide.httpclient
     

-
// dotnet add package ioxide ioxide.nghttp2 ioxide.httpclient
+
// dotnet add package ioxide ioxide.http2 ioxide.httpclient
 //   dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443
 //   PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H2ToH3
 //   curl -k --http2 https://127.0.0.1:8443/
@@ -5282,7 +5319,7 @@ 
HTTP/2 in · HTTP/3 out

 using System.Text;
 using ioxide;
 using ioxide.httpclient;
-using ioxide.nghttp2;
+using ioxide.http2;
 using ioxide.tls;
 
 // ── Knobs ────────────────────────────────────────────────────────────────────────────────────
@@ -5374,7 +5411,7 @@ 
HTTP/2 in · HTTP/3 out

 
             // Buffered + async: each stream dispatches with its body assembled, and the handler
             // may await - the upstream round trip resumes inline on this reactor.
-            await new Nghttp2Connection(pipe).RunBufferedAsync(async request =>
+            await new Http2Connection(pipe).RunBufferedAsync(async request =>
             {
                 try
                 {
@@ -5383,11 +5420,11 @@ 
HTTP/2 in · HTTP/3 out

                     using HttpClientResponse response = await client.SendAsync(new HttpClientRequest(
                         request.Method, request.Path) { Body = request.Body });
 
-                    // Copy before Dispose: the response arena is freed then, and nghttp2 copies
-                    // the h2 response only AFTER this handler returns. A real proxy would also
+                    // Copy before Dispose: the response arena is freed then, and the h2 response
+                    // is framed only AFTER this handler returns. A real proxy would also
                     // drop hop-by-hop headers - Connection, Keep-Alive, Transfer-Encoding are all
                     // illegal in h2 and would be a protocol error to forward.
-                    var proxied = new Nghttp2Response
+                    var proxied = new Http2Response
                     {
                         Status = response.Status,
                         Body = response.Body.ToArray(),
@@ -5403,7 +5440,7 @@ 
HTTP/2 in · HTTP/3 out

                     // Upstream down is a gateway error on this stream, not a dead h2 connection:
                     // every other stream on it keeps working. A refused certificate arrives the
                     // same way - the handshake is part of opening the upstream connection.
-                    return new Nghttp2Response
+                    return new Http2Response
                     {
                         Status = 502,
                         Body = Encoding.ASCII.GetBytes($"upstream failed: {e.Message}\n"),
diff --git a/docs/learn/overview.html b/docs/learn/overview.html
index ec5ea53..a277fd2 100644
--- a/docs/learn/overview.html
+++ b/docs/learn/overview.html
@@ -121,7 +121,7 @@ 
Speaking a protocol

     SslStream. HTTP/2 and HTTP/3 each come twice - a bundled native, or the same
     surface in pure C# with no native code at all:

     
    
-      
  • ioxide.nghttp2 / ioxide.http2 - HTTP/2. h2c with prior
+      
  • ioxide.http2 - HTTP/2, pure C#. h2c with prior
       knowledge, or h2 over TLS chosen by ALPN.
    • 
       
  • ioxide.nghttp3 / ioxide.http3 - HTTP/3, over
       QUIC from ioxide.ngtcp2.
    • 
diff --git a/docs/learn/tls.html b/docs/learn/tls.html
index 2890403..af5b064 100644
--- a/docs/learn/tls.html
+++ b/docs/learn/tls.html
@@ -289,7 +289,7 @@ 
    kTLS: performance
    
     size: 0.79× at 64 bytes is not the same server as 0.35× at 64 KiB.
    
     
    kTLS is behind on large single writes - the kernel has to split one 256 KiB write
     into sixteen records - and level once the protocol above it already chunks. Over HTTP/2, where
-    nghttp2 frames responses into 16 KiB DATA frames before they reach the socket, the two are
+    HTTP/2 frames responses into 16 KiB DATA frames before they reach the socket, the two are
     within noise of each other at every size.
    
 
     
    kTLS: where the cost actually is
    
diff --git a/scripts/gen-docs-panes.py b/scripts/gen-docs-panes.py
index 7c9ec1f..fd4febd 100644
--- a/scripts/gen-docs-panes.py
+++ b/scripts/gen-docs-panes.py
@@ -108,30 +108,20 @@
         "connection and pool state stay single-threaded without a lock even when a handler wanders."),
 
     # ── protocols ────────────────────────────────────────────────────────────────────────────
-    "h2tls": (
-        "Http2/Tls", "HTTP/2 · TLS & ALPN", "ioxide + ioxide.nghttp2",
+    "h2cstls": (
+        "Http2/Tls", "HTTP/2 · TLS & ALPN", "ioxide + ioxide.http2",
         ["curl -k --http2 https://127.0.0.1:8443/"],
         "How a browser actually reaches h2: over TLS, with the protocol chosen during the "
         "handshake. Alpn = [\"h2\", \"http/1.1\"] is an ordered preference, not a "
         "weighting - the server takes the first entry the client also offered, and this sample "
-        "then branches on what was agreed, so one port serves both. That is why the h2c samples "
-        "are the exception rather than the rule. TLS here is "
-        "OpenSSL both ways by default; the kernelTx/kernelRx "
-        "knobs at the top are what move either direction into the kernel."),
-    "h2cstls": (
-        "Http2/Tls", "HTTP/2 · pure C# over TLS", "ioxide + ioxide.http2",
-        ["curl -k --http2 https://127.0.0.1:8443/"],
-        "The pure-C# HTTP/2 server behind TLS, and the diff against "
-        " is three "
-        "type names and a package - no native library anywhere. Both take an "
-        "IDuplexPipe, so neither learns what is under it. "
-        "Measured on this rig at 2 reactors, 64 connections, 32 streams, a 2-byte body: pure C# "
-        "runs 2.46× nghttp2 over TLS and 2.73× cleartext, at about a third of "
-        "the CPU per request. That ordering does not generalize - at 32 connections over 4 "
-        "reactors nghttp2 is ahead instead - and a 2-byte body measures framing and HPACK rather "
-        "than moving data. TLS itself costs both the same 0.05µs per request."),
+        "then branches on what was agreed, so ONE port serves both. That is why the h2c samples "
+        "are the exception rather than the rule. What Http2Connection is handed is a "
+        "TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code "
+        "is byte-for-byte the h2c sample's. TLS is OpenSSL both ways by default; the "
+        "kernelTx/kernelRx knobs at the top move either direction into "
+        "the kernel, and cost the same 0.05µs per request either way."),
     "h2bcl": (
-        "Http2/SslStream", "HTTP/2 · over SslStream", "ioxide + ioxide.nghttp2",
+        "Http2/SslStream", "HTTP/2 · over SslStream", "ioxide + ioxide.http2",
         ["curl -k --http2 https://127.0.0.1:8443/"],
         "HTTP/2 over the BCL's SslStream, and the point is the ten-line "
         "Stream-to-IDuplexPipe adapter at the bottom. "
@@ -139,7 +129,7 @@
         "directly, over ioxide's TLS, or over SslStream - the transport is a "
         "constructor argument, not a branch inside the protocol."),
     "h3cs": (
-        "Http3/Buffered", "HTTP/3 · pure C#", "ioxide + ioxide.ngtcp2 + ioxide.http3",
+        "Http3/Buffered", "HTTP/3 · buffered", "ioxide + ioxide.ngtcp2 + ioxide.http3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "HTTP/3 with no native library above the transport - frames, QPACK and Huffman are "
         "all managed code, and only QUIC itself stays native. Drop-in for "
@@ -150,7 +140,7 @@
         "from sending the header frame and a short body in ONE call rather than two; the large-body "
         "lead is the native shim copying every response body at submit, which this never does."),
     "h3stream": (
-        "Http3/Nghttp3Response", "HTTP/3 · streamed response", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Response", "HTTP/3 · response streamed (nghttp3)", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/",
          "curl --http3-only -kN https://127.0.0.1:8443/feed   # never ends"],
         "The response body produced OVER TIME instead of handed over whole - each flush becomes a "
@@ -162,7 +152,7 @@
         "PULLS body bytes rather than accepting pushes, which is why this carries a resume and a "
         "drain the pure-C# writer does not need."),
     "h3csstream": (
-        "Http3/StreamedBoth", "HTTP/3 · streamed both ways", "ioxide + ioxide.ngtcp2 + ioxide.http3",
+        "Http3/StreamedBoth", "HTTP/3 · request + response streamed", "ioxide + ioxide.ngtcp2 + ioxide.http3",
         ["curl --http3-only -k https://127.0.0.1:8443/",
          "curl --http3-only -kN https://127.0.0.1:8443/feed        # never ends",
          "curl --http3-only -k --data-binary @big.bin https://127.0.0.1:8443/echo"],
@@ -178,7 +168,7 @@
         "resume and a drain because nghttp3 pulls instead; this measures 1.32× its "
         "throughput on the same 8×1 KiB response."),
     "h3buf": (
-        "Http3/Nghttp3Buffered", "HTTP/3 · buffered dispatch", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Buffered", "HTTP/3 · buffered (nghttp3)", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "The same server as  with the other "
         "dispatch mode - one method call is the whole difference. Buffered waits for "
@@ -232,16 +222,57 @@
         "appends across recvs into the same buffer, so a request split over several reads arrives "
         "contiguous. Costs memory per connection - see "
         "."),
-    "h2": (
-        "Http2/Nghttp2", "HTTP/2 · nghttp2", "ioxide + ioxide.nghttp2",
-        ["curl --http2-prior-knowledge http://127.0.0.1:8080/"],
-        'This is h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and there is no upgrade dance. For h2 over TLS see the  tab - the protocol code there is byte-for-byte identical, because Nghttp2Connection takes an IDuplexPipe and never learns what is under it.'),
     "h2cs": (
-        "Http2/Buffered", "HTTP/2 · pure C#", "ioxide + ioxide.http2",
+        "Http2/Buffered", "HTTP/2 · buffered", "ioxide + ioxide.http2",
         ["curl --http2-prior-knowledge http://127.0.0.1:8080/"],
-        "Which to take? They measure the same. Interleaved warm runs on one rig - h2load -n 300000 -c 32 -m 32, 4 reactors, 2-byte body - put the ratio between 0.98× and 1.09×, and a 1 KiB body holds the same. On a small response the cost is the loop and the syscalls, not the header codec. So take ioxide.http2 when shipping a native library is inconvenient, and ioxide.nghttp2 when you want the reference implementation's coverage of the protocol's darker corners. The same choice exists one version up: ioxide.http3 is the pure-C# drop-in for ioxide.nghttp3."),
+        "h2c with prior knowledge: the peer opens with the HTTP/2 connection preface and "
+        "there is no upgrade dance. For h2 over TLS see "
+        " - the protocol code "
+        "there is byte-for-byte this one, because Http2Connection takes an "
+        "IDuplexPipe and never learns what is under it. "
+        "BUFFERED is the dispatch mode: the handler runs once the request has fully arrived, so "
+        "request.Body holds the whole body and the answer is one "
+        "Http2Response. That is the right default, and the wrong one for a large "
+        "upload or an endless response - the three tabs after it are those cases."),
+    "h2sresp": (
+        "Http2/StreamedResponse", "HTTP/2 · response streamed", "ioxide + ioxide.http2",
+        ["curl --http2-prior-knowledge http://127.0.0.1:8080/",
+         "curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed   # never ends"],
+        "The RESPONSE body produced over time instead of returned whole - each flush becomes a "
+        "DATA frame. /feed is why the mode exists: an endless response has no final "
+        "byte, so a buffered API cannot express it at all. Http2ResponseWriter is an "
+        "IBufferWriter<byte>, so a serializer or a framework's response sink "
+        "writes into it unchanged. What HTTP/2 adds over "
+        " is that credit "
+        "is SHARED: every stream rides one TCP connection, so a flush waits on whichever of the "
+        "stream and connection windows runs out first, and a WINDOW_UPDATE for either wakes it."),
+    "h2sreq": (
+        "Http2/StreamedRequest", "HTTP/2 · request streamed", "ioxide + ioxide.http2",
+        ["head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \\",
+         "  http://127.0.0.1:8080/upload"],
+        "The other direction, and a different problem. StreamRequestBodies dispatches "
+        "at the HEADERS and hands the handler an Http2BodyReader, so it runs while "
+        "the upload is still arriving. What changes is what bounds memory: buffered holds the "
+        "whole body, so MaxRequestBytes is all that stands between a hostile peer and "
+        "the arena; streamed holds ONE flow-control window, because a chunk credits the peer's "
+        "window only as the handler reads it. Fall behind and the peer runs out of credit "
+        "and stops sending - backpressure the peer takes part in, rather than a buffer you hope is "
+        "big enough."),
+    "h2sboth": (
+        "Http2/StreamedBoth", "HTTP/2 · both directions streamed", "ioxide + ioxide.http2",
+        ["curl --http2-prior-knowledge -N http://127.0.0.1:8080/feed",
+         "head -c 50000000 /dev/zero | curl --http2-prior-knowledge --data-binary @- \\",
+         "  http://127.0.0.1:8080/echo"],
+        "Both at once, which is the shape a proxy needs: /echo reads a chunk and "
+        "writes a chunk, so neither the upload nor the download is ever held whole. The two "
+        "directions are separate switches - StreamRequestBodies for the read side, "
+        "RunAsync with a writer for the write side - because they solve different "
+        "problems and most servers want exactly one of them. Mirrors "
+        "; the difference is HTTP/2's shared connection window, which a handler "
+        "that stops reading holds down for every other stream on the connection."),
     "h3": (
-        "Http3/Nghttp3Request", "HTTP/3 · nghttp3", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
+        "Http3/Nghttp3Request", "HTTP/3 · request streamed (nghttp3)", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
         "HTTP/3 over QUIC, dispatched as the body streams. Compare "
         ", which waits for end-of-stream "
diff --git a/scripts/gen-docs-proxy-panes.py b/scripts/gen-docs-proxy-panes.py
index d16f173..d230d40 100644
--- a/scripts/gen-docs-proxy-panes.py
+++ b/scripts/gen-docs-proxy-panes.py
@@ -32,17 +32,17 @@
                 'PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H1ToH3',
                 'curl -k https://127.0.0.1:8443/']),
     "H2ToH1": ("h2 → h1", "HTTP/2 in · HTTP/1.1 out",
-               "ioxide + ioxide.nghttp2 + ioxide.httpclient",
+               "ioxide + ioxide.http2 + ioxide.httpclient",
                [
                 'PLAYGROUND_PORT=8444 dotnet run --project Playground/Tls/Ktls   # a TLS origin',
                 'curl -k --http2 https://127.0.0.1:8443/']),
     "H2ToH2": ("h2 → h2", "HTTP/2 in · HTTP/2 out",
-               "ioxide + ioxide.nghttp2 + ioxide.httpclient",
+               "ioxide + ioxide.http2 + ioxide.httpclient",
                [
                 'PLAYGROUND_PORT=8444 dotnet run --project Playground/Http2/Tls  # an h2-over-TLS origin',
                 'curl -k --http2 https://127.0.0.1:8443/']),
     "H2ToH3": ("h2 → h3", "HTTP/2 in · HTTP/3 out",
-               "ioxide + ioxide.nghttp2 + ioxide.httpclient",
+               "ioxide + ioxide.http2 + ioxide.httpclient",
                [
                 'dotnet run --project Playground/Http3/Nghttp3Request          # h3 origin on udp :8443',
                 'PLAYGROUND_UPSTREAM_PORT=8443 dotnet run --project Playground/Proxy/H2ToH3',
@@ -79,7 +79,7 @@
               "concurrency - a hundred h2 streams need a hundred h1 connections, because h1 has no "
               "multiplexing to borrow.",
     "H2ToH2": "The narrowest proxy in this set: two sockets per reactor no matter how many requests "
-              "are in flight. Two independent nghttp2 sessions are involved and they share nothing - "
+              "are in flight. Two independent HTTP/2 sessions are involved and they share nothing - "
               "HPACK state is per-connection, so a proxy always re-encodes. “Just splice the frames” "
               "is not a shortcut that exists.",
     "H2ToH3": "TCP in, QUIC out - the client half of a migration where the origin moved to HTTP/3 and "

From 5a39f54ea82da4beb2f445ada17430e825401fc8 Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 15:46:42 +0100
Subject: [PATCH 10/14] chore(release): 0.4.176

All ten packable projects move together, as they always have. The gate on
bumping was the async-workload numbers, and those exist now: an asynchronous
handler went from serving 0 bytes to serving, streamed responses measure 2.45x,
and buffered 8 KiB is unchanged.

ioxide.nghttp2 is not among them any more - it stays at 0.4.169 in dropped/,
which is genuinely its last published version. That also means 0.4.176 is the
first release where ioxide.httpclient does not pull it in: its dependencies are
ioxide, ioxide.http2, ioxide.nghttp3 and ioxide.ngtcp2, so the HTTP/2 half of
the client no longer ships a native library. The h3 half still does.

Verified by packing: every inter-package dependency resolves to 0.4.176, and
CI's pack steps match the ten projects exactly.
---
 src/clients/ioxide.file/ioxide.file.csproj             | 2 +-
 src/clients/ioxide.httpclient/ioxide.httpclient.csproj | 2 +-
 src/clients/ioxide.pg/ioxide.pg.csproj                 | 2 +-
 src/clients/ioxide.redis/ioxide.redis.csproj           | 2 +-
 src/ioxide/ioxide.csproj                               | 2 +-
 src/protocols/ioxide.http2/ioxide.http2.csproj         | 2 +-
 src/protocols/ioxide.http3/ioxide.http3.csproj         | 2 +-
 src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj     | 2 +-
 src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj       | 2 +-
 src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj       | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/clients/ioxide.file/ioxide.file.csproj b/src/clients/ioxide.file/ioxide.file.csproj
index d73d3cd..4802d6e 100644
--- a/src/clients/ioxide.file/ioxide.file.csproj
+++ b/src/clients/ioxide.file/ioxide.file.csproj
@@ -8,7 +8,7 @@
     ioxide.file
 
     ioxide.file
-    0.4.169
+    0.4.176
     MDA2AV
     File serving for the ioxide io_uring runtime: immutable asset snapshots with baked responses, pooled positional ring reads, atomic reloads.
     MIT
diff --git a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj
index 07d1ce7..d03bf3b 100644
--- a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj
+++ b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj
@@ -8,7 +8,7 @@
     ioxide.httpclient
 
     ioxide.httpclient
-    0.4.169
+    0.4.176
     MDA2AV
     The ring-native HTTP client for the ioxide io_uring runtime: HTTP/1.1, HTTP/2 and HTTP/3 behind one API, with the protocol chosen per origin via Alt-Svc. One package - the h1 parser, a pure-C# HTTP/2 client on ioxide.http2's framing, the nghttp3 bridge, client-side TLS (SNI, ALPN and certificate verification) for https:// origins, and the negotiating client - sharing one set of message types. Every response resumes the awaiting handler inline on its own reactor thread.
     MIT
diff --git a/src/clients/ioxide.pg/ioxide.pg.csproj b/src/clients/ioxide.pg/ioxide.pg.csproj
index 90acf90..e1c475f 100644
--- a/src/clients/ioxide.pg/ioxide.pg.csproj
+++ b/src/clients/ioxide.pg/ioxide.pg.csproj
@@ -8,7 +8,7 @@
     ioxide.pg
 
     ioxide.pg
-    0.4.169
+    0.4.176
     MDA2AV
     Postgres driver for the ioxide io_uring runtime: pooled ring-native connections per reactor, ring-native connect and handshake, inline completion resume.
     MIT
diff --git a/src/clients/ioxide.redis/ioxide.redis.csproj b/src/clients/ioxide.redis/ioxide.redis.csproj
index f2e63c6..df41b65 100644
--- a/src/clients/ioxide.redis/ioxide.redis.csproj
+++ b/src/clients/ioxide.redis/ioxide.redis.csproj
@@ -8,7 +8,7 @@
     ioxide.redis
 
     ioxide.redis
-    0.4.169
+    0.4.176
     MDA2AV
     Redis client for the ioxide io_uring runtime: pooled ring-native connections per reactor, full RESP2 protocol, a generic command API plus typed helpers (strings, keys, hashes, lists, sets, sorted sets, pub/sub, transactions, scripting), and pipelining. Inline completion resume.
     MIT
diff --git a/src/ioxide/ioxide.csproj b/src/ioxide/ioxide.csproj
index 75fdb69..ab62146 100644
--- a/src/ioxide/ioxide.csproj
+++ b/src/ioxide/ioxide.csproj
@@ -8,7 +8,7 @@
     ioxide
 
     ioxide
-    0.4.169
+    0.4.176
     MDA2AV
     A shared-nothing io_uring runtime for .NET: one ring per reactor thread, inline completions, zero native dependencies. The engine - reactor, connection, and the IRingHost client seam. Includes TLS termination: the OpenSSL handshake driven over the ring, then kernel TLS (kTLS) transmit offload, so handlers keep writing plaintext. TLS needs OpenSSL 3 and the Linux tls module; nothing else does, and neither is loaded unless you use it.
     MIT
diff --git a/src/protocols/ioxide.http2/ioxide.http2.csproj b/src/protocols/ioxide.http2/ioxide.http2.csproj
index eb318c3..e41b632 100644
--- a/src/protocols/ioxide.http2/ioxide.http2.csproj
+++ b/src/protocols/ioxide.http2/ioxide.http2.csproj
@@ -8,7 +8,7 @@
     ioxide.http2
 
     ioxide.http2
-    0.4.169
+    0.4.176
     MDA2AV
     Pure-C# HTTP/2 for the ioxide io_uring runtime: framing, HPACK (static and dynamic tables, Huffman) and flow control, with zero native code. Serves h2c with prior knowledge and h2 over TLS by ALPN, buffered or streamed in either direction, and the same framing drives ioxide.httpclient's HTTP/2 client.
     MIT
diff --git a/src/protocols/ioxide.http3/ioxide.http3.csproj b/src/protocols/ioxide.http3/ioxide.http3.csproj
index 8a6538c..f7669fe 100644
--- a/src/protocols/ioxide.http3/ioxide.http3.csproj
+++ b/src/protocols/ioxide.http3/ioxide.http3.csproj
@@ -8,7 +8,7 @@
     ioxide.http3
 
     ioxide.http3
-    0.4.169
+    0.4.176
     MDA2AV
     Pure C# HTTP/3 for the ioxide io_uring runtime: frame parsing, QPACK (static table + Huffman) and request dispatch with zero native dependencies. Rides any QuicConnection via its stream read surface - engine-agnostic, drop-in alternative to ioxide.nghttp3.
     MIT
diff --git a/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj b/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj
index 70ffb18..8007bf7 100644
--- a/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj
+++ b/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj
@@ -8,7 +8,7 @@
     ioxide.nghttp3
 
     ioxide.nghttp3
-    0.4.169
+    0.4.176
     MDA2AV
     HTTP/3 layer for the ioxide io_uring runtime: nghttp3 (H3 + QPACK) bundled as a single self-contained native library with no external dependencies. Rides any QuicConnection via its stream read surface - engine-agnostic, no ioxide.ngtcp2 dependency.
     MIT
diff --git a/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj b/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj
index 8feb361..16f7648 100644
--- a/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj
+++ b/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj
@@ -8,7 +8,7 @@
     ioxide.ngtcp2
 
     ioxide.ngtcp2
-    0.4.169
+    0.4.176
     MDA2AV
     QUIC engine for the ioxide io_uring runtime: ngtcp2 + picotls bundled as a single self-contained native library (only system dependency: libcrypto.so.3 / OpenSSL 3.x). Plugs into the reactor's QUIC transport via QuicConnection. Server side; engine bindings in progress.
     MIT
diff --git a/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj b/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj
index 5c0295d..5ff4358 100644
--- a/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj
+++ b/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj
@@ -8,7 +8,7 @@
     ioxide.Kestrel
 
     ioxide.Kestrel
-    0.4.169
+    0.4.176
     MDA2AV
     ASP.NET Core Kestrel transport backed by the ioxide io_uring runtime: one reactor (ring) per core, SO_REUSEPORT load-balanced, with Kestrel's HTTP request loop pinned to the reactor thread. Drop-in via UseIoxide().
     MIT

From b83e267641d0a5afff59debef609972d33c2cdf4 Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 16:24:17 +0100
Subject: [PATCH 11/14] fix(http2): close the two published h2 DoS vectors

Both were found by asking what nghttp2 was buying us beyond speed. The answer
was hardening, and these are the two most-published HTTP/2 denial-of-service
vectors of recent years - neither of which the managed server defended against.

MaxConcurrentStreams was advertised in SETTINGS and never enforced: the option
appeared exactly three times, none of which compared it to _streams.Count. So a
peer could open unbounded streams, each costing a PendingRequest and a pooled
arena, and "open a stream, reset it, repeat" (CVE-2023-44487) cost the peer
nothing. Streams past the limit are now refused with REFUSED_STREAM, which
RFC 9113 8.7 makes safe for the peer to retry elsewhere.

The header block was unbounded. MaxFrameSize caps one frame at 16 KiB, but
nothing capped how many CONTINUATION frames follow a HEADERS that never sets
END_HEADERS, so the accumulated block grew until the process died - the
CONTINUATION flood. MaxHeaderListSize bounds it, is advertised, and exceeding
it is a CONNECTION error rather than a stream one, because a block that stops
being decoded desynchronises HPACK for everything after it.

The subtlety in both: a refused or over-long block still has to be DECODED.
HPACK is one stream across the whole connection, so skipping a block would
desynchronise the table for every later request. A refused stream's block
decodes into a shared scratch and is thrown away, and that scratch is bounded
too - otherwise refusing a stream would itself be the way in.

The tests fail without the fix, which is the only reason to trust them: the
flood test wedges for the full 120s timeout, and the stream test sees no
RST_STREAM at all. Chaos 39 pass, 0 failed.
---
 .../ioxide.http2/Http2Connection.Frames.cs    | 63 ++++++++++++++++++-
 src/protocols/ioxide.http2/Http2Connection.cs |  6 ++
 src/protocols/ioxide.http2/Http2Options.cs    | 18 +++++-
 tests/Ioxide.Tests.Chaos/H2ChaosTests.cs      | 63 ++++++++++++++++++-
 tests/Ioxide.Tests.Chaos/H2cClient.cs         | 36 +++++++++++
 5 files changed, 181 insertions(+), 5 deletions(-)

diff --git a/src/protocols/ioxide.http2/Http2Connection.Frames.cs b/src/protocols/ioxide.http2/Http2Connection.Frames.cs
index 421883d..63a1e68 100644
--- a/src/protocols/ioxide.http2/Http2Connection.Frames.cs
+++ b/src/protocols/ioxide.http2/Http2Connection.Frames.cs
@@ -140,6 +140,17 @@ private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload)
 
         if (!_streams.TryGetValue(header.StreamId, out PendingRequest? pending))
         {
+            if (_streams.Count >= _options.MaxConcurrentStreams)
+            {
+                // Past the limit we advertised. The block still has to be DECODED - HPACK is one
+                // stream across the whole connection, so skipping it would desynchronise every
+                // later request - but it decodes into a scratch that is thrown away, and the peer
+                // is told REFUSED_STREAM, which RFC 9113 8.7 makes safe for it to retry elsewhere.
+                _discardingStream = header.StreamId;
+                DiscardHeaderBlock(header, block);
+                return;
+            }
+
             // Opens at what the peer's SETTINGS advertised, not at the RFC default. Streams are
             // created long after those SETTINGS arrive, so starting at 65535 and waiting for a
             // WINDOW_UPDATE means waiting for one the peer has no reason to send - it believes we
@@ -156,6 +167,12 @@ private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload)
         // piecewise - the whole block has to be in hand first.
         pending.AppendHeaderBlock(block);
 
+        if (pending.HeaderBlock.Length > _options.MaxHeaderListSize)
+        {
+            GoAway(Http2Error.EnhanceYourCalm);
+            return;
+        }
+
         if ((header.Flags & FrameFlags.EndHeaders) != 0)
         {
             if (!DecodeHeaderBlock(pending))
@@ -173,6 +190,34 @@ private void HandleHeaders(in FrameHeader header, ReadOnlySpan payload)
         }
     }
 
+    /// 
+    /// A refused stream's header block: decoded to keep the HPACK table in step with the peer, and
+    /// thrown away. Bounded like any other block, so refusing a stream cannot itself be the way in.
+    /// 
+    private void DiscardHeaderBlock(in FrameHeader header, ReadOnlySpan block)
+    {
+        _discardBlock.AppendHeaderBlock(block);
+
+        if (_discardBlock.HeaderBlock.Length > _options.MaxHeaderListSize)
+        {
+            GoAway(Http2Error.EnhanceYourCalm);
+            return;
+        }
+
+        if ((header.Flags & FrameFlags.EndHeaders) == 0)
+        {
+            return;   // more CONTINUATION to come
+        }
+
+        int streamId = _discardingStream;
+        _discardingStream = 0;
+
+        if (DecodeHeaderBlock(_discardBlock, discard: true))
+        {
+            ResetStream(streamId, Http2Error.RefusedStream);
+        }
+    }
+
     /// 
     /// Hand this request over the moment its headers are in, with the body still to come. The
     /// stream stays in _streams - unlike the buffered path, later DATA frames still have
@@ -191,6 +236,13 @@ private void BeginStreamedBody(PendingRequest pending, bool ended)
 
     private void HandleContinuation(in FrameHeader header, ReadOnlySpan payload)
     {
+        // The block belongs to a stream we refused; keep decoding it so HPACK stays in step.
+        if (header.StreamId == _discardingStream)
+        {
+            DiscardHeaderBlock(header, payload);
+            return;
+        }
+
         if (!_streams.TryGetValue(header.StreamId, out PendingRequest? pending))
         {
             GoAway(Http2Error.ProtocolError);
@@ -199,6 +251,12 @@ private void HandleContinuation(in FrameHeader header, ReadOnlySpan payloa
 
         pending.AppendHeaderBlock(payload);
 
+        if (pending.HeaderBlock.Length > _options.MaxHeaderListSize)
+        {
+            GoAway(Http2Error.EnhanceYourCalm);
+            return;
+        }
+
         if ((header.Flags & FrameFlags.EndHeaders) != 0)
         {
             if (!DecodeHeaderBlock(pending))
@@ -211,7 +269,7 @@ private void HandleContinuation(in FrameHeader header, ReadOnlySpan payloa
         }
     }
 
-    private bool DecodeHeaderBlock(PendingRequest pending)
+    private bool DecodeHeaderBlock(PendingRequest pending, bool discard = false)
     {
         try
         {
@@ -226,7 +284,8 @@ private bool DecodeHeaderBlock(PendingRequest pending)
             }
 
             PendingRequest target = pending;
-            _decoder.Decode(block, _headerScratch, (name, value) => target.AddHeader(name, value));
+            _decoder.Decode(block, _headerScratch,
+                discard ? static (_, _) => { } : (name, value) => target.AddHeader(name, value));
             pending.ClearHeaderBlock();
             pending.HeadersDone = true;
             return true;
diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs
index 4d9550c..4a0d23c 100644
--- a/src/protocols/ioxide.http2/Http2Connection.cs
+++ b/src/protocols/ioxide.http2/Http2Connection.cs
@@ -43,6 +43,12 @@ public sealed partial class Http2Connection : IDisposable
     // fired once it has unwound, so a resumed handler cannot re-enter the parser mid-frame.
     private readonly List _bodyWakes = [];
 
+    // The header block of a stream refused for exceeding MaxConcurrentStreams: decoded to keep
+    // HPACK in step with the peer, then thrown away. A block cannot interleave with another
+    // stream's frames, so one of these is enough.
+    private readonly PendingRequest _discardBlock = new();
+    private int _discardingStream;
+
     private bool _prefaceSeen;
     private bool _disposed;
     private bool _failed;
diff --git a/src/protocols/ioxide.http2/Http2Options.cs b/src/protocols/ioxide.http2/Http2Options.cs
index aca78b0..f00f9fd 100644
--- a/src/protocols/ioxide.http2/Http2Options.cs
+++ b/src/protocols/ioxide.http2/Http2Options.cs
@@ -15,9 +15,25 @@ public sealed record Http2Options
     /// Flow-control window advertised per stream.
     public int InitialWindowSize { get; init; } = 1 << 20;
 
-    /// Streams the peer may have open at once.
+    /// 
+    /// Streams the peer may have open at once. Advertised in SETTINGS and enforced: past
+    /// this, a new stream is refused with REFUSED_STREAM rather than allocated. It was advisory
+    /// once, which made "open a stream, reset it, repeat" (CVE-2023-44487) cost the server an
+    /// arena per cycle and the peer nothing.
+    /// 
     public int MaxConcurrentStreams { get; init; } = 1000;
 
+    /// 
+    /// Ceiling on one request's header block, accumulated across HEADERS and every CONTINUATION
+    /// that follows it.
+    ///
+    ///  bounds one frame; nothing bounds how many CONTINUATION frames a
+    /// peer may send, so without this a HEADERS that never sets END_HEADERS grows the block until
+    /// the process dies - the CONTINUATION flood. Exceeding it is a CONNECTION error, not a stream
+    /// one, because a block that stops being decoded desynchronises HPACK for everything after it.
+    /// 
+    public int MaxHeaderListSize { get; init; } = 64 * 1024;
+
     /// 
     /// Dispatch each request as soon as its HEADERS are in, with the body arriving through
     ///  instead of assembled into
diff --git a/tests/Ioxide.Tests.Chaos/H2ChaosTests.cs b/tests/Ioxide.Tests.Chaos/H2ChaosTests.cs
index ff4ecc2..fe61279 100644
--- a/tests/Ioxide.Tests.Chaos/H2ChaosTests.cs
+++ b/tests/Ioxide.Tests.Chaos/H2ChaosTests.cs
@@ -11,11 +11,11 @@ namespace Ioxide.Tests;
 /// 
 internal static class H2ChaosTests
 {
-    private static int StartH2c() => TestServer.Start(static async (_, conn) =>
+    private static int StartH2c(Http2Options? options = null) => TestServer.Start(async (_, conn) =>
     {
         try
         {
-            await new Http2Connection(conn).RunBufferedAsync(static _ => Http2Response.Text("ok"));
+            await new Http2Connection(conn, options).RunBufferedAsync(static _ => Http2Response.Text("ok"));
         }
         finally
         {
@@ -116,5 +116,64 @@ public static void Register(Runner runner)
             // burst without losing the connection.
             Assert.True(client.AwaitResponse(streamId: 1), "server dropped a multiplexed request burst");
         });
+
+        runner.Test("h2c: a CONTINUATION flood is cut off instead of growing without bound", () =>
+        {
+            // MaxFrameSize bounds one frame; nothing bounds how MANY continuations follow a HEADERS
+            // that never sets END_HEADERS. Unbounded, the accumulated block grows until the process
+            // dies - the flood disclosed in April 2024. The block is capped now, and exceeding it is
+            // a CONNECTION error because a block that stops being decoded desynchronises HPACK.
+            int port = StartH2c(new Http2Options { MaxHeaderListSize = 16 * 1024 });
+
+            using (var client = new H2cClient(port))
+            {
+                client.Open();
+                client.RequestHeadersOnly(streamId: 1, endHeaders: false, endStream: false);
+
+                byte[] filler = new byte[4096];
+                byte seen = 0;
+                for (int i = 0; i < 64 && seen == 0; i++)   // 256 KiB, far past the 16 KiB cap
+                {
+                    try
+                    {
+                        client.WriteFrame(H2cClient.Continuation, flags: 0, streamId: 1, filler);
+                    }
+                    catch (IOException)
+                    {
+                        seen = 0xFF;   // server closed on us, which is also a refusal
+                        break;
+                    }
+                    seen = client.AwaitAnyOf([0x7], timeoutMs: 50);   // GOAWAY
+                }
+
+                Assert.True(seen != 0, "server accepted an unbounded CONTINUATION block");
+            }
+
+            AssertServes(port);   // and the process is still there to serve the next connection
+        });
+
+        runner.Test("h2c: streams past MaxConcurrentStreams are refused, not allocated", () =>
+        {
+            // The limit was advertised in SETTINGS and never enforced, so "open a stream, reset it,
+            // repeat" cost the server an arena per cycle and the peer nothing (CVE-2023-44487).
+            // Streams past the limit now get REFUSED_STREAM, which RFC 9113 8.7 makes safe to retry.
+            int port = StartH2c(new Http2Options { MaxConcurrentStreams = 4 });
+
+            using var client = new H2cClient(port);
+            client.Open();
+
+            // END_HEADERS but NOT END_STREAM: each stream stays open, so the limit is reached.
+            for (int i = 0; i < 4; i++)
+            {
+                client.RequestHeadersOnly(1 + (i * 2), endHeaders: true, endStream: false);
+            }
+
+            client.RequestHeadersOnly(streamId: 101, endHeaders: true, endStream: false);
+            Assert.Equal(H2cClient.RstStream, client.AwaitAnyOf([H2cClient.RstStream], streamId: 101));
+
+            // Refusing must not have desynchronised HPACK - the block was still decoded - so a
+            // stream opened afterwards on the same connection still parses.
+            AssertServes(port);
+        });
     }
 }
diff --git a/tests/Ioxide.Tests.Chaos/H2cClient.cs b/tests/Ioxide.Tests.Chaos/H2cClient.cs
index 56de645..1d80b6d 100644
--- a/tests/Ioxide.Tests.Chaos/H2cClient.cs
+++ b/tests/Ioxide.Tests.Chaos/H2cClient.cs
@@ -18,6 +18,42 @@ public sealed class H2cClient : IDisposable
     private const byte Data = 0x0, Headers = 0x1, Settings = 0x4, GoAway = 0x7;
     private const byte EndStream = 0x1, EndHeaders = 0x4, Ack = 0x1;
 
+    public const byte RstStream = 0x3, Continuation = 0x9;
+
+    /// HEADERS that deliberately leaves the block OPEN, so CONTINUATION must follow.
+    public void RequestHeadersOnly(int streamId, bool endHeaders = true, bool endStream = true)
+        => WriteFrame(Headers, (byte)((endHeaders ? EndHeaders : 0) | (endStream ? EndStream : 0)),
+                      streamId, Hpack("GET", "/"));
+
+    /// 
+    /// Pump until the server sends one of  (0 = any stream), answering
+    /// SETTINGS as they arrive. Returns the frame type seen, or 0 on timeout or a closed connection.
+    /// 
+    public byte AwaitAnyOf(ReadOnlySpan wanted, int streamId = 0, int timeoutMs = 4000)
+    {
+        long deadline = Environment.TickCount64 + timeoutMs;
+        while (Environment.TickCount64 < deadline)
+        {
+            if (!TryReadFrame(out byte type, out byte flags, out int sid, out _))
+            {
+                return 0;
+            }
+            if (type == Settings && (flags & Ack) == 0)
+            {
+                WriteFrame(Settings, Ack, 0, ReadOnlySpan.Empty);
+                continue;
+            }
+            foreach (byte want in wanted)
+            {
+                if (type == want && (streamId == 0 || sid == streamId))
+                {
+                    return type;
+                }
+            }
+        }
+        return 0;
+    }
+
     private readonly TcpClient _tcp;
     private readonly NetworkStream _s;
 

From 2b3547e838fef81b0d64832d5458fca717f5d0eb Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 16:26:58 +0100
Subject: [PATCH 12/14] revert(nghttp2): bring the binding back as a supported
 alternative

Retiring it traded away something the benchmark could not see. It is the
reference implementation: continuously fuzzed, patched by people whose job it
is when the next HTTP/2 CVE lands, and carrying a decade of interop against
every other stack. The two DoS vectors closed in the previous commit are
exactly the class of thing that buys - both were vectors nghttp2 had defended
against for years and the managed server never had.

So it is back in src/, in the solution, in CI's pack list and at 0.4.176, and
dropped/ is gone with it - an empty folder documenting a decision that was
reversed is worse than no folder.

What is NOT restored is the client. ioxide.httpclient stays on the managed
stack, which measured 1.35x-1.39x the binding as a client and is where the
features are. So nothing depends on ioxide.nghttp2 now: it is a standalone
server-side option a user opts into, not something pulled in transitively.

The sample comes back as Playground/Http2/Nghttp2Buffered - the naming scheme
puts the library back on the tab now that there are two h2 implementations
again, and Buffered is the honest suffix because buffered is all it does. Its
pane says so, and says what that costs: no streamed response, no streamed
request, and the dispatch loop still waits for each handler in turn.

Unit 36, Chaos 39, Http 38 pass. The sample serves.
---
 .github/workflows/build.yml                   |   6 +-
 .../Playground.Http2.Nghttp2Buffered.csproj   |   4 +-
 .../Http2/Nghttp2Buffered}/Program.cs         |   0
 docs/assets/style.css                         |   3 +
 docs/index.html                               |  91 ++++++++++++++++++
 dropped/README.md                             |  33 -------
 ioxide.slnx                                   |   3 +
 {dropped => scripts}/build-nghttp2-native.sh  |   0
 scripts/gen-docs-panes.py                     |  14 +++
 src/protocols/ioxide.http2/Http2Connection.cs |   8 +-
 .../Connection/Nghttp2Connection.Callbacks.cs |   0
 .../Connection/Nghttp2Connection.Egress.cs    |   0
 .../Nghttp2Connection.RunBuffered.cs          |   0
 .../Connection/Nghttp2Connection.cs           |   0
 .../Connection/Nghttp2Options.cs              |   0
 .../ioxide.nghttp2/Http/CookieEnumerator.cs   |   0
 .../ioxide.nghttp2/Http/KeyValueList.cs       |   0
 .../ioxide.nghttp2/Http/Nghttp2Request.cs     |   0
 .../ioxide.nghttp2/Http/Nghttp2Response.cs    |   0
 .../ioxide.nghttp2/Interop/Nghttp2.cs         |   0
 .../ioxide.nghttp2/ioxide.nghttp2.csproj      |   2 +-
 .../native/ioxide_nghttp2_shim.c              |   0
 .../linux-x64/native/libioxide_nghttp2.so     | Bin
 23 files changed, 124 insertions(+), 40 deletions(-)
 rename dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj => Playground/Http2/Nghttp2Buffered/Playground.Http2.Nghttp2Buffered.csproj (80%)
 rename {dropped/Playground.Http2.Nghttp2 => Playground/Http2/Nghttp2Buffered}/Program.cs (100%)
 delete mode 100644 dropped/README.md
 rename {dropped => scripts}/build-nghttp2-native.sh (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Connection/Nghttp2Connection.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Connection/Nghttp2Options.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Http/CookieEnumerator.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Http/KeyValueList.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Http/Nghttp2Request.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Http/Nghttp2Response.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/Interop/Nghttp2.cs (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/ioxide.nghttp2.csproj (98%)
 rename {dropped => src/protocols}/ioxide.nghttp2/native/ioxide_nghttp2_shim.c (100%)
 rename {dropped => src/protocols}/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so (100%)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3147d61..620d220 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -86,7 +86,8 @@ jobs:
       # Paths mirror the src/ grouping in ioxide.slnx (core, protocols/, clients/, serving/).
       # Every project carrying a PackageId is packed. ioxide.httpclient is the whole client -
       # h1, h2 and h3 - and project-references http2/ngtcp2/nghttp3, so those ship as its NuGet
-      # dependencies.
+      # dependencies. ioxide.nghttp2 is packed too but nothing depends on it: it is a
+      # standalone server-side alternative now, not something the client pulls in.
       - name: Pack ioxide
         run: dotnet pack src/ioxide/ioxide.csproj --configuration Release --no-build --output ./artifacts
 
@@ -96,6 +97,9 @@ jobs:
       - name: Pack ioxide.nghttp3
         run: dotnet pack src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj --configuration Release --no-build --output ./artifacts
 
+      - name: Pack ioxide.nghttp2
+        run: dotnet pack src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj --configuration Release --no-build --output ./artifacts
+
       - name: Pack ioxide.http2
         run: dotnet pack src/protocols/ioxide.http2/ioxide.http2.csproj --configuration Release --no-build --output ./artifacts
 
diff --git a/dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj b/Playground/Http2/Nghttp2Buffered/Playground.Http2.Nghttp2Buffered.csproj
similarity index 80%
rename from dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj
rename to Playground/Http2/Nghttp2Buffered/Playground.Http2.Nghttp2Buffered.csproj
index 7a93f02..6004db4 100644
--- a/dropped/Playground.Http2.Nghttp2/Playground.Http2.Nghttp2.csproj
+++ b/Playground/Http2/Nghttp2Buffered/Playground.Http2.Nghttp2Buffered.csproj
@@ -6,8 +6,8 @@
     enable
     enable
     true
-    Playground.Http2.Nghttp2
-    Playground.Http2.Nghttp2
+    Playground.Http2.Nghttp2Buffered
+    Playground.Http2.Nghttp2Buffered
   
 
   
diff --git a/dropped/Playground.Http2.Nghttp2/Program.cs b/Playground/Http2/Nghttp2Buffered/Program.cs
similarity index 100%
rename from dropped/Playground.Http2.Nghttp2/Program.cs
rename to Playground/Http2/Nghttp2Buffered/Program.cs
diff --git a/docs/assets/style.css b/docs/assets/style.css
index ccb11b6..cf9af82 100644
--- a/docs/assets/style.css
+++ b/docs/assets/style.css
@@ -230,6 +230,7 @@ nav.top .links a.gh svg { display: block; }
 #tab-h2sresp:checked ~ .ex-menu label[for="tab-h2sresp"],
 #tab-h2sreq:checked ~ .ex-menu label[for="tab-h2sreq"],
 #tab-h2sboth:checked ~ .ex-menu label[for="tab-h2sboth"],
+#tab-h2ng:checked ~ .ex-menu label[for="tab-h2ng"],
 #tab-pxmatrix:checked ~ .ex-menu label[for="tab-pxmatrix"],
 #tab-pxh1toh1:checked ~ .ex-menu label[for="tab-pxh1toh1"],
 #tab-pxh1toh2:checked ~ .ex-menu label[for="tab-pxh1toh2"],
@@ -361,6 +362,7 @@ nav.top .links a.gh svg { display: block; }
 #tab-h2sresp:checked ~ .pane-h2sresp { display: block; }
 #tab-h2sreq:checked ~ .pane-h2sreq { display: block; }
 #tab-h2sboth:checked ~ .pane-h2sboth { display: block; }
+#tab-h2ng:checked ~ .pane-h2ng { display: block; }
 #tab-pxmatrix:checked ~ .pane-pxmatrix { display: block; }
 #tab-pxh1toh1:checked ~ .pane-pxh1toh1 { display: block; }
 #tab-pxh1toh2:checked ~ .pane-pxh1toh2 { display: block; }
@@ -497,6 +499,7 @@ nav.top .links a.gh svg { display: block; }
 #tab-h2sresp:checked ~ .ex-menu label[for="tab-h2sresp"],
 #tab-h2sreq:checked ~ .ex-menu label[for="tab-h2sreq"],
 #tab-h2sboth:checked ~ .ex-menu label[for="tab-h2sboth"],
+#tab-h2ng:checked ~ .ex-menu label[for="tab-h2ng"],
     #tab-pxmatrix:checked ~ .ex-menu label[for="tab-pxmatrix"],
   #tab-pxh1toh1:checked ~ .ex-menu label[for="tab-pxh1toh1"],
   #tab-pxh1toh2:checked ~ .ex-menu label[for="tab-pxh1toh2"],
diff --git a/docs/index.html b/docs/index.html
index eb2575f..11b9b21 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -43,6 +43,7 @@
   
   
   
+  
   
   
   
@@ -108,6 +109,7 @@
       
       
       
+      
       
       
     

@@ -3816,6 +3818,95 @@ 
HTTP/2 · both directions streamed

 }

     
Both at once, which is the shape a proxy needs: /echo reads a chunk and writes a chunk, so neither the upload nor the download is ever held whole. The two directions are separate switches - StreamRequestBodies for the read side, RunAsync with a writer for the write side - because they solve different problems and most servers want exactly one of them. Mirrors ; the difference is HTTP/2's shared connection window, which a handler that stops reading holds down for every other stream on the connection.

   
+  

+    

+      
HTTP/2 · nghttp2

+      ioxide + ioxide.nghttp2
+    

+
// dotnet add package ioxide
+// dotnet add package ioxide.nghttp2
+//   curl --http2-prior-knowledge http://127.0.0.1:8080/
+
+using System.Text;
+using ioxide;
+using ioxide.nghttp2;
+
+// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
+
+ushort port      = 8080;
+int    reactors  = Environment.ProcessorCount;
+int    bodyBytes = 2;
+
+
+// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The
+// handler code is identical either way; this only changes how recv buffers are handed out.
+bool incrementalBuffers = false;
+// ─────────────────────────────────────────────────────────────────────────────────────────────
+
+var config = new ServerConfig
+{
+    ReactorCount   = reactors,  // io_uring rings/threads - one per core
+    RingEntries    = 8192,                                                        // SQ/CQ depth per ring
+    DualStack      = false,                                                       // true = one IPv6 socket also accepts IPv4-mapped
+    RecvBufferSize = 32 * 1024,                                                   // bytes per shared recv buffer
+    RecvSlots      = 4096,                                                        // shared recv buffer-ring depth
+    Incremental    = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null,                                                        // per-connection recv rings (6.12+) - see Tcp/Incremental
+    Udp            = null,                                                        // no raw UDP sockets (TCP-only server)
+    Quic           = null,                                                        // no QUIC transport - see Http3/* and Quic/Alpn
+    Tcp = new TcpOptions
+    {
+        Port             = port,
+        ExtraPorts       = [],                             // extra listener ports (one handler, several doors)
+        ListenBacklog    = 1024,                           // accept-queue depth per SO_REUSEPORT listener
+        WriteSlabSize    = 16 * 1024,                      // per-connection write buffer before overflow kicks in
+        PoolMax          = 1024,                           // pooled connection objects kept per reactor
+        WriteOverflow    = WriteOverflowStrategy.Grow,     // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
+        ZeroCopySend     = false,                          // SEND_ZC: kernel copies less, wins on large writes
+        RecvQueueEntries = 64,                             // per-connection recv completion queue depth
+    },
+};
+
+byte[] body = bodyBytes == 2
+    ? "ok"u8.ToArray()
+    : [.. Enumerable.Repeat((byte)'x', bodyBytes)];
+
+var threads = new Thread[config.ReactorCount];
+
+for (int i = 0; i < threads.Length; i++)
+{
+    var reactor = new Reactor(i, config);
+
+    reactor.TcpHandle = async (r, conn) =>
+    {
+        try
+        {
+            // The connection owns the read loop from here: it feeds nghttp2, dispatches each
+            // request once its stream ends, and drains the egress once per batch.
+            await new Nghttp2Connection(conn).RunBufferedAsync(_ => new Nghttp2Response
+            {
+                Status = 200,
+                Body = body,
+            });
+        }
+        finally
+        {
+            conn.DecRef();
+        }
+    };
+
+    threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
+    threads[i].Start();
+}
+
+Console.WriteLine($"[nghttp2] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+                + $"{body.Length}-byte body (h2c prior knowledge)");
+
+foreach (Thread thread in threads)
+{
+    thread.Join();
+}

+    
The same h2c server on the reference implementation. Kept because being battle-tested is a property no amount of benchmarking substitutes for: nghttp2 is continuously fuzzed, patched by people whose job it is when the next HTTP/2 CVE lands, and has a decade of interop against every other stack. What it does not have is the streaming: it is buffered only, so there is no counterpart here to  or , and its dispatch loop still waits for each handler in turn. Measured as a client on this rig, the managed stack runs 1.35×-1.39× it. Take this one when you want the reference implementation's coverage; take  for everything else.

+  

 
 
   

diff --git a/dropped/README.md b/dropped/README.md
deleted file mode 100644
index e4f7c5b..0000000
--- a/dropped/README.md
+++ /dev/null
@@ -1,33 +0,0 @@
-# dropped
-
-Code that used to ship and no longer does. It is kept because it was real, it was measured, and the
-reasoning behind retiring it is easier to follow with the thing itself still readable.
-
-Nothing here is in `ioxide.slnx`, nothing here is built by CI, and nothing here is published to
-NuGet. It will not compile against the current tree forever, and that is expected - if you need it,
-take it from the last release tag that shipped it rather than from here.
-
-## ioxide.nghttp2
-
-The nghttp2 binding: HTTP/2 framing, HPACK and flow control from the reference C implementation,
-driven sans-I/O over an `IDuplexPipe`.
-
-It was replaced by `ioxide.http2`, which does the same job in pure C#. That started as the
-drop-in-without-a-native-library option and ended as the only one:
-
-- **It measured at least as well.** Interleaved warm runs put the two within `0.98x`-`1.09x` of each
-  other on a small body; where they diverged, the ordering depended on the connection-to-reactor
-  ratio rather than on the codec.
-- **It grew past what the binding could reach.** Streamed responses, streamed request bodies and
-  non-blocking dispatch all landed on the managed side. The binding kept the blocking dispatch loop
-  it was written with, where one slow handler held up every other stream on the connection.
-- **Two implementations of one protocol is a tax on every change**, paid in samples, docs, tests and
-  benchmark fixtures, and the second one was no longer buying coverage of the protocol's darker
-  corners - it was buying a native build step.
-
-The last thing holding it in the tree was `ioxide.httpclient`, whose HTTP/2 *client* was built on
-it. That client is pure C# now too, on the same framing and HPACK the server uses, so the binding
-had no callers left.
-
-`build-nghttp2-native.sh` built the native library it bound to. `Playground.Http2.Nghttp2` was its
-h2c sample; `Playground/Http2/Buffered` is the same server on the managed stack.
diff --git a/ioxide.slnx b/ioxide.slnx
index ca623ab..bc770cf 100644
--- a/ioxide.slnx
+++ b/ioxide.slnx
@@ -5,6 +5,7 @@
   
   
     
+    
     
     
     
@@ -55,6 +56,7 @@
 
   
     
+    
     
     
     
@@ -123,6 +125,7 @@
   
   
     
+    
     
   
 
diff --git a/dropped/build-nghttp2-native.sh b/scripts/build-nghttp2-native.sh
similarity index 100%
rename from dropped/build-nghttp2-native.sh
rename to scripts/build-nghttp2-native.sh
diff --git a/scripts/gen-docs-panes.py b/scripts/gen-docs-panes.py
index fd4febd..26216c5 100644
--- a/scripts/gen-docs-panes.py
+++ b/scripts/gen-docs-panes.py
@@ -271,6 +271,20 @@
         "; the difference is HTTP/2's shared connection window, which a handler "
         "that stops reading holds down for every other stream on the connection."),
+    "h2ng": (
+        "Http2/Nghttp2Buffered", "HTTP/2 · nghttp2", "ioxide + ioxide.nghttp2",
+        ["curl --http2-prior-knowledge http://127.0.0.1:8080/"],
+        "The same h2c server on the reference implementation. Kept because being battle-tested "
+        "is a property no amount of benchmarking substitutes for: nghttp2 is continuously fuzzed, "
+        "patched by people whose job it is when the next HTTP/2 CVE lands, and has a decade of "
+        "interop against every other stack. "
+        "What it does not have is the streaming: it is buffered only, so there is no "
+        "counterpart here to  "
+        "or , and its dispatch "
+        "loop still waits for each handler in turn. Measured as a client on this rig, the managed "
+        "stack runs 1.35×-1.39× it. Take this one when you want the reference "
+        "implementation's coverage; take "
+        " for everything else."),
     "h3": (
         "Http3/Nghttp3Request", "HTTP/3 · request streamed (nghttp3)", "ioxide + ioxide.ngtcp2 + ioxide.nghttp3",
         ["curl --http3-only -k https://127.0.0.1:8443/"],
diff --git a/src/protocols/ioxide.http2/Http2Connection.cs b/src/protocols/ioxide.http2/Http2Connection.cs
index 4a0d23c..42eeb56 100644
--- a/src/protocols/ioxide.http2/Http2Connection.cs
+++ b/src/protocols/ioxide.http2/Http2Connection.cs
@@ -12,9 +12,11 @@ namespace ioxide.http2;
 ///     new Http2Connection(conn).RunBufferedAsync(request => Http2Response.Text("hello"));
 /// 
 ///
-/// This is the only HTTP/2 in ioxide: the nghttp2 binding it began as an alternative to was
-/// retired once this measured level with it and then grew past it - see dropped/. The same
-/// framing and HPACK drive ioxide.httpclient's HTTP/2 client, pointed the other way round.
+/// The default HTTP/2 here, and the one the features land on: streamed responses, streamed
+/// request bodies and non-blocking dispatch are all this side. ioxide.nghttp2 remains as
+/// the battle-tested alternative - buffered only, and the reference implementation's coverage of
+/// the protocol's darker corners. Measured on this rig it runs 1.35x-1.39x the binding as a
+/// client; the same framing and HPACK drive ioxide.httpclient, pointed the other way round.
 ///
 /// It speaks to an  and knows nothing about TLS: hand it a
 /// TcpConnectionDualPipe for h2c or a TlsConnectionDualPipe for h2 over TLS, and the
diff --git a/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs
rename to src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Callbacks.cs
diff --git a/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
rename to src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
diff --git a/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
rename to src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
diff --git a/dropped/ioxide.nghttp2/Connection/Nghttp2Connection.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Connection/Nghttp2Connection.cs
rename to src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.cs
diff --git a/dropped/ioxide.nghttp2/Connection/Nghttp2Options.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Options.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Connection/Nghttp2Options.cs
rename to src/protocols/ioxide.nghttp2/Connection/Nghttp2Options.cs
diff --git a/dropped/ioxide.nghttp2/Http/CookieEnumerator.cs b/src/protocols/ioxide.nghttp2/Http/CookieEnumerator.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Http/CookieEnumerator.cs
rename to src/protocols/ioxide.nghttp2/Http/CookieEnumerator.cs
diff --git a/dropped/ioxide.nghttp2/Http/KeyValueList.cs b/src/protocols/ioxide.nghttp2/Http/KeyValueList.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Http/KeyValueList.cs
rename to src/protocols/ioxide.nghttp2/Http/KeyValueList.cs
diff --git a/dropped/ioxide.nghttp2/Http/Nghttp2Request.cs b/src/protocols/ioxide.nghttp2/Http/Nghttp2Request.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Http/Nghttp2Request.cs
rename to src/protocols/ioxide.nghttp2/Http/Nghttp2Request.cs
diff --git a/dropped/ioxide.nghttp2/Http/Nghttp2Response.cs b/src/protocols/ioxide.nghttp2/Http/Nghttp2Response.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Http/Nghttp2Response.cs
rename to src/protocols/ioxide.nghttp2/Http/Nghttp2Response.cs
diff --git a/dropped/ioxide.nghttp2/Interop/Nghttp2.cs b/src/protocols/ioxide.nghttp2/Interop/Nghttp2.cs
similarity index 100%
rename from dropped/ioxide.nghttp2/Interop/Nghttp2.cs
rename to src/protocols/ioxide.nghttp2/Interop/Nghttp2.cs
diff --git a/dropped/ioxide.nghttp2/ioxide.nghttp2.csproj b/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj
similarity index 98%
rename from dropped/ioxide.nghttp2/ioxide.nghttp2.csproj
rename to src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj
index fe1239f..cdd5b6b 100644
--- a/dropped/ioxide.nghttp2/ioxide.nghttp2.csproj
+++ b/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj
@@ -8,7 +8,7 @@
     ioxide.nghttp2
 
     ioxide.nghttp2
-    0.4.169
+    0.4.176
     MDA2AV
     HTTP/2 for the ioxide io_uring runtime: framing, HPACK and flow control from nghttp2, statically linked behind a small shim with no external dependencies beyond libc. Serves HTTP/2 over any TcpConnection - h2c with prior knowledge, or h2 over TLS via ALPN - and backs the HTTP/2 client in ioxide.httpclient from the same session code. nghttp2 is sans-I/O, so ioxide keeps the ring and the loop.
     MIT
diff --git a/dropped/ioxide.nghttp2/native/ioxide_nghttp2_shim.c b/src/protocols/ioxide.nghttp2/native/ioxide_nghttp2_shim.c
similarity index 100%
rename from dropped/ioxide.nghttp2/native/ioxide_nghttp2_shim.c
rename to src/protocols/ioxide.nghttp2/native/ioxide_nghttp2_shim.c
diff --git a/dropped/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so b/src/protocols/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so
similarity index 100%
rename from dropped/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so
rename to src/protocols/ioxide.nghttp2/runtimes/linux-x64/native/libioxide_nghttp2.so

From a0d249ece74dee0654dcc8622ae7bbaf18c8789f Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 16:37:57 +0100
Subject: [PATCH 13/14] fix(nghttp2): a slow handler no longer blocks every
 other stream

The binding kept the dispatch loop the managed stack had already been fixed
out of: DispatchReadyAsync awaited each handler in turn, so one request that
parked held up every other stream on that TCP connection - including responses
already submitted with nowhere to go. Two requests on one connection, one
sleeping a second, over nghttp:

                 /slow    /fast
  before         1.02s    1.02s
  after          1.01s    17.01ms

Handlers that answer synchronously stay inline, so they still submit in time
for the pass drain and allocate no Task.

The drain needed the guard first, and for the reason the managed stack needed a
whole write queue: a handler finishing late submits and drains from outside the
read loop, and two drains interleaving would write out of the single _egress
buffer while a flush was outstanding - which a PipeWriter refuses outright.
Here one flag is enough, because nghttp2 holds the queued frames itself: a
caller that arrives mid-drain sets _drainAgain and the in-flight drain loops
once more to pull what was just submitted.

The detached tail also has to do what the loop would have: submit, answer 500
and log if the handler threw, retire the request, and drain - nothing observes
that Task, so an escaping exception would leave the peer waiting on a stream
that never comes.

This is the prerequisite for streaming. Streaming on a dispatch loop that
blocks would mean a streamed response parking the whole connection.

Unit 36, Chaos 39, Http 38, E2E 46 pass.
---
 .../Connection/Nghttp2Connection.Egress.cs    | 50 +++++++++++----
 .../Nghttp2Connection.RunBuffered.cs          | 64 +++++++++++++++++--
 2 files changed, 97 insertions(+), 17 deletions(-)

diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
index ac60c81..2814e2c 100644
--- a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
+++ b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.Egress.cs
@@ -90,6 +90,12 @@ private static int WriteStatus(int status, Span destination)
     /// Pull everything nghttp2 has queued into the write slab and flush once. Looping until the
     /// session reports nothing left is what keeps a batch of responses to a single send.
     /// 
+    // One drain at a time. Both the read loop and a handler that finished late can start one, and a
+    // drain awaits a real flush - so without this they interleave writes out of the single _egress
+    // buffer, and a PipeWriter refuses a Write while a flush is outstanding anyway.
+    private bool _draining;
+    private bool _drainAgain;
+
     private async ValueTask FlushEgressAsync()
     {
         if (_handle == 0)
@@ -97,23 +103,45 @@ private async ValueTask FlushEgressAsync()
             return;
         }
 
-        bool staged = false;
+        if (_draining)
+        {
+            // A drain owns the pipe. Whatever was just submitted is sitting in nghttp2's own
+            // queue rather than in ours, so the loop below will pull it before it returns - which
+            // is why this needs no queue of its own, unlike the managed stack.
+            _drainAgain = true;
+            return;
+        }
 
-        while (true)
+        _draining = true;
+        try
         {
-            int produced = DrainOnce();
-            if (produced <= 0)
+            do
             {
-                break;
+                _drainAgain = false;
+
+                bool staged = false;
+                while (true)
+                {
+                    int produced = DrainOnce();
+                    if (produced <= 0)
+                    {
+                        break;
+                    }
+
+                    _pipe.Output.Write(_egress.AsSpan(0, produced));
+                    staged = true;
+                }
+
+                if (staged)
+                {
+                    await _pipe.Output.FlushAsync();
+                }
             }
-
-            _pipe.Output.Write(_egress.AsSpan(0, produced));
-            staged = true;
+            while (_drainAgain && _handle != 0);
         }
-
-        if (staged)
+        finally
         {
-            await _pipe.Output.FlushAsync();
+            _draining = false;
         }
     }
 
diff --git a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
index 6b96ba1..4548ac9 100644
--- a/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
+++ b/src/protocols/ioxide.nghttp2/Connection/Nghttp2Connection.RunBuffered.cs
@@ -88,18 +88,70 @@ private async ValueTask DispatchReadyAsync(Func inFlight;
+
             try
             {
-                Nghttp2Request request = pending.Freeze();
-                Nghttp2Response response = await handler(request);
-                SubmitResponse(pending.StreamId, response);
+                inFlight = handler(pending.Freeze());
             }
-            finally
+            catch
             {
-                // The arena backs the request's memories, so it can only go back once the handler
-                // has returned and the response is submitted (which copies).
                 pending.Dispose();
+                throw;
             }
+
+            // Answered synchronously, which nearly every handler does. Stay inline: the response is
+            // submitted in time for this pass's drain, so it still leaves with every other one, and
+            // there is no Task to allocate.
+            if (inFlight.IsCompletedSuccessfully)
+            {
+                try
+                {
+                    SubmitResponse(pending.StreamId, inFlight.Result);
+                }
+                finally
+                {
+                    // The arena backs the request's memories, so it can only go back once the
+                    // handler has returned and the response is submitted (which copies).
+                    pending.Dispose();
+                }
+                continue;
+            }
+
+            // It parked - a database, an upstream, a disk. Awaiting here would hold every OTHER
+            // stream on this connection behind it, including responses already submitted and
+            // waiting to go, because they all share this one dispatch loop and one TCP connection.
+            _ = CompleteAsync(inFlight, pending);
+        }
+    }
+
+    /// 
+    /// The tail of a handler that parked. Nothing awaits this, so everything the dispatch loop
+    /// would have done afterwards has to happen here: submitting, retiring the request, and
+    /// draining, since this pass's drain has long gone by.
+    /// 
+    private async Task CompleteAsync(ValueTask inFlight, PendingRequest pending)
+    {
+        try
+        {
+            SubmitResponse(pending.StreamId, await inFlight);
+        }
+        catch (Exception exception)
+        {
+            // Nobody can observe this task, so an escaping exception would vanish silently and the
+            // peer would wait on a stream that is never coming.
+            Console.Error.WriteLine(
+                $"[ioxide.nghttp2] request handler faulted: {exception.GetBaseException().Message}");
+
+            if (!IsBroken)
+            {
+                SubmitResponse(pending.StreamId, new Nghttp2Response { Status = 500 });
+            }
+        }
+        finally
+        {
+            pending.Dispose();
+            await FlushEgressAsync();
         }
     }
 }

From 62d65d29f9b779d34d01938a9e3057d2a72a5fd6 Mon Sep 17 00:00:00 2001
From: Diogo Martins 
Date: Mon, 10 Aug 2026 16:43:38 +0100
Subject: [PATCH 14/14] test(nghttp2): the binding is supported again, so it is
 tested again

Chaos covered only the managed Http2Connection, and the Http suite's h2 tests
moved to the managed client - so a package we just committed to supporting had
essentially no automated coverage, and the dispatch fix in the previous commit
had none at all.

The wire is identical, so H2cClient is shared and the same assaults point at
Nghttp2Connection: bad preface, oversize frame, unknown frame types, a frame
truncated mid-payload, and a CONTINUATION flood.

That last one is worth stating, because it answers what "battle-tested" buys in
something other than adjectives: nghttp2 refuses the flood with NO configuring,
where the managed server had to be taught MaxHeaderListSize this morning. The
same test, the same client, two implementations, one of which had the defence
already.

The head-of-line test asserts on ORDER rather than elapsed time - the first
response to come back must be /fast, not the /slow stream dispatched before it.
That is deterministic where a stopwatch is not, and it fails against the loop it
replaced: made blocking again, it reports "expected [3], got [1]".

Chaos 46, Unit 36, Http 38, E2E 46.
---
 tests/Ioxide.Tests.Chaos/H2cClient.cs         |  26 +++
 .../Ioxide.Tests.Chaos.csproj                 |   1 +
 tests/Ioxide.Tests.Chaos/Nghttp2ChaosTests.cs | 164 ++++++++++++++++++
 tests/Ioxide.Tests.Chaos/Program.cs           |   1 +
 4 files changed, 192 insertions(+)
 create mode 100644 tests/Ioxide.Tests.Chaos/Nghttp2ChaosTests.cs

diff --git a/tests/Ioxide.Tests.Chaos/H2cClient.cs b/tests/Ioxide.Tests.Chaos/H2cClient.cs
index 1d80b6d..617a637 100644
--- a/tests/Ioxide.Tests.Chaos/H2cClient.cs
+++ b/tests/Ioxide.Tests.Chaos/H2cClient.cs
@@ -20,6 +20,32 @@ public sealed class H2cClient : IDisposable
 
     public const byte RstStream = 0x3, Continuation = 0x9;
 
+    /// 
+    /// The stream id of the FIRST response to come back, whichever it is. Ordering rather than a
+    /// stopwatch is what makes a head-of-line test deterministic: if dispatch waits for each
+    /// handler in turn, the slow stream answers first because it was dispatched first.
+    /// 
+    public int AwaitFirstResponse(int timeoutMs = 8000)
+    {
+        long deadline = Environment.TickCount64 + timeoutMs;
+        while (Environment.TickCount64 < deadline)
+        {
+            if (!TryReadFrame(out byte type, out byte flags, out int sid, out _))
+            {
+                return -1;
+            }
+            if (type == Settings && (flags & Ack) == 0)
+            {
+                WriteFrame(Settings, Ack, 0, ReadOnlySpan.Empty);
+            }
+            else if (type == Headers)
+            {
+                return sid;
+            }
+        }
+        return -1;
+    }
+
     /// HEADERS that deliberately leaves the block OPEN, so CONTINUATION must follow.
     public void RequestHeadersOnly(int streamId, bool endHeaders = true, bool endStream = true)
         => WriteFrame(Headers, (byte)((endHeaders ? EndHeaders : 0) | (endStream ? EndStream : 0)),
diff --git a/tests/Ioxide.Tests.Chaos/Ioxide.Tests.Chaos.csproj b/tests/Ioxide.Tests.Chaos/Ioxide.Tests.Chaos.csproj
index c8d0d16..2e5d6b0 100644
--- a/tests/Ioxide.Tests.Chaos/Ioxide.Tests.Chaos.csproj
+++ b/tests/Ioxide.Tests.Chaos/Ioxide.Tests.Chaos.csproj
@@ -16,6 +16,7 @@
          server is the one surface it does not, so it is referenced directly. -->
     
     
+    
   
 
 
diff --git a/tests/Ioxide.Tests.Chaos/Nghttp2ChaosTests.cs b/tests/Ioxide.Tests.Chaos/Nghttp2ChaosTests.cs
new file mode 100644
index 0000000..2843d34
--- /dev/null
+++ b/tests/Ioxide.Tests.Chaos/Nghttp2ChaosTests.cs
@@ -0,0 +1,164 @@
+using ioxide;
+using ioxide.nghttp2;
+
+namespace Ioxide.Tests;
+
+/// 
+/// The same h2c assaults as , pointed at the nghttp2 binding instead of
+/// the managed server. It is a supported option again, so it is tested like one - and because the
+/// wire is identical, the client harness is shared and the two can be compared directly.
+///
+/// Two of these are the vectors the managed server had to be taught to defend against. Running
+/// them here answers what "battle-tested" is actually worth: the reference implementation should
+/// turn both away without being told to.
+/// 
+internal static class Nghttp2ChaosTests
+{
+    private static int StartH2c() => TestServer.Start(static async (_, conn) =>
+    {
+        try
+        {
+            await new Nghttp2Connection(conn).RunBufferedAsync(
+                static _ => new Nghttp2Response { Status = 200, Body = "ok"u8.ToArray() });
+        }
+        finally
+        {
+            conn.DecRef();
+        }
+    });
+
+    /// A server whose /slow handler parks for a second before answering.
+    private static int StartSlowH2c() => TestServer.Start(static async (_, conn) =>
+    {
+        try
+        {
+            await new Nghttp2Connection(conn).RunBufferedAsync(static async request =>
+            {
+                if (request.Path.Span.SequenceEqual("/slow"u8))
+                {
+                    await Task.Delay(1000);
+                }
+                return new Nghttp2Response { Status = 200, Body = "ok"u8.ToArray() };
+            });
+        }
+        finally
+        {
+            conn.DecRef();
+        }
+    });
+
+    private static void AssertServes(int port)
+    {
+        using var client = new H2cClient(port);
+        client.Open();
+        client.Request(streamId: 1);
+        Assert.True(client.AwaitResponse(streamId: 1), "server did not answer a well-formed h2c request");
+    }
+
+    public static void Register(Runner runner)
+    {
+        runner.Test("nghttp2: a well-formed request is answered", () => AssertServes(StartH2c()));
+
+        runner.Test("nghttp2: a bad connection preface is rejected, server survives", () =>
+        {
+            int port = StartH2c();
+
+            using (var bad = new H2cClient(port))
+            {
+                bad.WriteRaw("NOT-AN-HTTP2-PREFACE\r\n\r\n"u8);
+                bad.WriteRaw(new byte[64]);
+            }
+
+            AssertServes(port);
+        });
+
+        runner.Test("nghttp2: a frame larger than the max is rejected, server survives", () =>
+        {
+            int port = StartH2c();
+
+            using (var bad = new H2cClient(port))
+            {
+                bad.Open();
+                // Declare 1 MiB of DATA and send none: past SETTINGS_MAX_FRAME_SIZE, and a lie.
+                bad.WriteFrameHeader(0x0, 0, 1, 1024 * 1024, ReadOnlySpan.Empty);
+            }
+
+            AssertServes(port);
+        });
+
+        runner.Test("nghttp2: unknown frame types are ignored, the request still answers", () =>
+        {
+            int port = StartH2c();
+
+            using var client = new H2cClient(port);
+            client.Open();
+            client.WriteFrame(0x2A, flags: 0, streamId: 0, "ignore me"u8);   // no such frame type
+            client.Request(streamId: 1);
+
+            Assert.True(client.AwaitResponse(streamId: 1), "an unknown frame type broke the connection");
+        });
+
+        runner.Test("nghttp2: a frame truncated mid-payload is handled, server survives", () =>
+        {
+            int port = StartH2c();
+
+            using (var bad = new H2cClient(port))
+            {
+                bad.Open();
+                bad.WriteFrameHeader(0x0, 0, 1, declaredLen: 256, actual: new byte[32]);
+            }
+
+            AssertServes(port);
+        });
+
+        runner.Test("nghttp2: a CONTINUATION flood is refused, server survives", () =>
+        {
+            // The managed server needed MaxHeaderListSize taught to it. nghttp2 has defended
+            // against this since the flood was disclosed, so this should pass with no configuring -
+            // which is precisely the argument for keeping the binding around.
+            int port = StartH2c();
+
+            using (var client = new H2cClient(port))
+            {
+                client.Open();
+                client.RequestHeadersOnly(streamId: 1, endHeaders: false, endStream: false);
+
+                byte[] filler = new byte[4096];
+                byte seen = 0;
+                for (int i = 0; i < 128 && seen == 0; i++)
+                {
+                    try
+                    {
+                        client.WriteFrame(H2cClient.Continuation, flags: 0, streamId: 1, filler);
+                    }
+                    catch (IOException)
+                    {
+                        seen = 0xFF;   // closed on us, which is a refusal
+                        break;
+                    }
+                    seen = client.AwaitAnyOf([0x7], timeoutMs: 50);   // GOAWAY
+                }
+
+                Assert.True(seen != 0, "nghttp2 accepted an unbounded CONTINUATION block");
+            }
+
+            AssertServes(port);
+        });
+
+        runner.Test("nghttp2: a slow handler does not block another stream", () =>
+        {
+            // The binding kept the blocking dispatch loop long after the managed stack lost it:
+            // DispatchReadyAsync awaited each handler in turn, so /fast could not answer until
+            // /slow had. Asserting on ORDER rather than elapsed time keeps this deterministic -
+            // with a blocking loop the first response is always the stream dispatched first.
+            int port = StartSlowH2c();
+
+            using var client = new H2cClient(port);
+            client.Open();
+            client.Request(streamId: 1, path: "/slow");
+            client.Request(streamId: 3, path: "/fast");
+
+            Assert.Equal(3, client.AwaitFirstResponse());
+        });
+    }
+}
diff --git a/tests/Ioxide.Tests.Chaos/Program.cs b/tests/Ioxide.Tests.Chaos/Program.cs
index de3994a..b67bddb 100644
--- a/tests/Ioxide.Tests.Chaos/Program.cs
+++ b/tests/Ioxide.Tests.Chaos/Program.cs
@@ -20,6 +20,7 @@ private static int Main()
         TcpChaosTests.Register(runner);
         TlsChaosTests.Register(runner);
         H2ChaosTests.Register(runner);
+        Nghttp2ChaosTests.Register(runner);
         QuicChaosTests.Register(runner);
         H3ChaosTests.Register(runner);