From c73668a855d2529e4aee72037876fbf5925250d3 Mon Sep 17 00:00:00 2001 From: Diogo Martins Date: Mon, 10 Aug 2026 21:59:42 +0100 Subject: [PATCH] fix(nghttp3): a streamed handler that awaits anything no longer hangs Nghttp3ResponseWriter.FlushAsync parked on a pass waiter, and pass waiters are released only by DrainStreamed - a loop that runs while the peer is sending. A peer waiting for our response sends nothing, so a handler that resumed from outside that loop waited for a pass its own output was supposed to provoke. Which is every real handler. The existing streamed tests, the samples and the benchmarks all write their chunks in a tight loop, so the handler never leaves the pass that dispatched it and the drain resumes it inline. Await a file read, a query or an upstream first and you come back somewhere else: the first chunk stalls with the bytes sitting in nghttp3, and the second hangs outright waiting for the first to be taken. DrainStreamed now records that it is on the stack, and a flush that finds no drain running drives one itself instead of parking. Inside the pass nothing changes - the loop still resumes writers inline and a one-chunk response still pays no reactor round trip, which was the point of the original design. The regression test awaits between chunks, which no existing test does. Without the fix it reports "expected [chunkchunkchunkchunk], got []" - the same empty body a static file server hit when it read the file off io_uring mid-response. Also bumps all eleven packages to 0.4.178, since the fix is only useful to anyone consuming it from NuGet. Unit 36, Chaos 47, Http 38, E2E 47. --- src/clients/ioxide.file/ioxide.file.csproj | 2 +- .../ioxide.httpclient.csproj | 2 +- src/clients/ioxide.pg/ioxide.pg.csproj | 2 +- src/clients/ioxide.redis/ioxide.redis.csproj | 2 +- src/ioxide/ioxide.csproj | 2 +- .../ioxide.http2/ioxide.http2.csproj | 2 +- .../ioxide.http3/ioxide.http3.csproj | 2 +- .../ioxide.nghttp2/ioxide.nghttp2.csproj | 2 +- .../Connection/Nghttp3Connection.Streamed.cs | 47 +++++++++++++++++++ .../Http/Nghttp3ResponseWriter.cs | 2 + .../ioxide.nghttp3/ioxide.nghttp3.csproj | 2 +- .../ioxide.ngtcp2/ioxide.ngtcp2.csproj | 2 +- .../ioxide.Kestrel/ioxide.Kestrel.csproj | 2 +- tests/Ioxide.Tests.E2E/Protocols/H3Tests.cs | 42 +++++++++++++++++ 14 files changed, 102 insertions(+), 11 deletions(-) diff --git a/src/clients/ioxide.file/ioxide.file.csproj b/src/clients/ioxide.file/ioxide.file.csproj index dd8e597..0a98f9b 100644 --- a/src/clients/ioxide.file/ioxide.file.csproj +++ b/src/clients/ioxide.file/ioxide.file.csproj @@ -8,7 +8,7 @@ ioxide.file ioxide.file - 0.4.177 + 0.4.178 MDA2AV File serving for the ioxide io_uring runtime: immutable asset snapshots with baked responses, pooled positional ring reads, atomic reloads. MIT diff --git a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj index 467920a..c8d825b 100644 --- a/src/clients/ioxide.httpclient/ioxide.httpclient.csproj +++ b/src/clients/ioxide.httpclient/ioxide.httpclient.csproj @@ -8,7 +8,7 @@ ioxide.httpclient ioxide.httpclient - 0.4.177 + 0.4.178 MDA2AV The ring-native HTTP client for the ioxide io_uring runtime: HTTP/1.1, HTTP/2 and HTTP/3 behind one API, with the protocol chosen per origin via Alt-Svc. One package - the h1 parser, a pure-C# HTTP/2 client on ioxide.http2's framing, the nghttp3 bridge, client-side TLS (SNI, ALPN and certificate verification) for https:// origins, and the negotiating client - sharing one set of message types. Every response resumes the awaiting handler inline on its own reactor thread. MIT diff --git a/src/clients/ioxide.pg/ioxide.pg.csproj b/src/clients/ioxide.pg/ioxide.pg.csproj index 583a140..cfd3c7b 100644 --- a/src/clients/ioxide.pg/ioxide.pg.csproj +++ b/src/clients/ioxide.pg/ioxide.pg.csproj @@ -8,7 +8,7 @@ ioxide.pg ioxide.pg - 0.4.177 + 0.4.178 MDA2AV Postgres driver for the ioxide io_uring runtime: pooled ring-native connections per reactor, ring-native connect and handshake, inline completion resume. MIT diff --git a/src/clients/ioxide.redis/ioxide.redis.csproj b/src/clients/ioxide.redis/ioxide.redis.csproj index 01ffc0f..ea45241 100644 --- a/src/clients/ioxide.redis/ioxide.redis.csproj +++ b/src/clients/ioxide.redis/ioxide.redis.csproj @@ -8,7 +8,7 @@ ioxide.redis ioxide.redis - 0.4.177 + 0.4.178 MDA2AV Redis client for the ioxide io_uring runtime: pooled ring-native connections per reactor, full RESP2 protocol, a generic command API plus typed helpers (strings, keys, hashes, lists, sets, sorted sets, pub/sub, transactions, scripting), and pipelining. Inline completion resume. MIT diff --git a/src/ioxide/ioxide.csproj b/src/ioxide/ioxide.csproj index 7880a54..1e750fc 100644 --- a/src/ioxide/ioxide.csproj +++ b/src/ioxide/ioxide.csproj @@ -8,7 +8,7 @@ ioxide ioxide - 0.4.177 + 0.4.178 MDA2AV A shared-nothing io_uring runtime for .NET: one ring per reactor thread, inline completions, zero native dependencies. The engine - reactor, connection, and the IRingHost client seam. Includes TLS termination: the OpenSSL handshake driven over the ring, then kernel TLS (kTLS) transmit offload, so handlers keep writing plaintext. TLS needs OpenSSL 3 and the Linux tls module; nothing else does, and neither is loaded unless you use it. MIT diff --git a/src/protocols/ioxide.http2/ioxide.http2.csproj b/src/protocols/ioxide.http2/ioxide.http2.csproj index f45a6cf..4a62270 100644 --- a/src/protocols/ioxide.http2/ioxide.http2.csproj +++ b/src/protocols/ioxide.http2/ioxide.http2.csproj @@ -8,7 +8,7 @@ ioxide.http2 ioxide.http2 - 0.4.177 + 0.4.178 MDA2AV Pure-C# HTTP/2 for the ioxide io_uring runtime: framing, HPACK (static and dynamic tables, Huffman) and flow control, with zero native code. Serves h2c with prior knowledge and h2 over TLS by ALPN, buffered or streamed in either direction, and the same framing drives ioxide.httpclient's HTTP/2 client. MIT diff --git a/src/protocols/ioxide.http3/ioxide.http3.csproj b/src/protocols/ioxide.http3/ioxide.http3.csproj index d822da7..c01517c 100644 --- a/src/protocols/ioxide.http3/ioxide.http3.csproj +++ b/src/protocols/ioxide.http3/ioxide.http3.csproj @@ -8,7 +8,7 @@ ioxide.http3 ioxide.http3 - 0.4.177 + 0.4.178 MDA2AV Pure C# HTTP/3 for the ioxide io_uring runtime: frame parsing, QPACK (static table + Huffman) and request dispatch with zero native dependencies. Rides any QuicConnection via its stream read surface - engine-agnostic, drop-in alternative to ioxide.nghttp3. MIT diff --git a/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj b/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj index a066fee..c496b93 100644 --- a/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj +++ b/src/protocols/ioxide.nghttp2/ioxide.nghttp2.csproj @@ -8,7 +8,7 @@ ioxide.nghttp2 ioxide.nghttp2 - 0.4.177 + 0.4.178 MDA2AV HTTP/2 for the ioxide io_uring runtime: framing, HPACK and flow control from nghttp2, statically linked behind a small shim with no external dependencies beyond libc. Serves HTTP/2 over any TcpConnection - h2c with prior knowledge, or h2 over TLS via ALPN - and backs the HTTP/2 client in ioxide.httpclient from the same session code. nghttp2 is sans-I/O, so ioxide keeps the ring and the loop. MIT diff --git a/src/protocols/ioxide.nghttp3/Connection/Nghttp3Connection.Streamed.cs b/src/protocols/ioxide.nghttp3/Connection/Nghttp3Connection.Streamed.cs index 5c681c1..d4af26e 100644 --- a/src/protocols/ioxide.nghttp3/Connection/Nghttp3Connection.Streamed.cs +++ b/src/protocols/ioxide.nghttp3/Connection/Nghttp3Connection.Streamed.cs @@ -126,7 +126,44 @@ public async Task RunStreamedResponseAsync(Func + // True while DrainStreamed is on the stack. A writer that flushes from inside it may park and + // be resumed by the loop; one that flushes from anywhere else has to drive the drain itself, + // because no loop is running to do it. + private bool _inStreamedDrain; + private void DrainStreamed() + { + if (_inStreamedDrain) + { + return; // already draining; the loop below picks up whatever was just staged + } + + _inStreamedDrain = true; + try + { + DrainStreamedCore(); + } + finally + { + _inStreamedDrain = false; + } + } + + /// + /// Drive egress for a writer that resumed OUTSIDE the read pass - after a file read, a database + /// call, an upstream response. Inside the pass this does nothing, because the drain loop is + /// already running and will pump what was just staged; that path stays exactly as it was, which + /// is what keeps a one-chunk response from paying a reactor round trip it does not need. + /// + internal void PumpIfOutsidePass() + { + if (!_inStreamedDrain && !_protocolFailed) + { + DrainStreamed(); + } + } + + private void DrainStreamedCore() { while (!_protocolFailed) { @@ -282,6 +319,16 @@ internal Task PumpAsync() return Task.CompletedTask; } + // Outside the drain loop there is nobody to release a pass waiter: the loop only runs while + // the peer is sending, and a peer waiting for our response sends nothing. Parking here is + // what made a handler that awaits anything - a file read, a query, an upstream - stall on + // its first chunk and hang on its second. Drive the drain instead and carry on. + if (!_inStreamedDrain) + { + DrainStreamed(); + return Task.CompletedTask; + } + // NOT RunContinuationsAsynchronously: everything on a reactor resumes inline on the // reactor thread, and a parked writer is no different. Completing this runs the handler // to its next park right there, so the drain loop needs no scheduler round trip to let it diff --git a/src/protocols/ioxide.nghttp3/Http/Nghttp3ResponseWriter.cs b/src/protocols/ioxide.nghttp3/Http/Nghttp3ResponseWriter.cs index 118255b..28edeba 100644 --- a/src/protocols/ioxide.nghttp3/Http/Nghttp3ResponseWriter.cs +++ b/src/protocols/ioxide.nghttp3/Http/Nghttp3ResponseWriter.cs @@ -150,6 +150,7 @@ public async ValueTask FlushAsync() PromoteStagedChunk(); _connection.ResumeStreamedResponse(_streamId); + _connection.PumpIfOutsidePass(); } /// @@ -182,6 +183,7 @@ public async ValueTask CompleteAsync() // nothing to keep the handler around for. _completed = true; _connection.ResumeStreamedResponse(_streamId); + _connection.PumpIfOutsidePass(); } /// diff --git a/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj b/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj index 2e1b337..49c82e1 100644 --- a/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj +++ b/src/protocols/ioxide.nghttp3/ioxide.nghttp3.csproj @@ -8,7 +8,7 @@ ioxide.nghttp3 ioxide.nghttp3 - 0.4.177 + 0.4.178 MDA2AV HTTP/3 layer for the ioxide io_uring runtime: nghttp3 (H3 + QPACK) bundled as a single self-contained native library with no external dependencies. Rides any QuicConnection via its stream read surface - engine-agnostic, no ioxide.ngtcp2 dependency. MIT diff --git a/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj b/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj index 0b63487..6d18be9 100644 --- a/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj +++ b/src/protocols/ioxide.ngtcp2/ioxide.ngtcp2.csproj @@ -8,7 +8,7 @@ ioxide.ngtcp2 ioxide.ngtcp2 - 0.4.177 + 0.4.178 MDA2AV QUIC engine for the ioxide io_uring runtime: ngtcp2 + picotls bundled as a single self-contained native library (only system dependency: libcrypto.so.3 / OpenSSL 3.x). Plugs into the reactor's QUIC transport via QuicConnection. Server side; engine bindings in progress. MIT diff --git a/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj b/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj index 7f8cd84..7e1eafb 100644 --- a/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj +++ b/src/serving/ioxide.Kestrel/ioxide.Kestrel.csproj @@ -8,7 +8,7 @@ ioxide.Kestrel ioxide.Kestrel - 0.4.177 + 0.4.178 MDA2AV ASP.NET Core Kestrel transport backed by the ioxide io_uring runtime: one reactor (ring) per core, SO_REUSEPORT load-balanced, with Kestrel's HTTP request loop pinned to the reactor thread. Drop-in via UseIoxide(). MIT diff --git a/tests/Ioxide.Tests.E2E/Protocols/H3Tests.cs b/tests/Ioxide.Tests.E2E/Protocols/H3Tests.cs index eeb4e0e..df1e747 100644 --- a/tests/Ioxide.Tests.E2E/Protocols/H3Tests.cs +++ b/tests/Ioxide.Tests.E2E/Protocols/H3Tests.cs @@ -79,6 +79,48 @@ public static void Register(Runner runner) Assert.Equal("got 600000", text); }); + runner.Test("h3: a streamed response whose handler awaits between chunks still arrives", () => + { + // Every other streamed test writes its chunks in a tight loop, so the handler never + // leaves the pass that dispatched it and DrainStreamed resumes it inline. A REAL + // handler awaits something first - a file read, a query, an upstream - and comes back + // outside that pass. There the flush used to park on a pass waiter that only inbound + // packets release, while the peer sat waiting for the response that would have + // provoked them: first chunk stalled, second hung outright. + (string certPath, string keyPath) = TestCert.Ensure(); + using var engine = new QuicEngine(certPath, keyPath, cidLength: 8); + + (_, int udpPort) = TestServer.StartDatagram( + onDatagram: null, + quicFactory: engine.CreateFactory(), + quicHandle: static (_, conn) => new Nghttp3Connection(conn).RunStreamedResponseAsync( + static async (_, writer) => + { + writer.WriteHeaders(new Nghttp3Response { Status = 200 }); + + for (int i = 0; i < 4; i++) + { + // The point of the test. Task.Yield is the cheapest way to leave the + // dispatch pass; a file read or a database call lands in the same place. + await Task.Yield(); + + "chunk"u8.CopyTo(writer.GetSpan(5)); + writer.Advance(5); + await writer.FlushAsync(); + } + + await writer.CompleteAsync(); + })); + + using var client = new H3TestClient("127.0.0.1", udpPort); + client.Connect(); + Assert.True(client.CompleteHandshake(timeoutMs: 5000), "handshake did not complete"); + + (int status, string body) = client.Get("/streamed", timeoutMs: 10_000); + Assert.Equal(200, status); + Assert.Equal("chunkchunkchunkchunk", body); + }); + runner.Test("h3: buffered-async handler (whole body in req.Body, handler may await)", () => { (string certPath, string keyPath) = TestCert.Ensure();