Skip to content

tls: mutual TLS on the TCP side, for both OpenSSL and kTLS #185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MDA2AV merged 1 commit into from
Aug 12, 2026
Merged

tls: mutual TLS on the TCP side, for both OpenSSL and kTLS #185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions Playground/Tls/MtlsKtlsPipes/Playground.Tls.MtlsKtlsPipes.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net11.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<RootNamespace>Playground.Tls.MtlsKtlsPipes</RootNamespace>
<AssemblyName>Playground.Tls.MtlsKtlsPipes</AssemblyName>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="../../Shared/Playground.Shared.csproj" />
<ProjectReference Include="../../../src/ioxide/ioxide.csproj" />
</ItemGroup>

</Project>
226 changes: 226 additions & 0 deletions Playground/Tls/MtlsKtlsPipes/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
using System.Buffers;
using System.IO.Pipelines;
using System.Text;
using ioxide;
using ioxide.tls;
using Playground.Shared;

// ─────────────────────────────────────────────────────────────────────────────────────────────
// tls-mtls-ktls-pipes - TLS where the CLIENT proves who it is too, served through an
// IDuplexPipe, with the KERNEL encrypting outbound records.
//
// sudo modprobe tls # once per boot; kTLS needs the module
// PLAYGROUND_CLIENT_CA=ca.crt dotnet run -c Release --project Playground/Tls/MtlsKtlsPipes
// curl -k --cert client.crt --key client.key https://127.0.0.1:8443/
// curl -k https://127.0.0.1:8443/ # no certificate: answered as anonymous
//
// Make a CA and a certificate it signs:
//
// openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -out ca.crt -days 365 \
// -subj "/CN=my CA" -addext "basicConstraints=critical,CA:TRUE"
// openssl req -newkey rsa:2048 -nodes -keyout client.key -out client.csr -subj "/CN=alice"
// printf 'extendedKeyUsage=clientAuth\n' > c.cnf
// openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
// -out client.crt -days 365 -extfile c.cnf
//
// ClientCaPath is what turns this on. Leave it null and nothing is requested - the handshake is
// exactly the one Playground/Tls/KtlsPipes performs.
//
// RequireClientCertificate is the interesting knob. OFF (the default) is the mixed port: anyone
// connects, and the HANDLER decides what an unauthenticated peer may reach, which is why
// TlsSession.PeerSubject exists. ON refuses at the handshake, before a byte of request is read -
// cheaper, and blunter: there is no public route left.
//
// Either way a certificate that IS offered gets verified. "Optional" governs presenting nothing,
// not presenting anything.
//
// This file is Playground/Tls/MtlsOpenSslPipes with KernelTx = true and nothing else changed -
// diff them. That mTLS survives the swap untouched is the thing worth seeing: client
// authentication belongs to the handshake, and kTLS only takes over afterwards.
// Needs: ioxide
// ─────────────────────────────────────────────────────────────────────────────────────────────

// ── Knobs ────────────────────────────────────────────────────────────────────────────────────
// Edit these. That is the whole mechanism - there is no config file and nothing else to find.

ushort port = 8443; // https://127.0.0.1:8443/
int reactors = Environment.ProcessorCount; // one ring per reactor, one reactor per core
int bodyBytes = 8 * 1024; // TLS cost is per-byte, so a 2-byte "ok" would hide it

Env.Override(ref port, ref reactors, ref bodyBytes);

// The server's own certificate and key. Null generates a self-signed pair on first run.
string? certOverride = null;
string? keyOverride = null;

Env.OverrideCert(ref certOverride, ref keyOverride);

// The CA that CLIENT certificates are checked against - the switch that turns mTLS on.
string? clientCaPath = Environment.GetEnvironmentVariable("PLAYGROUND_CLIENT_CA");

// Refuse a client offering no certificate, during the handshake. Off by default: see the header.
bool requireClientCertificate = Environment.GetEnvironmentVariable("PLAYGROUND_REQUIRE_CLIENT_CERT") == "1";

// Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor.
bool incrementalBuffers = false;

Env.OverrideIncremental(ref incrementalBuffers);
// ─────────────────────────────────────────────────────────────────────────────────────────────

if (clientCaPath is null)
{
Console.Error.WriteLine(
"set PLAYGROUND_CLIENT_CA to a PEM bundle of the CA that signs your client certificates.");
return 1;
}

(string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride);

var config = new ServerConfig
{
ReactorCount = reactors, // io_uring rings/threads - one per core
RingEntries = 8192, // SQ/CQ depth per ring
DualStack = false, // true = one IPv6 socket also accepts IPv4-mapped
RecvBufferSize = 32 * 1024, // bytes per shared recv buffer
RecvSlots = 4096, // shared recv buffer-ring depth
Incremental = incrementalBuffers ? new IncrementalOptions { MaxConnections = 1024, RecvSlots = 8, RecvBufferSize = 16 * 1024 } : null,
Udp = null, // no raw UDP sockets (TCP-only server)
Quic = null, // no QUIC transport - see Http3/*
Tcp = new TcpOptions
{
Port = port,
ExtraPorts = [], // extra listener ports (one handler, several doors)
ListenBacklog = 1024, // accept-queue depth per SO_REUSEPORT listener
WriteSlabSize = 16 * 1024, // per-connection write buffer before overflow kicks in
PoolMax = 1024, // pooled connection objects kept per reactor
WriteOverflow = WriteOverflowStrategy.Grow, // Grow = realloc one slab; Segmented = chain + vectored SENDMSG
ZeroCopySend = false, // SEND_ZC: kernel copies less, wins on large writes
RecvQueueEntries = 64, // per-connection recv completion queue depth
},
};

var tlsOptions = new TlsOptions
{
CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory)
KeyPath = keyPath, // PEM key file (or KeyPem for in-memory)
Alpn = ["http/1.1"], // protocols this port serves, most-preferred first

ClientCaPath = clientCaPath, // trust anchors for CLIENT certificates (or ClientCaPem)
RequireClientCertificate = requireClientCertificate, // refuse a client with none, at the handshake

// The one line that differs from Playground/Tls/MtlsOpenSslPipes. The kernel encrypts outbound
// records, which needs `sudo modprobe tls` and pins this port to TLS 1.3, one ciphersuite, and
// no session resumption.
//
// It does NOT change anything above it. The client certificate is exchanged and verified during
// the HANDSHAKE, which OpenSSL performs either way - the kernel only takes over record crypto
// afterwards - so ClientCaPath, RequireClientCertificate and PeerSubject behave identically.
KernelTx = true,
KernelRx = false, // kTLS receive (experimental; requires KernelTx)
};

byte[] body = new byte[bodyBytes];
"ioxide-mtls "u8.CopyTo(body);
for (int i = "ioxide-mtls "u8.Length; i < bodyBytes; i++)
{
body[i] = (byte)('a' + (i % 26));
}

// What an unauthenticated peer gets. Only reachable with RequireClientCertificate off - with it on
// that client never completed a handshake, so nothing here ever runs for it.
const string denied = "client certificate required";
byte[] forbidden = Encoding.ASCII.GetBytes(
$"HTTP/1.1 403 Forbidden\r\nContent-Length: {denied.Length}\r\n\r\n{denied}");

var threads = new Thread[config.ReactorCount];

for (int i = 0; i < threads.Length; i++)
{
var reactor = new Reactor(i, config);

reactor.OnStart = r => TlsService.Start(r, tlsOptions);

reactor.TcpHandle = async (r, conn) =>
{
TlsSession? tls = null;
try
{
tls = await r.GetService<TlsService>().AcceptAsync(conn);

// Who connected. Null means the peer offered no certificate, which only happens when
// RequireClientCertificate is off - a certificate that failed to verify never reaches
// here, because that fails the handshake.
//
// This is the whole point of the feature: enforcing an identity is half of it, and a
// server that can only enforce cannot authorise.
byte[] response = tls.PeerSubject is { } subject
? [.. Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Length: {bodyBytes}\r\nX-Client: {subject}\r\n\r\n"),
.. body]
: forbidden;

await using var pipe = new TlsConnectionDualPipe(conn, tls, ownsSession: false);

while (true)
{
ReadResult read = await pipe.Input.ReadAsync();

// Answer per REQUEST, not per read. TLS hands back RECORDS, so one request split
// across two records would otherwise draw two responses.
int answered = 0;
SequencePosition consumed = read.Buffer.Start;

var reader = new SequenceReader<byte>(read.Buffer);
while (reader.TryReadTo(out ReadOnlySequence<byte> _, "\r\n\r\n"u8, advancePastDelimiter: true))
{
consumed = reader.Position;
answered++;
}

// Consumed only whole requests; examined everything, so a partial head parks
// until more arrives instead of spinning on the same bytes.
pipe.Input.AdvanceTo(consumed, read.Buffer.End);

for (int n = 0; n < answered; n++)
{
// The identity is a property of the CONNECTION, so it is the same for every
// request on it - decided once, above the loop, not per request.
pipe.Output.Write(response);
}

if (answered > 0)
{
await pipe.Output.FlushAsync();
}

if (read.IsCompleted || read.IsCanceled)
{
return;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine($"[tls-mtls-ktls-pipes] connection failed: {e.Message}");
}
finally
{
tls?.Dispose();
conn.DecRef();
}
};

threads[i] = new Thread(reactor.Run) { Name = $"reactor-{i}" };
threads[i].Start();
}

Console.WriteLine($"[tls-mtls-ktls-pipes] {config.ReactorCount} reactors on :{config.Tcp!.Port}, "
+ $"client CA {clientCaPath}, "
+ $"{(requireClientCertificate ? "certificate REQUIRED" : "certificate optional")}, tx=kernel");

foreach (Thread thread in threads)
{
thread.Join();
}

return 0;
18 changes: 18 additions & 0 deletions Playground/Tls/MtlsOpenSslPipes/Playground.Tls.MtlsOpenSslPipes.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net11.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<RootNamespace>Playground.Tls.MtlsOpenSslPipes</RootNamespace>
<AssemblyName>Playground.Tls.MtlsOpenSslPipes</AssemblyName>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="../../Shared/Playground.Shared.csproj" />
<ProjectReference Include="../../../src/ioxide/ioxide.csproj" />
</ItemGroup>

</Project>
Loading
Loading