diff --git a/Playground/Clients/File/Program.cs b/Playground/Clients/File/Program.cs index e990c1d8..b22f0efa 100644 --- a/Playground/Clients/File/Program.cs +++ b/Playground/Clients/File/Program.cs @@ -59,6 +59,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideFile(ref dir, ref buffered, ref tlsMode); @@ -70,7 +71,7 @@ Env.Override(ref port, ref reactors); // ───────────────────────────────────────────────────────────────────────────────────────────── -// The constraint this sample exists to show, enforced instead of described: with OpenSSL owning +// The constraint this sample exists to show, enforced instead of described: with BoringSSL owning // transmit the slab must hold CIPHERTEXT, so a file read straight into it would go out in the // clear. There is no slab path for that backend - refuse rather than serve cleartext. if (tlsMode == "openssl" && !buffered) @@ -139,8 +140,8 @@ // The whole point: with transmit in the kernel the slab carries PLAINTEXT and the // kernel makes the records, so ReadFileAsync may still read the file straight into - // it. With this false, OpenSSL encrypts and the slab must hold ciphertext - which - // is why the slab path is refused above. Receive is OpenSSL either way. + // it. With this false, BoringSSL encrypts and the slab must hold ciphertext - which + // is why the slab path is refused above. Receive is BoringSSL either way. KernelTx = ktls, }); } @@ -333,9 +334,9 @@ static unsafe void WriteNative(TcpConnection conn, TlsSession? tls, nint data, i => Emit(conn, tls, new ReadOnlySpan((void*)data, length)); // Every response byte goes through here. TlsSession.Write is correct under BOTH backends - it -// writes plaintext to the slab when the kernel encrypts, and encrypts into the slab when OpenSSL +// writes plaintext to the slab when the kernel encrypts, and encrypts into the slab when BoringSSL // does - so no call site has to know which one is in play. A bare conn.Write would be right only -// for cleartext and kTLS, and silently wrong for OpenSSL. +// for cleartext and kTLS, and silently wrong for BoringSSL. static void Emit(TcpConnection conn, TlsSession? tls, ReadOnlySpan bytes) { if (tls is null) conn.Write(bytes); diff --git a/Playground/Clients/GatewayTlsPipes/Program.cs b/Playground/Clients/GatewayTlsPipes/Program.cs index 075bf6e9..e8518000 100644 --- a/Playground/Clients/GatewayTlsPipes/Program.cs +++ b/Playground/Clients/GatewayTlsPipes/Program.cs @@ -38,6 +38,7 @@ // A real PEM pair for the front door, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/Dockerfile b/Playground/Dockerfile index 753a16a4..e9f95fba 100644 --- a/Playground/Dockerfile +++ b/Playground/Dockerfile @@ -33,8 +33,8 @@ RUN csproj="$(ls Playground/${SAMPLE}/*.csproj)" \ && dotnet publish "$csproj" -c Release -o /app \ && ln -s "/app/$(basename "$csproj" .csproj)" /app/playground -# The ngtcp2 bundle's only external dependency is OpenSSL 3 (libcrypto.so.3), which the Debian-based -# runtime image already carries. +# The native bundles (BoringSSL, ngtcp2, nghttp2, nghttp3) depend on nothing beyond libc; the runtime +# image's own OpenSSL is .NET's, for its crypto APIs. FROM mcr.microsoft.com/dotnet/runtime:11.0-preview WORKDIR /app COPY --from=build /app . diff --git a/Playground/Http2/Sni/Program.cs b/Playground/Http2/Sni/Program.cs index 2fdec433..73dc7bf3 100644 --- a/Playground/Http2/Sni/Program.cs +++ b/Playground/Http2/Sni/Program.cs @@ -52,6 +52,7 @@ // A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); @@ -116,7 +117,7 @@ var reactor = new Reactor(i, config); // One TlsService per reactor. It owns the contexts - the default and one per name - and picks - // between them during each handshake on this ring. Each entry costs one OpenSSL context, built + // between them during each handshake on this ring. Each entry costs one BoringSSL context, built // once at startup; choosing one per handshake is a lookup that allocates nothing, so serving // ten names costs what serving one does. reactor.OnStart = r => TlsService.Start(r, tlsOptions); diff --git a/Playground/Http2/SslStream/Program.cs b/Playground/Http2/SslStream/Program.cs index bc016cba..0b067e0d 100644 --- a/Playground/Http2/SslStream/Program.cs +++ b/Playground/Http2/SslStream/Program.cs @@ -34,6 +34,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/Http2/Tls/Program.cs b/Playground/Http2/Tls/Program.cs index 62b36d8c..4e33322c 100644 --- a/Playground/Http2/Tls/Program.cs +++ b/Playground/Http2/Tls/Program.cs @@ -41,9 +41,10 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // Hand OUTBOUND encryption to the kernel: the handler writes plaintext and the kernel makes the -// records. Off by default - OpenSSL both ways is the portable path, and on loopback the kernel +// records. Off by default - BoringSSL both ways is the portable path, and on loopback the kernel // is not faster. Its real payoff is sendfile and NIC offload, which a benchmark here cannot see. bool kernelTx = false; @@ -109,7 +110,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Length: {body.Length}\r\ KeyPath = keyPath, // PEM private key file KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["h2", "http/1.1"], // ORDERED, most preferred first - a client offering both gets h2 - KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = OpenSSL both ways) + KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = BoringSSL both ways) KernelRx = kernelRx, // kTLS decrypt: needs KernelTx; experimental (see the knob above) }); diff --git a/Playground/Http3/ManagedBuffered/Program.cs b/Playground/Http3/ManagedBuffered/Program.cs index 1f145812..d46b79d7 100644 --- a/Playground/Http3/ManagedBuffered/Program.cs +++ b/Playground/Http3/ManagedBuffered/Program.cs @@ -38,6 +38,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Playground/Http3/ManagedStreamedBoth/Program.cs b/Playground/Http3/ManagedStreamedBoth/Program.cs index 004b124a..c3ab4c3d 100644 --- a/Playground/Http3/ManagedStreamedBoth/Program.cs +++ b/Playground/Http3/ManagedStreamedBoth/Program.cs @@ -47,6 +47,7 @@ string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Playground/Http3/MutualTls/Program.cs b/Playground/Http3/MutualTls/Program.cs index cf025d68..fe3bb0cf 100644 --- a/Playground/Http3/MutualTls/Program.cs +++ b/Playground/Http3/MutualTls/Program.cs @@ -37,6 +37,7 @@ // The server's own certificate and key. Null generates a self-signed pair on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); diff --git a/Playground/Http3/Nghttp3Buffered/Program.cs b/Playground/Http3/Nghttp3Buffered/Program.cs index a1a39f16..788e195c 100644 --- a/Playground/Http3/Nghttp3Buffered/Program.cs +++ b/Playground/Http3/Nghttp3Buffered/Program.cs @@ -71,6 +71,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/Http3/Nghttp3Request/Program.cs b/Playground/Http3/Nghttp3Request/Program.cs index 751a19de..b9d45f89 100644 --- a/Playground/Http3/Nghttp3Request/Program.cs +++ b/Playground/Http3/Nghttp3Request/Program.cs @@ -51,6 +51,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Playground/Http3/Nghttp3Response/Program.cs b/Playground/Http3/Nghttp3Response/Program.cs index 94253284..04a3a5c5 100644 --- a/Playground/Http3/Nghttp3Response/Program.cs +++ b/Playground/Http3/Nghttp3Response/Program.cs @@ -50,6 +50,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Playground/Http3/Nghttp3StreamedBoth/Program.cs b/Playground/Http3/Nghttp3StreamedBoth/Program.cs index 65fba04e..a6e03ce0 100644 --- a/Playground/Http3/Nghttp3StreamedBoth/Program.cs +++ b/Playground/Http3/Nghttp3StreamedBoth/Program.cs @@ -56,6 +56,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── diff --git a/Playground/Http3/Sni/Program.cs b/Playground/Http3/Sni/Program.cs index 9d807420..a90ebc9c 100644 --- a/Playground/Http3/Sni/Program.cs +++ b/Playground/Http3/Sni/Program.cs @@ -55,6 +55,7 @@ // A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // The names this port answers for, beside the default. Add a line and it is served. string[] hosts = ["alpha.test", "beta.test"]; diff --git a/Playground/Proxy/H1ToH1/Program.cs b/Playground/Proxy/H1ToH1/Program.cs index cbb4287e..3b65bd8d 100644 --- a/Playground/Proxy/H1ToH1/Program.cs +++ b/Playground/Proxy/H1ToH1/Program.cs @@ -17,7 +17,7 @@ // dotnet run -c Release --project Playground/Proxy/H1ToH1 // curl -k https://127.0.0.1:8443/ // -// Two TLS stacks are in play. INBOUND is ioxide's own termination, OpenSSL in both directions +// Two TLS stacks are in play. INBOUND is ioxide's own termination, BoringSSL in both directions // by default. OUTBOUND is the client pool's: each upstream connection wraps its own // TlsClientStream. Neither shows up in the proxying code. // @@ -104,7 +104,7 @@ KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem) KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["http/1.1"], // protocols offered, most preferred first - KernelTx = false, // kTLS encrypt (off = OpenSSL both ways) + KernelTx = false, // kTLS encrypt (off = BoringSSL both ways) KernelRx = false, // kTLS decrypt; requires KernelTx, experimental }); diff --git a/Playground/Proxy/H2ToH1/Program.cs b/Playground/Proxy/H2ToH1/Program.cs index c74c932f..bba2fd1b 100644 --- a/Playground/Proxy/H2ToH1/Program.cs +++ b/Playground/Proxy/H2ToH1/Program.cs @@ -113,7 +113,7 @@ KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem) KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["h2"], // protocols offered, most preferred first - KernelTx = false, // kTLS encrypt (off = OpenSSL both ways) + KernelTx = false, // kTLS encrypt (off = BoringSSL both ways) KernelRx = false, // kTLS decrypt; requires KernelTx, experimental }); diff --git a/Playground/Quic/Alpn/Program.cs b/Playground/Quic/Alpn/Program.cs index 2ba43186..2b52acfd 100644 --- a/Playground/Quic/Alpn/Program.cs +++ b/Playground/Quic/Alpn/Program.cs @@ -43,6 +43,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/Quic/Pipe/Program.cs b/Playground/Quic/Pipe/Program.cs index 0f626c86..361f1b53 100644 --- a/Playground/Quic/Pipe/Program.cs +++ b/Playground/Quic/Pipe/Program.cs @@ -43,6 +43,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/Quic/Raw/Program.cs b/Playground/Quic/Raw/Program.cs index dc2a1c5b..2e6755c9 100644 --- a/Playground/Quic/Raw/Program.cs +++ b/Playground/Quic/Raw/Program.cs @@ -42,6 +42,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/Playground/README.md b/Playground/README.md index 32b25b5a..1f93e483 100644 --- a/Playground/README.md +++ b/Playground/README.md @@ -28,7 +28,7 @@ delete it when you copy the file out and the literals are the whole configuratio ## Certificates The samples that are about the certificate rather than about the transport. TLS exists twice in -ioxide and they are not one stack with two doors: on TCP, OpenSSL terminates **above** the +ioxide and they are not one stack with two doors: on TCP, BoringSSL terminates **above** the transport and each reactor owns its own `TlsService`; on QUIC, TLS 1.3 lives **inside** the transport (picotls, bundled with ngtcp2) and one `QuicEngine` is shared by every reactor. The same feature therefore reads differently on each side, which is why each has its own sample. @@ -53,13 +53,13 @@ trust. ## TLS over TCP -The certificate is terminated above the transport, by OpenSSL or by the kernel. All of these serve +The certificate is terminated above the transport, by BoringSSL or by the kernel. All of these serve HTTP/1.1 framed by hand. | Project | Lines | Shows | Packages | | --- | --- | --- | --- | -| [`Tls.OpenSsl`](Tls/OpenSsl/Program.cs) | 243 | **The default**: OpenSSL both ways - no kernel module, and TLS 1.2 / any suite / resumption. | `ioxide` | -| [`Tls.KtlsTx`](Tls/KtlsTx/Program.cs) | 225 | The deployable kernel mode: kernel TX (the handler writes plaintext), OpenSSL RX. `Tls/Ktls` minus one line. `modprobe tls`. | `ioxide` | +| [`Tls.OpenSsl`](Tls/OpenSsl/Program.cs) | 243 | **The default**: BoringSSL both ways - no kernel module, and TLS 1.2 / any suite / resumption. | `ioxide` | +| [`Tls.KtlsTx`](Tls/KtlsTx/Program.cs) | 225 | The deployable kernel mode: kernel TX (the handler writes plaintext), BoringSSL RX. `Tls/Ktls` minus one line. `modprobe tls`. | `ioxide` | | [`Tls.Ktls`](Tls/Ktls/Program.cs) | 225 | **Full kernel TLS** - both directions in the kernel. Experimental: a TLS 1.3 KeyUpdate cannot be read through `IORING_OP_RECV`. | `ioxide` | | [`Tls.OpenSslPipes`](Tls/OpenSslPipes/Program.cs) | 186 | The default backend through an `IDuplexPipe`. | `ioxide` | | [`Tls.KtlsPipes`](Tls/KtlsPipes/Program.cs) | 184 | Full kTLS through an `IDuplexPipe`. Its serve loop is byte-identical to `Tls/OpenSslPipes` - over a pipe the backend is invisible. | `ioxide` | diff --git a/Playground/Shared/Env.cs b/Playground/Shared/Env.cs index cff6df4b..db2e41a1 100644 --- a/Playground/Shared/Env.cs +++ b/Playground/Shared/Env.cs @@ -68,6 +68,16 @@ public static void OverrideQuic(ref ushort quicPort, ref int reactors) reactors = Int("PLAYGROUND_REACTORS", reactors); } + /// + /// PLAYGROUND_TLS_CERT and PLAYGROUND_TLS_KEY: a real PEM pair in place of the self-signed + /// localhost one a sample generates when its own literals are null. + /// + public static void OverrideTlsCert(ref string? certOverride, ref string? keyOverride) + { + certOverride = StrOrNull("PLAYGROUND_TLS_CERT") ?? certOverride; + keyOverride = StrOrNull("PLAYGROUND_TLS_KEY") ?? keyOverride; + } + /// /// The kTLS knobs, same escape hatch. These exist so bench/tls-matrix.sh can measure one build /// in every backend combination instead of rebuilding per cell - a sample that only ever runs diff --git a/Playground/Tls/Ktls/Program.cs b/Playground/Tls/Ktls/Program.cs index 0c958449..3f1d6480 100644 --- a/Playground/Tls/Ktls/Program.cs +++ b/Playground/Tls/Ktls/Program.cs @@ -5,12 +5,12 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// tls-ktls - FULL kernel TLS: the OpenSSL handshake runs over the ring, then both directions +// tls-ktls - FULL kernel TLS: the BoringSSL handshake runs over the ring, then both directions // are handed to the kernel - the handler writes plaintext, the kernel makes the records, and // recv delivers plaintext straight into ring memory. Experimental (see KernelRx below); -// Tls/KtlsTx is the deployable drop-back - kernel TX, OpenSSL RX - and Tls/OpenSsl the default. +// Tls/KtlsTx is the deployable drop-back - kernel TX, BoringSSL RX - and Tls/OpenSsl the default. // -// sudo modprobe tls # needs the Linux 'tls' module + OpenSSL 3 +// sudo modprobe tls # needs the Linux 'tls' module // dotnet run -c Release --project Playground/Tls/Ktls // curl -ks https://127.0.0.1:8443/ | head -c 40 // @@ -33,6 +33,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); @@ -68,14 +69,14 @@ KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the + // NOT the default - TLS is BoringSSL both ways unless you ask for this. It is what lets the // handler below write PLAINTEXT to the connection: the kernel turns it into records on send. // Remove this line and conn.Write would put cleartext on the wire. KernelTx = true, // Full kTLS: the kernel decrypts inbound too. Experimental - about one first connection in // twelve fails outright, and a client sending a TLS 1.3 KeyUpdate loses the connection. - // Tls/KtlsTx is this same server without this line: kernel TX, OpenSSL RX, deployable today. + // Tls/KtlsTx is this same server without this line: kernel TX, BoringSSL RX, deployable today. KernelRx = true, }; @@ -97,7 +98,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCont { var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => diff --git a/Playground/Tls/KtlsPipes/Program.cs b/Playground/Tls/KtlsPipes/Program.cs index 23a8ffed..8539ef2c 100644 --- a/Playground/Tls/KtlsPipes/Program.cs +++ b/Playground/Tls/KtlsPipes/Program.cs @@ -18,7 +18,7 @@ // // TlsConnectionDualPipe composes rather than implements. Each direction has two possible halves: // -// kernel OpenSSL +// kernel BoringSSL // read TcpConnectionPipeReader TlsDecryptingPipeReader // (plaintext is already in (decrypts into a Pipe // ring memory - zero copy) it owns) @@ -33,7 +33,7 @@ // // Compare the raw-ring pair, Playground/Tls/Ktls and Playground/Tls/OpenSsl. Those two DO differ // in the handler, because at that level the backend is visible: kTLS writes plaintext straight -// to the connection, OpenSSL goes through TlsSession.Write. Over a pipe it is not. Needs: ioxide +// to the connection, BoringSSL goes through TlsSession.Write. Over a pipe it is not. Needs: ioxide // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -50,6 +50,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── @@ -86,7 +87,7 @@ KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. + // NOT the default - TLS is BoringSSL both ways unless you ask for this. // Playground/Tls/OpenSslPipes is the same server without these two lines. KernelTx = true, diff --git a/Playground/Tls/KtlsTx/Program.cs b/Playground/Tls/KtlsTx/Program.cs index fd795619..0393925e 100644 --- a/Playground/Tls/KtlsTx/Program.cs +++ b/Playground/Tls/KtlsTx/Program.cs @@ -5,12 +5,12 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// tls-ktls-tx - the deployable kernel mode: the OpenSSL handshake runs over the ring, transmit +// tls-ktls-tx - the deployable kernel mode: the BoringSSL handshake runs over the ring, transmit // is handed to KERNEL TLS (the handler writes plaintext, the kernel makes the records), and -// receive stays in OpenSSL. Tls/Ktls is this plus kernel receive (experimental); Tls/OpenSsl +// receive stays in BoringSSL. Tls/Ktls is this plus kernel receive (experimental); Tls/OpenSsl // is the default, with the kernel in neither direction. // -// sudo modprobe tls # needs the Linux 'tls' module + OpenSSL 3 +// sudo modprobe tls # needs the Linux 'tls' module // dotnet run -c Release --project Playground/Tls/KtlsTx // curl -ks https://127.0.0.1:8443/ | head -c 40 // @@ -33,6 +33,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The // handler code is identical either way; this only changes how recv buffers are handed out. bool incrementalBuffers = false; @@ -73,7 +74,7 @@ KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the + // NOT the default - TLS is BoringSSL both ways unless you ask for this. It is what lets the // handler below write PLAINTEXT to the connection: the kernel turns it into records on send. // Remove this line and conn.Write would put cleartext on the wire. KernelTx = true, @@ -99,7 +100,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCont { var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => diff --git a/Playground/Tls/MtlsKtlsPipes/Program.cs b/Playground/Tls/MtlsKtlsPipes/Program.cs index acf67e72..8c5552e3 100644 --- a/Playground/Tls/MtlsKtlsPipes/Program.cs +++ b/Playground/Tls/MtlsKtlsPipes/Program.cs @@ -52,6 +52,7 @@ // The server's own certificate and key. Null generates a self-signed pair on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); @@ -115,7 +116,7 @@ // no session resumption. // // It does NOT change anything above it. The client certificate is exchanged and verified during - // the HANDSHAKE, which OpenSSL performs either way - the kernel only takes over record crypto + // the HANDSHAKE, which BoringSSL performs either way - the kernel only takes over record crypto // afterwards - so ClientCaPath, RequireClientCertificate and PeerSubject behave identically. KernelTx = true, KernelRx = false, // kTLS receive (experimental; requires KernelTx) diff --git a/Playground/Tls/MtlsOpenSslPipes/Program.cs b/Playground/Tls/MtlsOpenSslPipes/Program.cs index dc344528..ad8b2ac9 100644 --- a/Playground/Tls/MtlsOpenSslPipes/Program.cs +++ b/Playground/Tls/MtlsOpenSslPipes/Program.cs @@ -7,7 +7,7 @@ // ───────────────────────────────────────────────────────────────────────────────────────────── // tls-mtls-openssl-pipes - TLS where the CLIENT proves who it is too, served through an -// IDuplexPipe, with OpenSSL doing the crypto in userspace. +// IDuplexPipe, with BoringSSL doing the crypto in userspace. // // PLAYGROUND_CLIENT_CA=ca.crt dotnet run -c Release --project Playground/Tls/MtlsOpenSslPipes // curl -k --cert client.crt --key client.key https://127.0.0.1:8443/ @@ -49,6 +49,7 @@ // The server's own certificate and key. Null generates a self-signed pair on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); Env.OverrideCert(ref certOverride, ref keyOverride); @@ -107,9 +108,9 @@ ClientCaPath = clientCaPath, // trust anchors for CLIENT certificates (or ClientCaPem) RequireClientCertificate = requireClientCertificate, // refuse a client with none, at the handshake - // KernelTx stays false (the default). OpenSSL encrypts and decrypts, so nothing here needs the + // KernelTx stays false (the default). BoringSSL encrypts and decrypts, so nothing here needs the // 'tls' kernel module. Client verification is unaffected by that choice either way - the - // certificate is exchanged during the handshake, which OpenSSL always performs. + // certificate is exchanged during the handshake, which BoringSSL always performs. KernelTx = false, KernelRx = false, // kTLS receive (experimental; requires KernelTx) }; diff --git a/Playground/Tls/MultiPort/Program.cs b/Playground/Tls/MultiPort/Program.cs index 1e862135..c978ad2d 100644 --- a/Playground/Tls/MultiPort/Program.cs +++ b/Playground/Tls/MultiPort/Program.cs @@ -7,7 +7,7 @@ // ───────────────────────────────────────────────────────────────────────────────────────────── // tls-multiport - one server, two doors: plaintext on :8080, TLS on :8081, and ONE serve loop -// for both. The TLS door terminates with the default backend (OpenSSL both ways, no kernel +// for both. The TLS door terminates with the default backend (BoringSSL both ways, no kernel // module) and serves through a TlsConnectionDualPipe; the plaintext door pairs the connection's // own pipe halves. The loop reads an IDuplexPipe and cannot tell which door it got - that is // the point. @@ -33,6 +33,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); @@ -62,7 +63,7 @@ }, }; -// Default backend: OpenSSL in both directions. Only connections on the TLS door go near this. +// Default backend: BoringSSL in both directions. Only connections on the TLS door go near this. var tlsOptions = new TlsOptions { CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory) diff --git a/Playground/Tls/OpenSsl/Program.cs b/Playground/Tls/OpenSsl/Program.cs index 93278af8..ae8e5b1b 100644 --- a/Playground/Tls/OpenSsl/Program.cs +++ b/Playground/Tls/OpenSsl/Program.cs @@ -5,7 +5,7 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// tls-openssl - the same TLS termination as Tls/Ktls with kernel TLS turned OFF. OpenSSL does +// tls-openssl - the same TLS termination as Tls/Ktls with kernel TLS turned OFF. BoringSSL does // both directions: it decrypts what the ring delivers, and it encrypts responses before they // reach the write slab. // @@ -15,20 +15,21 @@ // Why choose this over kTLS. It drops every constraint kTLS imposes: no 'tls' kernel module (so // it runs in a container that does not have it), no TLS-1.3-only restriction, no single-suite // restriction, session resumption is available again, and there is no handshake-alignment -// problem. The cost on this rig is nothing - see the table below - though a deployment using -// sendfile or a NIC that offloads TLS would see a real difference kTLS cannot give up. +// problem. On this rig it is also cheaper than kTLS at every size - see the table below - though +// a deployment using sendfile or a NIC that offloads TLS would see a real difference kTLS cannot +// give up. // // The one thing the handler must do differently is on the write side: with kTLS the kernel makes -// the records so you write plaintext, and without it OpenSSL has to encrypt first. +// the records so you write plaintext, and without it BoringSSL has to encrypt first. // Everything else - the handshake, ALPN, the read loop, the framing - is identical. // -// Measured here, 4 reactors, wrk -t4 -c64, against the plaintext Tcp/Raw baseline: +// Measured here, requests/s, 4 reactors, wrk -t4 -c64, against the plaintext Tcp/Raw baseline: // -// response plaintext kTLS OpenSSL -// 64 B 1,379,980 1,091,224 0.79x 1,093,361 0.79x -// 8 KiB 1,109,287 749,833 0.68x 802,325 0.72x -// 64 KiB 448,093 157,644 0.35x 196,486 0.44x -// 256 KiB 133,097 41,691 0.31x 49,801 0.37x +// response plaintext kTLS TX BoringSSL +// 64 B 1,245,989 937,322 0.75x 976,752 0.78x +// 8 KiB 991,826 582,376 0.59x 676,158 0.68x +// 64 KiB 344,094 134,112 0.39x 153,538 0.45x +// 256 KiB 113,412 38,786 0.34x 40,250 0.35x // // Read that as: which TLS backend you pick is worth far less than the cost of TLS itself. // @@ -50,6 +51,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The // handler code is identical either way; this only changes how recv buffers are handed out. bool incrementalBuffers = false; @@ -91,7 +93,7 @@ Alpn = ["http/1.1"], // protocols this port serves, most-preferred first // KernelTx stays false (the default), so this sample IS the stock configuration: no TLS - // ULP on the socket, OpenSSL encrypts and decrypts, MSG_WAITALL stays on, and nothing here + // ULP on the socket, BoringSSL encrypts and decrypts, MSG_WAITALL stays on, and nothing here // needs the 'tls' kernel module. Tls/Ktls is the sample that opts into the kernel path. KernelTx = false, KernelRx = false, // kTLS receive (experimental; requires KernelTx) @@ -115,7 +117,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCont { var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => @@ -132,7 +134,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCont { // The handshake reads and writes through this same connection. Unlike Tls/Ktls, no // ULP is attached afterwards - the socket stays an ordinary TCP socket and every - // record is made and opened by OpenSSL. + // record is made and opened by BoringSSL. tls = await r.GetService().AcceptAsync(conn); // A request can ride in with the handshake's final flight - answer it before parking @@ -151,7 +153,7 @@ .. Encoding.ASCII.GetBytes($"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nCont { if (item.HasBuffer) { - // Records in, plaintext out - OpenSSL decrypts what the ring received. + // Records in, plaintext out - BoringSSL decrypts what the ring received. Decrypt(tls, in item, carry); conn.ReturnBuffer(in item); } @@ -208,7 +210,7 @@ static bool Answer(TcpConnection conn, TlsSession tls, Carry carry, ReadOnlyMemo carry.Consume(end + 4); // The one line that differs from Tls/Ktls. There, kTLS is producing the records so the - // handler writes plaintext; here OpenSSL has to encrypt before anything reaches the slab. + // handler writes plaintext; here BoringSSL has to encrypt before anything reaches the slab. tls.Write(conn, response.Span); wrote = true; diff --git a/Playground/Tls/OpenSslPipes/Program.cs b/Playground/Tls/OpenSslPipes/Program.cs index dbd3199d..b419b71b 100644 --- a/Playground/Tls/OpenSslPipes/Program.cs +++ b/Playground/Tls/OpenSslPipes/Program.cs @@ -6,7 +6,7 @@ using Playground.Shared; // ───────────────────────────────────────────────────────────────────────────────────────────── -// tls-openssl-pipes - TLS served through an IDuplexPipe, with OpenSSL doing the crypto in userspace. +// tls-openssl-pipes - TLS served through an IDuplexPipe, with BoringSSL doing the crypto in userspace. // // dotnet run -c Release --project Playground/Tls/OpenSslPipes # NO modprobe needed // curl -ks https://127.0.0.1:8443/ | head -c 40 @@ -17,7 +17,7 @@ // // TlsConnectionDualPipe composes rather than implements. Each direction has two possible halves: // -// kernel OpenSSL +// kernel BoringSSL // read TcpConnectionPipeReader TlsDecryptingPipeReader // (plaintext is already in (decrypts into a Pipe // ring memory - zero copy) it owns) @@ -32,7 +32,7 @@ // // Compare the raw-ring pair, Playground/Tls/Ktls and Playground/Tls/OpenSsl. Those two DO differ // in the handler, because at that level the backend is visible: kTLS writes plaintext straight to -// the connection, OpenSSL goes through TlsSession.Write. Over a pipe it is not. Needs: ioxide +// the connection, BoringSSL goes through TlsSession.Write. Over a pipe it is not. Needs: ioxide // ───────────────────────────────────────────────────────────────────────────────────────────── // ── Knobs ──────────────────────────────────────────────────────────────────────────────────── @@ -49,6 +49,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // Per-connection recv buffer rings (kernel 6.12+) instead of one shared ring per reactor. The // handler code is identical either way; this only changes how recv buffers are handed out. bool incrementalBuffers = false; @@ -89,7 +90,7 @@ KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // KernelTx stays false (the default). No TLS ULP on the socket, so OpenSSL encrypts and + // KernelTx stays false (the default). No TLS ULP on the socket, so BoringSSL encrypts and // decrypts and nothing here needs the 'tls' kernel module - which also keeps TLS 1.2, any // ciphersuite and session resumption. Playground/Tls/KtlsPipes adds the one line that // changes this. diff --git a/Playground/Tls/Sni/Program.cs b/Playground/Tls/Sni/Program.cs index 7452aa8d..ac5596cc 100644 --- a/Playground/Tls/Sni/Program.cs +++ b/Playground/Tls/Sni/Program.cs @@ -32,7 +32,7 @@ // names you want to answer for. Keys are ASCII host names only: SNI carries no port, an IP // address is not a legal value, and an international name belongs here in its xn-- form. // -// Each entry costs one OpenSSL context, built once at startup. Choosing one per handshake is a +// Each entry costs one BoringSSL context, built once at startup. Choosing one per handshake is a // lookup on the reactor thread and allocates nothing, so the cost of serving ten names is the // same as serving one. // @@ -54,6 +54,7 @@ // A real PEM pair for the DEFAULT certificate, or null to generate a self-signed localhost one. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // The names this port answers for, beside the default. Add a line and it is served. string[] hosts = ["alpha.test", "beta.test"]; diff --git a/Playground/Tls/SslStream/Program.cs b/Playground/Tls/SslStream/Program.cs index 4ab2e130..fdaba8a0 100644 --- a/Playground/Tls/SslStream/Program.cs +++ b/Playground/Tls/SslStream/Program.cs @@ -32,6 +32,7 @@ // A real PEM pair, or null to generate a self-signed localhost cert on first run. string? certOverride = null; string? keyOverride = null; +Env.OverrideTlsCert(ref certOverride, ref keyOverride); // ───────────────────────────────────────────────────────────────────────────────────────────── (string certPath, string keyPath) = QuicCert.Ensure(certOverride, keyOverride); diff --git a/README.md b/README.md index 4201b3ff..a44bb1bc 100644 --- a/README.md +++ b/README.md @@ -40,14 +40,14 @@ runnable code side by side. Every package shares one version and depends only on the runtime. Start with **[`ioxide`](https://www.nuget.org/packages/ioxide/)** - reactors, TCP/UDP transports, connections, -the ring-native client seam, and TLS termination (OpenSSL handshake over the ring, then kTLS, so +the ring-native client seam, and TLS termination (BoringSSL handshake over the ring, then kTLS, so handlers keep writing plaintext) - then add exactly what you use. **Protocols.** HTTP/2 and HTTP/3 each come twice: a bundled native, or the same surface in pure C#. | Package | What it does | | --- | --- | -| [`ioxide.ngtcp2`](https://www.nuget.org/packages/ioxide.ngtcp2/) | QUIC. ngtcp2 + picotls as one bundled native; only system dependency is OpenSSL 3. | +| [`ioxide.ngtcp2`](https://www.nuget.org/packages/ioxide.ngtcp2/) | QUIC. ngtcp2 + picotls on BoringSSL's libcrypto as one bundled native; needs nothing from the system. | | [`ioxide.nghttp3`](https://www.nuget.org/packages/ioxide.nghttp3/) | HTTP/3 + QPACK (nghttp3), bundled native. Rides any QUIC connection. | | [`ioxide.http3`](https://www.nuget.org/packages/ioxide.http3/) | HTTP/3 in pure C# - frames, QPACK, Huffman. Zero native code, drop-in for `ioxide.nghttp3`. | | [`ioxide.nghttp2`](https://www.nuget.org/packages/ioxide.nghttp2/) | HTTP/2 (nghttp2), bundled native. h2c, or h2 over TLS via ALPN. | diff --git a/bench/README.md b/bench/README.md index fef2aaa9..c46e0c4a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -87,12 +87,12 @@ Anything missing **skips with a note** instead of failing the run. These answer one question each, holding everything else constant, and exist because the suite deliberately does not. -- **`tls-matrix.sh`** - h1 and h2 across plaintext / OpenSSL / kTLS-TX / kTLS-both, at 64 B and +- **`tls-matrix.sh`** - h1 and h2 across plaintext / BoringSSL / kTLS-TX / kTLS-both, at 64 B and 8 KB. There is no kTLS-RX-only cell: RX is programmed at the same handoff as TX, so asking for - it alone silently gets you OpenSSL. + it alone is refused at startup. - **`file-matrix.sh`** - static files, reading straight into the write slab versus through a - buffer, crossed with cleartext / kTLS / OpenSSL. Whether the slab path is legal at all follows - from what the slab must hold when it is sent, so there is no OpenSSL+slab cell. Runs **one + buffer, crossed with cleartext / kTLS / BoringSSL. Whether the slab path is legal at all follows + from what the slab must hold when it is sent, so there is no BoringSSL+slab cell. Runs **one saturated reactor**: an earlier four-reactor version reported an 11% win where the single-reactor method measures 22%. - **`h2-matrix.sh`** - nghttp2 versus the pure-C# HTTP/2, with and without TLS, on one load @@ -114,7 +114,7 @@ deliberately does not. Measured here at 2 reactors, 64 connections, 32 streams, a 2-byte body, the pure-C# server runs 2.7x nghttp2 cleartext and 2.5x over TLS, at about a third of the CPU per request. TLS costs -**both** the same 0.05us per request - identical absolute cost from the same OpenSSL - so it only +**both** the same 0.05us per request - identical absolute cost from the same TLS library - so it only looks like a bigger hit on the faster server. That ordering narrows as reactors rise and connections-per-reactor falls (2.76x at 2 reactors, diff --git a/bench/h2-matrix.sh b/bench/h2-matrix.sh index e75ec6f1..76d2d0ca 100755 --- a/bench/h2-matrix.sh +++ b/bench/h2-matrix.sh @@ -7,7 +7,7 @@ # constant except the implementation - same h2load invocation, same reactor count, same body, same # TLS backend, interleaved so drift hits both arms equally. # -# TLS here is OpenSSL both ways, which is the default. Which backend does not matter for this +# TLS here is BoringSSL both ways, which is the default. Which backend does not matter for this # question: both arms carry the identical TLS cost, so it cancels. set -uo pipefail cd "$(dirname "$0")/.." diff --git a/bench/run.sh b/bench/run.sh index af836c0a..d9460e26 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -82,7 +82,7 @@ play Tls/OpenSsl PLAYGROUND_REACTORS=$R PLAYGROUND_PORT=18443 \ && note "tls-openssl" "${R}r" "$(wrk_h1 https://127.0.0.1:18443/) req/s" stop_play if [ -d /sys/module/tls ]; then - # The deployable hybrid (kernel TX, OpenSSL RX) - matches the matrix's ktls-tx cell. + # The deployable hybrid (kernel TX, BoringSSL RX) - matches the matrix's ktls-tx cell. play Tls/KtlsTx PLAYGROUND_REACTORS=$R PLAYGROUND_PORT=18443 \ && note "tls-ktls-tx" "${R}r" "$(wrk_h1 https://127.0.0.1:18443/) req/s" stop_play diff --git a/bench/tls-matrix.sh b/bench/tls-matrix.sh index 3c5ce992..dcf52129 100755 --- a/bench/tls-matrix.sh +++ b/bench/tls-matrix.sh @@ -2,8 +2,8 @@ # The TLS backend matrix: h1 and h2, each in plaintext and in all three TLS configurations. # # plaintext no TLS at all - the baseline everything else is measured against -# openssl OpenSSL encrypts and decrypts. KernelTx=0 KernelRx=0 (default) -# ktls-tx kernel encrypts, OpenSSL decrypts. KernelTx=1 KernelRx=0 +# openssl BoringSSL encrypts and decrypts. KernelTx=0 KernelRx=0 (default) +# ktls-tx kernel encrypts, BoringSSL decrypts. KernelTx=1 KernelRx=0 # ktls kernel does both. KernelTx=1 KernelRx=1 # # There is no ktls-rx-only cell: RX is programmed at the same handoff as TX, so TlsService gates diff --git a/docs/how-ioxide-does-h3.html b/docs/how-ioxide-does-h3.html index 836a6fcc..b1bee5dd 100644 --- a/docs/how-ioxide-does-h3.html +++ b/docs/how-ioxide-does-h3.html @@ -87,7 +87,7 @@

The pieces

TLS here is not the TLS you configured for TCP

ioxide terminates TLS twice, through two entirely separate stacks, and this catches people - out. Over TCP it is OpenSSL, configured through TlsService. Over QUIC it is + out. Over TCP it is BoringSSL, configured through TlsService. Over QUIC it is picotls, driven by ngtcp2 and configured on QuicEngine. They share no code and no configuration object, deliberately - QUIC does not use TLS records at all, it carries the handshake in its own CRYPTO frames, so the two have almost nothing in common below the diff --git a/docs/index.html b/docs/index.html index 7e902d41..5c8828b8 100644 --- a/docs/index.html +++ b/docs/index.html @@ -355,7 +355,7 @@

SNI · a certificate per host

{ t.Join(); } -

One port, several names, a certificate each - chosen inside the handshake, before a byte of your protocol exists. The certificate at the top stays the DEFAULT: a client that sends no name (anything connecting by IP) or asks for one that is not registered gets it rather than a dead connection. Each entry is one OpenSSL context built at startup, so choosing one per handshake is a lookup that allocates nothing - ten names cost what one does. The same feature on h2 is , and on QUIC .

+

One port, several names, a certificate each - chosen inside the handshake, before a byte of your protocol exists. The certificate at the top stays the DEFAULT: a client that sends no name (anything connecting by IP) or asks for one that is not registered gets it rather than a dead connection. Each entry is one BoringSSL context built at startup, so choosing one per handshake is a lookup that allocates nothing - ten names cost what one does. The same feature on h2 is , and on QUIC .

@@ -437,9 +437,9 @@

mutual TLS · openssl · pipes

ClientCaPath = clientCaPath, // trust anchors for CLIENT certificates (or ClientCaPem) RequireClientCertificate = requireClientCertificate, // refuse a client with none, at the handshake - // KernelTx stays false (the default). OpenSSL encrypts and decrypts, so nothing here needs the + // KernelTx stays false (the default). BoringSSL encrypts and decrypts, so nothing here needs the // 'tls' kernel module. Client verification is unaffected by that choice either way - the - // certificate is exchanged during the handshake, which OpenSSL always performs. + // certificate is exchanged during the handshake, which BoringSSL always performs. KernelTx = false, KernelRx = false, // kTLS receive (experimental; requires KernelTx) }; @@ -637,7 +637,7 @@

mutual TLS · kernel tx · pipes

// no session resumption. // // It does NOT change anything above it. The client certificate is exchanged and verified during - // the HANDSHAKE, which OpenSSL performs either way - the kernel only takes over record crypto + // the HANDSHAKE, which BoringSSL performs either way - the kernel only takes over record crypto // afterwards - so ClientCaPath, RequireClientCertificate and PeerSubject behave identically. KernelTx = true, KernelRx = false, // kTLS receive (experimental; requires KernelTx) @@ -836,7 +836,7 @@

h2 · a certificate per host

var reactor = new Reactor(i, config); // One TlsService per reactor. It owns the contexts - the default and one per name - and picks - // between them during each handshake on this ring. Each entry costs one OpenSSL context, built + // between them during each handshake on this ring. Each entry costs one BoringSSL context, built // once at startup; choosing one per handshake is a lookup that allocates nothing, so serving // ten names costs what serving one does. reactor.OnStart = r => TlsService.Start(r, tlsOptions); @@ -1524,7 +1524,7 @@

plaintext + TLS · one server

}, }; -// Default backend: OpenSSL in both directions. Only connections on the TLS door go near this. +// Default backend: BoringSSL in both directions. Only connections on the TLS door go near this. var tlsOptions = new TlsOptions { CertificatePath = certPath, // PEM chain file (or CertificatePem for in-memory) @@ -1638,7 +1638,7 @@

plaintext + TLS · one server

// The plaintext door's IDuplexPipe: the connection's own reader and writer, paired. sealed record PlainPipe(PipeReader Input, PipeWriter Output) : IDuplexPipe; -

One server, two doors: plaintext on :8080, TLS on :8081, and ONE serve loop for both. The branch on ListenerPort is the entire difference - the TLS door builds a TlsConnectionDualPipe, the plaintext door pairs the connection's own reader and writer, and the loop reads an IDuplexPipe without knowing which it got. Ports come from multi-port; TLS is the default backend, OpenSSL both ways.

+

One server, two doors: plaintext on :8080, TLS on :8081, and ONE serve loop for both. The branch on ListenerPort is the entire difference - the TLS door builds a TlsConnectionDualPipe, the plaintext door pairs the connection's own reader and writer, and the loop reads an IDuplexPipe without knowing which it got. Ports come from multi-port; TLS is the default backend, BoringSSL both ways.

@@ -1757,7 +1757,7 @@

TLS · SslStream

{ thread.Join(); } -

The third TLS backend, and the portable one: the BCL's SslStream over TcpConnectionStream. Fully managed, full-featured, and the slowest of the three, because the bytes are copied through a Stream both ways. Reach for it for client certificates - the ring-native path does not do mTLS - or anything else OpenSSL-on-the-ring does not expose; for TLS 1.2 and resumption the default already has you covered: .

+

The third TLS backend, and the portable one: the BCL's SslStream over TcpConnectionStream. Fully managed, full-featured, and the slowest of the three, because the bytes are copied through a Stream both ways. Reach for it for client certificates - the ring-native path does not do mTLS - or anything else BoringSSL-on-the-ring does not expose; for TLS 1.2 and resumption the default already has you covered: .

@@ -2093,7 +2093,7 @@

HTTP/2 · TLS & ALPN

// Hand OUTBOUND encryption to the kernel: the handler writes plaintext and the kernel makes the -// records. Off by default - OpenSSL both ways is the portable path, and on loopback the kernel +// records. Off by default - BoringSSL both ways is the portable path, and on loopback the kernel // is not faster. Its real payoff is sendfile and NIC offload, which a benchmark here cannot see. bool kernelTx = false; @@ -2157,7 +2157,7 @@

HTTP/2 · TLS & ALPN

KeyPath = keyPath, // PEM private key file KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["h2", "http/1.1"], // ORDERED, most preferred first - a client offering both gets h2 - KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = OpenSSL both ways) + KernelTx = kernelTx, // kTLS encrypt: kernel makes the records (off = BoringSSL both ways) KernelRx = kernelRx, // kTLS decrypt: needs KernelTx; experimental (see the knob above) }); @@ -2282,7 +2282,7 @@

HTTP/2 · TLS & ALPN

_len -= count; } } -

How a browser actually reaches h2: over TLS, with the protocol chosen during the handshake. Alpn = ["h2", "http/1.1"] is an ordered preference, not a weighting - the server takes the first entry the client also offered, and this sample then branches on what was agreed, so ONE port serves both. That is why the h2c samples are the exception rather than the rule. What Http2Connection is handed is a TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code is byte-for-byte the h2c sample's. TLS is OpenSSL both ways by default; the kernelTx/kernelRx knobs at the top move either direction into the kernel, and cost the same 0.05µs per request either way.

+

How a browser actually reaches h2: over TLS, with the protocol chosen during the handshake. Alpn = ["h2", "http/1.1"] is an ordered preference, not a weighting - the server takes the first entry the client also offered, and this sample then branches on what was agreed, so ONE port serves both. That is why the h2c samples are the exception rather than the rule. What Http2Connection is handed is a TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code is byte-for-byte the h2c sample's. TLS is BoringSSL both ways by default; the kernelTx/kernelRx knobs at the top move either direction into the kernel, and cost the same 0.05µs per request either way.

@@ -4257,7 +4257,7 @@

kTLS · raw ring

ioxide
// dotnet add package ioxide
-//   sudo modprobe tls        # kTLS needs the Linux 'tls' module + OpenSSL 3
+//   sudo modprobe tls        # kTLS needs the Linux 'tls' module
 //   curl -k https://127.0.0.1:8443/
 
 using System.Text;
@@ -4308,14 +4308,14 @@ 

kTLS · raw ring

KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the + // NOT the default - TLS is BoringSSL both ways unless you ask for this. It is what lets the // handler below write PLAINTEXT to the connection: the kernel turns it into records on send. // Remove this line and conn.Write would put cleartext on the wire. KernelTx = true, // Full kTLS: the kernel decrypts inbound too. Experimental - about one first connection in // twelve fails outright, and a client sending a TLS 1.3 KeyUpdate loses the connection. - // Tls/KtlsTx is this same server without this line: kernel TX, OpenSSL RX, deployable today. + // Tls/KtlsTx is this same server without this line: kernel TX, BoringSSL RX, deployable today. KernelRx = true, }; @@ -4337,7 +4337,7 @@

kTLS · raw ring

{ var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => @@ -4526,7 +4526,7 @@

kTLS · pipes

KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. + // NOT the default - TLS is BoringSSL both ways unless you ask for this. // Playground/Tls/OpenSslPipes is the same server without these two lines. KernelTx = true, @@ -4686,7 +4686,7 @@

hybrid · raw ring

KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // NOT the default - TLS is OpenSSL both ways unless you ask for this. It is what lets the + // NOT the default - TLS is BoringSSL both ways unless you ask for this. It is what lets the // handler below write PLAINTEXT to the connection: the kernel turns it into records on send. // Remove this line and conn.Write would put cleartext on the wire. KernelTx = true, @@ -4712,7 +4712,7 @@

hybrid · raw ring

{ var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => @@ -4838,11 +4838,11 @@

hybrid · raw ring

_len -= count; } }
-

The deployable kernel mode: kernel TX, OpenSSL RX. The handler still writes plaintext - the kernel makes the records on send, which is what keeps sendfile and NIC offload reachable - while receive takes the well-trodden userspace path instead of kernel RX's experimental one. is this plus kernel receive; is the default, with the kernel in neither direction.

+

The deployable kernel mode: kernel TX, BoringSSL RX. The handler still writes plaintext - the kernel makes the records on send, which is what keeps sendfile and NIC offload reachable - while receive takes the well-trodden userspace path instead of kernel RX's experimental one. is this plus kernel receive; is the default, with the kernel in neither direction.

-

OpenSSL · raw ring

+

BoringSSL · raw ring

ioxide
// dotnet add package ioxide
@@ -4900,7 +4900,7 @@ 

OpenSSL · raw ring

Alpn = ["http/1.1"], // protocols this port serves, most-preferred first // KernelTx stays false (the default), so this sample IS the stock configuration: no TLS - // ULP on the socket, OpenSSL encrypts and decrypts, MSG_WAITALL stays on, and nothing here + // ULP on the socket, BoringSSL encrypts and decrypts, MSG_WAITALL stays on, and nothing here // needs the 'tls' kernel module. Tls/Ktls is the sample that opts into the kernel path. KernelTx = false, KernelRx = false, // kTLS receive (experimental; requires KernelTx) @@ -4924,7 +4924,7 @@

OpenSSL · raw ring

{ var reactor = new Reactor(i, config); - // One TlsService per reactor: it owns the OpenSSL contexts and drives handshakes on this ring. + // One TlsService per reactor: it owns the BoringSSL contexts and drives handshakes on this ring. reactor.OnStart = r => TlsService.Start(r, tlsOptions); reactor.TcpHandle = async (r, conn) => @@ -4941,7 +4941,7 @@

OpenSSL · raw ring

{ // The handshake reads and writes through this same connection. Unlike Tls/Ktls, no // ULP is attached afterwards - the socket stays an ordinary TCP socket and every - // record is made and opened by OpenSSL. + // record is made and opened by BoringSSL. tls = await r.GetService<TlsService>().AcceptAsync(conn); // A request can ride in with the handshake's final flight - answer it before parking @@ -4960,7 +4960,7 @@

OpenSSL · raw ring

{ if (item.HasBuffer) { - // Records in, plaintext out - OpenSSL decrypts what the ring received. + // Records in, plaintext out - BoringSSL decrypts what the ring received. Decrypt(tls, in item, carry); conn.ReturnBuffer(in item); } @@ -5017,7 +5017,7 @@

OpenSSL · raw ring

carry.Consume(end + 4); // The one line that differs from Tls/Ktls. There, kTLS is producing the records so the - // handler writes plaintext; here OpenSSL has to encrypt before anything reaches the slab. + // handler writes plaintext; here BoringSSL has to encrypt before anything reaches the slab. tls.Write(conn, response.Span); wrote = true; @@ -5052,12 +5052,12 @@

OpenSSL · raw ring

_len -= count; } }
-

The default. No TLS ULP is attached at all - OpenSSL encrypts and decrypts, and the response goes through TlsSession.Write, which is correct whichever backend the session ended up with. That drops every constraint kTLS imposes: no kernel module, TLS 1.2, any ciphersuite, session resumption back, no handshake-alignment problem. What it gives up is sendfile and NIC offload. It costs nothing measurable here - against the plaintext baseline, 4 reactors, wrk -t4 -c64: 0.79× for both at a 64-byte response, 0.35× kTLS vs 0.44× OpenSSL at 64 KiB. Which backend you pick matters far less than the cost of TLS.

+

The default. No TLS ULP is attached at all - BoringSSL encrypts and decrypts, and the response goes through TlsSession.Write, which is correct whichever backend the session ended up with. That drops every constraint kTLS imposes: no kernel module, TLS 1.2, any ciphersuite, session resumption back, no handshake-alignment problem. What it gives up is sendfile and NIC offload. It is cheaper than kTLS here - against the plaintext baseline, 4 reactors, wrk -t4 -c64: 0.75× kTLS TX vs 0.78× BoringSSL at a 64-byte response, 0.39× vs 0.45× at 64 KiB. Which backend you pick matters far less than the cost of TLS.

-

OpenSSL · pipes

+

BoringSSL · pipes

ioxide
// dotnet add package ioxide
@@ -5115,7 +5115,7 @@ 

OpenSSL · pipes

KeyPath = keyPath, // PEM key file (or KeyPem for in-memory) Alpn = ["http/1.1"], // protocols this port serves, most-preferred first - // KernelTx stays false (the default). No TLS ULP on the socket, so OpenSSL encrypts and + // KernelTx stays false (the default). No TLS ULP on the socket, so BoringSSL encrypts and // decrypts and nothing here needs the 'tls' kernel module - which also keeps TLS 1.2, any // ciphersuite and session resumption. Playground/Tls/KtlsPipes adds the one line that // changes this. @@ -6397,7 +6397,7 @@

The nine combinations

call. Nothing else changes between the nine samples below - not the reactor, not the config, not the handler's shape.

Every hop is TLS, which is the only way an h2 frontend is reachable from a browser - at all. It is reached two different ways: kTLS facing (OpenSSL handshake over the + at all. It is reached two different ways: kTLS facing (BoringSSL handshake over the ring, then the kernel owns transmit, so the handler writes plaintext), and a TlsClientContext upstream. The h3 frontend needs neither - QUIC has no cleartext mode, so TLS 1.3 is already inside the transport.

@@ -6514,7 +6514,7 @@

HTTP/1.1 in · HTTP/1.1 out

KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem) KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["http/1.1"], // protocols offered, most preferred first - KernelTx = false, // kTLS encrypt (off = OpenSSL both ways) + KernelTx = false, // kTLS encrypt (off = BoringSSL both ways) KernelRx = false, // kTLS decrypt; requires KernelTx, experimental }); @@ -6750,7 +6750,7 @@

HTTP/2 in · HTTP/1.1 out

KeyPath = keyPath, // PEM private key file (set exactly one of Path/Pem) KeyPem = null, // in-memory PEM alternative to KeyPath Alpn = ["h2"], // protocols offered, most preferred first - KernelTx = false, // kTLS encrypt (off = OpenSSL both ways) + KernelTx = false, // kTLS encrypt (off = BoringSSL both ways) KernelRx = false, // kTLS decrypt; requires KernelTx, experimental }); @@ -8036,7 +8036,7 @@

Client · static files

ushort port = tlsOn ? (ushort)8443 : (ushort)8080; // ───────────────────────────────────────────────────────────────────────────────────────────── -// The constraint this sample exists to show, enforced instead of described: with OpenSSL owning +// The constraint this sample exists to show, enforced instead of described: with BoringSSL owning // transmit the slab must hold CIPHERTEXT, so a file read straight into it would go out in the // clear. There is no slab path for that backend - refuse rather than serve cleartext. if (tlsMode == "openssl" && !buffered) @@ -8103,8 +8103,8 @@

Client · static files

// The whole point: with transmit in the kernel the slab carries PLAINTEXT and the // kernel makes the records, so ReadFileAsync may still read the file straight into - // it. With this false, OpenSSL encrypts and the slab must hold ciphertext - which - // is why the slab path is refused above. Receive is OpenSSL either way. + // it. With this false, BoringSSL encrypts and the slab must hold ciphertext - which + // is why the slab path is refused above. Receive is BoringSSL either way. KernelTx = ktls, }); } @@ -8297,9 +8297,9 @@

Client · static files

=> Emit(conn, tls, new ReadOnlySpan<byte>((void*)data, length)); // Every response byte goes through here. TlsSession.Write is correct under BOTH backends - it -// writes plaintext to the slab when the kernel encrypts, and encrypts into the slab when OpenSSL +// writes plaintext to the slab when the kernel encrypts, and encrypts into the slab when BoringSSL // does - so no call site has to know which one is in play. A bare conn.Write would be right only -// for cleartext and kTLS, and silently wrong for OpenSSL. +// for cleartext and kTLS, and silently wrong for BoringSSL. static void Emit(TcpConnection conn, TlsSession? tls, ReadOnlySpan<byte> bytes) { if (tls is null) conn.Write(bytes); @@ -8386,7 +8386,7 @@

Client · static files

_length -= count; } }
-

Every file under the root is opened once and the descriptors shared across reactors, read positionally off the ring - nothing locked, nothing cached in memory. The knob worth reading is tlsMode, because it decides whether the fast path is even legal, and the whole question is what is the write slab supposed to hold when it is sent? Cleartext: the slab IS the wire. kTLS: the slab holds plaintext and the kernel makes the records - so ReadFileAsync can still read the file straight into it and the bytes are never copied. OpenSSL: the slab must hold ciphertext, so the body has to pass through TlsSession.Write, which needs a source buffer - and that buffer IS the copy. The sample refuses that combination rather than serving the file in the clear. Measured on one saturated reactor at 4 KiB, skipping the copy is worth 1.22× cleartext and 1.21× under kTLS; at 64 KiB the kernel's per-byte crypto overtakes it and OpenSSL wins outright.

+

Every file under the root is opened once and the descriptors shared across reactors, read positionally off the ring - nothing locked, nothing cached in memory. The knob worth reading is tlsMode, because it decides whether the fast path is even legal, and the whole question is what is the write slab supposed to hold when it is sent? Cleartext: the slab IS the wire. kTLS: the slab holds plaintext and the kernel makes the records - so ReadFileAsync can still read the file straight into it and the bytes are never copied. BoringSSL: the slab must hold ciphertext, so the body has to pass through TlsSession.Write, which needs a source buffer - and that buffer IS the copy. The sample refuses that combination rather than serving the file in the clear. Measured on one saturated reactor at 4 KiB, skipping the copy is worth 1.22× cleartext and 1.21× under kTLS; at 64 KiB the kernel's per-byte crypto overtakes it and BoringSSL wins outright.

diff --git a/docs/learn/dev-tls.html b/docs/learn/dev-tls.html index dd121094..0902d868 100644 --- a/docs/learn/dev-tls.html +++ b/docs/learn/dev-tls.html @@ -39,16 +39,16 @@

TLS internals

-

ioxide.tls terminates TLS with OpenSSL in both directions by - default: the handshake runs over memory BIOs, inbound records decrypt in userspace, and - responses are encrypted before they reach the write slab. Kernel TLS is a per-direction +

ioxide.tls terminates TLS with BoringSSL, bundled in the package, + in both directions by default: the handshake runs over memory BIOs, inbound records decrypt in + userspace, and responses are encrypted before they reach the write slab. Kernel TLS is a per-direction opt-in (TlsOptions.KernelTx / KernelRx) for the deployments where sendfile or NIC offload pays for its constraints.

TlsService · TlsService.cs

Per-reactor TLS termination, holding one SSL_CTX shared across that reactor's - connections (created in OnStart). By default the context keeps OpenSSL's own - posture - TLS 1.2 and 1.3, any ciphersuite, session tickets on. Only when + connections (created in OnStart). By default the context keeps BoringSSL's own + posture - TLS 1.2 and 1.3, its default ciphersuites, session tickets on. Only when KernelTx opts into kTLS is it pinned to TLS 1.3 with TLS_AES_128_GCM_SHA256 and tickets disabled: the kernel needs a known key layout, and a ticket would advance the record sequence after the handshake and break the handoff. @@ -64,14 +64,14 @@

The handshake

TlsService.cs · the handshake as an ordinary reactor await loop

while (true)
 {
-    int ret = OpenSsl.SSL_accept(ssl);
-    int err = ret == 1 ? 0 : OpenSsl.SSL_get_error(ssl, ret);   // read the error immediately
+    int ret = BoringSsl.SSL_accept(ssl);
+    int err = ret == 1 ? 0 : BoringSsl.SSL_get_error(ssl, ret);   // read the error immediately
 
     await FlushOutbound(conn, wbio);    // drain server flights into the slab, then io_uring send
 
     if (ret == 1) break;
-    if (err != OpenSsl.SSL_ERROR_WANT_READ)
-        throw new IOException($"TLS handshake failed: {OpenSsl.LastError()}");
+    if (err != BoringSsl.SSL_ERROR_WANT_READ)
+        throw new IOException($"TLS handshake failed: {BoringSsl.LastError()}");
 
     RecvSnapshot snapshot = await conn.ReadAsync();   // WANT_READ: pull more ciphertext
     bool fed = FeedInbound(conn, rbio, snapshot);
@@ -83,22 +83,22 @@ 

The handshake

Per-session secret via ex_data (no global map)

kTLS keys are derived from the TLS 1.3 traffic secrets (server for TX, client for RX), which - OpenSSL surfaces through a keylog callback. The callback is a static function pointer with no + BoringSSL surfaces through a keylog callback. The callback is a static function pointer with no instance, so to find the right session it reads a GCHandle to the TlsSession stored on the SSL via SSL_set_ex_data / SSL_get_ex_data (the index allocated once with - CRYPTO_get_ex_new_index), and stores the secret on that session directly - no + SSL_get_ex_new_index), and stores the secret on that session directly - no process-global, recycled-pointer-keyed map, so two reactors that get the same recycled SSL address can't collide. On every path that does not program the kernel the secrets are zeroed at the handoff decision, and again on TlsSession.Dispose, which also frees the handle.

TlsService.cs · one ex_data index, allocated once; each SSL* carries a GCHandle to its session

private static readonly int SslSessionIndex =
-    OpenSsl.CRYPTO_get_ex_new_index(OpenSsl.CRYPTO_EX_INDEX_SSL, 0, 0, 0, 0, 0);
+    BoringSsl.SSL_get_ex_new_index(0, 0, 0, 0, 0);
 
 // In AcceptAsync, bind this SSL* to its managed session:
 GCHandle handle = GCHandle.Alloc(session);
-OpenSsl.SSL_set_ex_data(ssl, SslSessionIndex, GCHandle.ToIntPtr(handle));
+BoringSsl.SSL_set_ex_data(ssl, SslSessionIndex, GCHandle.ToIntPtr(handle));
 session.AttachHandle(handle);

TlsService.cs · the static keylog callback resolves its session through ex_data - no global map

[UnmanagedCallersOnly]
@@ -113,7 +113,7 @@ 

Per-session secret via ex_data (no global map)

if (!server && !client) return; - nint data = OpenSsl.SSL_get_ex_data(ssl, SslSessionIndex); + nint data = BoringSsl.SSL_get_ex_data(ssl, SslSessionIndex); if (data == 0 || GCHandle.FromIntPtr(data).Target is not TlsSession session) return; @@ -208,8 +208,9 @@

Ktls · Ktls.cs

} }
-

OpenSsl · OpenSsl.cs

-

The minimal OpenSSL 3 P/Invoke surface (via [LibraryImport] source-gen): context +

BoringSsl · BoringSsl.cs

+

The minimal P/Invoke surface over libioxide_boringssl.so, the BoringSSL build the + package ships (via [LibraryImport] source-gen): context setup (SSL_CTX_new, protocol/ciphersuite/ticket controls, cert/key file loads, the keylog and ALPN callback setters), the per-connection objects (SSL_new/_free, SSL_set_bio/_accept_state/_accept/_read/_write/_shutdown/_get_error), diff --git a/docs/learn/overview.html b/docs/learn/overview.html index d66b6471..d50b73ad 100644 --- a/docs/learn/overview.html +++ b/docs/learn/overview.html @@ -122,7 +122,7 @@

Speaking a protocol

knowledge, or h2 over TLS chosen by ALPN.
  • ioxide.nghttp3 / ioxide.http3 - HTTP/3, over QUIC from ioxide.ngtcp2.
  • -
  • TLS - OpenSSL on the ring, kernel TLS as an opt-in, plus a +
  • TLS - BoringSSL on the ring, kernel TLS as an opt-in, plus a portable SslStream path. In the core package; no reference to add.
  • diff --git a/docs/learn/tls.html b/docs/learn/tls.html index 19b241f8..7482cea7 100644 --- a/docs/learn/tls.html +++ b/docs/learn/tls.html @@ -41,8 +41,8 @@

    TLS

    ioxide gives you two ways to serve TLS, with a deliberate trade-off between them. ioxide.tls is the native path: the handshake runs over the ring, - OpenSSL does the record crypto in both directions by default, and kernel TLS transmit offload - is one flag away for the deployments that can use it. SslStream over + BoringSSL (bundled in the package) does the record crypto in both directions by default, and + kernel TLS transmit offload is one flag away for the deployments that can use it. SslStream over TcpConnectionStream is the portable path: fully managed, full-featured, and a bit slower. Both terminate on a dedicated port and need nothing from the core engine beyond what is already public - and both ship inside the ioxide @@ -54,11 +54,11 @@

    TLS

    - + - +
    ioxide.tlsSslStream + TcpConnectionStream
    cryptoOpenSSL; kernel TX offload opt-infully managed
    cryptoBoringSSL, bundled; kernel TX offload opt-infully managed
    copiesone staged copy; zero-copy send under kTLS TXbuffered both ways
    TLS versions1.2 and 1.3 (1.3 only under kTLS)1.2 and 1.3
    featuresALPN; resumption in the default mode (tickets are per-reactor)client certs, resumption, everything
    dependencyOpenSSL 3 (tls module only for kTLS)none - portable
    dependencynone - BoringSSL ships in the package (tls module only for kTLS)none - portable
    throughputbaseline~0.65× (measured)
    rolefast pathcompatibility path
    @@ -68,14 +68,14 @@

    kTLS: the model

    The catch is that kTLS only handles the record layer - it does not do the handshake. So ioxide.tls splits the work:

      -
    • Handshake in userspace, over the ring. OpenSSL runs the TLS 1.3 handshake through +
    • Handshake in userspace, over the ring. BoringSSL runs the TLS 1.3 handshake through memory BIOs; the ciphertext flows over the connection's normal recv/send. Handshake bytes ride the same io_uring as everything else, and the handshake resumes inline like any await.
    • Transmit offloaded to the kernel. Once the handshake completes, the negotiated keys are programmed into the socket. From then on your handler writes plaintext and the kernel produces the TLS records on the existing io_uring send path - no managed crypto, no extra copy.
    • -
    • Receive stays in userspace. Inbound records are decrypted by OpenSSL. Requests are +
    • Receive stays in userspace. Inbound records are decrypted by BoringSSL. Requests are small, so this is cheap; the heavy direction (responses) is the one the kernel handles.
    @@ -175,7 +175,7 @@

    The handler

    kTLS: how the hand-off works

    After SSL_accept completes over the memory BIOs, AcceptAsync:

      -
    1. captures the TLS 1.3 server traffic secret from OpenSSL's keylog callback;
    2. +
    3. captures the TLS 1.3 server traffic secret from BoringSSL's keylog callback;
    4. derives the AES-128-GCM key and IV with HKDF-Expand-Label (RFC 8446);
    5. programs them into the socket - setsockopt(TCP_ULP, "tls") then setsockopt(SOL_TLS, TLS_TX, ...) with record sequence 0;
    6. @@ -224,7 +224,7 @@

      ALPN: choosing the protocol

      kTLS: receive, and the one limitation

      The transmit side is kernel-offloaded; the receive side is userspace. Each inbound - slice is fed to OpenSSL and decrypted with SSL_read (that is what + slice is fed to BoringSSL and decrypted with SSL_read (that is what TlsSession.Decrypt does). This is the deliberate boundary of the implementation:

      • Why not offload RX too? kTLS RX can only take over at a clean TLS record boundary @@ -245,25 +245,26 @@

        kTLS: constraints and requirements

      • No session resumption (tickets disabled, see above).
      • ALPN is selected from the client's offer, by server preference (see above).
      • Linux tls kernel module (modprobe tls; standard on - mainstream distros) and OpenSSL 3 (present in the .NET runtime images).
      • + mainstream distros). The TLS library needs nothing installed: BoringSSL ships in the + package.
      -

      Choosing a backend: kTLS, or OpenSSL both ways

      -

      OpenSSL is the default in both directions. TlsOptions.KernelTx turns +

      Choosing a backend: kTLS, or BoringSSL both ways

      +

      BoringSSL is the default in both directions. TlsOptions.KernelTx turns kernel encryption on; it is off unless you ask, so the socket gets no TLS ULP and nothing requires the tls module. Everything else - the handshake, ALPN, the read loop - is identical either way.

      It used to default the other way, which made ioxide's TLS asymmetric: the kernel encrypted - while OpenSSL decrypted. That asymmetry is now something you opt into, because it is not free + while userspace decrypted. That asymmetry is now something you opt into, because it is not free and it was not faster. Handlers should write through TlsSession.Write(connection, plaintext), which is correct in both modes - a bare connection.Write is only safe with kTLS on, and puts cleartext on the wire without it.

      - + - + @@ -272,30 +273,30 @@

      Choosing a backend: kTLS, or OpenSSL both ways

      see - they are loopback, with no NIC and no sendfile anywhere in the path.

      kTLS: performance

      -

      Measured against plaintext, not against each other. 4 reactors, - wrk -t4 -c64, HTTP/1.1, Tcp/Raw as the baseline:

      +

      Measured against plaintext, not against each other. Requests per second, 4 reactors, + wrk -t4 -c64, HTTP/1.1, Tcp/Raw as the baseline + (bench/tls-matrix.sh, best of two passes):

      KernelTx = truedefault (OpenSSL)
      KernelTx = truedefault (BoringSSL)
      tls kernel modulerequirednot needed
      TLS versions1.3 only1.2 and 1.3
      ciphersuitesone (AES-128-GCM)any
      ciphersuitesone (AES-128-GCM)any BoringSSL implements
      session resumptiondisabledavailable
      handshake alignmentconstrains the handoffirrelevant
      sendfile / NIC offloadyesno
      - - - - - + + + + +
      responseplaintextkTLSOpenSSL
      64 B1,379,9801,091,224  0.79×1,093,361  0.79×
      8 KiB1,109,287749,833  0.68×802,325  0.72×
      64 KiB448,093157,644  0.35×196,486  0.44×
      256 KiB133,09741,691  0.31×49,801  0.37×
      responseplaintextkTLS TXBoringSSL
      64 B1,245,989937,322  0.75×976,752  0.78×
      8 KiB991,826582,376  0.59×676,158  0.68×
      64 KiB344,094134,112  0.39×153,538  0.45×
      256 KiB113,41238,786  0.34×40,250  0.35×

      Two things worth taking from that. TLS costs far more than the choice of backend does - - at 64 KiB you are at a third of plaintext either way, so the gap between kTLS and OpenSSL is a - rounding error next to the cost of encrypting at all. And the ratio depends on response - size: 0.79× at 64 bytes is not the same server as 0.35× at 64 KiB.

      -

      kTLS is behind on large single writes - the kernel has to split one 256 KiB write - into sixteen records - and level once the protocol above it already chunks. Over HTTP/2, where - HTTP/2 frames responses into 16 KiB DATA frames before they reach the socket, the two are - within noise of each other at every size.

      + at 64 KiB you are under half of plaintext either way, so the gap between kTLS and BoringSSL is + small next to the cost of encrypting at all. And the ratio depends on response size: + 0.78× at 64 bytes is not the same server as 0.35× at 256 KiB.

      +

      kTLS is behind over HTTP/1.1 at every size, most between 8 and 64 KiB, and level once the + protocol above it already chunks: over HTTP/2, which frames responses into 16 KiB DATA frames + before they reach the socket, the two are within 5% of each other at 64 B and 8 KiB.

      kTLS: where the cost actually is

      The remainder of this section is about kTLS specifically. The cost is dominated by the inherent work of encrypting each response - which the kernel does, on the existing send path, with no userspace copy - plus the small fixed cost of decrypting each request. Skipping the userspace copy does not make it faster than - OpenSSL on the ring: the table above has OpenSSL level or ahead at every size, because the + BoringSSL on the ring: the table above has BoringSSL ahead at every size, because the crypto dominates and the kernel path gives up batching the userspace one keeps. What the kernel path uniquely enables is sendfile and NIC offload - which loopback numbers cannot show. The remaining gap to plaintext is the AES-GCM work itself, which no TLS @@ -348,16 +349,16 @@

      Which to use

      path does not do mTLS, so that still means SslStream.

      • kTLS (KernelTx = true is the deployable hybrid - kernel TX, - OpenSSL RX; add KernelRx = true for both directions, which is experimental) + BoringSSL RX; add KernelRx = true for both directions, which is experimental) → you are on Linux 6+ with the tls module, TLS 1.3 and ALPN are enough, and you want sendfile or a NIC that offloads TLS. Sample: Tls/Ktls.
      • -
      • OpenSSL both ways (the default) → no kernel module, so it runs in a container +
      • BoringSSL both ways (the default) → no kernel module, so it runs in a container that lacks one; TLS 1.2 available, any ciphersuite, resumption back, and no - handshake-alignment constraint. Costs nothing measurable here - see the table above. Sample: + handshake-alignment constraint. Cheaper than kTLS at every size here - see the table above. Sample: Tls/OpenSsl.
      • -
      • SslStream over TcpConnectionStream → now only for what OpenSSL itself - cannot give you, or when you want the BCL's stack specifically. It is the slowest of the +
      • SslStream over TcpConnectionStream → now only for what BoringSSL + itself cannot give you, or when you want the BCL's stack specifically. It is the slowest of the three, because every byte is encrypted in userspace and copied through a Stream.
      diff --git a/fuzz/Ioxide.Fuzz/Program.cs b/fuzz/Ioxide.Fuzz/Program.cs index bdc2b9d6..d55515af 100644 --- a/fuzz/Ioxide.Fuzz/Program.cs +++ b/fuzz/Ioxide.Fuzz/Program.cs @@ -4,7 +4,7 @@ // Fuzz the managed parsers that read bytes chosen by whoever connected. // // This is where ioxide's parsing actually lives. The QUIC transport is vendored ngtcp2 and the -// TLS is OpenSSL or picotls, so the hand-written attack surface is small and almost all of it is +// TLS is BoringSSL or picotls, so the hand-written attack surface is small and almost all of it is // C#: QPACK field sections, HTTP/3 varints, frame headers. Those decode straight from the wire // into a request a handler then trusts. // diff --git a/fuzz/README.md b/fuzz/README.md index 33772271..4e1f9e4e 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -29,7 +29,7 @@ bash fuzz/shim/build.sh && fuzz/shim/build/fuzz_client_hello # the one C parser ## `Ioxide.Fuzz/` - the managed parsers This is where ioxide's parsing lives, so this is where most of the effort goes. The QUIC transport -is vendored ngtcp2 and the TLS is OpenSSL or picotls, so the hand-written attack surface is small +is vendored ngtcp2 and the TLS is BoringSSL or picotls, so the hand-written attack surface is small and almost all of it is C#: HTTP/3 varints, QPACK prefixed integers, and whole QPACK field sections decoded straight into the request object a handler is then handed. diff --git a/scripts/build-boringssl-native.sh b/scripts/build-boringssl-native.sh new file mode 100755 index 00000000..f6f35477 --- /dev/null +++ b/scripts/build-boringssl-native.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# +# Build the TLS native bundle for ioxide: BoringSSL (libssl + libcrypto) statically linked into ONE +# shared library, libioxide_boringssl.so, whose only external dependency is libc. Both TLS columns +# use it - the userspace one for every record, the kTLS one for the handshake - and so does +# ioxide.httpclient. Output lands in the package's runtimes/ dir so the nupkg carries everything. +# +# scripts/build-boringssl-native.sh # build with the pinned ref below +# BORINGSSL_REF= scripts/build-boringssl-native.sh +# +set -euo pipefail +cd "$(dirname "$0")/.." + +# PINNED, like the other native scripts: a security-critical library compiled into a binary that +# ships in a nupkg has to be reproducible. BoringSSL has no releases; this is the main commit the +# shipped .so was built from (2026-10-11). It needs SSL_CTX_set1_tls13_ciphers, which is recent. +BORINGSSL_REF=${BORINGSSL_REF:-5b83e5d1abf0ae0033ae52ab579d906b67633b2c} +WORK=${WORK:-/tmp/ioxide-boringssl-native} +OUT=src/ioxide/runtimes/linux-x64/native + +rm -rf "$WORK" && mkdir -p "$WORK" "$OUT" +cd "$WORK" + +echo "==> fetching boringssl ($BORINGSSL_REF)" +mkdir -p boringssl +git -C boringssl init -q +git -C boringssl remote add origin https://github.com/google/boringssl +git -C boringssl fetch -q --depth 1 origin "$BORINGSSL_REF" || { + echo "could not fetch $BORINGSSL_REF" >&2 + exit 1 +} +git -C boringssl checkout -q FETCH_HEAD + +echo "==> building libssl + libcrypto (static, PIC, public API exported)" +# BORINGSSL_SHARED_LIBRARY gives OPENSSL_EXPORT default visibility in the static archives; every +# internal symbol stays hidden. +cmake -S boringssl -B boringssl/build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_C_FLAGS=-DBORINGSSL_SHARED_LIBRARY -DCMAKE_CXX_FLAGS=-DBORINGSSL_SHARED_LIBRARY >/dev/null +cmake --build boringssl/build --target ssl crypto -j"$(nproc)" >/dev/null + +echo "==> linking libioxide_boringssl.so" +# std:: vtables and typeinfo that BoringSSL's own objects instantiate stay local too. +echo '{ local: _ZSt*; _ZNSt*; _ZNKSt*; _ZTISt*; _ZTSSt*; _ZTVSt*; };' >std-local.map +# -Bsymbolic: .NET loads the system's OpenSSL into the same process for its own crypto, and the +# names are the same; every call inside this library has to stay inside it. BoringSSL is C++ built +# without exceptions or RTTI: the little it needs from the C++ runtime is linked in, and hidden. +g++ -shared -o libioxide_boringssl.so -Wl,-soname,libioxide_boringssl.so \ + -Wl,--whole-archive boringssl/build/libssl.a boringssl/build/libcrypto.a -Wl,--no-whole-archive \ + -Wl,-Bsymbolic -Wl,--no-undefined -static-libstdc++ -static-libgcc -lpthread \ + -Wl,--exclude-libs,libstdc++.a:libgcc.a:libgcc_eh.a -Wl,--version-script=std-local.map +strip --strip-unneeded libioxide_boringssl.so + +echo "==> verifying" +ldd libioxide_boringssl.so | grep -vE 'vdso|libc\.|libm\.|ld-linux' && { + echo "unexpected dependency"; exit 1; } || true +nm -D --defined-only libioxide_boringssl.so | awk '{print $3}' >exports.txt +for symbol in SSL_CTX_new SSL_CTX_set1_tls13_ciphers SSL_CTX_set_min_proto_version \ + SSL_CTX_set_tlsext_servername_callback SSL_CTX_set_keylog_callback SSL_get_ex_new_index \ + BIO_s_mem PEM_read_bio_PrivateKey X509_verify_cert_error_string; do + grep -qx "$symbol" exports.txt || { echo "missing export $symbol"; exit 1; } +done +[ "$(grep -c '^SSL_' exports.txt)" -gt 300 ] || { echo "libssl exports missing"; exit 1; } +[ "$(grep -c '^EVP_' exports.txt)" -gt 200 ] || { echo "libcrypto exports missing"; exit 1; } +c++filt /dev/null +cp "$WORK/libioxide_boringssl.so" "$OUT/libioxide_boringssl.so" +echo "==> done: $OUT/libioxide_boringssl.so ($(du -h "$OUT/libioxide_boringssl.so" | cut -f1))" diff --git a/scripts/build-ngtcp2-native.sh b/scripts/build-ngtcp2-native.sh index c2e4b9a4..f7442e26 100755 --- a/scripts/build-ngtcp2-native.sh +++ b/scripts/build-ngtcp2-native.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # # Build the self-contained QUIC native bundle for ioxide.ngtcp2: -# ngtcp2 (QUIC engine) + its picotls crypto backend + picotls (TLS 1.3 handshake), -# statically linked into ONE shared library whose only external dependency is -# libcrypto.so.3 (picotls uses OpenSSL's libcrypto primitives, so any OpenSSL 3.x -# works - no OpenSSL 3.5 QUIC API needed). Output lands in the package's -# runtimes/ dir so the nupkg carries everything. +# ngtcp2 (QUIC engine) + its picotls crypto backend + picotls (TLS 1.3 handshake) + +# BoringSSL's libcrypto (the primitives picotls's OpenSSL backend calls), +# statically linked into ONE shared library whose only external dependency is libc. BoringSSL's +# symbols stay hidden inside it. Output lands in the package's runtimes/ dir so the nupkg carries +# everything. # # scripts/build-ngtcp2-native.sh # build with pinned refs below # NGTCP2_REF=v1.14.0 scripts/build-ngtcp2-native.sh @@ -21,13 +21,15 @@ cd "$(dirname "$0")/.." # build. These are the commits the shipped .so was built from. NGTCP2_REF=${NGTCP2_REF:-7f7e24b1deb1dc94985c2d6345ca68cc387b4709} PICOTLS_REF=${PICOTLS_REF:-f07f1c8c68b237f1468bc1f1fe1b68aba3ff23b4} +# The commit scripts/build-boringssl-native.sh pins; kept the same so ioxide ships one BoringSSL. +BORINGSSL_REF=${BORINGSSL_REF:-5b83e5d1abf0ae0033ae52ab579d906b67633b2c} WORK=${WORK:-/tmp/ioxide-quic-native} OUT=src/protocols/ioxide.ngtcp2/runtimes/linux-x64/native rm -rf "$WORK" && mkdir -p "$WORK" "$OUT" cd "$WORK" -echo "==> fetching picotls ($PICOTLS_REF) + ngtcp2 ($NGTCP2_REF)" +echo "==> fetching boringssl ($BORINGSSL_REF) + picotls ($PICOTLS_REF) + ngtcp2 ($NGTCP2_REF)" # Fetch the exact ref rather than cloning a branch: --branch takes tags and branches, and these are # commits. Falling back to a plain clone - as this used to - would quietly build something else. @@ -43,44 +45,72 @@ fetch_at() { # url, ref, dir git -C "$3" submodule update -q --init --depth 1 --recursive } +fetch_at https://github.com/google/boringssl "$BORINGSSL_REF" boringssl fetch_at https://github.com/h2o/picotls "$PICOTLS_REF" picotls fetch_at https://github.com/ngtcp2/ngtcp2 "$NGTCP2_REF" ngtcp2 -echo "==> building picotls (static, PIC)" +echo "==> building boringssl (static, PIC, symbols hidden)" +cmake -S boringssl -B boringssl/build -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON >/dev/null +cmake --build boringssl/build --target crypto ssl decrepit -j"$(nproc)" >/dev/null +# The layout picotls's FIND_PACKAGE(OpenSSL) expects; it recognises BoringSSL by openssl/base.h. +mkdir -p boringssl-root/lib +ln -s "$WORK/boringssl/include" boringssl-root/include +cp boringssl/build/libcrypto.a boringssl/build/libssl.a boringssl/build/libdecrepit.a boringssl-root/lib/ + +echo "==> building picotls (static, PIC, on BoringSSL)" cmake -S picotls -B picotls/build -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DWITH_FUSION=OFF >/dev/null -cmake --build picotls/build -j"$(nproc)" >/dev/null + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DWITH_FUSION=OFF \ + -DOPENSSL_ROOT_DIR="$WORK/boringssl-root" -DOPENSSL_USE_STATIC_LIBS=TRUE >/dev/null +# Only the two archives: the test programs would link libssl, which is C++. +cmake --build picotls/build --target picotls-core picotls-openssl -j"$(nproc)" >/dev/null echo "==> building ngtcp2 + crypto_picotls (static, PIC, default visibility)" # ngtcp2 pins hidden visibility on its static targets; the bundle needs the API exported. sed -i 's/C_VISIBILITY_PRESET hidden/C_VISIBILITY_PRESET default/' \ ngtcp2/lib/CMakeLists.txt ngtcp2/crypto/picotls/CMakeLists.txt +# ngtcp2 finds "OpenSSL" for the picotls backend too; pointed at BoringSSL, whose version lives in +# base.h where FindOpenSSL does not look, so the 1.1.1 floor would refuse it. -I BoringSSL first: +# picotls/openssl.h includes , which would otherwise find the system's headers. +sed -i 's/find_package(OpenSSL 1.1.1 REQUIRED)/find_package(OpenSSL REQUIRED)/' ngtcp2/CMakeLists.txt cmake -S ngtcp2 -B ngtcp2/build -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TESTING=OFF \ + -DCMAKE_C_FLAGS="-I$WORK/boringssl/include" \ + -DOPENSSL_ROOT_DIR="$WORK/boringssl-root" -DOPENSSL_USE_STATIC_LIBS=TRUE \ -DENABLE_PICOTLS=ON -DENABLE_OPENSSL=OFF -DENABLE_GNUTLS=OFF \ -DENABLE_SHARED_LIB=OFF -DENABLE_STATIC_LIB=ON \ -DPICOTLS_INCLUDE_DIR="$WORK/picotls/include" \ - "-DPICOTLS_LIBRARIES=$WORK/picotls/build/libpicotls-openssl.a;$WORK/picotls/build/libpicotls-core.a;crypto" >/dev/null + "-DPICOTLS_LIBRARIES=$WORK/picotls/build/libpicotls-openssl.a;$WORK/picotls/build/libpicotls-core.a;$WORK/boringssl-root/lib/libdecrepit.a;$WORK/boringssl-root/lib/libcrypto.a;stdc++" >/dev/null cmake --build ngtcp2/build -j"$(nproc)" >/dev/null echo "==> compiling the C# facade shim" SHIM="$OLDPWD/src/protocols/ioxide.ngtcp2/native/ioxide_ngtcp2_shim.c" -gcc -c -O2 -fPIC -o shim.o "$SHIM" \ +gcc -c -O2 -fPIC -o shim.o "$SHIM" -I"$WORK/boringssl/include" \ -Ingtcp2/lib/includes -Ingtcp2/build/lib/includes \ -Ingtcp2/crypto/includes -Ipicotls/include echo "==> linking libioxide_ngtcp2.so" -gcc -shared -o libioxide_ngtcp2.so shim.o \ +# std:: vtables and typeinfo that BoringSSL's own objects instantiate stay local too. +echo '{ local: _ZSt*; _ZNSt*; _ZNKSt*; _ZTISt*; _ZTSSt*; _ZTVSt*; };' >std-local.map +# picotls's Blowfish (CID encryption) is in BoringSSL's libdecrepit. BoringSSL is C++ built without +# exceptions or RTTI: the little it needs from the C++ runtime is linked in, and hidden. +g++ -shared -o libioxide_ngtcp2.so shim.o \ -Wl,--whole-archive \ ngtcp2/build/lib/libngtcp2.a \ ngtcp2/build/crypto/picotls/libngtcp2_crypto_picotls.a \ picotls/build/libpicotls-openssl.a \ picotls/build/libpicotls-core.a \ - -Wl,--no-whole-archive -lcrypto -Wl,--no-undefined + -Wl,--no-whole-archive boringssl-root/lib/libdecrepit.a boringssl-root/lib/libcrypto.a \ + -static-libstdc++ -static-libgcc -lpthread -Wl,-Bsymbolic -Wl,--no-undefined \ + -Wl,--exclude-libs,libstdc++.a:libgcc.a:libgcc_eh.a -Wl,--version-script=std-local.map +strip --strip-unneeded libioxide_ngtcp2.so # BoringSSL builds with -ggdb echo "==> verifying" -ldd libioxide_ngtcp2.so | grep -vE 'vdso|libcrypto|libc\.|ld-linux' && { +ldd libioxide_ngtcp2.so | grep -vE 'vdso|libc\.|ld-linux' && { echo "unexpected dependency"; exit 1; } || true +# BoringSSL is linked in, not offered: its symbols must not be exported for anything else to bind. +nm -D --defined-only libioxide_ngtcp2.so | grep -qE ' ((EVP|X509|SSL|BIO)_|_Z)' && { + echo "BoringSSL or C++ symbols exported"; exit 1; } || true [ "$(nm -D --defined-only libioxide_ngtcp2.so | grep -c ' ngtcp2_')" -gt 500 ] || { echo "ngtcp2 exports missing"; exit 1; } [ "$(nm -D --defined-only libioxide_ngtcp2.so | grep -c ' ptls_')" -gt 100 ] || { diff --git a/scripts/gen-docs-panes.py b/scripts/gen-docs-panes.py index f117845c..5cbe1984 100644 --- a/scripts/gen-docs-panes.py +++ b/scripts/gen-docs-panes.py @@ -21,7 +21,7 @@ "One port, several names, a certificate each - chosen inside the handshake, before a byte " "of your protocol exists. The certificate at the top stays the DEFAULT: a client that sends " "no name (anything connecting by IP) or asks for one that is not registered gets it rather " - "than a dead connection. Each entry is one OpenSSL context built at startup, so choosing " + "than a dead connection. Each entry is one BoringSSL context built at startup, so choosing " "one per handshake is a lookup that allocates nothing - ten names cost what one does. " "The same feature on h2 is " ", and on QUIC " @@ -87,7 +87,7 @@ "switches back to stock sockets so the same app can be measured both ways."), "tls": ( "Tls/Ktls", "kTLS · raw ring", "ioxide", - ["sudo modprobe tls # kTLS needs the Linux 'tls' module + OpenSSL 3", + ["sudo modprobe tls # kTLS needs the Linux 'tls' module", "curl -k https://127.0.0.1:8443/"], "Opt-in - and FULL kTLS: both directions in the kernel, set right on the options. " "Kernel RX is experimental, so Tls/KtlsTx in the repo is this server minus " @@ -101,7 +101,7 @@ "Tls/KtlsTx", "hybrid · raw ring", "ioxide", ["sudo modprobe tls # the kernel half still needs the module", "curl -k https://127.0.0.1:8443/"], - "The deployable kernel mode: kernel TX, OpenSSL RX. The handler still writes " + "The deployable kernel mode: kernel TX, BoringSSL RX. The handler still writes " "plaintext - the kernel makes the records on send, which is what keeps sendfile " "and NIC offload reachable - while receive takes the well-trodden userspace path instead of " "kernel RX's experimental one. " @@ -109,16 +109,17 @@ "receive; is the " "default, with the kernel in neither direction."), "tlsossl": ( - "Tls/OpenSsl", "OpenSSL · raw ring", "ioxide", + "Tls/OpenSsl", "BoringSSL · raw ring", "ioxide", ["curl -k https://127.0.0.1:8443/ # no modprobe needed"], - "The default. No TLS ULP is attached at all - OpenSSL encrypts and decrypts, and the " + "The default. No TLS ULP is attached at all - BoringSSL encrypts and decrypts, and the " "response goes through TlsSession.Write, which is correct whichever backend the " "session ended up with. That drops every constraint kTLS imposes: no kernel module, TLS 1.2, any " "ciphersuite, session resumption back, no handshake-alignment problem. What it gives up is " "sendfile and NIC offload. " - "It costs nothing measurable here - against the plaintext baseline, 4 reactors, " - "wrk -t4 -c64: 0.79× for both at a 64-byte response, 0.35× kTLS vs " - "0.44× OpenSSL at 64 KiB. Which backend you pick matters far less than the cost of TLS."), + "It is cheaper than kTLS here - against the plaintext baseline, 4 reactors, " + "wrk -t4 -c64: 0.75× kTLS TX vs 0.78× BoringSSL at a 64-byte " + "response, 0.39× vs 0.45× at 64 KiB. Which backend you pick matters far less than " + "the cost of TLS."), "tlspipe": ( "Tls/KtlsPipes", "kTLS · pipes", "ioxide", ["sudo modprobe tls", "curl -k https://127.0.0.1:8443/"], @@ -128,7 +129,7 @@ "because TlsConnectionDualPipe composes its halves from the session rather than " "from configuration."), "tlsosslpipe": ( - "Tls/OpenSslPipes", "OpenSSL · pipes", "ioxide", + "Tls/OpenSslPipes", "BoringSSL · pipes", "ioxide", ["curl -k https://127.0.0.1:8443/ # no modprobe needed"], "Diff this against and " "the only functional difference is KernelTx; the rest is the banner and the log " @@ -145,7 +146,7 @@ "the TLS door builds a TlsConnectionDualPipe, the plaintext door pairs the " "connection's own reader and writer, and the loop reads an IDuplexPipe without " "knowing which it got. Ports come from multi-port; " - "TLS is the default backend, OpenSSL both ways."), + "TLS is the default backend, BoringSSL both ways."), # ── transport ──────────────────────────────────────────────────────────────────────────── "tlsbcl": ( @@ -155,7 +156,7 @@ "TcpConnectionStream. Fully managed, full-featured, and the slowest of the " "three, because the bytes are copied through a Stream both ways. Reach for it " "for client certificates - the ring-native path does not do mTLS - or anything else " - "OpenSSL-on-the-ring does not expose; for TLS 1.2 and resumption the default already has " + "BoringSSL-on-the-ring does not expose; for TLS 1.2 and resumption the default already has " "you covered: ."), "big": ( "Tcp/Big", "TCP · large responses", "ioxide", @@ -189,7 +190,7 @@ "then branches on what was agreed, so ONE port serves both. That is why the h2c samples " "are the exception rather than the rule. What Http2Connection is handed is a " "TlsConnectionDualPipe: it never learns TLS is involved, so the protocol code " - "is byte-for-byte the h2c sample's. TLS is OpenSSL both ways by default; the " + "is byte-for-byte the h2c sample's. TLS is BoringSSL both ways by default; the " "kernelTx/kernelRx knobs at the top move either direction into " "the kernel, and cost the same 0.05µs per request either way."), "h2bcl": ( @@ -300,12 +301,12 @@ "the whole question is what is the write slab supposed to hold when it is sent? " "Cleartext: the slab IS the wire. kTLS: the slab holds plaintext and the kernel makes " "the records - so ReadFileAsync can still read the file straight into it and the " - "bytes are never copied. OpenSSL: the slab must hold ciphertext, so the body has to " + "bytes are never copied. BoringSSL: the slab must hold ciphertext, so the body has to " "pass through TlsSession.Write, which needs a source buffer - and that buffer IS " "the copy. The sample refuses that combination rather than serving the file in the clear. " "Measured on one saturated reactor at 4 KiB, skipping the copy is worth 1.22× " "cleartext and 1.21× under kTLS; at 64 KiB the kernel's per-byte crypto " - "overtakes it and OpenSSL wins outright."), + "overtakes it and BoringSSL wins outright."), # ── the last hand-written code panes, now generated too ────────────────────────────────── "pipe": ( diff --git a/src/clients/ioxide.httpclient/Tls/OpenSsl.cs b/src/clients/ioxide.httpclient/Tls/BoringSsl.cs similarity index 80% rename from src/clients/ioxide.httpclient/Tls/OpenSsl.cs rename to src/clients/ioxide.httpclient/Tls/BoringSsl.cs index caddc218..6c7a3880 100644 --- a/src/clients/ioxide.httpclient/Tls/OpenSsl.cs +++ b/src/clients/ioxide.httpclient/Tls/BoringSsl.cs @@ -3,44 +3,43 @@ namespace ioxide.httpclient; /// -/// The OpenSSL 3 surface a TLS CLIENT needs: connect state, SNI, peer verification, the ALPN offer, -/// and the memory-BIO plumbing the handshake rides on. +/// The BoringSSL surface a TLS CLIENT needs: connect state, SNI, peer verification, the ALPN offer, +/// and the memory-BIO plumbing the handshake rides on. The library is the one the ioxide package +/// ships, libioxide_boringssl.so. /// /// /// Deliberately separate from the one in ioxide.tls rather than shared. That package terminates TLS /// for inbound connections and its whole shape serves kernel-TLS offload - TLS 1.3 only, a single /// ciphersuite, no session tickets, a keylog callback to extract the traffic secret. None of that /// applies to a client, and the two halves have zero logic in common: what they genuinely share is -/// about sixteen ABI declarations of memory-BIO boilerplate that have not changed since OpenSSL -/// 3.0. Duplicating those is cheaper than making a client package depend on a server one. +/// about sixteen ABI declarations of memory-BIO boilerplate. Duplicating those is cheaper than +/// making a client package depend on a server one. /// -internal static unsafe partial class OpenSsl +internal static unsafe partial class BoringSsl { - private const string Ssl = "libssl.so.3"; - private const string Crypto = "libcrypto.so.3"; + private const string Ssl = "libioxide_boringssl.so"; + private const string Crypto = "libioxide_boringssl.so"; public const int SSL_ERROR_NONE = 0; public const int SSL_ERROR_WANT_READ = 2; public const int SSL_ERROR_WANT_WRITE = 3; public const int SSL_ERROR_ZERO_RETURN = 6; - public const int SSL_CTRL_SET_TLSEXT_HOSTNAME = 55; - public const int SSL_CTRL_SET_MIN_PROTO_VERSION = 123; - public const int TLSEXT_NAMETYPE_host_name = 0; - public const int SSL_VERIFY_NONE = 0x00; public const int SSL_VERIFY_PEER = 0x01; public const long X509_V_OK = 0; - public const uint X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS = 0x4; + public const long X509_V_ERR_INVALID_CALL = 65; // --- context --------------------------------------------------------------------------------- [LibraryImport(Ssl)] public static partial nint TLS_client_method(); [LibraryImport(Ssl)] public static partial nint SSL_CTX_new(nint method); [LibraryImport(Ssl)] public static partial void SSL_CTX_free(nint ctx); - [LibraryImport(Ssl)] public static partial long SSL_CTX_ctrl(nint ctx, int cmd, long arg, nint parg); + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_min_proto_version(nint ctx, ushort version); - /// Trust anchors from the system store (/etc/ssl/certs and friends). + /// + /// Trust anchors from the system store: /etc/ssl/certs, or SSL_CERT_FILE / SSL_CERT_DIR when set. + /// [LibraryImport(Ssl)] public static partial int SSL_CTX_set_default_verify_paths(nint ctx); [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] @@ -68,7 +67,7 @@ internal static unsafe partial class OpenSsl public const int SSL_FILETYPE_PEM = 1; /// Protocols we offer, as a length-prefixed wire list. Returns 0 on SUCCESS. - [LibraryImport(Ssl)] public static partial int SSL_CTX_set_alpn_protos(nint ctx, byte* protos, uint length); + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_alpn_protos(nint ctx, byte* protos, nuint length); // --- connection ------------------------------------------------------------------------------ @@ -92,7 +91,7 @@ internal static unsafe partial class OpenSsl /// The clear, the call and the classify are one operation and are deliberately not offered /// apart - the raw entry points above are private so that they cannot be. /// - /// OpenSSL's error queue belongs to the THREAD, which here is a reactor shared by every pooled + /// The error queue belongs to the THREAD, which here is a reactor shared by every pooled /// upstream connection on it, and SSL_get_error consults that queue BEFORE asking the SSL /// whether it merely wants more data. So a residue left by any other connection - most /// reliably by a teardown calling SSL_shutdown on a handshake that never finished - is read as @@ -123,7 +122,10 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) error = n > 0 ? SSL_ERROR_NONE : SSL_get_error(ssl, n); return n; } - [LibraryImport(Ssl)] public static partial long SSL_ctrl(nint ssl, int cmd, long larg, nint parg); + + /// The name sent as SNI. Returns 0 for a name that cannot be one. + [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] + public static partial int SSL_set_tlsext_host_name(nint ssl, string name); /// /// The hostname the peer certificate must match. Distinct from SNI: SNI tells the server which @@ -132,8 +134,12 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] public static partial int SSL_set1_host(nint ssl, string hostname); - /// How 's name is matched: X509_CHECK_FLAG_* bits. - [LibraryImport(Ssl)] public static partial void SSL_set_hostflags(nint ssl, uint flags); + /// + /// An address the peer certificate must carry as an iPAddress name. + /// checks DNS names only, so an origin reached by its address is verified through this. + /// + [LibraryImport(Crypto)] public static partial int X509_VERIFY_PARAM_set1_ip(nint param, byte* ip, nuint length); + [LibraryImport(Ssl)] public static partial nint SSL_get0_param(nint ssl); [LibraryImport(Ssl)] public static partial long SSL_get_verify_result(nint ssl); [LibraryImport(Ssl)] public static partial void SSL_get0_alpn_selected(nint ssl, byte** data, uint* length); @@ -149,15 +155,15 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) // --- errors ---------------------------------------------------------------------------------- - [LibraryImport(Crypto)] public static partial ulong ERR_get_error(); - [LibraryImport(Crypto)] public static partial void ERR_error_string_n(ulong e, byte* buf, nuint len); + [LibraryImport(Crypto)] public static partial uint ERR_get_error(); + [LibraryImport(Crypto)] public static partial void ERR_error_string_n(uint e, byte* buf, nuint len); public static string LastError() { - ulong code = ERR_get_error(); + uint code = ERR_get_error(); if (code == 0) { - return "no OpenSSL error"; + return "no BoringSSL error"; } Span buffer = stackalloc byte[256]; diff --git a/src/clients/ioxide.httpclient/Tls/TlsClientContext.cs b/src/clients/ioxide.httpclient/Tls/TlsClientContext.cs index 47344b58..f1a4f6f6 100644 --- a/src/clients/ioxide.httpclient/Tls/TlsClientContext.cs +++ b/src/clients/ioxide.httpclient/Tls/TlsClientContext.cs @@ -36,7 +36,7 @@ private TlsClientContext(nint ctx, TlsClientOptions options) Options = options; } - /// Build the context. Throws if OpenSSL rejects any of the configuration. + /// Build the context. Throws if BoringSSL rejects any of the configuration. public static TlsClientContext Create(TlsClientOptions options) { if (options.ServerName is { } name && string.IsNullOrWhiteSpace(name)) @@ -46,21 +46,22 @@ public static TlsClientContext Create(TlsClientOptions options) nameof(options)); } - nint ctx = OpenSsl.SSL_CTX_new(OpenSsl.TLS_client_method()); + nint ctx = BoringSsl.SSL_CTX_new(BoringSsl.TLS_client_method()); if (ctx == 0) { - throw new IOException($"SSL_CTX_new: {OpenSsl.LastError()}"); + throw new IOException($"SSL_CTX_new: {BoringSsl.LastError()}"); } try { - // Checked, because ssl3_ctx_ctrl returns 0 for a version it will not accept and applies - // NOTHING - silently, with no error queued. A caller pinning a floor got OpenSSL's - // default instead and no way to find out. - if (OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_SET_MIN_PROTO_VERSION, options.MinimumVersion, 0) != 1) + // Checked, because a version the library will not accept leaves its default floor in + // place, and a caller pinning one would have no way to find out. The range check comes + // first: the int would otherwise wrap into a ushort that happens to be a version. + if (options.MinimumVersion is < 0 or > ushort.MaxValue || + BoringSsl.SSL_CTX_set_min_proto_version(ctx, (ushort)options.MinimumVersion) != 1) { throw new IOException( - $"set_min_proto_version({options.MinimumVersion:X}): OpenSSL refused the version."); + $"set_min_proto_version({options.MinimumVersion:X}): BoringSSL refused the version."); } ConfigureVerification(ctx, options); @@ -71,7 +72,7 @@ public static TlsClientContext Create(TlsClientOptions options) } catch { - OpenSsl.SSL_CTX_free(ctx); + BoringSsl.SSL_CTX_free(ctx); throw; } } @@ -83,22 +84,22 @@ private static void ConfigureVerification(nint ctx, TlsClientOptions options) // Encrypted but unauthenticated: anything in the path can present its own certificate // and read or rewrite the whole connection. TlsClientOptions documents the one case // this is for. - OpenSsl.SSL_CTX_set_verify(ctx, OpenSsl.SSL_VERIFY_NONE, 0); + BoringSsl.SSL_CTX_set_verify(ctx, BoringSsl.SSL_VERIFY_NONE, 0); return; } // SSL_VERIFY_PEER with no callback makes a failed chain fail the handshake outright, which // is what we want - there is no prompt to fall back to. - OpenSsl.SSL_CTX_set_verify(ctx, OpenSsl.SSL_VERIFY_PEER, 0); + BoringSsl.SSL_CTX_set_verify(ctx, BoringSsl.SSL_VERIFY_PEER, 0); int loaded = options.CaFile is { } caFile - ? OpenSsl.SSL_CTX_load_verify_locations(ctx, caFile, null) - : OpenSsl.SSL_CTX_set_default_verify_paths(ctx); + ? BoringSsl.SSL_CTX_load_verify_locations(ctx, caFile, null) + : BoringSsl.SSL_CTX_set_default_verify_paths(ctx); if (loaded != 1) { string source = options.CaFile ?? "the system trust store"; - throw new IOException($"could not load trust anchors from {source}: {OpenSsl.LastError()}"); + throw new IOException($"could not load trust anchors from {source}: {BoringSsl.LastError()}"); } } @@ -125,25 +126,25 @@ private static void ConfigureClientCertificate(nint ctx, TlsClientOptions option nameof(options)); } - if (OpenSsl.SSL_CTX_use_certificate_chain_file(ctx, options.CertificateFile) != 1) + if (BoringSsl.SSL_CTX_use_certificate_chain_file(ctx, options.CertificateFile) != 1) { throw new IOException( - $"could not load client certificate '{options.CertificateFile}': {OpenSsl.LastError()}"); + $"could not load client certificate '{options.CertificateFile}': {BoringSsl.LastError()}"); } - if (OpenSsl.SSL_CTX_use_PrivateKey_file(ctx, options.PrivateKeyFile, OpenSsl.SSL_FILETYPE_PEM) != 1) + if (BoringSsl.SSL_CTX_use_PrivateKey_file(ctx, options.PrivateKeyFile, BoringSsl.SSL_FILETYPE_PEM) != 1) { throw new IOException( - $"could not load client private key '{options.PrivateKeyFile}': {OpenSsl.LastError()}"); + $"could not load client private key '{options.PrivateKeyFile}': {BoringSsl.LastError()}"); } // Catch a mismatched pair here rather than during a handshake, where it surfaces as an // opaque failure against a specific origin. - if (OpenSsl.SSL_CTX_check_private_key(ctx) != 1) + if (BoringSsl.SSL_CTX_check_private_key(ctx) != 1) { throw new IOException( $"client private key '{options.PrivateKeyFile}' does not match certificate " - + $"'{options.CertificateFile}': {OpenSsl.LastError()}"); + + $"'{options.CertificateFile}': {BoringSsl.LastError()}"); } } @@ -158,13 +159,13 @@ private static void ConfigureAlpn(nint ctx, TlsClientOptions options) int result; fixed (byte* p = wire) { - result = OpenSsl.SSL_CTX_set_alpn_protos(ctx, p, (uint)wire.Length); + result = BoringSsl.SSL_CTX_set_alpn_protos(ctx, p, (nuint)wire.Length); } // Inverted against every other function here: this one returns 0 for success. if (result != 0) { - throw new IOException($"SSL_CTX_set_alpn_protos: {OpenSsl.LastError()}"); + throw new IOException($"SSL_CTX_set_alpn_protos: {BoringSsl.LastError()}"); } } @@ -219,6 +220,6 @@ public void Dispose() return; } _disposed = true; - OpenSsl.SSL_CTX_free(_ctx); + BoringSsl.SSL_CTX_free(_ctx); } } diff --git a/src/clients/ioxide.httpclient/Tls/TlsClientOptions.cs b/src/clients/ioxide.httpclient/Tls/TlsClientOptions.cs index 3b2f0a0f..1ea69714 100644 --- a/src/clients/ioxide.httpclient/Tls/TlsClientOptions.cs +++ b/src/clients/ioxide.httpclient/Tls/TlsClientOptions.cs @@ -59,7 +59,7 @@ public sealed record TlsClientOptions public int HandshakeTimeoutMs { get; init; } = 10_000; } -/// TLS protocol version numbers, so callers need not know the OpenSSL constants. +/// TLS protocol version numbers, so callers need not know the wire values. public static class OpenSslVersions { public const int Tls12 = 0x0303; diff --git a/src/clients/ioxide.httpclient/Tls/TlsClientStream.cs b/src/clients/ioxide.httpclient/Tls/TlsClientStream.cs index 90f2d106..79f55c83 100644 --- a/src/clients/ioxide.httpclient/Tls/TlsClientStream.cs +++ b/src/clients/ioxide.httpclient/Tls/TlsClientStream.cs @@ -7,14 +7,14 @@ namespace ioxide.httpclient; /// (pointer, length) shape as itself and mean the same thing, so a /// client that already speaks to a raw socket can speak to this instead with no other change. /// -/// OpenSSL sees only memory BIOs; every byte that reaches the network still goes through the +/// BoringSSL sees only memory BIOs; every byte that reaches the network still goes through the /// reactor's ring, and every await resumes inline on the reactor thread like the rest of ioxide. /// /// /// Reactor thread only, and one operation at a time in each direction - the same contract the /// underlying has, since each direction owns a single completion source. /// -/// The OpenSSL calls live in small synchronous helpers rather than inline: an unsafe method +/// The BoringSSL calls live in small synchronous helpers rather than inline: an unsafe method /// cannot contain an await, and the alternative (pointers held across a suspension) is exactly the /// shape that goes wrong when a continuation resumes on a different stack. /// @@ -26,8 +26,8 @@ public sealed class TlsClientStream : IDisposable private readonly RingSocket _socket; private readonly nint _ssl; - private readonly nint _rbio; // ciphertext IN: socket -> OpenSSL - private readonly nint _wbio; // ciphertext OUT: OpenSSL -> socket + private readonly nint _rbio; // ciphertext IN: socket -> BoringSSL + private readonly nint _wbio; // ciphertext OUT: BoringSSL -> socket // One staging buffer PER DIRECTION, which is not an optimisation - it is required. RingSocket // deliberately allows one send and one recv to be in flight at once (separate completion @@ -66,34 +66,34 @@ internal static async ValueTask HandshakeAsync(RingSocket socke // Everything from here to the constructor is unowned: the stream that would free it does // not exist yet, and ClientTransport hands the socket over on the understanding that the // stream closes it on failure. So each step that can throw unwinds explicitly. - nint ssl = OpenSsl.SSL_new(context.Handle); + nint ssl = BoringSsl.SSL_new(context.Handle); if (ssl == 0) { socket.Dispose(); - throw new IOException($"SSL_new: {OpenSsl.LastError()}"); + throw new IOException($"SSL_new: {BoringSsl.LastError()}"); } TlsClientStream stream; try { - nint rbio = OpenSsl.BIO_new(OpenSsl.BIO_s_mem()); - nint wbio = OpenSsl.BIO_new(OpenSsl.BIO_s_mem()); + nint rbio = BoringSsl.BIO_new(BoringSsl.BIO_s_mem()); + nint wbio = BoringSsl.BIO_new(BoringSsl.BIO_s_mem()); if (rbio == 0 || wbio == 0) { // Free whichever one succeeded; SSL_set_bio never ran, so neither is owned yet. if (rbio != 0) { - OpenSsl.BIO_free(rbio); + BoringSsl.BIO_free(rbio); } if (wbio != 0) { - OpenSsl.BIO_free(wbio); + BoringSsl.BIO_free(wbio); } - throw new IOException($"BIO_new: {OpenSsl.LastError()}"); + throw new IOException($"BIO_new: {BoringSsl.LastError()}"); } - OpenSsl.SSL_set_bio(ssl, rbio, wbio); // ssl owns both BIOs from here - OpenSsl.SSL_set_connect_state(ssl); + BoringSsl.SSL_set_bio(ssl, rbio, wbio); // ssl owns both BIOs from here + BoringSsl.SSL_set_connect_state(ssl); ConfigureNames(ssl, options, serverName); // throws when SSL_set1_host rejects the name @@ -101,7 +101,7 @@ internal static async ValueTask HandshakeAsync(RingSocket socke } catch { - OpenSsl.SSL_free(ssl); // frees the BIOs too, once set_bio has run + BoringSsl.SSL_free(ssl); // frees the BIOs too, once set_bio has run socket.Dispose(); throw; } @@ -121,26 +121,39 @@ internal static async ValueTask HandshakeAsync(RingSocket socke // SNI and the verification name are two different things and both are needed: SNI tells the // origin which certificate to send, SSL_set1_host checks the one it actually sent. Setting only // the first is the classic way to end up with encryption and no authentication. - private static void ConfigureNames(nint ssl, TlsClientOptions options, string serverName) + private static unsafe void ConfigureNames(nint ssl, TlsClientOptions options, string serverName) { - nint name = Marshal.StringToHGlobalAnsi(serverName); - try + // An address is not a host name: it is checked against the certificate's iPAddress names + // (SSL_set1_host would compare it with dNSNames) and not sent as SNI, which RFC 6066 3 + // reserves for host names. Only the full forms count, so a numeric host name stays a name. + if ((serverName.Contains(':') || serverName.Count(c => c == '.') == 3) + && System.Net.IPAddress.TryParse(serverName, out System.Net.IPAddress? address)) { - OpenSsl.SSL_ctrl(ssl, OpenSsl.SSL_CTRL_SET_TLSEXT_HOSTNAME, - OpenSsl.TLSEXT_NAMETYPE_host_name, name); + if (options.VerifyCertificate) + { + byte[] bytes = address.GetAddressBytes(); + fixed (byte* p = bytes) + { + if (BoringSsl.X509_VERIFY_PARAM_set1_ip(BoringSsl.SSL_get0_param(ssl), p, (nuint)bytes.Length) != 1) + { + throw new IOException($"X509_VERIFY_PARAM_set1_ip('{serverName}'): {BoringSsl.LastError()}"); + } + } + } + return; } - finally + + if (BoringSsl.SSL_set_tlsext_host_name(ssl, serverName) != 1) { - Marshal.FreeHGlobal(name); + throw new IOException($"SSL_set_tlsext_host_name('{serverName}'): {BoringSsl.LastError()}"); } - if (options.VerifyCertificate && OpenSsl.SSL_set1_host(ssl, serverName) != 1) + // BoringSSL matches a wildcard only as a whole leftmost label, so 'ww*.example.com' never + // matches 'www.example.com': a partial wildcard names a prefix, not a host. + if (options.VerifyCertificate && BoringSsl.SSL_set1_host(ssl, serverName) != 1) { - throw new IOException($"SSL_set1_host('{serverName}'): {OpenSsl.LastError()}"); + throw new IOException($"SSL_set1_host('{serverName}'): {BoringSsl.LastError()}"); } - - // 'ww*.example.com' must not match 'www.example.com': a partial wildcard names a prefix, not a host. - OpenSsl.SSL_set_hostflags(ssl, OpenSsl.X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); } private async ValueTask RunHandshakeAsync(TlsClientOptions options, string serverName) @@ -150,14 +163,22 @@ private async ValueTask RunHandshakeAsync(TlsClientOptions options, string serve while (true) { - int ret = OpenSsl.Connect(_ssl, out int err); + int ret = BoringSsl.Connect(_ssl, out int err); - // Read before the alert's flush, during which other connections' OpenSSL calls clear this thread's queue. - string? failure = ret != 1 && err != OpenSsl.SSL_ERROR_WANT_READ && err != OpenSsl.SSL_ERROR_WANT_WRITE + // Read before the alert's flush, during which other connections' BoringSSL calls clear this thread's queue. + string? failure = ret != 1 && err != BoringSsl.SSL_ERROR_WANT_READ && err != BoringSsl.SSL_ERROR_WANT_WRITE ? HandshakeFailure(options) : null; - await FlushCiphertextAsync(); + try + { + await FlushCiphertextAsync(); + } + catch (IOException) when (failure is not null) + { + // The alert after a failed handshake is best-effort: a peer that already closed + // refuses it, and that refusal must not stand in for the handshake's own failure. + } if (ret == 1) { @@ -185,8 +206,8 @@ private async ValueTask RunHandshakeAsync(TlsClientOptions options, string serve if (options.VerifyCertificate) { - long verify = OpenSsl.SSL_get_verify_result(_ssl); - if (verify != OpenSsl.X509_V_OK) + long verify = BoringSsl.SSL_get_verify_result(_ssl); + if (verify != BoringSsl.X509_V_OK) { throw new IOException( $"certificate for '{serverName}' was not accepted (X509 verify result {verify})"); @@ -197,12 +218,14 @@ private async ValueTask RunHandshakeAsync(TlsClientOptions options, string serve } // The queue says the handshake failed; a refused certificate's verify result says which check refused it. + // BoringSSL's error for that is CERTIFICATE_VERIFY_FAILED, so the words callers match on are written out. + // INVALID_CALL is not a verdict: the handshake failed before there was a session to verify. private string HandshakeFailure(TlsClientOptions options) { - string error = OpenSsl.LastError(); - long verify = OpenSsl.SSL_get_verify_result(_ssl); - return options.VerifyCertificate && verify != OpenSsl.X509_V_OK - ? $"{error}: {OpenSsl.VerifyResultText(verify)}" + string error = BoringSsl.LastError(); + long verify = BoringSsl.SSL_get_verify_result(_ssl); + return options.VerifyCertificate && verify is not (BoringSsl.X509_V_OK or BoringSsl.X509_V_ERR_INVALID_CALL) + ? $"certificate verify failed: {BoringSsl.VerifyResultText(verify)} ({error})" : error; } @@ -226,14 +249,14 @@ public async ValueTask SendAsync(nint buffer, int length) // a recv from the send path would alias that source, which is the same class of bug as // sharing a staging buffer. Whoever owns the read side feeds the BIO; the caller // retries. - if (err == OpenSsl.SSL_ERROR_WANT_READ) + if (err == BoringSsl.SSL_ERROR_WANT_READ) { throw new IOException( "SSL_write needs inbound data first (post-handshake message); retry once the " + "read side has been pumped"); } - throw new IOException($"SSL_write failed (error {err}): {OpenSsl.LastError()}"); + throw new IOException($"SSL_write failed (error {err}): {BoringSsl.LastError()}"); } // The records exist only in the write BIO until this runs, so it is not optional. @@ -258,21 +281,21 @@ public async ValueTask RecvAsync(nint buffer, int length) return n; } - if (err == OpenSsl.SSL_ERROR_ZERO_RETURN) + if (err == BoringSsl.SSL_ERROR_ZERO_RETURN) { _peerClosed = true; return 0; // clean close_notify } - if (err == OpenSsl.SSL_ERROR_WANT_WRITE) + if (err == BoringSsl.SSL_ERROR_WANT_WRITE) { // A key update or renegotiation needs to put bytes on the wire before it can // produce more plaintext. await FlushCiphertextAsync(); continue; } - if (err != OpenSsl.SSL_ERROR_WANT_READ) + if (err != BoringSsl.SSL_ERROR_WANT_READ) { - throw new IOException($"SSL_read failed (error {err}): {OpenSsl.LastError()}"); + throw new IOException($"SSL_read failed (error {err}): {BoringSsl.LastError()}"); } // Incomplete record: pull more ciphertext. @@ -289,7 +312,7 @@ public async ValueTask RecvAsync(nint buffer, int length) } } - // Drain whatever OpenSSL staged into the write BIO out to the socket. Partial sends loop: + // Drain whatever BoringSSL staged into the write BIO out to the socket. Partial sends loop: // the ring reports what it took, not what we asked for. private async ValueTask FlushCiphertextAsync() { @@ -321,24 +344,24 @@ private async ValueTask FlushCiphertextAsync() private static unsafe nint AllocateCiphertextBuffer() => (nint)NativeMemory.Alloc(CiphertextBufferSize); private unsafe void FeedCiphertext(int length) - => OpenSsl.BIO_write(_rbio, (byte*)_inbound, length); + => BoringSsl.BIO_write(_rbio, (byte*)_inbound, length); private unsafe int TakeCiphertext() - => OpenSsl.BIO_read(_wbio, (byte*)_outbound, CiphertextBufferSize); + => BoringSsl.BIO_read(_wbio, (byte*)_outbound, CiphertextBufferSize); - private unsafe nuint PendingCiphertext() => OpenSsl.BIO_ctrl_pending(_wbio); + private unsafe nuint PendingCiphertext() => BoringSsl.BIO_ctrl_pending(_wbio); private unsafe int SslRead(nint buffer, int length, out int error) - => OpenSsl.Read(_ssl, (byte*)buffer, length, out error); + => BoringSsl.Read(_ssl, (byte*)buffer, length, out error); private unsafe int SslWrite(nint buffer, int length, out int error) - => OpenSsl.Write(_ssl, (byte*)buffer, length, out error); + => BoringSsl.Write(_ssl, (byte*)buffer, length, out error); private static unsafe string? ReadNegotiatedAlpn(nint ssl) { byte* data; uint length; - OpenSsl.SSL_get0_alpn_selected(ssl, &data, &length); + BoringSsl.SSL_get0_alpn_selected(ssl, &data, &length); return data == null || length == 0 ? null : System.Text.Encoding.ASCII.GetString(data, (int)length); @@ -361,15 +384,15 @@ public void Dispose() // SSL_shutdown says so by failing and leaving "shutdown while in init" on the error queue - // which belongs to the reactor, not to this connection, so it becomes the next pooled // connection's fatal error. - if (!_peerClosed && OpenSsl.SSL_in_init(_ssl) == 0) + if (!_peerClosed && BoringSsl.SSL_in_init(_ssl) == 0) { - if (OpenSsl.SSL_shutdown(_ssl) < 0) + if (BoringSsl.SSL_shutdown(_ssl) < 0) { - OpenSsl.ERR_clear_error(); // whatever it queued is ours to clean up + BoringSsl.ERR_clear_error(); // whatever it queued is ours to clean up } } - OpenSsl.SSL_free(_ssl); // frees both BIOs + BoringSsl.SSL_free(_ssl); // frees both BIOs FreeCiphertextBuffer(_inbound); FreeCiphertextBuffer(_outbound); _socket.Dispose(); @@ -383,7 +406,7 @@ private sealed class HandshakeDeadline(TlsClientStream stream) : IRingCompletion public void Complete(int result) { // -ETIME: the wait ran its course. Disposed is checked first: by then SSL_free has run. - if (result is -62 or 0 && !stream._disposed && OpenSsl.SSL_in_init(stream._ssl) != 0) + if (result is -62 or 0 && !stream._disposed && BoringSsl.SSL_in_init(stream._ssl) != 0) { Expired = true; Native.shutdown(stream._socket.Fd, Native.SHUT_RDWR); diff --git a/src/ioxide/Tls/Interop/OpenSsl.cs b/src/ioxide/Tls/Interop/BoringSsl.cs similarity index 70% rename from src/ioxide/Tls/Interop/OpenSsl.cs rename to src/ioxide/Tls/Interop/BoringSsl.cs index 5946a4c7..08be2b26 100644 --- a/src/ioxide/Tls/Interop/OpenSsl.cs +++ b/src/ioxide/Tls/Interop/BoringSsl.cs @@ -2,31 +2,28 @@ namespace ioxide.tls; -/// Minimal OpenSSL 3 surface: server context, memory-BIO handshake, keylog. -internal static unsafe partial class OpenSsl +/// +/// Minimal BoringSSL surface: server context, memory-BIO handshake, keylog. The library ships with +/// this package as libioxide_boringssl.so (scripts/build-boringssl-native.sh): libssl and libcrypto +/// in one object, so both names below are the same file. +/// +internal static unsafe partial class BoringSsl { - private const string Ssl = "libssl.so.3"; - private const string Crypto = "libcrypto.so.3"; + private const string Ssl = "libioxide_boringssl.so"; + private const string Crypto = "libioxide_boringssl.so"; public const int SSL_FILETYPE_PEM = 1; public const int SSL_ERROR_WANT_READ = 2; public const int SSL_ERROR_WANT_WRITE = 3; public const int SSL_ERROR_ZERO_RETURN = 6; - public const int SSL_CTRL_SET_MIN_PROTO_VERSION = 123; - public const int SSL_CTRL_SET_MAX_PROTO_VERSION = 124; - public const int SSL_CTRL_EXTRA_CHAIN_CERT = 14; - public const int TLS1_2_VERSION = 0x0303; - public const int TLS1_3_VERSION = 0x0304; + public const ushort TLS1_2_VERSION = 0x0303; + public const ushort TLS1_3_VERSION = 0x0304; + + /// The RFC 8446 protocol id of the one suite kTLS is programmed for. + public const ushort TLS_AES_128_GCM_SHA256 = 0x1301; public const int SSL_TLSEXT_ERR_OK = 0; public const int SSL_TLSEXT_ERR_NOACK = 3; - - // Server Name Indication (RFC 6066 3.1). The callback fires mid-handshake, after the - // ClientHello is parsed and before a certificate is chosen, which is the only point at which - // the name is known and the choice is still open. - public const int SSL_CTRL_SET_TLSEXT_SERVERNAME_CB = 53; - public const int SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG = 54; public const int TLSEXT_NAMETYPE_host_name = 0; - public const int CRYPTO_EX_INDEX_SSL = 0; // Client-certificate verification (mutual TLS). public const int SSL_VERIFY_PEER = 0x01; @@ -39,27 +36,52 @@ internal static unsafe partial class OpenSsl public static partial int SSL_CTX_use_certificate_chain_file(nint ctx, string file); [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] public static partial int SSL_CTX_use_PrivateKey_file(nint ctx, string file, int type); - /// TLS 1.3 suites, e.g. "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384". - [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] - public static partial int SSL_CTX_set_ciphersuites(nint ctx, string list); - /// TLS 1.2 and below. A separate list from the 1.3 one, in OpenSSL and here. + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_min_proto_version(nint ctx, ushort version); + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_max_proto_version(nint ctx, ushort version); + + /// + /// TLS 1.3 suites by protocol id, in server preference order. Only the listed suites are + /// negotiated; an empty list means BoringSSL's built-in order, so callers refuse empty first. + /// + [LibraryImport(Ssl)] public static partial int SSL_CTX_set1_tls13_ciphers(nint ctx, ushort* ids, uint* flags, nuint count); + + [LibraryImport(Ssl)] public static partial nint SSL_get_cipher_by_value(ushort value); + [LibraryImport(Ssl)] public static partial nint SSL_CIPHER_standard_name(nint cipher); + + /// + /// The protocol id of the TLS 1.3 suite called (its RFC 8446 name), or + /// null when this BoringSSL does not implement it. Asks the library rather than a table here: + /// the TLS 1.3 ids are 0x1301-0x1305, and BoringSSL knows the three it offers. + /// + public static ushort? Tls13SuiteId(string name) + { + for (ushort id = 0x1301; id <= 0x1305; id++) + { + nint cipher = SSL_get_cipher_by_value(id); + if (cipher != 0 && Marshal.PtrToStringUTF8(SSL_CIPHER_standard_name(cipher)) == name) + { + return id; + } + } + return null; + } + + /// TLS 1.2 and below. A separate list from the 1.3 one, in BoringSSL and here. [LibraryImport(Ssl, StringMarshalling = StringMarshalling.Utf8)] public static partial int SSL_CTX_set_cipher_list(nint ctx, string list); - [LibraryImport(Ssl)] public static partial long SSL_CTX_ctrl(nint ctx, int cmd, long arg, nint parg); [LibraryImport(Ssl)] public static partial int SSL_CTX_set_num_tickets(nint ctx, nuint num); - /// - /// Renegotiation, which TLS 1.3 does not have at all and which no current client asks a server - /// for over TLS 1.2. Refusing it closes a request-flood amplifier, and - since a renegotiation - /// carries a second ClientHello - a second chance to pick a different certificate mid-connection. - /// - public const ulong SSL_OP_NO_RENEGOTIATION = 1UL << 30; - public const ulong SSL_OP_CIPHER_SERVER_PREFERENCE = 0x00400000UL; + + /// Takes ownership of on success. + [LibraryImport(Ssl)] public static partial int SSL_CTX_add_extra_chain_cert(nint ctx, nint x509); + + public const uint SSL_OP_CIPHER_SERVER_PREFERENCE = 0x00400000; /// PEM's "the BIO holds no further object", i.e. an ordinary end of bundle. - public const int PEM_R_NO_START_LINE = 108; + public const int PEM_R_NO_START_LINE = 110; + public const int ERR_LIB_PEM = 9; - [LibraryImport(Ssl)] public static partial ulong SSL_CTX_set_options(nint ctx, ulong options); + [LibraryImport(Ssl)] public static partial uint SSL_CTX_set_options(nint ctx, uint options); /// /// Drops one reference. Contexts that are SERVING are never freed - a handshake may be between @@ -71,17 +93,16 @@ internal static unsafe partial class OpenSsl [LibraryImport(Ssl)] public static partial void SSL_CTX_set_keylog_callback(nint ctx, nint cb); [LibraryImport(Ssl)] public static partial void SSL_CTX_set_alpn_select_cb(nint ctx, nint cb, nint arg); - /// - /// Takes a function pointer rather than a value, which is why it is not SSL_CTX_ctrl: the - /// servername callback is installed through this and its argument through SSL_CTX_ctrl. - /// - [LibraryImport(Ssl)] public static partial long SSL_CTX_callback_ctrl(nint ctx, int cmd, nint fp); + // Server Name Indication (RFC 6066 3.1). The callback fires mid-handshake, after the + // ClientHello is parsed and before a certificate is chosen, which is the only point at which + // the name is known and the choice is still open. + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_tlsext_servername_callback(nint ctx, nint cb); + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_tlsext_servername_arg(nint ctx, nint arg); /// - /// Install the callback OpenSSL asks for a private key's passphrase. Passing none is NOT "no - /// password": the default is PEM_def_callback, which READS THE TERMINAL - so a server started - /// from a tty with an encrypted key blocks on a prompt instead of failing, and a rotation - /// thread would block while still serving. + /// Install the callback asked for a private key's passphrase. ioxide never has one, and says so + /// explicitly rather than relying on the library default, so an encrypted key always fails the + /// same way: at load, with an error, instead of whatever a default might try. /// [LibraryImport(Ssl)] public static partial void SSL_CTX_set_default_passwd_cb(nint ctx, nint cb); @@ -131,14 +152,21 @@ internal static unsafe partial class OpenSsl /// Zero when two names are the same name, compared canonically. [LibraryImport(Crypto)] public static partial int X509_NAME_cmp(nint a, nint b); - /// The peer's leaf certificate, borrowed - no reference taken, so it is not freed. - [LibraryImport(Ssl)] public static partial nint SSL_get0_peer_certificate(nint ssl); + [LibraryImport(Ssl)] private static partial nint SSL_get_peer_certificate(nint ssl); + + /// The peer's leaf certificate, borrowed: the session keeps its own reference. + public static nint PeerCertificate(nint ssl) + { + nint cert = SSL_get_peer_certificate(ssl); + X509_free(cert); + return cert; + } // What the handshake actually settled on, for callers that have to REPORT it rather than // assume it - ASP.NET reads both off ITlsHandshakeFeature. [LibraryImport(Ssl)] public static partial int SSL_version(nint ssl); [LibraryImport(Ssl)] public static partial nint SSL_get_current_cipher(nint ssl); - [LibraryImport(Ssl)] public static partial uint SSL_CIPHER_get_protocol_id(nint cipher); + [LibraryImport(Ssl)] public static partial ushort SSL_CIPHER_get_protocol_id(nint cipher); [LibraryImport(Crypto)] public static partial int i2d_X509(nint x509, byte** outBuf); /// X509_V_OK, or why the chain was rejected. @@ -148,16 +176,15 @@ internal static unsafe partial class OpenSsl /// /// The legacy one-line rendering of a DN. Called with a NULL buffer it allocates the exact - /// size and the caller frees it with ; called with a buffer too small + /// size and the caller frees it with ; called with a buffer too small /// it reports SUCCESS and writes a truncated prefix, which is why nothing here passes one. /// [LibraryImport(Crypto)] public static partial nint X509_NAME_oneline(nint name, byte* buf, int size); - /// OPENSSL_free. The macro passes __FILE__/__LINE__, which may be NULL/0. - [LibraryImport(Crypto)] public static partial void CRYPTO_free(nint ptr, nint file, int line); + [LibraryImport(Crypto)] public static partial void OPENSSL_free(nint ptr); - /// The element at of an OpenSSL stack, borrowed, or 0 past its end. - [LibraryImport(Crypto)] public static partial nint OPENSSL_sk_value(nint stack, int index); + /// The element at of a stack, borrowed, or 0 past its end. + [LibraryImport(Crypto)] public static partial nint OPENSSL_sk_value(nint stack, nuint index); // Structural access to one attribute of a DN, so an identity can be compared as a value // instead of matched inside a rendered string. See TlsSession.PeerCommonName. @@ -181,19 +208,19 @@ internal static unsafe partial class OpenSsl [LibraryImport(Ssl)] public static partial int SSL_shutdown(nint ssl); /// - /// The three OpenSSL operations whose result has to be classified, each performed from a clean - /// error queue and returning what the classification said. + /// The three operations whose result has to be classified, each performed from a clean error + /// queue and returning what the classification said. /// /// /// The clear, the call and the classify are one operation and are deliberately not offered /// apart - the raw entry points above are private so that they cannot be. /// - /// The reason is that OpenSSL's error queue belongs to the THREAD, and here a thread is a - /// reactor shared by every connection it serves. SSL_get_error consults that queue BEFORE it - /// asks the SSL whether it merely wants more data, so a residue any other connection left - - /// an aborted handshake, a refused client, a port scan - is read as THIS connection's fatal - /// error and kills it. Clearing at each call site works until someone adds the next call site; - /// this shape means there is no next call site to forget. + /// The reason is that the error queue belongs to the THREAD, and here a thread is a reactor + /// shared by every connection it serves. SSL_get_error consults that queue BEFORE it asks the + /// SSL whether it merely wants more data, so a residue any other connection left - an aborted + /// handshake, a refused client, a port scan - is read as THIS connection's fatal error and + /// kills it. Clearing at each call site works until someone adds the next call site; this shape + /// means there is no next call site to forget. /// public static unsafe int Accept(nint ssl, out int error) { @@ -228,12 +255,7 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) /// /// Scopes a session to this server, so a resumption offered to a different one is declined. /// - /// - /// Not optional once client certificates are asked for: OpenSSL treats an unset id context - /// under SSL_VERIFY_PEER as fatal to the HANDSHAKE, not merely to the resumption, so a client - /// that returns with a ticket is refused outright. - /// - [LibraryImport(Ssl)] public static partial int SSL_CTX_set_session_id_context(nint ctx, byte* sid, uint len); + [LibraryImport(Ssl)] public static partial int SSL_CTX_set_session_id_context(nint ctx, byte* sid, nuint len); [LibraryImport(Ssl)] private static partial int SSL_get_error(nint ssl, int ret); @@ -244,11 +266,11 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) // callback can find the right session without a global, pointer-keyed map. [LibraryImport(Ssl)] public static partial int SSL_set_ex_data(nint ssl, int idx, nint data); [LibraryImport(Ssl)] public static partial nint SSL_get_ex_data(nint ssl, int idx); - [LibraryImport(Crypto)] public static partial int CRYPTO_get_ex_new_index(int classIndex, long argl, nint argp, nint newFunc, nint dupFunc, nint freeFunc); + [LibraryImport(Ssl)] public static partial int SSL_get_ex_new_index(long argl, nint argp, nint unused, nint dupUnused, nint freeFunc); [LibraryImport(Crypto)] public static partial nint BIO_new(nint type); [LibraryImport(Crypto)] public static partial nint BIO_s_mem(); - [LibraryImport(Crypto)] public static partial nint BIO_new_mem_buf(byte* buf, int len); + [LibraryImport(Crypto)] public static partial nint BIO_new_mem_buf(byte* buf, nint len); [LibraryImport(Crypto)] public static partial int BIO_free(nint bio); // PEM-from-memory loading: the same certificate/key material a chain file carries, read @@ -275,8 +297,21 @@ public static unsafe int Write(nint ssl, byte* buf, int num, out int error) public const int BIO_CTRL_INFO = 3; [LibraryImport(Crypto)] public static partial long BIO_ctrl(nint bio, int cmd, long larg, byte** parg); - [LibraryImport(Crypto)] public static partial ulong ERR_get_error(); - [LibraryImport(Crypto)] public static partial void ERR_error_string_n(ulong e, byte* buf, nuint len); + [LibraryImport(Crypto)] public static partial uint ERR_get_error(); + [LibraryImport(Crypto)] private static partial uint ERR_peek_error(); + [LibraryImport(Crypto)] public static partial void ERR_error_string_n(uint e, byte* buf, nuint len); + + private const int ERR_LIB_X509 = 11; + private const int X509_R_KEY_TYPE_MISMATCH = 115; + private const int X509_R_KEY_VALUES_MISMATCH = 116; + + /// Whether the queued error says a key does not belong to the certificate loaded. + public static bool ErrorIsKeyMismatch() + { + uint code = ERR_peek_error(); + return (code >> 24) == ERR_LIB_X509 + && (code & 0xFFF) is X509_R_KEY_TYPE_MISMATCH or X509_R_KEY_VALUES_MISMATCH; + } /// /// Drains the queue and reports whether it held ONLY "no start line" - PEM's way of saying the @@ -289,10 +324,10 @@ public static bool ErrorIsEndOfPem() bool sawEnd = false; bool sawOther = false; - ulong code; + uint code; while ((code = ERR_get_error()) != 0) { - if ((int)(code & 0xFFF) == PEM_R_NO_START_LINE) + if ((code >> 24) == ERR_LIB_PEM && (code & 0xFFF) == PEM_R_NO_START_LINE) { sawEnd = true; } @@ -307,8 +342,8 @@ public static bool ErrorIsEndOfPem() public static string LastError() { - ulong code = ERR_get_error(); - if (code == 0) return "no OpenSSL error"; + uint code = ERR_get_error(); + if (code == 0) return "no BoringSSL error"; Span buf = stackalloc byte[256]; string message; diff --git a/src/ioxide/Tls/Interop/Ktls.cs b/src/ioxide/Tls/Interop/Ktls.cs index 8175d544..b3b912a0 100644 --- a/src/ioxide/Tls/Interop/Ktls.cs +++ b/src/ioxide/Tls/Interop/Ktls.cs @@ -71,7 +71,7 @@ private struct Msghdr /// decrypts inbound records and an ordinary recv returns plaintext. /// /// - /// How many application-data records the client has already sent that OpenSSL consumed during + /// How many application-data records the client has already sent that BoringSSL consumed during /// the handshake. This is the one thing that cannot be guessed: the sequence number is part of /// the AEAD nonce, so starting the kernel at the wrong one makes every record fail to /// authenticate. The client's first request routinely rides in with its Finished flight, and diff --git a/src/ioxide/Tls/Pipes/TlsDecryptingPipeReader.cs b/src/ioxide/Tls/Pipes/TlsDecryptingPipeReader.cs index 34e36162..7bb1b934 100644 --- a/src/ioxide/Tls/Pipes/TlsDecryptingPipeReader.cs +++ b/src/ioxide/Tls/Pipes/TlsDecryptingPipeReader.cs @@ -7,7 +7,7 @@ namespace ioxide.tls; /// -/// The read half when OpenSSL decrypts. Nothing runs in the background: a read decrypts whatever +/// The read half when BoringSSL decrypts. Nothing runs in the background: a read decrypts whatever /// ciphertext the ring has queued for this connection, and when that is not enough it parks on the /// connection and decrypts what arrives. The connection therefore has a read armed only while the /// caller is waiting, exactly as with . diff --git a/src/ioxide/Tls/Pipes/TlsEncryptingPipeWriter.cs b/src/ioxide/Tls/Pipes/TlsEncryptingPipeWriter.cs index 8b1f77f0..80bd4c2f 100644 --- a/src/ioxide/Tls/Pipes/TlsEncryptingPipeWriter.cs +++ b/src/ioxide/Tls/Pipes/TlsEncryptingPipeWriter.cs @@ -6,7 +6,7 @@ namespace ioxide.tls; /// -/// The write half when the kernel is not encrypting: plaintext is staged here, OpenSSL turns +/// The write half when the kernel is not encrypting: plaintext is staged here, BoringSSL turns /// it into records on flush, and the records go into the connection's slab. /// /// The counterpart is , which is what a kTLS-TX connection diff --git a/src/ioxide/Tls/Pipes/TlsProloguePipeReader.cs b/src/ioxide/Tls/Pipes/TlsProloguePipeReader.cs index 1667fc66..f65f0350 100644 --- a/src/ioxide/Tls/Pipes/TlsProloguePipeReader.cs +++ b/src/ioxide/Tls/Pipes/TlsProloguePipeReader.cs @@ -21,10 +21,10 @@ public sealed class TlsProloguePipeReader : PipeReader /// /// How far the carry may grow before the connection is faulted. While the carry is live this /// class copies ring bytes into it and hands the ring buffers straight back, so recv never - /// stalls - there is no backpressure on this path at all, unlike the OpenSSL column where the + /// stalls - there is no backpressure on this path at all, unlike the BoringSSL column where the /// inbound Pipe's pause threshold applies. A peer that sends a partial head and then dribbles /// forever kept a caller examining without consuming, and the carry grew for the life of the - /// connection. Generous next to the 64 KB the OpenSSL column pauses at, because a legitimate + /// connection. Generous next to the 64 KB the BoringSSL column pauses at, because a legitimate /// caller does examine a whole request head without consuming it - but bounded. /// private const int MaxCarryBytes = 1 << 20; @@ -60,13 +60,12 @@ public TlsProloguePipeReader(PipeReader inner, ReadOnlySpan prologue) /// True once this is a pass-through and costs nothing. public bool Drained => _drained; - public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) - { - if (_drained) - { - return await _inner.ReadAsync(cancellationToken); - } + // Not async: an async pass-through allocates its state machine on every read that has to wait. + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) => + _drained ? _inner.ReadAsync(cancellationToken) : ReadCarryAsync(cancellationToken); + private async ValueTask ReadCarryAsync(CancellationToken cancellationToken) + { // A cancel that arrived while the carry was live is served here, not latched into the // inner reader to pop out as a surprise after the carry drains. if (_cancelRequested) @@ -215,7 +214,7 @@ private void Append(in ReadOnlySequence more) if (_length + extra > MaxCarryBytes) { - // Faulting rather than silently growing. The OpenSSL column would simply stop reading + // Faulting rather than silently growing. The BoringSSL column would simply stop reading // here, but the ring buffers behind these bytes have already been handed back, so there // is nothing left to apply backpressure with - the only honest answers are to bound it // or to keep buffering, and this transport does not get to keep buffering. diff --git a/src/ioxide/Tls/README.md b/src/ioxide/Tls/README.md index 6d4c1b4a..fc68002b 100644 --- a/src/ioxide/Tls/README.md +++ b/src/ioxide/Tls/README.md @@ -2,8 +2,8 @@ TLS termination for [ioxide](https://github.com/MDA2AV/ioxide). Part of the `ioxide` package - there is nothing extra to install, and none of it loads unless you call it. The handshake runs -through OpenSSL over the reactor's ring; after it, OpenSSL encrypts and decrypts in both -directions by default. Kernel TLS transmit offload (`TlsOptions.KernelTx`) is opt-in, for the +through BoringSSL, which ships inside the package, over the reactor's ring; after it, BoringSSL +encrypts and decrypts in both directions by default. Kernel TLS transmit offload (`TlsOptions.KernelTx`) is opt-in, for the deployments where `sendfile` or NIC crypto pays for its constraints. ```csharp @@ -33,7 +33,7 @@ the first. ## How it works -- **By default OpenSSL owns both directions.** Responses go through `TlsSession.Write`, which +- **By default BoringSSL owns both directions.** Responses go through `TlsSession.Write`, which encrypts into the connection's write slab; inbound records decrypt through `TlsSession.Decrypt`. No kernel module, TLS 1.2 and 1.3, any ciphersuite, session resumption available (each reactor issues its own ticket keys, so a ticket resumes on the reactor that @@ -45,12 +45,13 @@ the first. no resumption. The payoff is `sendfile` and NIC offload, not loopback throughput. - **`KernelRx = true`** (requires `KernelTx`, experimental) hands receive to the kernel as well; see the `TlsOptions.KernelRx` doc for why it is a research toggle. -- **The handshake** rides the ring like any other I/O: OpenSSL drives memory BIOs, the service +- **The handshake** rides the ring like any other I/O: BoringSSL drives memory BIOs, the service pumps ciphertext through the connection's recv/send. Resumes inline on the reactor. ## Requirements -- **OpenSSL 3** - present in the .NET runtime container images. +- **Nothing for TLS itself** - BoringSSL ships in the package as `libioxide_boringssl.so` + (linux-x64), built by `scripts/build-boringssl-native.sh` from a pinned commit. - **Linux `tls` kernel module** - only for the kTLS opt-in (`modprobe tls`; standard on mainstream distros). diff --git a/src/ioxide/Tls/TlsConnectionDualPipe.cs b/src/ioxide/Tls/TlsConnectionDualPipe.cs index 985e4ea2..362e8020 100644 --- a/src/ioxide/Tls/TlsConnectionDualPipe.cs +++ b/src/ioxide/Tls/TlsConnectionDualPipe.cs @@ -10,12 +10,12 @@ namespace ioxide.tls; /// one applies is not a choice the caller makes - it is what the handshake actually achieved: /// /// -/// kTLS (opt-in) OpenSSL (default) +/// kTLS (opt-in) BoringSSL (default) /// read TcpConnectionPipeReader TlsDecryptingPipeReader /// write TcpConnectionPipeWriter TlsEncryptingPipeWriter /// /// The kTLS column is the plaintext connection's own reader and writer, reused unchanged: with the -/// kernel doing the crypto there is nothing for a TLS-specific half to do. The OpenSSL column is +/// kernel doing the crypto there is nothing for a TLS-specific half to do. The BoringSSL column is /// the pair that has to exist - one decrypts into a Pipe it owns, the other encrypts before the /// bytes reach the slab. /// @@ -33,7 +33,7 @@ public sealed class TlsConnectionDualPipe : IDuplexPipe, IAsyncDisposable private readonly TlsSession _tls; private readonly bool _ownsSession; - private readonly TlsDecryptingPipeReader? _decrypting; // only when OpenSSL decrypts + private readonly TlsDecryptingPipeReader? _decrypting; // only when BoringSSL decrypts private readonly PipeWriter _writer; /// @@ -69,7 +69,7 @@ public TlsConnectionDualPipe(TcpConnection connection, TlsSession session, bool Input = _decrypting; } - // Default is OpenSSL: encrypt before the bytes reach the slab. With kTLS opted in the + // Default is BoringSSL: encrypt before the bytes reach the slab. With kTLS opted in the // kernel makes the records instead, so the plaintext connection's own writer is enough. _writer = session.KernelTx ? new TcpConnectionPipeWriter(connection) diff --git a/src/ioxide/Tls/TlsOptions.cs b/src/ioxide/Tls/TlsOptions.cs index 5021273b..d04d37d0 100644 --- a/src/ioxide/Tls/TlsOptions.cs +++ b/src/ioxide/Tls/TlsOptions.cs @@ -39,7 +39,7 @@ public sealed class TlsOptions /// refused, naming it, rather than built and never served. So are two keys differing only in /// case, since only the first could ever be served. /// - /// Each entry costs one OpenSSL context. Selection at handshake time is a short linear scan + /// Each entry costs one BoringSSL context. Selection at handshake time is a short linear scan /// over the names as UTF-8 bytes, on the reactor thread, with no managed allocation - a /// dictionary would mean decoding the name into a string first, which is the allocation the /// shape exists to avoid. Contexts are rebuilt whenever the certificates are replaced. @@ -78,7 +78,7 @@ public sealed class TlsOptions /// /// Lowest TLS version this server will negotiate. /// - /// keeps OpenSSL's own floor, which on a current + /// keeps BoringSSL's own floor, which on a current /// build is TLS 1.2. turns the server into 1.3-only, /// which is what a deployment that has to state a posture usually wants: it removes every /// TLS 1.2 ciphersuite, renegotiation, and static-RSA key exchange in one setting rather than @@ -91,10 +91,10 @@ public sealed class TlsOptions public TlsProtocolVersion MinProtocolVersion { get; init; } = TlsProtocolVersion.Default; /// - /// TLS 1.3 ciphersuites to offer, in preference order and in OpenSSL's naming, e.g. - /// "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384". Null keeps OpenSSL's defaults. + /// TLS 1.3 ciphersuites to offer, in preference order and in BoringSSL's naming, e.g. + /// "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384". Null keeps BoringSSL's defaults. /// - /// Separate from because OpenSSL keeps the two lists separate: a 1.3 + /// Separate from because BoringSSL keeps the two lists separate: a 1.3 /// suite named here does not restrict a 1.2 handshake and vice versa, so a server that means /// to constrain both has to say both - or set to /// and be done. @@ -105,7 +105,7 @@ public sealed class TlsOptions public string? CipherSuites { get; init; } /// - /// Ciphers for TLS 1.2 and below, in OpenSSL's cipher-list syntax. Null keeps OpenSSL's + /// Ciphers for TLS 1.2 and below, in BoringSSL's cipher-list syntax. Null keeps BoringSSL's /// defaults. Has no effect on a 1.3 handshake - see . /// public string? CipherList { get; init; } @@ -140,10 +140,10 @@ public sealed class TlsOptions /// re-read whenever a context is built, so replacing the file changes who is trusted at the next /// rotation, whereas PEM text is a value and does not move. /// - /// This route parses the PEM itself rather than handing the bytes to OpenSSL's file loader, and - /// reads a bundle as a path is read: CERTIFICATE and TRUSTED CERTIFICATE blocks - /// are both trusted, a byte-order mark is ignored, an issuer the bundle repeats is hinted once, - /// and a malformed block is refused outright rather than silently ending the bundle. One + /// This route parses the PEM itself rather than handing the bytes to BoringSSL's file loader, + /// and reads a bundle as a path is read: CERTIFICATE and TRUSTED CERTIFICATE + /// blocks are both trusted, a byte-order mark is ignored, an issuer the bundle repeats is hinted + /// once, and a malformed block is refused outright rather than silently ending the bundle. One /// difference remains: an anchor from a TRUSTED CERTIFICATE block is also hinted here, /// where the file route's hint loader skips that block. /// @@ -175,7 +175,7 @@ public sealed class TlsOptions /// Two reasons it is opt-in even where is. /// /// The handoff must land on a record boundary. Whatever the handshake already pulled off the - /// socket is invisible to the kernel: OpenSSL decrypts it, and the record sequence the kernel + /// socket is invisible to the kernel: BoringSSL decrypts it, and the record sequence the kernel /// starts at accounts for it. When what arrived ends inside a record, the handshake waits for /// the rest of that record before handing over; a connection whose records keep arriving cut /// past 1 MiB stays on the userspace path. @@ -190,14 +190,14 @@ public sealed class TlsOptions public bool KernelRx { get; init; } /// - /// Let the kernel encrypt outbound records. Off by default - TLS is OpenSSL in both + /// Let the kernel encrypt outbound records. Off by default - TLS is BoringSSL in both /// directions unless you turn this on. /// - /// It used to default on, which made ioxide's TLS asymmetric: the kernel encrypted, OpenSSL + /// It used to default on, which made ioxide's TLS asymmetric: the kernel encrypted, the library /// decrypted. That asymmetry is now something you opt into rather than something you inherit, /// because it is not free and it was not faster. Measured on loopback, HTTP/1.1, 4 reactors: - /// kTLS trails OpenSSL by roughly 20-25% on large single writes and costs about 20% more CPU - /// per request, while a small response shows no difference either way. What it constrains is + /// kTLS costs 12-29% more CPU per request than BoringSSL at 8-64 KiB responses and about 5% + /// more at 64 B and 256 KiB - never less. What it constrains is /// the part that matters: the Linux tls module has to be present, TLS 1.3 only, one /// ciphersuite, and session resumption is disabled because a ticket would consume a record /// sequence number and desynchronise the handoff. @@ -206,7 +206,7 @@ public sealed class TlsOptions /// of which those measurements exercise. /// /// It changes what a handler must write. With kTLS the kernel produces the records, so - /// a handler writes plaintext straight to the connection; without it OpenSSL has to encrypt + /// a handler writes plaintext straight to the connection; without it BoringSSL has to encrypt /// first. is correct either way and is what samples use - a bare /// connection.Write on a session with this off puts CLEARTEXT on the wire. /// diff --git a/src/ioxide/Tls/TlsProtocolVersion.cs b/src/ioxide/Tls/TlsProtocolVersion.cs index 101a38c2..1af0cf58 100644 --- a/src/ioxide/Tls/TlsProtocolVersion.cs +++ b/src/ioxide/Tls/TlsProtocolVersion.cs @@ -3,7 +3,7 @@ namespace ioxide.tls; /// Lowest TLS version a server will negotiate. See . public enum TlsProtocolVersion { - /// Whatever OpenSSL's own floor is - TLS 1.2 on a current build. + /// Whatever BoringSSL's own floor is - TLS 1.2 on a current build. Default = 0, /// Accept TLS 1.2 and above. diff --git a/src/ioxide/Tls/TlsService.cs b/src/ioxide/Tls/TlsService.cs index 5a7ab2fb..07e3d609 100644 --- a/src/ioxide/Tls/TlsService.cs +++ b/src/ioxide/Tls/TlsService.cs @@ -6,9 +6,9 @@ namespace ioxide.tls; /// /// Per-reactor TLS termination. drives the TLS 1.3 handshake through -/// OpenSSL memory BIOs - handshake bytes ride the same ring recv/send as everything else - and +/// BoringSSL memory BIOs - handshake bytes ride the same ring recv/send as everything else - and /// hands back the that carries the connection from there. By default -/// OpenSSL owns both directions: the handler writes through and +/// BoringSSL owns both directions: the handler writes through and /// passes each ciphertext slice through . Kernel TLS is opt-in /// per direction ( / ), and the /// handoff happens here, right after the handshake. @@ -83,7 +83,7 @@ private void SweepHandshakes() } } - // Identifies this server to OpenSSL's session cache. Any stable value works - what matters is + // Identifies this server to BoringSSL's session cache. Any stable value works - what matters is // that it is set at all, and that it does not change between the contexts of one service. private static readonly byte[] SessionIdContext = "ioxide"u8.ToArray(); @@ -92,12 +92,11 @@ private void SweepHandshakes() /// /// How much the kTLS RX handoff holds while it waits for a cut record's rest. Past it the - /// connection stays on OpenSSL, which decrypts what was held. + /// connection stays on BoringSSL, which decrypts what was held. /// private const nuint MaxHandoffBytes = 1 << 20; - private static readonly int SslSessionIndex = - OpenSsl.CRYPTO_get_ex_new_index(OpenSsl.CRYPTO_EX_INDEX_SSL, 0, 0, 0, 0, 0); + private static readonly int SslSessionIndex = BoringSsl.SSL_get_ex_new_index(0, 0, 0, 0, 0); private TlsService(CertificateSlot slot, GCHandle slotHandle, GCHandle alpnHandle, TlsOptions options) { @@ -360,7 +359,7 @@ public static TlsService Start(Reactor reactor, TlsOptions options, bool registe /// certificate material, which is exactly why this method is not allowed to change anything else. /// /// The contexts it replaces are kept, not freed. A handshake may be between reading the table - /// and using what it found, and OpenSSL gives no way to wait that out - so they are retained + /// and using what it found, and BoringSSL gives no way to wait that out - so they are retained /// for the life of the service, as its contexts already were. A rotation is an operational /// event, not a hot path; rotating a handful of names a few times a year costs kilobytes. /// @@ -406,10 +405,10 @@ private static Certificates BuildCertificates(TlsOptions options, TlsCertificate // server serving the wrong certificate rather than one that failed to start. delegate* unmanaged servername = &ServerNameCallback; - if (OpenSsl.SSL_CTX_callback_ctrl(ctx, OpenSsl.SSL_CTRL_SET_TLSEXT_SERVERNAME_CB, (nint)servername) != 1 || - OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_SET_TLSEXT_SERVERNAME_ARG, 0, GCHandle.ToIntPtr(slotHandle)) != 1) + if (BoringSsl.SSL_CTX_set_tlsext_servername_callback(ctx, (nint)servername) != 1 || + BoringSsl.SSL_CTX_set_tlsext_servername_arg(ctx, GCHandle.ToIntPtr(slotHandle)) != 1) { - throw new IOException($"could not install the SNI callback: {OpenSsl.LastError()}"); + throw new IOException($"could not install the SNI callback: {BoringSsl.LastError()}"); } } } @@ -432,17 +431,17 @@ private static void Release(nint ctx, HostTable? byHost) { if (ctx != 0) { - OpenSsl.SSL_CTX_free(ctx); + BoringSsl.SSL_CTX_free(ctx); } foreach (nint hostCtx in byHost?.Contexts ?? []) { - OpenSsl.SSL_CTX_free(hostCtx); + BoringSsl.SSL_CTX_free(hostCtx); } } /// - /// One OpenSSL context per SNI name, built here so that selecting one at handshake time is a + /// One BoringSSL context per SNI name, built here so that selecting one at handshake time is a /// lookup and nothing more. Null when no alternatives were configured. /// /// @@ -450,19 +449,19 @@ private static void Release(nint ctx, HostTable? byHost) /// setting is repeated here - the two halves are worth telling apart, because only one of them /// is load-bearing: /// - /// Settings OpenSSL copies into the SSL when it is created, from the DEFAULT context, and never + /// Settings BoringSSL copies into the SSL when it is created, from the DEFAULT context, and never /// re-reads: the verify mode, SSL_OP_* options, min/max protocol version, and the ticket count. /// A swap cannot touch these, so a host context cannot weaken them - it is why SNI cannot be /// used to escape mutual TLS or the kTLS protocol floor. Setting them below is inert. /// - /// Settings a host context therefore has to carry itself, because OpenSSL takes them from it: + /// Settings a host context therefore has to carry itself, because BoringSSL takes them from it: /// the client trust anchors and CA list, and the ALPN and keylog callbacks, each read back off /// the context after the swap - and EVERYTHING IN ITS CERT, which the swap duplicates wholesale /// over the connection's own. That is the certificate and key, and with them the security /// level, the signature-algorithm lists, and the certificate callback. Omitting any of these /// changes how the connection behaves: dropping the anchors would fail every mutual-TLS client /// of a named host, and a security level or sigalg list configured only on the default would be - /// silently reset to OpenSSL's own for every named one. + /// silently reset to BoringSSL's own for every named one. /// /// That split is the invariant to preserve: anything added to the default context in future /// belongs in the second group unless it is known to be in the first. @@ -538,7 +537,7 @@ private static void Release(nint ctx, HostTable? byHost) // had already built. for (int i = 0; i < next; i++) { - OpenSsl.SSL_CTX_free(contexts[i]); + BoringSsl.SSL_CTX_free(contexts[i]); } throw; @@ -548,12 +547,12 @@ private static void Release(nint ctx, HostTable? byHost) } /// - /// One configured OpenSSL context. The default certificate and every SNI alternative are built + /// One configured BoringSSL context. The default certificate and every SNI alternative are built /// HERE and nowhere else. /// /// /// That is the point of the method rather than a tidiness argument. A per-host context has to - /// carry everything OpenSSL reads back off the context after a handshake swaps to it, and the + /// carry everything BoringSSL reads back off the context after a handshake swaps to it, and the /// two were previously configured by two pieces of code that had to be kept in step by hand - /// so a setting added to one and forgotten on the other would apply to clients that sent no /// name and silently not to clients that did. Built from one body, they cannot drift. @@ -563,10 +562,10 @@ private static void Release(nint ctx, HostTable? byHost) /// private static nint NewContext(TlsOptions options, TlsCertificate certificate, string what, GCHandle alpnHandle) { - nint ctx = OpenSsl.SSL_CTX_new(OpenSsl.TLS_server_method()); + nint ctx = BoringSsl.SSL_CTX_new(BoringSsl.TLS_server_method()); if (ctx == 0) { - throw new IOException($"SSL_CTX_new{what}: {OpenSsl.LastError()}"); + throw new IOException($"SSL_CTX_new{what}: {BoringSsl.LastError()}"); } try @@ -578,7 +577,7 @@ private static nint NewContext(TlsOptions options, TlsCertificate certificate, s // The context is this method's to release until it returns one. A caller cannot do it: // the assignment it would free through never happened, which is exactly how a renewal // retried against a half-written PEM leaked a certificate and key per attempt. - OpenSsl.SSL_CTX_free(ctx); + BoringSsl.SSL_CTX_free(ctx); throw; } @@ -589,50 +588,37 @@ private static nint NewContext(TlsOptions options, TlsCertificate certificate, s /// exactly one owner and one release path. private static void Configure(nint ctx, TlsOptions options, TlsCertificate certificate, string what, GCHandle alpnHandle) { - // Before any key is read. OpenSSL's default passphrase callback is PEM_def_callback, which - // reads the TERMINAL - so an encrypted key makes Start, or ReplaceCertificates on a - // rotation thread while the server is still answering, block on a prompt rather than fail. - // TlsOptions has no passphrase, so the only correct answer is that none is available, which - // turns a hang into the ordinary refusal every other unusable key already gets. + // Before any key is read. TlsOptions has no passphrase, so the only correct answer is that + // none is available: an encrypted key gets the ordinary refusal every other unusable key + // gets, whatever the library's default callback would have tried (OpenSSL's read the + // terminal, which made Start - or a rotation thread - block on a prompt). unsafe { delegate* unmanaged refuse = &NoPassphrase; - OpenSsl.SSL_CTX_set_default_passwd_cb(ctx, (nint)refuse); + BoringSsl.SSL_CTX_set_default_passwd_cb(ctx, (nint)refuse); } - - // OpenSSL copies a context's options onto each SSL as it is created and never re-reads - // them, so this holds for every connection whether or not it asked for a name. TLS 1.3 has - // no renegotiation to refuse; over TLS 1.2 refusing it costs nothing anyone uses and takes - // away a second ClientHello - which, with SNI, would be a second go at the certificate. - OpenSsl.SSL_CTX_set_options(ctx, OpenSsl.SSL_OP_NO_RENEGOTIATION); + // No renegotiation to refuse: a BoringSSL server never renegotiates. // Caller-stated posture first; the kTLS block below then pins what it must, and Start has // already refused the combinations where the two would disagree. if (options.MinProtocolVersion != TlsProtocolVersion.Default) { - long floor = options.MinProtocolVersion == TlsProtocolVersion.Tls13 - ? OpenSsl.TLS1_3_VERSION - : OpenSsl.TLS1_2_VERSION; + ushort floor = options.MinProtocolVersion == TlsProtocolVersion.Tls13 + ? BoringSsl.TLS1_3_VERSION + : BoringSsl.TLS1_2_VERSION; - if (OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_SET_MIN_PROTO_VERSION, floor, 0) != 1) + if (BoringSsl.SSL_CTX_set_min_proto_version(ctx, floor) != 1) { - throw new IOException($"set_min_proto_version{what}: {OpenSsl.LastError()}"); + throw new IOException($"set_min_proto_version{what}: {BoringSsl.LastError()}"); } } if (options.CipherSuites is { } suites) { - // OpenSSL is no help here on its own. It accepts an EMPTY 1.3 list and returns success, - // which leaves no TLS 1.3 suite enabled at all - so with a 1.3 floor the server starts - // clean and fails every handshake. And it ignores names it does not know as long as one - // in the list is valid, so a single typo silently drops the suite the operator meant to - // pin. Neither shows up until a client cannot connect, and neither is a mistake the - // caller can see. Note CipherList (1.2) behaves the opposite way and refuses both. - // - // Each name is offered on its own first, because a one-name list that OpenSSL does not - // recognise IS an all-unknown list, which it does refuse. The full list is applied - // afterwards so ordering is the caller's. + // Every name has to be a TLS 1.3 suite this BoringSSL implements, so a typo or a TLS 1.2 + // name fails here rather than at the first handshake. Empty is refused as well: BoringSSL + // reads an empty list as its built-in order, which is what null already means. string[] names = suites.Split(':', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); if (names.Length == 0) { @@ -640,72 +626,77 @@ private static void Configure(nint ctx, TlsOptions options, TlsCertificate certi $"CipherSuites{what} is empty, which disables every TLS 1.3 suite. Leave it null for the default."); } - foreach (string name in names) + ushort[] ids = new ushort[names.Length]; + for (int i = 0; i < names.Length; i++) { - if (OpenSsl.SSL_CTX_set_ciphersuites(ctx, name) != 1) - { - OpenSsl.ERR_clear_error(); - throw new IOException($"set_ciphersuites{what}: OpenSSL does not know the suite '{name}'."); - } + ids[i] = BoringSsl.Tls13SuiteId(names[i]) ?? throw new IOException( + $"CipherSuites{what}: '{names[i]}' is not a TLS 1.3 suite BoringSSL implements; " + + "TLS 1.2 ciphers go in CipherList."); } - if (OpenSsl.SSL_CTX_set_ciphersuites(ctx, suites) != 1) + unsafe { - throw new IOException($"set_ciphersuites{what}: {OpenSsl.LastError()}"); + fixed (ushort* p = ids) + { + if (BoringSsl.SSL_CTX_set1_tls13_ciphers(ctx, p, null, (nuint)ids.Length) != 1) + { + throw new IOException($"set1_tls13_ciphers{what}: {BoringSsl.LastError()}"); + } + } } } // Empty is not "no restriction" - it is a list nothing matches, which would fail every // TLS 1.2 handshake at the point of use rather than here. - if (options.CipherList is { } ciphers && OpenSsl.SSL_CTX_set_cipher_list(ctx, ciphers) != 1) + if (options.CipherList is { } ciphers && BoringSsl.SSL_CTX_set_cipher_list(ctx, ciphers) != 1) { - throw new IOException($"set_cipher_list{what}: {OpenSsl.LastError()}"); + throw new IOException($"set_cipher_list{what}: {BoringSsl.LastError()}"); } - // A stated list is documented as being in preference order, and OpenSSL's default is to let - // the CLIENT's order win - so without this the operator's order was decoration. Set only - // when they actually stated one: with no list configured the default order is OpenSSL's to - // choose, and forcing server preference there would change which suite existing clients - // negotiate for no one's benefit. + // A stated list is documented as being in preference order. A TLS 1.3 list set above always + // is; for TLS 1.2 the default lets the CLIENT's order win, so without this the operator's + // order was decoration. Set only when they actually stated one: with no list configured the + // default order is BoringSSL's to choose. if (options.CipherSuites is not null || options.CipherList is not null) { - OpenSsl.SSL_CTX_set_options(ctx, OpenSsl.SSL_OP_CIPHER_SERVER_PREFERENCE); + BoringSsl.SSL_CTX_set_options(ctx, BoringSsl.SSL_OP_CIPHER_SERVER_PREFERENCE); } // The kTLS constraints are the kernel path's, not TLS's - so they apply only when the - // kernel path was asked for. The default keeps OpenSSL's own defaults: TLS 1.2 and 1.3, + // kernel path was asked for. The default keeps BoringSSL's own defaults: TLS 1.2 and 1.3, // any ciphersuite, session tickets on. This is the table in TlsOptions.KernelTx made true. if (options.KernelTx) { // TLS 1.3 only, one suite: kTLS needs AES-128-GCM keys and a known layout. - OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_SET_MIN_PROTO_VERSION, OpenSsl.TLS1_3_VERSION, 0); - OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_SET_MAX_PROTO_VERSION, OpenSsl.TLS1_3_VERSION, 0); - if (OpenSsl.SSL_CTX_set_ciphersuites(ctx, "TLS_AES_128_GCM_SHA256") != 1) + unsafe { - throw new IOException($"set_ciphersuites{what}: {OpenSsl.LastError()}"); + ushort aes128Gcm = BoringSsl.TLS_AES_128_GCM_SHA256; + if (BoringSsl.SSL_CTX_set_min_proto_version(ctx, BoringSsl.TLS1_3_VERSION) != 1 || + BoringSsl.SSL_CTX_set_max_proto_version(ctx, BoringSsl.TLS1_3_VERSION) != 1 || + BoringSsl.SSL_CTX_set1_tls13_ciphers(ctx, &aes128Gcm, null, 1) != 1) + { + throw new IOException($"kTLS posture{what}: {BoringSsl.LastError()}"); + } } // No session tickets: they would consume record sequence numbers after the // handshake and break the kTLS handoff (which programs the record sequence). - OpenSsl.SSL_CTX_set_num_tickets(ctx, 0); + BoringSsl.SSL_CTX_set_num_tickets(ctx, 0); } LoadCertificate(ctx, certificate, what); ConfigureClientVerification(ctx, options); - // Scope sessions to this server. Required rather than tidy: with SSL_VERIFY_PEER set and no - // id context, OpenSSL treats a resumption attempt as fatal to the whole HANDSHAKE, not just - // to the resumption - so an mTLS port issues tickets and then refuses every client that - // comes back with one, which is roughly every second connection from anything that caches. + // Scope sessions to this server, so a ticket another server issued is not resumed here. // Constant per process: every context of this service must accept the others' tickets, or a // rotation would invalidate every outstanding session. unsafe { fixed (byte* id = SessionIdContext) { - if (OpenSsl.SSL_CTX_set_session_id_context(ctx, id, (uint)SessionIdContext.Length) != 1) + if (BoringSsl.SSL_CTX_set_session_id_context(ctx, id, (nuint)SessionIdContext.Length) != 1) { - throw new IOException($"could not set the session id context{what}: {OpenSsl.LastError()}"); + throw new IOException($"could not set the session id context{what}: {BoringSsl.LastError()}"); } } } @@ -720,13 +711,13 @@ private static void Configure(nint ctx, TlsOptions options, TlsCertificate certi if (options.KernelTx) { delegate* unmanaged keylog = &KeylogCallback; - OpenSsl.SSL_CTX_set_keylog_callback(ctx, (nint)keylog); + BoringSsl.SSL_CTX_set_keylog_callback(ctx, (nint)keylog); } // ALPN is settled against whichever context the handshake ends on, so every context // needs the callback - reading the same wire bytes through the same handle. delegate* unmanaged alpn = &AlpnSelectCallback; - OpenSsl.SSL_CTX_set_alpn_select_cb(ctx, (nint)alpn, GCHandle.ToIntPtr(alpnHandle)); + BoringSsl.SSL_CTX_set_alpn_select_cb(ctx, (nint)alpn, GCHandle.ToIntPtr(alpnHandle)); } } @@ -784,7 +775,7 @@ private static void RequireOneSourceEach(TlsCertificate certificate, string what /// listed, and would report it as a dead connection rather than a certificate mismatch. /// /// A name that WAS matched returns OK, and any other outcome returns NOACK. The difference is - /// what the server says about it: OK acknowledges the extension, so OpenSSL echoes the name + /// what the server says about it: OK acknowledges the extension, so BoringSSL echoes the name /// back and records it on the session. Acknowledging a name this server does not hold would be /// claiming one it cannot serve (RFC 6066 3), and would file the client's own string against /// the session. NOACK declines the extension WITHOUT failing the handshake, which is exactly @@ -794,9 +785,9 @@ private static void RequireOneSourceEach(TlsCertificate certificate, string what /// bytes already in native memory, against a table built when the service started. /// /// - /// "There is no passphrase." Zero written bytes makes OpenSSL fail the key load with an - /// ordinary error rather than prompting on the terminal, which is what it does when no - /// callback is installed at all. + /// "There is no passphrase." Zero written bytes fails the key load with an ordinary error. + /// BoringSSL's default callback never prompts either; this states the answer rather than + /// relying on that. /// [UnmanagedCallersOnly] private static int NoPassphrase(nint buf, int size, int rwflag, nint userdata) => 0; @@ -811,42 +802,42 @@ private static unsafe int ServerNameCallback(nint ssl, nint alert, nint arg) if (arg == 0 || GCHandle.FromIntPtr(arg).Target is not CertificateSlot slot || slot.Current.ByHost is not { } byHost) { - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } - nint namePtr = OpenSsl.SSL_get_servername(ssl, OpenSsl.TLSEXT_NAMETYPE_host_name); + nint namePtr = BoringSsl.SSL_get_servername(ssl, BoringSsl.TLSEXT_NAMETYPE_host_name); if (namePtr == 0) { // No SNI extension. The default certificate is the answer, which is what it is for. - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } - // OpenSSL's own copy of the name, NUL-terminated and valid for this handshake - read - // where it lies rather than decoded into a string only to be thrown away. OpenSSL has + // BoringSSL's own copy of the name, NUL-terminated and valid for this handshake - read + // where it lies rather than decoded into a string only to be thrown away. BoringSSL has // already refused an over-long name and one carrying a NUL, so the span is bounded. nint hostCtx = byHost.Find(MemoryMarshal.CreateReadOnlySpanFromNullTerminated((byte*)namePtr)); if (hostCtx == 0) { - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } // Fails only when the certificate cannot be duplicated, and the default context is // still a working answer - but then this name was NOT the one being served, so it must // not be acknowledged as though it were. - if (OpenSsl.SSL_set_SSL_CTX(ssl, hostCtx) == 0) + if (BoringSsl.SSL_set_SSL_CTX(ssl, hostCtx) == 0) { - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } } catch { // A throw here would cross native frames. The default certificate is a working answer, // so failing to select a better one is not a reason to lose the connection. - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } - return OpenSsl.SSL_TLSEXT_ERR_OK; + return BoringSsl.SSL_TLSEXT_ERR_OK; } /// @@ -860,7 +851,7 @@ private static unsafe int ServerNameCallback(nint ssl, nint alert, nint arg) /// from a handshake that already happened. /// /// Orthogonal to kTLS. The certificate is exchanged and validated during the handshake, which - /// OpenSSL performs either way - the kernel only takes over record crypto afterwards - so this + /// BoringSSL performs either way - the kernel only takes over record crypto afterwards - so this /// composes with and /// unchanged. /// @@ -871,70 +862,76 @@ private static unsafe void ConfigureClientVerification(nint ctx, TlsOptions opti return; } - if (options.ClientCaPath is not null) + if (options.ClientCaPath is not null && ReadAfterBom(options.ClientCaPath) is { } bundle) { - if (OpenSsl.SSL_CTX_load_verify_locations(ctx, options.ClientCaPath, null) != 1) + // BoringSSL's file loader does not skip a UTF-8 BOM, so the bundle goes in without it. + AddTrustAnchorsPem(ctx, System.Text.Encoding.UTF8.GetString(bundle), + $"client CA '{options.ClientCaPath}'"); + } + else if (options.ClientCaPath is not null) + { + if (BoringSsl.SSL_CTX_load_verify_locations(ctx, options.ClientCaPath, null) != 1) { throw new IOException( - $"could not load client CA '{options.ClientCaPath}': {OpenSsl.LastError()}"); + $"could not load client CA '{options.ClientCaPath}': {BoringSsl.LastError()}"); } // Tell the client which issuers we accept. Failure here is not fatal - it costs the - // hint, not the verification - so a CA file OpenSSL can trust but not enumerate still + // hint, not the verification - so a CA file BoringSSL can trust but not enumerate still // works for a client with a single certificate. - nint names = OpenSsl.SSL_load_client_CA_file(options.ClientCaPath); + nint names = BoringSsl.SSL_load_client_CA_file(options.ClientCaPath); if (names != 0) { - OpenSsl.SSL_CTX_set_client_CA_list(ctx, names); + BoringSsl.SSL_CTX_set_client_CA_list(ctx, names); } else { - OpenSsl.ERR_clear_error(); + BoringSsl.ERR_clear_error(); } } else { - AddTrustAnchorsPem(ctx, options.ClientCaPem!); + AddTrustAnchorsPem(ctx, options.ClientCaPem!, "ClientCaPem"); } - int mode = OpenSsl.SSL_VERIFY_PEER; + int mode = BoringSsl.SSL_VERIFY_PEER; if (options.RequireClientCertificate) { - mode |= OpenSsl.SSL_VERIFY_FAIL_IF_NO_PEER_CERT; + mode |= BoringSsl.SSL_VERIFY_FAIL_IF_NO_PEER_CERT; } // No callback: a chain that does not validate fails the handshake. There is no prompt to // fall back to and no partial trust worth inventing. - OpenSsl.SSL_CTX_set_verify(ctx, mode, 0); + BoringSsl.SSL_CTX_set_verify(ctx, mode, 0); } // Trust anchors from PEM text, added straight to the context's store - the in-memory mirror of // load_verify_locations, for hosts that carry a CA bundle as data rather than as a file. - private static unsafe void AddTrustAnchorsPem(nint ctx, string pem) + private static unsafe void AddTrustAnchorsPem(nint ctx, string pem, string source) { - nint store = OpenSsl.SSL_CTX_get_cert_store(ctx); + nint store = BoringSsl.SSL_CTX_get_cert_store(ctx); if (store == 0) { - throw new IOException($"SSL_CTX_get_cert_store: {OpenSsl.LastError()}"); + throw new IOException($"SSL_CTX_get_cert_store: {BoringSsl.LastError()}"); } - // UTF-8, so a byte-order mark arrives as the three bytes OpenSSL strips, as from a file. - byte[] bytes = System.Text.Encoding.UTF8.GetBytes(pem); + // UTF-8, minus any leading byte-order mark: BoringSSL's reader does not skip one. + byte[] bytes = System.Text.Encoding.UTF8.GetBytes(pem.TrimStart('\uFEFF')); int added = 0; fixed (byte* p = bytes) { - nint bio = OpenSsl.BIO_new_mem_buf(p, bytes.Length); + nint bio = BoringSsl.BIO_new_mem_buf(p, bytes.Length); if (bio == 0) { - throw new IOException($"BIO_new_mem_buf: {OpenSsl.LastError()}"); + throw new IOException($"BIO_new_mem_buf: {BoringSsl.LastError()}"); } try { while (true) { - nint cert = OpenSsl.PEM_read_bio_X509_AUX(bio, 0, 0, 0); + nint cert = BoringSsl.PEM_read_bio_X509_AUX(bio, 0, 0, 0); if (cert == 0) { // Exhausted, or stopped on a malformed block. Only the first is an end of @@ -943,49 +940,49 @@ private static unsafe void AddTrustAnchorsPem(nint ctx, string pem) // later anchor are refused with nothing said server-side. The file route // refuses such a bundle outright, and TlsOptions documents the two sources // as equivalent. - if (!OpenSsl.ErrorIsEndOfPem()) + if (!BoringSsl.ErrorIsEndOfPem()) { throw new IOException( - $"ClientCaPem is malformed after {added} usable certificate(s); " + $"{source} is malformed after {added} usable certificate(s); " + "the rest of the bundle would have been ignored."); } break; } - int ok = OpenSsl.X509_STORE_add_cert(store, cert); + int ok = BoringSsl.X509_STORE_add_cert(store, cert); // Same issuer hint the file route sends. Without it a client holding several // certificates has nothing to choose by and offers whichever comes first, so // the two anchor sources would accept different populations of client - which // is not something the choice between a path and PEM text should decide. // Not fatal on failure: it costs the hint, not the verification. - if (ok == 1 && !Hinted(ctx, cert) && OpenSsl.SSL_CTX_add_client_CA(ctx, cert) != 1) + if (ok == 1 && !Hinted(ctx, cert) && BoringSsl.SSL_CTX_add_client_CA(ctx, cert) != 1) { - OpenSsl.ERR_clear_error(); + BoringSsl.ERR_clear_error(); } - OpenSsl.X509_free(cert); // the store and the CA list took their own references + BoringSsl.X509_free(cert); // the store and the CA list took their own references if (ok != 1) { - throw new IOException($"X509_STORE_add_cert: {OpenSsl.LastError()}"); + throw new IOException($"X509_STORE_add_cert: {BoringSsl.LastError()}"); } added++; } } finally { - OpenSsl.BIO_free(bio); + BoringSsl.BIO_free(bio); } } // Running off the end of the last certificate leaves a "no start line" error behind. - OpenSsl.ERR_clear_error(); + BoringSsl.ERR_clear_error(); if (added == 0) { - throw new IOException("ClientCaPem contained no certificates."); + throw new IOException($"{source} contained no certificates."); } } @@ -993,13 +990,13 @@ private static unsafe void AddTrustAnchorsPem(nint ctx, string pem) // each name once, so a bundle that repeats an anchor must not grow the CertificateRequest here. private static bool Hinted(nint ctx, nint cert) { - nint names = OpenSsl.SSL_CTX_get_client_CA_list(ctx); - nint subject = OpenSsl.X509_get_subject_name(cert); + nint names = BoringSsl.SSL_CTX_get_client_CA_list(ctx); + nint subject = BoringSsl.X509_get_subject_name(cert); nint name; - for (int i = 0; (name = OpenSsl.OPENSSL_sk_value(names, i)) != 0; i++) + for (nuint i = 0; (name = BoringSsl.OPENSSL_sk_value(names, i)) != 0; i++) { - if (OpenSsl.X509_NAME_cmp(name, subject) == 0) + if (BoringSsl.X509_NAME_cmp(name, subject) == 0) { return true; } @@ -1008,83 +1005,130 @@ private static bool Hinted(nint ctx, nint cert) return false; } - // Certificate and key, from whichever source the caller carries. The file route is OpenSSL's + // Certificate and key, from whichever source the caller carries. The file route is BoringSSL's // own loaders; the in-memory route reads the same PEM through a memory BIO. Which of the two // is set was settled by RequireOneSourceEach before any context existed. private static void LoadCertificate(nint ctx, TlsCertificate certificate, string what) { - if (certificate.CertificatePath is not null) + if (certificate.CertificatePath is { } certificatePath) { - if (OpenSsl.SSL_CTX_use_certificate_chain_file(ctx, certificate.CertificatePath) != 1) + // BoringSSL's PEM reader, unlike OpenSSL's, does not skip a UTF-8 BOM: such a file goes + // through the in-memory route with the mark removed. + if (ReadAfterBom(certificatePath) is { } text) { - throw new IOException($"certificate{what} '{certificate.CertificatePath}': {OpenSsl.LastError()}"); + LoadCertificatePem(ctx, text, $"certificate{what} '{certificatePath}'"); + } + else if (BoringSsl.SSL_CTX_use_certificate_chain_file(ctx, certificatePath) != 1) + { + throw new IOException($"certificate{what} '{certificatePath}': {BoringSsl.LastError()}"); } } else { - LoadCertificatePem(ctx, certificate.CertificatePem!); + LoadCertificatePem(ctx, System.Text.Encoding.ASCII.GetBytes(certificate.CertificatePem!), "CertificatePem"); } - if (certificate.KeyPath is not null) + if (certificate.KeyPath is { } keyPath) { - if (OpenSsl.SSL_CTX_use_PrivateKey_file(ctx, certificate.KeyPath, OpenSsl.SSL_FILETYPE_PEM) != 1) + if (ReadAfterBom(keyPath) is { } text) + { + try + { + LoadKeyPem(ctx, text, $"private key{what} '{keyPath}'", what); + } + finally + { + System.Security.Cryptography.CryptographicOperations.ZeroMemory(text); + } + } + else if (BoringSsl.SSL_CTX_use_PrivateKey_file(ctx, keyPath, BoringSsl.SSL_FILETYPE_PEM) != 1) { - throw new IOException($"private key{what} '{certificate.KeyPath}': {OpenSsl.LastError()}"); + throw BoringSsl.ErrorIsKeyMismatch() + ? PairMismatch(what) + : new IOException($"private key{what} '{keyPath}': {BoringSsl.LastError()}"); } } else { - LoadKeyPem(ctx, certificate.KeyPem!); + LoadKeyPem(ctx, System.Text.Encoding.ASCII.GetBytes(certificate.KeyPem!), "KeyPem", what); } - // The pair has to be checked explicitly, because OpenSSL only checks it for us when the two - // land in the SAME algorithm slot. A certificate and key that are both RSA and unrelated - // are refused by use_PrivateKey itself - but an RSA certificate with an EC key is not: the - // key goes into the EC slot, which has no certificate, and the RSA slot keeps a certificate - // with no key. Nothing complains, the server starts, and every handshake afterwards fails - // with no shared cipher. That is the shape of an operator moving a server from RSA to EC - // and updating one of the two files. - if (OpenSsl.SSL_CTX_check_private_key(ctx) != 1) + // The pair is checked explicitly rather than trusting the key load to notice a mismatch: + // whether it does depends on the library and the key types (OpenSSL put an EC key next to + // an RSA certificate without complaint, and every handshake then failed with no shared + // cipher). That is the shape of an operator moving a server from RSA to EC and updating one + // of the two files. + if (BoringSsl.SSL_CTX_check_private_key(ctx) != 1) { - OpenSsl.ERR_clear_error(); - throw new IOException( - $"the certificate and private key{what} do not go together - a certificate of one " - + "algorithm with a key of another leaves the context with neither a usable pair."); + throw PairMismatch(what); } } - private static unsafe void LoadCertificatePem(nint ctx, string pem) + private static IOException PairMismatch(string what) + { + BoringSsl.ERR_clear_error(); + return new IOException( + $"the certificate and private key{what} do not go together - the key is not the one the " + + "certificate was issued for."); + } + + /// + /// The file's bytes after a leading UTF-8 BOM, or null when it has none - or cannot be read, so + /// that BoringSSL's own loader reports the file as it reports every other unusable one. + /// + private static byte[]? ReadAfterBom(string path) + { + try + { + using FileStream file = File.OpenRead(path); + Span head = stackalloc byte[3]; + if (file.ReadAtLeast(head, 3, throwOnEndOfStream: false) < 3 || !head.SequenceEqual(Utf8Bom)) + { + return null; + } + byte[] rest = new byte[file.Length - 3]; + file.ReadExactly(rest); + return rest; + } + catch (Exception e) when (e is IOException or UnauthorizedAccessException) + { + return null; + } + } + + private static ReadOnlySpan Utf8Bom => [0xEF, 0xBB, 0xBF]; + + private static unsafe void LoadCertificatePem(nint ctx, byte[] bytes, string source) { - byte[] bytes = System.Text.Encoding.ASCII.GetBytes(pem); fixed (byte* p = bytes) { // The memory BIO reads the pinned buffer in place, so the fixed block spans every read. - nint bio = OpenSsl.BIO_new_mem_buf(p, bytes.Length); + nint bio = BoringSsl.BIO_new_mem_buf(p, bytes.Length); if (bio == 0) { - throw new IOException($"CertificatePem: {OpenSsl.LastError()}"); + throw new IOException($"{source}: {BoringSsl.LastError()}"); } try { // First PEM block is the leaf; use_certificate takes its own reference. - nint leaf = OpenSsl.PEM_read_bio_X509(bio, 0, 0, 0); + nint leaf = BoringSsl.PEM_read_bio_X509(bio, 0, 0, 0); if (leaf == 0) { - throw new IOException($"CertificatePem carries no certificate: {OpenSsl.LastError()}"); + throw new IOException($"{source} carries no certificate: {BoringSsl.LastError()}"); } - int ok = OpenSsl.SSL_CTX_use_certificate(ctx, leaf); - OpenSsl.X509_free(leaf); + int ok = BoringSsl.SSL_CTX_use_certificate(ctx, leaf); + BoringSsl.X509_free(leaf); if (ok != 1) { - throw new IOException($"CertificatePem: {OpenSsl.LastError()}"); + throw new IOException($"{source}: {BoringSsl.LastError()}"); } // Any further blocks are the chain, closest-to-leaf first, exactly as in a chain // file. add_extra_chain_cert takes ownership on success, so no free there. while (true) { - nint extra = OpenSsl.PEM_read_bio_X509(bio, 0, 0, 0); + nint extra = BoringSsl.PEM_read_bio_X509(bio, 0, 0, 0); if (extra == 0) { // Exhausted, or stopped on a malformed block - the same distinction @@ -1096,58 +1140,61 @@ private static unsafe void LoadCertificatePem(nint ctx, string pem) // the rotation throw - says it went fine. A renewal read mid-write is the // ordinary way to produce exactly this file. SSL_CTX_use_certificate_chain_file, // which the path route uses and this mirrors, refuses it outright. - if (!OpenSsl.ErrorIsEndOfPem()) + if (!BoringSsl.ErrorIsEndOfPem()) { throw new IOException( - "CertificatePem is malformed after the leaf; the rest of the chain " + $"{source} is malformed after the leaf; the rest of the chain " + "would have been ignored and the certificate served without it."); } - OpenSsl.ERR_clear_error(); // end-of-data queues PEM_R_NO_START_LINE + BoringSsl.ERR_clear_error(); // end-of-data queues PEM_R_NO_START_LINE break; } - if (OpenSsl.SSL_CTX_ctrl(ctx, OpenSsl.SSL_CTRL_EXTRA_CHAIN_CERT, 0, extra) != 1) + if (BoringSsl.SSL_CTX_add_extra_chain_cert(ctx, extra) != 1) { - OpenSsl.X509_free(extra); - throw new IOException($"CertificatePem chain: {OpenSsl.LastError()}"); + BoringSsl.X509_free(extra); + throw new IOException($"{source} chain: {BoringSsl.LastError()}"); } } } finally { - OpenSsl.BIO_free(bio); + BoringSsl.BIO_free(bio); } } } - private static unsafe void LoadKeyPem(nint ctx, string pem) + private static unsafe void LoadKeyPem(nint ctx, byte[] bytes, string source, string what) { - byte[] bytes = System.Text.Encoding.ASCII.GetBytes(pem); fixed (byte* p = bytes) { - nint bio = OpenSsl.BIO_new_mem_buf(p, bytes.Length); + nint bio = BoringSsl.BIO_new_mem_buf(p, bytes.Length); if (bio == 0) { - throw new IOException($"KeyPem: {OpenSsl.LastError()}"); + throw new IOException($"{source}: {BoringSsl.LastError()}"); } try { - nint key = OpenSsl.PEM_read_bio_PrivateKey(bio, 0, 0, 0); + nint key = BoringSsl.PEM_read_bio_PrivateKey(bio, 0, 0, 0); if (key == 0) { - throw new IOException($"KeyPem carries no private key: {OpenSsl.LastError()}"); + throw new IOException($"{source} carries no private key: {BoringSsl.LastError()}"); + } + int ok = BoringSsl.SSL_CTX_use_PrivateKey(ctx, key); + BoringSsl.EVP_PKEY_free(key); + if (ok != 1 && BoringSsl.ErrorIsKeyMismatch()) + { + throw PairMismatch(what); } - int ok = OpenSsl.SSL_CTX_use_PrivateKey(ctx, key); - OpenSsl.EVP_PKEY_free(key); if (ok != 1) { - throw new IOException($"KeyPem: {OpenSsl.LastError()}"); + throw new IOException($"{source}: {BoringSsl.LastError()}"); } } finally { - OpenSsl.BIO_free(bio); + BoringSsl.BIO_free(bio); } } } @@ -1163,23 +1210,23 @@ public async ValueTask AcceptAsync(TcpConnection conn) // Read once, here: whichever certificates are in force when this connection starts are the // ones it uses. SSL_new takes its own reference, so a rotation a moment later cannot pull // the context out from under a handshake already running on it. - nint ssl = OpenSsl.SSL_new(_slot.Current.Default); + nint ssl = BoringSsl.SSL_new(_slot.Current.Default); if (ssl == 0) { - throw new IOException($"SSL_new: {OpenSsl.LastError()}"); + throw new IOException($"SSL_new: {BoringSsl.LastError()}"); } - nint rbio = OpenSsl.BIO_new(OpenSsl.BIO_s_mem()); - nint wbio = OpenSsl.BIO_new(OpenSsl.BIO_s_mem()); - OpenSsl.SSL_set_bio(ssl, rbio, wbio); // ssl owns both BIOs now - OpenSsl.SSL_set_accept_state(ssl); + nint rbio = BoringSsl.BIO_new(BoringSsl.BIO_s_mem()); + nint wbio = BoringSsl.BIO_new(BoringSsl.BIO_s_mem()); + BoringSsl.SSL_set_bio(ssl, rbio, wbio); // ssl owns both BIOs now + BoringSsl.SSL_set_accept_state(ssl); var session = new TlsSession(ssl, rbio, wbio); // Associate the session with this SSL so the keylog callback writes the secret onto it // directly (no global map). The handle is freed in TlsSession.Dispose. GCHandle handle = GCHandle.Alloc(session); - OpenSsl.SSL_set_ex_data(ssl, SslSessionIndex, GCHandle.ToIntPtr(handle)); + BoringSsl.SSL_set_ex_data(ssl, SslSessionIndex, GCHandle.ToIntPtr(handle)); session.AttachHandle(handle); session.AttachConnection(conn); // the teardown close_notify goes out on it, both modes @@ -1199,10 +1246,10 @@ public async ValueTask AcceptAsync(TcpConnection conn) { while (true) { - int ret = OpenSsl.Accept(ssl, out int err); + int ret = BoringSsl.Accept(ssl, out int err); - // Read before the alert's flush, during which other connections' OpenSSL calls clear this thread's queue. - string? failure = ret != 1 && err != OpenSsl.SSL_ERROR_WANT_READ ? OpenSsl.LastError() : null; + // Read before the alert's flush, during which other connections' BoringSSL calls clear this thread's queue. + string? failure = ret != 1 && err != BoringSsl.SSL_ERROR_WANT_READ ? BoringSsl.LastError() : null; await FlushOutbound(conn, wbio); // server flights stage into the slab @@ -1232,8 +1279,8 @@ public async ValueTask AcceptAsync(TcpConnection conn) session.CapturePeerCertificate(); // kTLS RX takes over at a record boundary. The records the handshake already pulled off - // the socket are invisible to the kernel, so OpenSSL decrypts them and the kernel's - // record sequence starts after them. -1 keeps this connection on OpenSSL. + // the socket are invisible to the kernel, so BoringSSL decrypts them and the kernel's + // record sequence starts after them. -1 keeps this connection on BoringSSL. int handedRecords = -1; if (_options.KernelTx && _options.KernelRx && session.ClientSecret is not null) { @@ -1280,7 +1327,7 @@ public async ValueTask AcceptAsync(TcpConnection conn) { throw new IOException("connection closed inside a TLS record"); } - if (OpenSsl.BIO_ctrl_pending(rbio) > MaxHandoffBytes) + if (BoringSsl.BIO_ctrl_pending(rbio) > MaxHandoffBytes) { break; // its records keep arriving cut } @@ -1312,7 +1359,7 @@ public async ValueTask AcceptAsync(TcpConnection conn) } else { - // No TLS ULP on this socket at all: OpenSSL encrypts, the reactor sends ordinary + // No TLS ULP on this socket at all: BoringSSL encrypts, the reactor sends ordinary // bytes, and MSG_WAITALL stays as accepted: there is no kTLS to reject it. The raw // secrets have no further use - and a TLS 1.2 handshake never produced them. session.ZeroSecrets(); @@ -1327,7 +1374,7 @@ public async ValueTask AcceptAsync(TcpConnection conn) } else if (session.ClientSecret is not null) { - // The client secret exists only for that handoff. OpenSSL keeps its own key + // The client secret exists only for that handoff. BoringSSL keeps its own key // schedule, so on every path that did not program the kernel the raw secret is // just key material sitting on the heap - scrub it now, not at GC's leisure. CryptographicOperations.ZeroMemory(session.ClientSecret); @@ -1355,7 +1402,7 @@ public async ValueTask AcceptAsync(TcpConnection conn) private static async ValueTask FlushOutbound(TcpConnection conn, nint wbio) { - int pending = (int)OpenSsl.BIO_ctrl_pending(wbio); + int pending = (int)BoringSsl.BIO_ctrl_pending(wbio); while (pending > 0) { int n = StageOutbound(conn, wbio, Math.Min(pending, 8 * 1024)); @@ -1374,7 +1421,7 @@ private static unsafe int StageOutbound(TcpConnection conn, nint wbio, int chunk int n; fixed (byte* p = dst) { - n = OpenSsl.BIO_read(wbio, p, chunk); + n = BoringSsl.BIO_read(wbio, p, chunk); } if (n > 0) { @@ -1390,7 +1437,7 @@ private static unsafe bool FeedInbound(TcpConnection conn, nint rbio, in RecvSna { if (item.HasBuffer) { - OpenSsl.BIO_write(rbio, item.Ptr, item.Len); + BoringSsl.BIO_write(rbio, item.Ptr, item.Len); conn.ReturnBuffer(in item); any = true; } @@ -1405,7 +1452,7 @@ private static unsafe int TakeWaiting(int fd, nint rbio, int count) int n = Sockets.ReceiveWaiting(fd, buffer, Math.Min(count, 16 * 1024)); if (n > 0) { - OpenSsl.BIO_write(rbio, buffer, n); + BoringSsl.BIO_write(rbio, buffer, n); } return n; } @@ -1441,7 +1488,7 @@ private void End() [UnmanagedCallersOnly] private static void KeylogCallback(nint ssl, nint line) { - // Guarded for the same reason ServerNameCallback is: OpenSSL's C frame sits between here + // Guarded for the same reason ServerNameCallback is: BoringSSL's C frame sits between here // and any managed caller, so an escaping exception aborts the PROCESS rather than faulting // one connection - uncatchably, and with no stack to read. The body parses a string and // allocates, so it is not throw-free by inspection. @@ -1471,7 +1518,7 @@ private static void KeylogCore(nint ssl, nint line) } // Resolve the session from the SSL's ex_data (set in AcceptAsync) - no global pointer map. - nint data = OpenSsl.SSL_get_ex_data(ssl, SslSessionIndex); + nint data = BoringSsl.SSL_get_ex_data(ssl, SslSessionIndex); if (data == 0 || GCHandle.FromIntPtr(data).Target is not TlsSession session) { return; @@ -1498,7 +1545,7 @@ private static void KeylogCore(nint ssl, nint line) [UnmanagedCallersOnly] private static unsafe int AlpnSelectCallback(nint ssl, nint outPtr, nint outLen, nint inPtr, uint inLen, nint arg) { - // Same reason as the other two: an exception crossing OpenSSL's frame aborts the process. + // Same reason as the other two: an exception crossing BoringSSL's frame aborts the process. // NOACK on failure is the safe answer - it declines ALPN and lets the handshake continue, // which is what this returns for "nothing in common" anyway. try @@ -1508,7 +1555,7 @@ private static unsafe int AlpnSelectCallback(nint ssl, nint outPtr, nint outLen, catch (Exception e) { Console.Error.WriteLine($"[tls] alpn callback faulted: {e}"); - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } } @@ -1516,7 +1563,7 @@ private static unsafe int AlpnSelectCore(nint outPtr, nint outLen, nint inPtr, u { if (arg == 0 || GCHandle.FromIntPtr(arg).Target is not byte[] wire) { - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } // SERVER preference: walk OUR list in order and take the first the client also offered, so @@ -1546,7 +1593,7 @@ private static unsafe int AlpnSelectCore(nint outPtr, nint outLen, nint inPtr, u { *(nint*)outPtr = inPtr + theirs + 1; *(byte*)outLen = (byte)haveLength; - return OpenSsl.SSL_TLSEXT_ERR_OK; + return BoringSsl.SSL_TLSEXT_ERR_OK; } theirs += 1 + haveLength; } @@ -1556,7 +1603,7 @@ private static unsafe int AlpnSelectCore(nint outPtr, nint outLen, nint inPtr, u // Nothing in common. NOACK continues without the extension rather than failing the // handshake - the client may still speak HTTP/1.1 quite happily. - return OpenSsl.SSL_TLSEXT_ERR_NOACK; + return BoringSsl.SSL_TLSEXT_ERR_NOACK; } /// diff --git a/src/ioxide/Tls/TlsSession.cs b/src/ioxide/Tls/TlsSession.cs index f161ec64..0aecea4a 100644 --- a/src/ioxide/Tls/TlsSession.cs +++ b/src/ioxide/Tls/TlsSession.cs @@ -5,7 +5,7 @@ namespace ioxide.tls; /// -/// The per-connection TLS state once the handshake is done: OpenSSL decrypts inbound records and +/// The per-connection TLS state once the handshake is done: BoringSSL decrypts inbound records and /// encrypts outbound ones, except for whichever direction was handed to the kernel (opt-in, see /// ). One session per connection, reactor-thread only. /// @@ -54,7 +54,7 @@ public sealed unsafe class TlsSession : IDisposable /// public bool Closed { - // The OpenSSL reader asks per recv, so this stays two field reads there. + // The BoringSSL reader asks per recv, so this stays two field reads there. get => _closed || (KernelRx && PeekKernelCloseNotify()); private set => _closed = value; } @@ -85,7 +85,7 @@ internal void CaptureAlpn() { byte* data; uint length; - OpenSsl.SSL_get0_alpn_selected(_ssl, &data, &length); + BoringSsl.SSL_get0_alpn_selected(_ssl, &data, &length); NegotiatedAlpn = data == null || length == 0 ? null : System.Text.Encoding.ASCII.GetString(data, (int)length); @@ -136,20 +136,20 @@ internal void CapturePeerCertificate() PeerCertificateDer = null; // Borrowed, not owned - get0 takes no reference, so there is nothing to free. - nint cert = OpenSsl.SSL_get0_peer_certificate(_ssl); + nint cert = BoringSsl.PeerCertificate(_ssl); if (cert == 0) { return; } - // Belt and braces: with SSL_VERIFY_PEER and no callback OpenSSL has already failed the + // Belt and braces: with SSL_VERIFY_PEER and no callback BoringSSL has already failed the // handshake on a bad chain, so this cannot report a name that was not verified. - if (OpenSsl.SSL_get_verify_result(_ssl) != OpenSsl.X509_V_OK) + if (BoringSsl.SSL_get_verify_result(_ssl) != BoringSsl.X509_V_OK) { return; } - nint name = OpenSsl.X509_get_subject_name(cert); + nint name = BoringSsl.X509_get_subject_name(cert); if (name == 0) { return; @@ -175,10 +175,10 @@ internal void CapturePeerCertificate() private static unsafe byte[]? ExportDer(nint cert) { byte* der = null; - int length = OpenSsl.i2d_X509(cert, &der); + int length = BoringSsl.i2d_X509(cert, &der); if (length <= 0 || der is null) { - OpenSsl.ERR_clear_error(); + BoringSsl.ERR_clear_error(); return null; } @@ -188,15 +188,15 @@ internal void CapturePeerCertificate() } finally { - OpenSsl.CRYPTO_free((nint)der, 0, 0); // i2d_X509 allocates when handed a null target + BoringSsl.OPENSSL_free((nint)der); // i2d_X509 allocates when handed a null target } } /// - /// The protocol version this handshake settled on, as OpenSSL's wire value (0x0303 = TLS 1.2, + /// The protocol version this handshake settled on, as BoringSSL's wire value (0x0303 = TLS 1.2, /// 0x0304 = TLS 1.3), or 0 before the handshake completes. /// - public int NegotiatedProtocolVersion => OpenSsl.SSL_version(_ssl); + public int NegotiatedProtocolVersion => BoringSsl.SSL_version(_ssl); /// /// The negotiated ciphersuite's IANA id, or 0 if there is none yet. The numbering is the same @@ -206,8 +206,8 @@ public ushort NegotiatedCipherSuiteId { get { - nint cipher = OpenSsl.SSL_get_current_cipher(_ssl); - return cipher == 0 ? (ushort)0 : (ushort)OpenSsl.SSL_CIPHER_get_protocol_id(cipher); + nint cipher = BoringSsl.SSL_get_current_cipher(_ssl); + return cipher == 0 ? (ushort)0 : (ushort)BoringSsl.SSL_CIPHER_get_protocol_id(cipher); } } @@ -221,7 +221,7 @@ public ushort NegotiatedCipherSuiteId /// private static string? RenderSubject(nint name) { - nint rendered = OpenSsl.X509_NAME_oneline(name, null, 0); + nint rendered = BoringSsl.X509_NAME_oneline(name, null, 0); if (rendered == 0) { return null; @@ -233,7 +233,7 @@ public ushort NegotiatedCipherSuiteId } finally { - OpenSsl.CRYPTO_free(rendered, 0, 0); + BoringSsl.OPENSSL_free(rendered); } } @@ -242,9 +242,9 @@ public ushort NegotiatedCipherSuiteId // The LAST CN, matching what every X.509 consumer does with a multi-CN subject: -1 starts // the scan, and each result seeds the next until there are no more. int index = -1; - for (int next = OpenSsl.X509_NAME_get_index_by_NID(name, OpenSsl.NID_commonName, -1); + for (int next = BoringSsl.X509_NAME_get_index_by_NID(name, BoringSsl.NID_commonName, -1); next >= 0; - next = OpenSsl.X509_NAME_get_index_by_NID(name, OpenSsl.NID_commonName, next)) + next = BoringSsl.X509_NAME_get_index_by_NID(name, BoringSsl.NID_commonName, next)) { index = next; } @@ -254,13 +254,13 @@ public ushort NegotiatedCipherSuiteId return null; } - nint entry = OpenSsl.X509_NAME_get_entry(name, index); + nint entry = BoringSsl.X509_NAME_get_entry(name, index); if (entry == 0) { return null; } - nint asn1 = OpenSsl.X509_NAME_ENTRY_get_data(entry); + nint asn1 = BoringSsl.X509_NAME_ENTRY_get_data(entry); if (asn1 == 0) { return null; @@ -269,7 +269,7 @@ public ushort NegotiatedCipherSuiteId // Decoded rather than read raw: a CN is PrintableString, UTF8String or BMPString depending // on the issuer, and only this normalises all three to UTF-8. nint utf8; - int length = OpenSsl.ASN1_STRING_to_UTF8(&utf8, asn1); + int length = BoringSsl.ASN1_STRING_to_UTF8(&utf8, asn1); if (length < 0) { return null; @@ -305,7 +305,7 @@ public ushort NegotiatedCipherSuiteId } finally { - OpenSsl.CRYPTO_free(utf8, 0, 0); + BoringSsl.OPENSSL_free(utf8); } } @@ -338,14 +338,14 @@ internal void MarkTxEnabled(int fd) public ReadOnlySpan Decrypt(byte* ciphertext, int length) { // With kTLS RX the kernel already decrypted this; recv handed us plaintext. Feeding it to - // OpenSSL would be feeding plaintext to a record parser. + // BoringSSL would be feeding plaintext to a record parser. if (KernelRx) { return new ReadOnlySpan(ciphertext, length); } _plainLen = 0; - OpenSsl.BIO_write(_rbio, ciphertext, length); + BoringSsl.BIO_write(_rbio, ciphertext, length); DrainPending(); return _plain.AsSpan(0, _plainLen); } @@ -361,7 +361,7 @@ public ReadOnlySpan Decrypt(byte* ciphertext, int length) /// /// The point of this over is one fewer copy. has to /// land the plaintext in _plain and hand back a span, so a caller feeding a pipe copies - /// it a second time; here OpenSSL writes into the pipe's own memory and there is no + /// it a second time; here BoringSSL writes into the pipe's own memory and there is no /// intermediate. Callers with nowhere better to put the bytes still want . /// public int DecryptInto(byte* ciphertext, int length, PipeWriter writer) @@ -373,7 +373,7 @@ public int DecryptInto(byte* ciphertext, int length, PipeWriter writer) return length; } - OpenSsl.BIO_write(_rbio, ciphertext, length); + BoringSsl.BIO_write(_rbio, ciphertext, length); int total = 0; while (true) @@ -386,7 +386,7 @@ public int DecryptInto(byte* ciphertext, int length, PipeWriter writer) fixed (byte* p = destination) { // Synchronous, so the pin lasts only for the call. - n = OpenSsl.Read(_ssl, p, destination.Length, out error); + n = BoringSsl.Read(_ssl, p, destination.Length, out error); } if (n > 0) @@ -420,7 +420,7 @@ internal int CountPendingRecords(out int missing) missing = 0; byte* buffer; - long available = OpenSsl.BIO_ctrl(_rbio, OpenSsl.BIO_CTRL_INFO, 0, &buffer); + long available = BoringSsl.BIO_ctrl(_rbio, BoringSsl.BIO_CTRL_INFO, 0, &buffer); if (available <= 0 || buffer == null) { return 0; @@ -450,12 +450,12 @@ internal int CountPendingRecords(out int missing) /// /// Stage into the connection's write slab, correctly for whichever /// backend this session ended up with: under kTLS TX it goes in as-is and the kernel makes the - /// records on send; otherwise OpenSSL encrypts here and the records go in. The one write call + /// records on send; otherwise BoringSSL encrypts here and the records go in. The one write call /// that is right in both modes - handlers should not branch on . /// /// - /// The OpenSSL branch is where the extra copy the kTLS write path avoids actually lives: - /// OpenSSL encrypts into its write BIO first, and BIO_read then moves the records into the + /// The BoringSSL branch is where the extra copy the kTLS write path avoids actually lives: + /// BoringSSL encrypts into its write BIO first, and BIO_read then moves the records into the /// slab. Two passes over the bytes instead of one - though BIO_read at least writes directly /// into the slab rather than through an intermediate of ours. /// @@ -474,11 +474,11 @@ public void Write(TcpConnection connection, ReadOnlySpan plaintext) return; } - // An OpenSSL session is not thread-safe, and the reactor decrypts with this one (#249). + // A BoringSSL session is not thread-safe, and the reactor decrypts with this one (#249). if (!connection.Reactor.OnReactorThread) { throw new InvalidOperationException( - "TlsSession.Write encrypts with OpenSSL, which the reactor uses for this session: call it on the reactor thread, or write through TlsConnectionDualPipe, whose flush is handed to the reactor."); + "TlsSession.Write encrypts with BoringSSL, which the reactor uses for this session: call it on the reactor thread, or write through TlsConnectionDualPipe, whose flush is handed to the reactor."); } WriteEncrypted(connection, plaintext); @@ -490,15 +490,15 @@ internal void WriteEncrypted(TcpConnection connection, ReadOnlySpan plaint { fixed (byte* p = plaintext) { - if (OpenSsl.Write(_ssl, p, plaintext.Length, out _) <= 0) + if (BoringSsl.Write(_ssl, p, plaintext.Length, out _) <= 0) { - throw new IOException($"TLS encrypt failed: {OpenSsl.LastError()}"); + throw new IOException($"TLS encrypt failed: {BoringSsl.LastError()}"); } } while (true) { - int pending = (int)OpenSsl.BIO_ctrl_pending(_wbio); + int pending = (int)BoringSsl.BIO_ctrl_pending(_wbio); if (pending <= 0) { return; @@ -510,7 +510,7 @@ internal void WriteEncrypted(TcpConnection connection, ReadOnlySpan plaint int n; fixed (byte* d = destination) { - n = OpenSsl.BIO_read(_wbio, d, chunk); + n = BoringSsl.BIO_read(_wbio, d, chunk); } if (n <= 0) @@ -538,16 +538,16 @@ private bool ShouldKeepReading(int err) { switch (err) { - case OpenSsl.SSL_ERROR_WANT_READ: - case OpenSsl.SSL_ERROR_WANT_WRITE: + case BoringSsl.SSL_ERROR_WANT_READ: + case BoringSsl.SSL_ERROR_WANT_WRITE: return false; // normal: the record is not complete yet - case OpenSsl.SSL_ERROR_ZERO_RETURN: + case BoringSsl.SSL_ERROR_ZERO_RETURN: Closed = true; // clean close_notify return false; default: - throw new IOException($"TLS decrypt failed (error {err}): {OpenSsl.LastError()}"); + throw new IOException($"TLS decrypt failed (error {err}): {BoringSsl.LastError()}"); } } @@ -567,7 +567,7 @@ internal void DrainPending() int n, error; fixed (byte* p = _plain) { - n = OpenSsl.Read(_ssl, p + _plainLen, _plain.Length - _plainLen, out error); + n = BoringSsl.Read(_ssl, p + _plainLen, _plain.Length - _plainLen, out error); } if (n > 0) @@ -592,10 +592,11 @@ public void Dispose() _disposed = true; // Clean server-side teardown: send close_notify so the peer can tell end-of-stream from a - // truncation, whichever backend owns the write side. Skip if the peer already closed, and - // where a raw send cannot go out cleanly: into a send the ring still has, or past the FIN - - // which a reactor teardown marks too, once the fd may be someone else's. - if (!Closed && _fd >= 0 && (_conn is null || (!_conn.FlushOutstanding && !_conn.FinSent))) + // truncation, whichever backend owns the write side - also when the peer closed first, which + // RFC 8446 6.1 asks of both sides. Skip where a raw send cannot go out cleanly: into a send + // the ring still has, or past the FIN - which a reactor teardown marks too, once the fd may + // be someone else's. + if (_fd >= 0 && (_conn is null || (!_conn.FlushOutstanding && !_conn.FinSent))) { if (_txEnabled) { @@ -603,7 +604,7 @@ public void Dispose() } else { - SendCloseNotifyOpenSsl(); + SendCloseNotifyBoringSsl(); } } @@ -611,7 +612,7 @@ public void Dispose() // into the kernel (and zeroed there) or were never needed. Do not leave them on the heap. ZeroSecrets(); - OpenSsl.SSL_free(_ssl); // frees both BIOs + BoringSsl.SSL_free(_ssl); // frees both BIOs if (_handle.IsAllocated) { _handle.Free(); @@ -633,25 +634,25 @@ internal void ZeroSecrets() } /// - /// Best-effort close_notify when OpenSSL owns the write side: SSL_shutdown produces the record + /// Best-effort close_notify when BoringSSL owns the write side: SSL_shutdown produces the record /// into the write BIO, and a plain send puts it on the wire. The mirror of /// - teardown-time, non-fatal, the peer may already be gone. /// - private void SendCloseNotifyOpenSsl() + private void SendCloseNotifyBoringSsl() { // Nothing to close down politely if the handshake never completed - there is no session // to end, and SSL_shutdown says so by failing and leaving "shutdown while in init" on the // shared queue, where it becomes the next connection's problem. - if (OpenSsl.SSL_in_init(_ssl) != 0) + if (BoringSsl.SSL_in_init(_ssl) != 0) { return; } - if (OpenSsl.SSL_shutdown(_ssl) < 0) + if (BoringSsl.SSL_shutdown(_ssl) < 0) { // Whatever it queued is ours to clean up; leaving it makes the next connection on this // reactor report an error it never caused. - OpenSsl.ERR_clear_error(); + BoringSsl.ERR_clear_error(); return; } @@ -667,7 +668,7 @@ private void SendCloseNotifyOpenSsl() byte* record = stackalloc byte[512]; while (true) { - int n = OpenSsl.BIO_read(_wbio, record, 512); + int n = BoringSsl.BIO_read(_wbio, record, 512); if (n <= 0) { break; diff --git a/src/ioxide/ioxide.csproj b/src/ioxide/ioxide.csproj index 0ebdda4d..1c65395e 100644 --- a/src/ioxide/ioxide.csproj +++ b/src/ioxide/ioxide.csproj @@ -10,7 +10,7 @@ ioxide 0.17.340 MDA2AV - A shared-nothing io_uring runtime for .NET: one ring per reactor thread, inline completions, zero native dependencies. The engine - reactor, connection, and the IRingHost client seam. Includes TLS termination: the OpenSSL handshake driven over the ring, then kernel TLS (kTLS) transmit offload, so handlers keep writing plaintext. TLS needs OpenSSL 3 and the Linux tls module; nothing else does, and neither is loaded unless you use it. + A shared-nothing io_uring runtime for .NET: one ring per reactor thread, inline completions, no system dependencies. The engine - reactor, connection, and the IRingHost client seam. Includes TLS termination on a bundled BoringSSL (libioxide_boringssl.so): the handshake driven over the ring, then optional kernel TLS (kTLS) offload, so handlers keep writing plaintext. kTLS needs the Linux tls module; nothing else does, and neither is loaded unless you use it. MIT https://mda2av.github.io/ioxide/ https://github.com/MDA2AV/ioxide @@ -21,6 +21,12 @@ + +