Bound chunked decoding: stream chunk data, cap the trailer section

README.md

@@ -145,6 +145,10 @@ var r = stream.TryReadChunk(input, out int consumed, out int dataOffset, out int
 // r: Chunk | Completed | NeedMoreData | Error
 ```
 
+Payload is handed out as it arrives (a `Chunk` result may be part of a chunk), so a peer's
+declared chunk size never decides how much you buffer; the trailer section is capped by
+`ParserLimits.MaxTotalTrailerBytes` (16 KiB by default; `new ChunkedBodyStream(limits)`).
+
 ## Benchmarks
 
 `linux-x64`, best-of-5, ns/op (lower is better). `Glyph11.Native` / `Glyph11.Pico` are the

SECURITY.md

@@ -121,6 +121,7 @@ All limits are configurable via `ParserLimits`:
 | `MaxQueryParameterCount` | 128 | Query parameter flooding |
 | `MaxMethodLength` | 16 | Oversized method strings |
 | `MaxTotalHeaderBytes` | 1048576 | Total header section DoS |
+| `MaxTotalTrailerBytes` | 16384 | Endless chunked trailer section (applies to `ChunkedBodyStream` / `ChunkedBodyValidator`) |
 
 ---
 

site/glyph11.html

@@ -145,7 +145,8 @@ <h2 id="limits">Limits &amp; configuration</h2>
 };
 
 // Available knobs (all int): MaxHeaderCount, MaxHeaderNameLength, MaxHeaderValueLength,
-// MaxUrlLength, MaxQueryParameterCount, MaxMethodLength, MaxTotalHeaderBytes.</code></pre>
+// MaxUrlLength, MaxQueryParameterCount, MaxMethodLength, MaxTotalHeaderBytes,
+// MaxTotalTrailerBytes (chunked trailer section; pass limits to new ChunkedBodyStream(limits)).</code></pre>
 
     <h2 id="pipereader">Reading from a PipeReader</h2>
     <p>The real-world loop: read, try to parse, and if the buffer doesn't yet hold a full header block,
@@ -186,7 +187,9 @@ <h2 id="pipereader">Reading from a PipeReader</h2>
     <h2 id="chunked">Chunked bodies</h2>
     <p>After the headers, a <code>Transfer-Encoding: chunked</code> body is decoded with
     <code>ChunkedBodyStream</code> — a small struct state machine you feed bytes to. It tells you where each
-    decoded chunk's payload sits, without copying.</p>
+    decoded chunk's payload sits, without copying. Payload is handed out as it arrives, so a peer's
+    declared chunk size never decides how much you buffer, and the trailer section is capped by
+    <code>ParserLimits.MaxTotalTrailerBytes</code>.</p>
 <pre><code class="language-csharp">using Glyph11.Parser;
 
 var decoder = new ChunkedBodyStream();
@@ -204,13 +207,13 @@ <h2 id="chunked">Chunked bodies</h2>
 
     switch (r)
     {
-        case ChunkResult.Chunk:        // a payload chunk is ready
+        case ChunkResult.Chunk:        // payload bytes are ready: a whole chunk or the part that arrived
             body.Write(input.AsSpan(offset + dataOffset, dataLength));
             offset += consumed;
             continue;
         case ChunkResult.Completed:    // terminal 0-length chunk seen → body done
             break;
-        case ChunkResult.NeedMoreData: // partial chunk — read more bytes, then call again
+        case ChunkResult.NeedMoreData: // incomplete framing — read more bytes, then call again
             break;
         case ChunkResult.Error:        // malformed chunk framing → reject the request
             throw new InvalidOperationException("bad chunked encoding");

src/Glyph11/Parser/ChunkedBodyStream.cs

@@ -10,7 +10,10 @@ public enum ChunkResult : byte
     /// <summary>Incomplete data — wait for more bytes from the pipe.</summary>
     NeedMoreData,
 
-    /// <summary>A complete chunk's decoded payload is available.</summary>
+    /// <summary>
+    /// Decoded payload bytes are available: a whole chunk, or the part of one that has arrived so
+    /// far (never empty). Chunk boundaries carry no meaning, so append the bytes and call again.
+    /// </summary>
     Chunk,
 
     /// <summary>Terminal chunk (and any trailers) consumed — body is done.</summary>
@@ -21,6 +24,11 @@ public enum ChunkResult : byte
 /// Stateful streaming parser for chunked transfer-encoding (RFC 9112 §7.1).
 /// Validates chunks incrementally so the caller can advance a PipeReader
 /// after each call without re-parsing from byte 0.
+/// <para>
+/// Chunk data is handed out as it arrives rather than once the whole chunk is buffered, so the
+/// caller's buffer is bounded by its read window, not by the size a peer declares. The trailer
+/// section is capped by <see cref="ParserLimits.MaxTotalTrailerBytes"/>.
+/// </para>
 /// </summary>
 public struct ChunkedBodyStream
 {
@@ -37,12 +45,30 @@ private enum Phase : byte
         Complete,
     }
 
+    // The limit fields sit in the padding after the byte-sized phase, keeping the state at the
+    // 32 bytes it had: every call copies it (the NeedMoreData rollback snapshot).
     private Phase _phase;          // starts at ChunkSize
+    private bool _hasLimits;       // false for new ChunkedBodyStream(): ParserLimits.Default applies
+    private int _maxTrailerBytes;  // trailer section cap, when constructed with limits
     private long _chunkSize;       // parsed size for current chunk
     private long _remaining;       // bytes left in ChunkData phase
     private int _digitCount;       // hex digits seen in ChunkSize
     private int _extensionBytes;   // extension bytes (4096 limit)
 
+    /// <summary>
+    /// Creates a decoder that enforces <paramref name="limits"/> (today:
+    /// <see cref="ParserLimits.MaxTotalTrailerBytes"/>). <c>new ChunkedBodyStream()</c> applies
+    /// <see cref="ParserLimits.Default"/>.
+    /// </summary>
+    public ChunkedBodyStream(in ParserLimits limits) : this()
+    {
+        _maxTrailerBytes = limits.MaxTotalTrailerBytes;
+        _hasLimits = true;
+    }
+
+    private readonly int MaxTrailerBytes
+        => _hasLimits ? _maxTrailerBytes : ParserLimits.Default.MaxTotalTrailerBytes;
+
     /// <summary>
     /// Reads the next chunk from <paramref name="input"/>.
     /// Call in a loop after each PipeReader.ReadAsync.
@@ -52,11 +78,16 @@ private enum Phase : byte
     /// <param name="dataOffset">Offset within <paramref name="input"/> where chunk payload starts (valid when result is <see cref="ChunkResult.Chunk"/>).</param>
     /// <param name="dataLength">Length of chunk payload bytes (valid when result is <see cref="ChunkResult.Chunk"/>).</param>
     /// <returns>
-    /// <see cref="ChunkResult.Chunk"/>: payload at input[dataOffset..dataOffset+dataLength], advance by bytesConsumed.<br/>
+    /// <see cref="ChunkResult.Chunk"/>: payload at input[dataOffset..dataOffset+dataLength] — a whole chunk or the
+    /// part of one that has arrived, never empty — advance by bytesConsumed.<br/>
     /// <see cref="ChunkResult.Completed"/>: terminal chunk consumed, body is done.<br/>
-    /// <see cref="ChunkResult.NeedMoreData"/>: incomplete, wait for more from pipe.
+    /// <see cref="ChunkResult.NeedMoreData"/>: incomplete, nothing consumed — wait for more from pipe and
+    /// re-present the same bytes.
     /// </returns>
-    /// <exception cref="HttpParseException">Thrown on malformed chunked encoding.</exception>
+    /// <exception cref="HttpParseException">
+    /// Thrown on malformed chunked encoding, or (status 431) when the trailer section exceeds
+    /// <see cref="ParserLimits.MaxTotalTrailerBytes"/>.
+    /// </exception>
     public ChunkResult TryReadChunk(
         ReadOnlySpan<byte> input,
         out int bytesConsumed,
@@ -67,6 +98,7 @@ public ChunkResult TryReadChunk(
         dataOffset = 0;
         dataLength = 0;
         int pos = 0;
+        int trailerStart = 0;   // where the trailer section begins; it is always parsed within one call
 
         // Snapshot state so we can roll back on NeedMoreData.
         // The caller re-presents the same bytes, so partial state must not persist.
@@ -169,6 +201,7 @@ public ChunkResult TryReadChunk(
                     {
                         // Terminal chunk — move to trailers
                         _phase = Phase.Trailers;
+                        trailerStart = pos;
                         continue;
                     }
 
@@ -180,16 +213,22 @@ public ChunkResult TryReadChunk(
 
                 case Phase.ChunkData:
                 {
-                    int available = input.Length - pos;
+                    int available = input.Length - pos;   // >= 1: the loop only runs while input remains
                     if (available < _remaining)
                     {
-                        this = snapshot;
-                        return ChunkResult.NeedMoreData;
+                        // Hand out what has arrived instead of waiting for the whole chunk: the
+                        // caller's buffer is then bounded by its read window, not by whatever size
+                        // the peer declared. The rest of the chunk follows on later calls.
+                        dataOffset = pos;
+                        dataLength = available;
+                        _remaining -= available;
+                        bytesConsumed = input.Length;
+                        return ChunkResult.Chunk;
                     }
 
-                    // All chunk data is present
+                    // The rest of the chunk's data is present
                     dataOffset = pos;
-                    dataLength = (int)_chunkSize;
+                    dataLength = (int)_remaining;
                     pos += (int)_remaining;
                     _remaining = 0;
                     _phase = Phase.DataCrlf;
@@ -200,6 +239,14 @@ public ChunkResult TryReadChunk(
                 {
                     if (pos + 1 >= input.Length)
                     {
+                        // The CRLF after the data has not arrived. Data read by this call is
+                        // handed out anyway — the CRLF is checked on the next call.
+                        if (dataLength > 0)
+                        {
+                            bytesConsumed = pos;
+                            return ChunkResult.Chunk;
+                        }
+
                         this = snapshot;
                         return ChunkResult.NeedMoreData;
                     }
@@ -218,8 +265,14 @@ public ChunkResult TryReadChunk(
                     _digitCount = 0;
                     _extensionBytes = 0;
 
-                    bytesConsumed = pos;
-                    return ChunkResult.Chunk;
+                    if (dataLength > 0)
+                    {
+                        bytesConsumed = pos;
+                        return ChunkResult.Chunk;
+                    }
+
+                    // The data was handed out by an earlier call: go on to the next chunk.
+                    continue;
                 }
 
                 case Phase.Trailers:
@@ -254,6 +307,16 @@ public ChunkResult TryReadChunk(
                         pos++;
                     }
 
+                    // The trailer section is re-presented from its start until it completes, so
+                    // cap the bytes seen so far — finished or not — or an endless section is
+                    // buffered forever. A CR that ends the input may be half of the final CRLF,
+                    // so it is not counted yet.
+                    int seen = pos - trailerStart;
+                    if (pos >= input.Length && input[pos - 1] == (byte)'\r')
+                        seen--;
+                    if (seen > MaxTrailerBytes)
+                        throw new HttpParseException("Chunked trailer section exceeds limit.", statusCode: 431);
+
                     if (pos >= input.Length)
                     {
                         this = snapshot;
@@ -271,7 +334,14 @@ public ChunkResult TryReadChunk(
             }
         }
 
-        // Ran out of input
+        // Ran out of input. A chunk whose data is all here but whose CRLF is not is still
+        // handed out (the CRLF is checked on the next call); anything else waits.
+        if (dataLength > 0)
+        {
+            bytesConsumed = pos;
+            return ChunkResult.Chunk;
+        }
+
         this = snapshot;
         return ChunkResult.NeedMoreData;
     }
@@ -280,9 +350,14 @@ public ChunkResult TryReadChunk(
     /// Multi-segment overload of <see cref="TryReadChunk(ReadOnlySpan{byte}, out int, out int, out int)"/>.
     /// Reads the next chunk directly from a <see cref="ReadOnlySequence{T}"/> — <b>no linearization</b>.
     /// On <see cref="ChunkResult.Chunk"/>, <paramref name="chunkData"/> is the decoded payload as a
-    /// (possibly multi-segment) slice of <paramref name="input"/>; copy it out before advancing the
-    /// reader past <paramref name="bytesConsumed"/>.
+    /// (possibly multi-segment) slice of <paramref name="input"/> — a whole chunk or the part of one
+    /// that has arrived, never empty; copy it out before advancing the reader past
+    /// <paramref name="bytesConsumed"/>.
     /// </summary>
+    /// <exception cref="HttpParseException">
+    /// Thrown on malformed chunked encoding, or (status 431) when the trailer section exceeds
+    /// <see cref="ParserLimits.MaxTotalTrailerBytes"/>.
+    /// </exception>
     public ChunkResult TryReadChunk(
         in ReadOnlySequence<byte> input,
         out long bytesConsumed,
@@ -294,6 +369,8 @@ public ChunkResult TryReadChunk(
         var reader = new SequenceReader<byte>(input);
         var snapshot = this;                 // roll back on NeedMoreData — caller re-presents the bytes
         ReadOnlySequence<byte> pending = default;
+        bool hasPending = false;          // cheaper to test than pending.IsEmpty, once per chunk
+        long trailerStart = 0;               // the trailer section is always parsed within one call
 
         while (!reader.End)
         {
@@ -374,6 +451,7 @@ public ChunkResult TryReadChunk(
                     if (_chunkSize == 0)
                     {
                         _phase = Phase.Trailers;
+                        trailerStart = reader.Consumed;
                         continue;
                     }
                     _remaining = _chunkSize;
@@ -385,10 +463,17 @@ public ChunkResult TryReadChunk(
                 {
                     if (reader.Remaining < _remaining)
                     {
-                        this = snapshot;
-                        return ChunkResult.NeedMoreData;
+                        // Hand out what has arrived (at least one byte: the loop runs only while
+                        // input remains) instead of buffering the whole declared chunk.
+                        long available = reader.Remaining;
+                        chunkData = reader.UnreadSequence;
+                        reader.Advance(available);
+                        _remaining -= available;
+                        bytesConsumed = reader.Consumed;
+                        return ChunkResult.Chunk;
                     }
                     pending = reader.UnreadSequence.Slice(0, _remaining);
+                    hasPending = true;
                     reader.Advance(_remaining);
                     _remaining = 0;
                     _phase = Phase.DataCrlf;
@@ -399,6 +484,15 @@ public ChunkResult TryReadChunk(
                 {
                     if (reader.Remaining < 2)
                     {
+                        // The CRLF after the data has not arrived: hand out data read by this call
+                        // anyway — the CRLF is checked on the next call.
+                        if (hasPending)
+                        {
+                            bytesConsumed = reader.Consumed;
+                            chunkData = pending;
+                            return ChunkResult.Chunk;
+                        }
+
                         this = snapshot;
                         return ChunkResult.NeedMoreData;
                     }
@@ -413,9 +507,15 @@ public ChunkResult TryReadChunk(
                     _digitCount = 0;
                     _extensionBytes = 0;
 
-                    bytesConsumed = reader.Consumed;
-                    chunkData = pending;
-                    return ChunkResult.Chunk;
+                    if (hasPending)
+                    {
+                        bytesConsumed = reader.Consumed;
+                        chunkData = pending;
+                        return ChunkResult.Chunk;
+                    }
+
+                    // The data was handed out by an earlier call: go on to the next chunk.
+                    continue;
                 }
 
                 case Phase.Trailers:
@@ -435,11 +535,23 @@ public ChunkResult TryReadChunk(
                     // a trailer line, up to CRLF; a bare LF inside it is invalid
                     if (!reader.TryReadTo(out ReadOnlySequence<byte> line, ParserConstants.Crlf, advancePastDelimiter: true))
                     {
+                        // Re-presented from the start of the section until it completes: cap the
+                        // bytes seen so far, or an endless trailer line is buffered forever. A CR
+                        // that ends the input may be half of the final CRLF: not counted yet.
+                        var unread = reader.UnreadSequence;
+                        long seen = reader.Consumed - trailerStart + unread.Length;
+                        if (unread.Slice(unread.Length - 1).FirstSpan[0] == (byte)'\r')
+                            seen--;
+                        if (seen > MaxTrailerBytes)
+                            throw new HttpParseException("Chunked trailer section exceeds limit.", statusCode: 431);
+
                         this = snapshot;
                         return ChunkResult.NeedMoreData;
                     }
                     if (line.PositionOf((byte)'\n') is not null)
                         throw new HttpParseException("Bare LF in chunked trailer.");
+                    if (reader.Consumed - trailerStart > MaxTrailerBytes)
+                        throw new HttpParseException("Chunked trailer section exceeds limit.", statusCode: 431);
                     continue;
                 }
 
@@ -451,6 +563,15 @@ public ChunkResult TryReadChunk(
             }
         }
 
+        // Ran out of input. A chunk whose data is all here but whose CRLF is not is still
+        // handed out (the CRLF is checked on the next call); anything else waits.
+        if (hasPending)
+        {
+            bytesConsumed = reader.Consumed;
+            chunkData = pending;
+            return ChunkResult.Chunk;
+        }
+
         this = snapshot;
         return ChunkResult.NeedMoreData;
     }

src/Glyph11/Parser/ParserLimits.cs

@@ -36,6 +36,14 @@ public readonly record struct ParserLimits
     /// <summary>Maximum total size of the header block including request line and terminators (default 1048576).</summary>
     public int MaxTotalHeaderBytes { get; init; }
 
+    /// <summary>
+    /// Maximum total size of a chunked body's trailer section in bytes: every trailer field line
+    /// after the last chunk, including its CRLF (default 16384). Enforced by
+    /// <see cref="ChunkedBodyStream"/> and <see cref="Validation.ChunkedBodyValidator"/>; exceeding
+    /// it is a 431, like an oversized header block.
+    /// </summary>
+    public int MaxTotalTrailerBytes { get; init; }
+
     /// <summary>Returns a <see cref="ParserLimits"/> with safe production defaults.</summary>
     public static ParserLimits Default => new()
     {
@@ -46,6 +54,7 @@ public readonly record struct ParserLimits
         MaxQueryParameterCount = 128,
         MaxMethodLength = 16,
         MaxReasonPhraseLength = 512,
-        MaxTotalHeaderBytes = 1_048_576
+        MaxTotalHeaderBytes = 1_048_576,
+        MaxTotalTrailerBytes = 16_384
     };
 }