test(core): re-vendor bin16 width-boundary vectors (LAB-868)

tests/dual_decode.rs

@@ -20,10 +20,9 @@
 //! attached to the decision memo on that issue); throughput evidence lives in
 //! the LAB-510 harness (`benches/hot_path.rs`, 64 MiB incompressible group).
 //!
-//! Width boundaries: the fixture's `*_bin` twins all fit in a bin8 header
-//! (`0xc4`, ≤ 255 B). The `width_boundary_*` tests cover the bin16 (`0xc5`)
-//! and bin32 (`0xc6`) headers with real LZ4 output from the real writer, per
-//! the MUST recorded in the decision record.
+//! Width boundaries: the fixture pins both bin8 (`0xc4`) and bin16 (`0xc5`)
+//! twins. The `width_boundary_*` tests additionally cover the bin32 (`0xc6`)
+//! header with real LZ4 output from the real writer, per the decision record.
 
 #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
 
@@ -123,27 +122,29 @@ fn dual_decode_matrix_against_legacy_vectors() {
             v.name
         );
         assert_eq!(
-            // 0xc4 = msgpack bin8. Every pinned twin's compressed_data is small
-            // enough to be bin8; a wider marker here is a width-selection
-            // regression that would break cross-SDK byte-identity.
             new_wire[1],
-            0xc4,
-            "[{}] compressed_data not bin8-encoded: 0x{:02x}",
+            match new_from_old.compressed_data.len() {
+                0..=255 => 0xc4,
+                256..=65_535 => 0xc5,
+                _ => 0xc6,
+            },
+            "[{}] compressed_data does not use the shortest bin marker: 0x{:02x}",
             v.name,
             new_wire[1]
         );
         // Documented micro-regression: bin costs at most +1 B, and only on the
         // smallest envelopes. A <=15 B compressed_data goes from a 1 B fixarray
         // header to a 2 B bin8 header (+1); any array >=16 B already carried a
-        // 3 B array16 header, so bin8's 2 B header shrinks it. Measured worst
-        // case on the pinned set is exactly +1 (`empty`, 25 -> 26 B); every
-        // other vector ties or shrinks. This +1 B ceiling is the micro-
+        // 3 B array16 header, so bin8's 2 B header shrinks it. The bin16 tier
+        // uses the same three-byte header as array16, so it cannot grow either.
+        // Measured worst case on the pinned set is exactly +1 (`empty`, 25 ->
+        // 26 B); every other vector ties or shrinks. This +1 B ceiling is the micro-
         // regression contract accepted by the writer flip (LAB-764/LAB-866),
         // so keep it tight — a +2/+3 bloat regression must not slip through
         // green.
         assert!(
             new_wire.len() <= old_wire.len() + 1,
-            "[{}] new wire exceeds the +1 B bin8-header ceiling",
+            "[{}] new wire exceeds the +1 B bin-header ceiling",
             v.name
         );
 
@@ -212,9 +213,8 @@ fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) {
     assert_all_readers_decode(name, &wire, &payload);
 }
 
-/// bin8/bin16 boundary: the fixture twins never leave bin8 territory (all
-/// compressed_data <= 255 B), so sweep real LZ4 outputs across the 255/256 B
-/// boundary and prove both header widths decode through every reader shape.
+/// bin8/bin16 boundary: the fixture pins one bin16 twin, and this sweep covers
+/// real LZ4 outputs across the 255/256 B boundary through every reader shape.
 /// LZ4 adds a few bytes of literal-run overhead on incompressible input, so
 /// the sweep brackets the boundary regardless of the exact overhead.
 #[test]

tests/vectors/wire-format.json

@@ -2,7 +2,7 @@
   "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input",
   "compression": "LZ4 block format (NOT frame)",
   "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.",
-  "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)",
+  "generator": "six original legacy vectors: cachekit-core v0.2.0; width_boundary_bin16 legacy vector: cachekit-core v0.5.0 real writer; *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)",
   "limits": {
     "max_compressed_size": 536870912,
     "max_compression_ratio": 1000,
@@ -63,6 +63,15 @@
       "input_size": 1,
       "name": "single_byte"
     },
+    {
+      "description": "300-byte deterministic xorshift64* incompressible payload (seed 0x9e3779b97f4a7c15, multiplier 0x2545f4914f6cdd1d; little-endian u64 output truncated to 300 bytes; real writer bin16 width boundary)",
+      "envelope_hex": "94dc012fccf0ccff1ecc9925cc9acc8ccca2ccd103745ecccbcc8eccab39ccf370ccdd1ecca10e523348cc977d244dcc80cca41841cc8f54ccbc38cc97142004ccd6ccfbccb5ccf12e5acc91cc9c5528733454ccc2ccc652cce2ccfbcc82cccccce2ccebccff51ccd6ccadccbfcca8cc8d011a0627cca416cc9fcca54a5ecc9a66ccdf61cc82cce87571ccbd25ccf7571a68cc8e1221ccd257cca97230cc906acca41ccca1cce6ccf76dcca7cce1ccfc0ecc9ecc83cc96324bcc8720ccef20cce3ccd6ccd77a713bcc951eccd8ccfa38ccfc6bccf8ccf6ccca4177102dcca9211f26ccf3ccba2c20cca2ccf0ccd1ccaaccd5095b302838192b5922226acca7027c3c13cce03268cc90ccd1ccaaccecccf2ccce1c442445cc916a26cc8c72cce4cced381a3b69cc9accc8ccb0cca9cc935614ccd4ccdccc83ccd6ccde44ccb1462747ccf7ccf8711fcc9a27234fcca6cc9c3c5f5ccc8319ccc6ccb239cc9d2dccde6a290831cc86cca46b3accb03b1ecca8ccdbccb4cc98ccdecce5ccc1cc97ccbe39ccfdccb8ccf1cccb324b542bcca01a6131ccbcccd67540273accc90fccd9164941ccd42669404a5dcc85ccefccc6cc9f2f2fccde1c78ccd2ccef404accf205ccefccd96f98cc8ecccfccde50524bcc886dcd012ca76d73677061636b",
+      "envelope_size": 478,
+      "format": "msgpack",
+      "input_hex": "99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f",
+      "input_size": 300,
+      "name": "width_boundary_bin16"
+    },
     {
       "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)",
       "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b",
@@ -128,7 +137,18 @@
       "name": "single_byte_bin",
       "envelope_encoding": "bin",
       "derived_from": "single_byte"
+    },
+    {
+      "description": "bin-encoded twin of 'width_boundary_bin16': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)",
+      "envelope_hex": "94c5012ff0ff1e99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f98cc8ecccfccde50524bcc886dcd012ca76d73677061636b",
+      "envelope_size": 331,
+      "format": "msgpack",
+      "input_hex": "99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f",
+      "input_size": 300,
+      "name": "width_boundary_bin16_bin",
+      "envelope_encoding": "bin",
+      "derived_from": "width_boundary_bin16"
     }
   ],
-  "version": "1.1.0"
+  "version": "1.1.1"
 }

tests/wire_format_vectors.rs

@@ -17,12 +17,14 @@
 //!
 //! Fixture provenance: vendored from
 //! <https://github.com/cachekit-io/protocol> `test-vectors/wire-format.json`
-//! at commit `6863495d3565ccd9f28827baa1351782575e770d`, integrity-pinned by
+//! at commit `5be35d5240817617275e3983de486a5826a587af`, integrity-pinned by
 //! sha256 below. To update: copy the file from a newer protocol ref, update
 //! `FIXTURE_SHA256` and this comment's commit hash together.
 
 #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))]
 
+use std::collections::{HashMap, HashSet};
+
 use cachekit_core::{ByteStorage, StorageEnvelope};
 use sha2::{Digest, Sha256};
 
@@ -31,7 +33,7 @@ use sha2::{Digest, Sha256};
 const FIXTURE: &str = include_str!("vectors/wire-format.json");
 
 /// sha256 of the vendored fixture — must match the protocol repo's copy.
-const FIXTURE_SHA256: &str = "d6184a945d8cb22491b5c937414fe4d01e682d084ccec9dae9526d0ffba650cb";
+const FIXTURE_SHA256: &str = "b902db88fb9b2c4a2d0def7266f8199a858fcb921262c1eaf2c2c03412b5b56a";
 
 #[derive(serde::Deserialize)]
 struct WireFormatFixture {
@@ -87,17 +89,56 @@ fn fixture_integrity_pinned_sha256() {
 #[test]
 fn fixture_is_current_version_with_vectors() {
     let fixture = load_fixture();
-    assert_eq!(fixture.version, "1.1.0");
-    let legacy = fixture.vectors.iter().filter(|v| v.is_legacy()).count();
-    let bin = fixture.vectors.len() - legacy;
+    assert_eq!(fixture.version, "1.1.1");
+    let legacy: HashSet<&str> = fixture
+        .vectors
+        .iter()
+        .filter(|v| v.is_legacy())
+        .map(|v| v.name.as_str())
+        .collect();
+    let bin = fixture.vectors.len() - legacy.len();
     assert!(
-        legacy >= 6,
-        "legacy vectors are retained forever as legacy-read proof; expected at least the original 6, got {legacy}"
+        legacy.len() >= 6,
+        "legacy vectors are retained forever as legacy-read proof; expected at least the original 6, got {}",
+        legacy.len()
     );
     assert!(
         bin >= 6,
         "protocol 1.1 pins a *_bin twin per legacy vector; expected at least 6, got {bin}"
     );
+    // Equal totals alone would admit duplicate twins or orphaned parents, so
+    // key every *_bin vector by `derived_from` and require exactly one twin
+    // per legacy vector.
+    let mut twins: HashMap<&str, usize> = HashMap::new();
+    for vector in fixture.vectors.iter().filter(|v| !v.is_legacy()) {
+        let parent = vector.derived_from.as_deref().unwrap_or_else(|| {
+            panic!(
+                "[{}] bin vector must name its legacy parent in derived_from",
+                vector.name
+            )
+        });
+        assert!(
+            legacy.contains(parent),
+            "[{}] derived_from {parent:?} names no legacy vector",
+            vector.name
+        );
+        *twins.entry(parent).or_insert(0) += 1;
+    }
+    for name in &legacy {
+        assert_eq!(
+            twins.get(name).copied().unwrap_or(0),
+            1,
+            "legacy vector {name:?} must have exactly one *_bin twin"
+        );
+    }
+    // LAB-868's deliverable, pinned by name: the bin16 width-boundary pair
+    // must survive any future re-vendor.
+    for required in ["width_boundary_bin16", "width_boundary_bin16_bin"] {
+        assert!(
+            fixture.vectors.iter().any(|v| v.name == required),
+            "fixture must retain the {required:?} vector"
+        );
+    }
 }
 
 #[test]
@@ -259,24 +300,26 @@ fn bin_twins_are_bin_encoded_and_field_identical_to_legacy() {
             });
 
         let twin_bytes = hex::decode(&twin.envelope_hex).expect("envelope_hex must decode");
+        let twin_env: StorageEnvelope = rmp_serde::from_slice(&twin_bytes)
+            .unwrap_or_else(|e| panic!("[{}] twin must deserialize: {e}", twin.name));
         assert_eq!(
             twin_bytes[0], 0x94,
             "[{}] outer fixarray(4) marker",
             twin.name
         );
         assert_eq!(
-            // 0xc4 = msgpack bin8. Every pinned twin is small enough to be
-            // bin8; a wider marker here is a width-selection regression.
             twin_bytes[1],
-            0xc4,
-            "[{}] compressed_data is not bin8-encoded: marker 0x{:02x}",
+            match twin_env.compressed_data.len() {
+                0..=255 => 0xc4,
+                256..=65_535 => 0xc5,
+                _ => 0xc6,
+            },
+            "[{}] compressed_data does not use the shortest bin marker: 0x{:02x}",
             twin.name,
             twin_bytes[1]
         );
 
         let parent_bytes = hex::decode(&parent.envelope_hex).expect("envelope_hex must decode");
-        let twin_env: StorageEnvelope = rmp_serde::from_slice(&twin_bytes)
-            .unwrap_or_else(|e| panic!("[{}] twin must deserialize: {e}", twin.name));
         let parent_env: StorageEnvelope = rmp_serde::from_slice(&parent_bytes)
             .unwrap_or_else(|e| panic!("[{parent_name}] parent must deserialize: {e}"));
         assert_eq!(