diff --git a/tests/dual_decode.rs b/tests/dual_decode.rs index 598aa8e..91dfe7c 100644 --- a/tests/dual_decode.rs +++ b/tests/dual_decode.rs @@ -20,10 +20,9 @@ //! attached to the decision memo on that issue); throughput evidence lives in //! the LAB-510 harness (`benches/hot_path.rs`, 64 MiB incompressible group). //! -//! Width boundaries: the fixture's `*_bin` twins all fit in a bin8 header -//! (`0xc4`, ≤ 255 B). The `width_boundary_*` tests cover the bin16 (`0xc5`) -//! and bin32 (`0xc6`) headers with real LZ4 output from the real writer, per -//! the MUST recorded in the decision record. +//! Width boundaries: the fixture pins both bin8 (`0xc4`) and bin16 (`0xc5`) +//! twins. The `width_boundary_*` tests additionally cover the bin32 (`0xc6`) +//! header with real LZ4 output from the real writer, per the decision record. #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] @@ -123,27 +122,29 @@ fn dual_decode_matrix_against_legacy_vectors() { v.name ); assert_eq!( - // 0xc4 = msgpack bin8. Every pinned twin's compressed_data is small - // enough to be bin8; a wider marker here is a width-selection - // regression that would break cross-SDK byte-identity. new_wire[1], - 0xc4, - "[{}] compressed_data not bin8-encoded: 0x{:02x}", + match new_from_old.compressed_data.len() { + 0..=255 => 0xc4, + 256..=65_535 => 0xc5, + _ => 0xc6, + }, + "[{}] compressed_data does not use the shortest bin marker: 0x{:02x}", v.name, new_wire[1] ); // Documented micro-regression: bin costs at most +1 B, and only on the // smallest envelopes. A <=15 B compressed_data goes from a 1 B fixarray // header to a 2 B bin8 header (+1); any array >=16 B already carried a - // 3 B array16 header, so bin8's 2 B header shrinks it. Measured worst - // case on the pinned set is exactly +1 (`empty`, 25 -> 26 B); every - // other vector ties or shrinks. This +1 B ceiling is the micro- + // 3 B array16 header, so bin8's 2 B header shrinks it. The bin16 tier + // uses the same three-byte header as array16, so it cannot grow either. + // Measured worst case on the pinned set is exactly +1 (`empty`, 25 -> + // 26 B); every other vector ties or shrinks. This +1 B ceiling is the micro- // regression contract accepted by the writer flip (LAB-764/LAB-866), // so keep it tight — a +2/+3 bloat regression must not slip through // green. assert!( new_wire.len() <= old_wire.len() + 1, - "[{}] new wire exceeds the +1 B bin8-header ceiling", + "[{}] new wire exceeds the +1 B bin-header ceiling", v.name ); @@ -212,9 +213,8 @@ fn assert_width_tier(name: &str, payload_len: usize, expected_marker: u8) { assert_all_readers_decode(name, &wire, &payload); } -/// bin8/bin16 boundary: the fixture twins never leave bin8 territory (all -/// compressed_data <= 255 B), so sweep real LZ4 outputs across the 255/256 B -/// boundary and prove both header widths decode through every reader shape. +/// bin8/bin16 boundary: the fixture pins one bin16 twin, and this sweep covers +/// real LZ4 outputs across the 255/256 B boundary through every reader shape. /// LZ4 adds a few bytes of literal-run overhead on incompressible input, so /// the sweep brackets the boundary regardless of the exact overhead. #[test] diff --git a/tests/vectors/wire-format.json b/tests/vectors/wire-format.json index 9be2ba0..fdc3cc5 100644 --- a/tests/vectors/wire-format.json +++ b/tests/vectors/wire-format.json @@ -2,7 +2,7 @@ "checksum": "xxHash3-64, 8 bytes big-endian, computed on uncompressed input", "compression": "LZ4 block format (NOT frame)", "envelope_format": "MessagePack positional array (rmp_serde::to_vec): [compressed_data, checksum, original_size, format]. Vectors without an 'envelope_encoding' field use the legacy array-of-integers encoding for compressed_data and are retained forever as legacy-read proof; their '*_bin' twins use msgpack bin (canonical for protocol 1.1+ writers). checksum always encodes as an array of 8 integers. Normative rules: spec/wire-format.md, 'Byte Layout' / 'Encoding compatibility'.", - "generator": "legacy vectors: cachekit-core v0.2.0 (unchanged since); *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", + "generator": "six original legacy vectors: cachekit-core v0.2.0; width_boundary_bin16 legacy vector: cachekit-core v0.5.0 real writer; *_bin vectors: tools/wire-format-reference.py generate - deterministic re-encode of the legacy fields, byte-verified against rmp-serde 1.3.1 + serde_bytes 0.11.19 output (LAB-783)", "limits": { "max_compressed_size": 536870912, "max_compression_ratio": 1000, @@ -63,6 +63,15 @@ "input_size": 1, "name": "single_byte" }, + { + "description": "300-byte deterministic xorshift64* incompressible payload (seed 0x9e3779b97f4a7c15, multiplier 0x2545f4914f6cdd1d; little-endian u64 output truncated to 300 bytes; real writer bin16 width boundary)", + "envelope_hex": "94dc012fccf0ccff1ecc9925cc9acc8ccca2ccd103745ecccbcc8eccab39ccf370ccdd1ecca10e523348cc977d244dcc80cca41841cc8f54ccbc38cc97142004ccd6ccfbccb5ccf12e5acc91cc9c5528733454ccc2ccc652cce2ccfbcc82cccccce2ccebccff51ccd6ccadccbfcca8cc8d011a0627cca416cc9fcca54a5ecc9a66ccdf61cc82cce87571ccbd25ccf7571a68cc8e1221ccd257cca97230cc906acca41ccca1cce6ccf76dcca7cce1ccfc0ecc9ecc83cc96324bcc8720ccef20cce3ccd6ccd77a713bcc951eccd8ccfa38ccfc6bccf8ccf6ccca4177102dcca9211f26ccf3ccba2c20cca2ccf0ccd1ccaaccd5095b302838192b5922226acca7027c3c13cce03268cc90ccd1ccaaccecccf2ccce1c442445cc916a26cc8c72cce4cced381a3b69cc9accc8ccb0cca9cc935614ccd4ccdccc83ccd6ccde44ccb1462747ccf7ccf8711fcc9a27234fcca6cc9c3c5f5ccc8319ccc6ccb239cc9d2dccde6a290831cc86cca46b3accb03b1ecca8ccdbccb4cc98ccdecce5ccc1cc97ccbe39ccfdccb8ccf1cccb324b542bcca01a6131ccbcccd67540273accc90fccd9164941ccd42669404a5dcc85ccefccc6cc9f2f2fccde1c78ccd2ccef404accf205ccefccd96f98cc8ecccfccde50524bcc886dcd012ca76d73677061636b", + "envelope_size": 478, + "format": "msgpack", + "input_hex": "99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f", + "input_size": 300, + "name": "width_boundary_bin16" + }, { "description": "bin-encoded twin of 'empty': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", "envelope_hex": "94c40100982d06cc800538ccd3cc94ccc200a76d73677061636b", @@ -128,7 +137,18 @@ "name": "single_byte_bin", "envelope_encoding": "bin", "derived_from": "single_byte" + }, + { + "description": "bin-encoded twin of 'width_boundary_bin16': identical fields, compressed_data as msgpack bin (canonical writer encoding, protocol 1.1+)", + "envelope_hex": "94c5012ff0ff1e99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f98cc8ecccfccde50524bcc886dcd012ca76d73677061636b", + "envelope_size": 331, + "format": "msgpack", + "input_hex": "99259a8ca2d103745ecb8eab39f370dd1ea10e523348977d244d80a418418f54bc3897142004d6fbb5f12e5a919c5528733454c2c652e2fb82cce2ebff51d6adbfa88d011a0627a4169fa54a5e9a66df6182e87571bd25f7571a688e1221d257a97230906aa41ca1e6f76da7e1fc0e9e8396324b8720ef20e3d6d77a713b951ed8fa38fc6bf8f6ca4177102da9211f26f3ba2c20a2f0d1aad5095b302838192b5922226aa7027c3c13e0326890d1aaecf2ce1c442445916a268c72e4ed381a3b699ac8b0a9935614d4dc83d6de44b1462747f7f8711f9a27234fa69c3c5f5c8319c6b2399d2dde6a29083186a46b3ab03b1ea8dbb498dee5c197be39fdb8f1cb324b542ba01a6131bcd67540273ac90fd9164941d42669404a5d85efc69f2f2fde1c78d2ef404af205efd96f", + "input_size": 300, + "name": "width_boundary_bin16_bin", + "envelope_encoding": "bin", + "derived_from": "width_boundary_bin16" } ], - "version": "1.1.0" + "version": "1.1.1" } diff --git a/tests/wire_format_vectors.rs b/tests/wire_format_vectors.rs index 03698cf..d164ea9 100644 --- a/tests/wire_format_vectors.rs +++ b/tests/wire_format_vectors.rs @@ -17,12 +17,14 @@ //! //! Fixture provenance: vendored from //! `test-vectors/wire-format.json` -//! at commit `6863495d3565ccd9f28827baa1351782575e770d`, integrity-pinned by +//! at commit `5be35d5240817617275e3983de486a5826a587af`, integrity-pinned by //! sha256 below. To update: copy the file from a newer protocol ref, update //! `FIXTURE_SHA256` and this comment's commit hash together. #![cfg(all(feature = "compression", feature = "checksum", feature = "messagepack"))] +use std::collections::{HashMap, HashSet}; + use cachekit_core::{ByteStorage, StorageEnvelope}; use sha2::{Digest, Sha256}; @@ -31,7 +33,7 @@ use sha2::{Digest, Sha256}; const FIXTURE: &str = include_str!("vectors/wire-format.json"); /// sha256 of the vendored fixture — must match the protocol repo's copy. -const FIXTURE_SHA256: &str = "d6184a945d8cb22491b5c937414fe4d01e682d084ccec9dae9526d0ffba650cb"; +const FIXTURE_SHA256: &str = "b902db88fb9b2c4a2d0def7266f8199a858fcb921262c1eaf2c2c03412b5b56a"; #[derive(serde::Deserialize)] struct WireFormatFixture { @@ -87,17 +89,56 @@ fn fixture_integrity_pinned_sha256() { #[test] fn fixture_is_current_version_with_vectors() { let fixture = load_fixture(); - assert_eq!(fixture.version, "1.1.0"); - let legacy = fixture.vectors.iter().filter(|v| v.is_legacy()).count(); - let bin = fixture.vectors.len() - legacy; + assert_eq!(fixture.version, "1.1.1"); + let legacy: HashSet<&str> = fixture + .vectors + .iter() + .filter(|v| v.is_legacy()) + .map(|v| v.name.as_str()) + .collect(); + let bin = fixture.vectors.len() - legacy.len(); assert!( - legacy >= 6, - "legacy vectors are retained forever as legacy-read proof; expected at least the original 6, got {legacy}" + legacy.len() >= 6, + "legacy vectors are retained forever as legacy-read proof; expected at least the original 6, got {}", + legacy.len() ); assert!( bin >= 6, "protocol 1.1 pins a *_bin twin per legacy vector; expected at least 6, got {bin}" ); + // Equal totals alone would admit duplicate twins or orphaned parents, so + // key every *_bin vector by `derived_from` and require exactly one twin + // per legacy vector. + let mut twins: HashMap<&str, usize> = HashMap::new(); + for vector in fixture.vectors.iter().filter(|v| !v.is_legacy()) { + let parent = vector.derived_from.as_deref().unwrap_or_else(|| { + panic!( + "[{}] bin vector must name its legacy parent in derived_from", + vector.name + ) + }); + assert!( + legacy.contains(parent), + "[{}] derived_from {parent:?} names no legacy vector", + vector.name + ); + *twins.entry(parent).or_insert(0) += 1; + } + for name in &legacy { + assert_eq!( + twins.get(name).copied().unwrap_or(0), + 1, + "legacy vector {name:?} must have exactly one *_bin twin" + ); + } + // LAB-868's deliverable, pinned by name: the bin16 width-boundary pair + // must survive any future re-vendor. + for required in ["width_boundary_bin16", "width_boundary_bin16_bin"] { + assert!( + fixture.vectors.iter().any(|v| v.name == required), + "fixture must retain the {required:?} vector" + ); + } } #[test] @@ -259,24 +300,26 @@ fn bin_twins_are_bin_encoded_and_field_identical_to_legacy() { }); let twin_bytes = hex::decode(&twin.envelope_hex).expect("envelope_hex must decode"); + let twin_env: StorageEnvelope = rmp_serde::from_slice(&twin_bytes) + .unwrap_or_else(|e| panic!("[{}] twin must deserialize: {e}", twin.name)); assert_eq!( twin_bytes[0], 0x94, "[{}] outer fixarray(4) marker", twin.name ); assert_eq!( - // 0xc4 = msgpack bin8. Every pinned twin is small enough to be - // bin8; a wider marker here is a width-selection regression. twin_bytes[1], - 0xc4, - "[{}] compressed_data is not bin8-encoded: marker 0x{:02x}", + match twin_env.compressed_data.len() { + 0..=255 => 0xc4, + 256..=65_535 => 0xc5, + _ => 0xc6, + }, + "[{}] compressed_data does not use the shortest bin marker: 0x{:02x}", twin.name, twin_bytes[1] ); let parent_bytes = hex::decode(&parent.envelope_hex).expect("envelope_hex must decode"); - let twin_env: StorageEnvelope = rmp_serde::from_slice(&twin_bytes) - .unwrap_or_else(|e| panic!("[{}] twin must deserialize: {e}", twin.name)); let parent_env: StorageEnvelope = rmp_serde::from_slice(&parent_bytes) .unwrap_or_else(|e| panic!("[{parent_name}] parent must deserialize: {e}")); assert_eq!(