diff --git a/Cargo.lock b/Cargo.lock index e1eeaaf..d0697d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -340,6 +340,7 @@ dependencies = [ "tracing", "tracing-subscriber", "uuid", + "zip", ] [[package]] @@ -398,6 +399,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" version = "0.3.13" @@ -2281,6 +2291,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -2664,6 +2680,18 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/consolebook-server/Cargo.toml b/crates/consolebook-server/Cargo.toml index 0f6773c..5cfbfa5 100644 --- a/crates/consolebook-server/Cargo.toml +++ b/crates/consolebook-server/Cargo.toml @@ -33,6 +33,7 @@ uuid = { version = "1", features = ["v4"] } # so local-time resolution has the one-executable semantics ADR 0009 # documents, on every platform. jiff = { version = "0.2", default-features = false, features = ["std", "tzdb-bundle-always"] } +zip = { version = "8.6.0", default-features = false } [dev-dependencies] http-body-util = "0.1" diff --git a/crates/consolebook-server/src/audit.rs b/crates/consolebook-server/src/audit.rs index be17e17..866d1f5 100644 --- a/crates/consolebook-server/src/audit.rs +++ b/crates/consolebook-server/src/audit.rs @@ -48,6 +48,7 @@ pub enum EventKind { AcknowledgmentRecorded, AmendmentOpened, TaskSignoffRecorded, + RecordExported, } impl EventKind { @@ -90,6 +91,7 @@ impl EventKind { Self::AcknowledgmentRecorded => "acknowledgment_recorded", Self::AmendmentOpened => "amendment_opened", Self::TaskSignoffRecorded => "task_signoff_recorded", + Self::RecordExported => "record_exported", } } } diff --git a/crates/consolebook-server/src/export_verify.rs b/crates/consolebook-server/src/export_verify.rs new file mode 100644 index 0000000..1004e28 --- /dev/null +++ b/crates/consolebook-server/src/export_verify.rs @@ -0,0 +1,733 @@ +//! Verification of record export archives from their bytes alone +//! (ADR 0014; docs/formats/record-export.md; #45). +//! +//! `record_export` produces archives; this module owns the independent +//! check: typed findings, per-unit and per-archive reports, the +//! container walk (including the central directory, which the `zip` +//! reader collapses by name), and the normative check list of the +//! format document. The verdict is consistency with the stated +//! fingerprints, never tamper-proofing (ADR 0010, ADR 0011). Split out +//! of `record_export` when that module crossed the reorganization +//! threshold (AGENTS.md). + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::io::{Cursor, Read}; + +use serde::Serialize; +use serde_json::Value; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; + +use crate::canonical; +use crate::record_envelope; +use crate::record_export::{ + ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, FORMAT_VERSION, RECORD_FILE, Scope, + UNIT_FORMAT, UNIT_MANIFEST_FILE, UnitEntry, UnitManifest, canonical_json, unit_path, +}; + +/// One thing a verifier found wrong. The verdict derives from the +/// absence of findings; wording is presentation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Finding { + NotAnArchive { + detail: String, + }, + ArchiveManifestMissing, + ArchiveManifestUnreadable { + detail: String, + }, + UnsupportedFormat { + format: String, + format_version: i64, + }, + /// A manifest's bytes are not the canonical serialization of what + /// they parse to: a member is missing, reordered, or reformatted. + ManifestNotCanonical { + path: String, + }, + /// Units are not strictly ascending by (record, version). + UnitsOutOfOrder, + /// The manifest lists no unit; the format refuses empty exports. + NoUnits, + /// The declared scope calls for a different number of units. + ScopeCardinality { + expected: usize, + listed: usize, + }, + /// A listed unit's identity contradicts the declared scope. + UnitOutsideScope { + path: String, + }, + /// The container's central directory could not be walked. + CentralDirectoryUnreadable { + detail: String, + }, + /// The central directory names one entry more than once; extraction + /// tools disagree on which copy they take. + DuplicateEntry { + path: String, + }, + UnitPathUnexpected { + path: String, + expected: String, + }, + /// The container holds an entry the manifest does not name. + UnlistedEntry { + path: String, + }, + MissingEntry { + path: String, + }, + EntryUnreadable { + path: String, + detail: String, + }, + UnitManifestUnreadable { + detail: String, + }, + /// The unit manifest and the archive manifest disagree on a member. + UnitManifestDisagrees { + member: &'static str, + }, + ContentHashMismatch, + /// `record.json` is not canonical bytes (or not JSON at all). + NotCanonical { + detail: String, + }, + /// The envelope's own identity members disagree with the manifest. + EnvelopeDisagrees { + member: &'static str, + }, + /// The bytes are not an envelope of any known record schema: a + /// member missing, unnamed by the schema, or of the wrong type. + EnvelopeInvalid { + detail: String, + }, + ChainHashMismatch, + /// A hash member is not 64 lowercase hex characters. + HashNotCanonical { + member: &'static str, + }, + /// A first version with a predecessor, or a later one without. + LineageShape, + /// `record_id` and `version_number` are positive integers. + IdentityOutOfRange { + member: &'static str, + }, + /// The predecessor is in the archive and its content hash is not + /// what this unit's chain was computed over. + PredecessorMismatch, +} + +impl fmt::Display for Finding { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::HashNotCanonical { member } => { + write!(f, "{member} is not 64 lowercase hex characters") + } + Self::IdentityOutOfRange { member } => { + write!(f, "{member} is not a positive integer") + } + Self::EnvelopeInvalid { detail } => { + write!(f, "the record bytes are not a valid envelope: {detail}") + } + Self::NotAnArchive { detail } => write!(f, "not a readable ZIP archive: {detail}"), + Self::ArchiveManifestMissing => f.write_str("the archive manifest is missing"), + Self::ArchiveManifestUnreadable { detail } => { + write!(f, "the archive manifest is unreadable: {detail}") + } + Self::UnsupportedFormat { + format, + format_version, + } => write!(f, "unsupported format '{format}' version {format_version}"), + Self::ManifestNotCanonical { path } => { + write!(f, "{path} is not canonical JSON") + } + Self::UnitsOutOfOrder => { + f.write_str("units are not strictly ascending by record and version") + } + Self::NoUnits => f.write_str("the manifest lists no unit"), + Self::ScopeCardinality { expected, listed } => write!( + f, + "the declared scope calls for {expected} unit(s); the manifest lists {listed}" + ), + Self::UnitOutsideScope { path } => { + write!(f, "unit {path} is outside the declared scope") + } + Self::CentralDirectoryUnreadable { detail } => { + write!(f, "the central directory could not be walked: {detail}") + } + Self::DuplicateEntry { path } => { + write!( + f, + "entry {path} appears more than once in the central directory" + ) + } + Self::UnitPathUnexpected { path, expected } => { + write!(f, "unit path {path} should be {expected}") + } + Self::UnlistedEntry { path } => write!(f, "entry {path} is not listed by the manifest"), + Self::MissingEntry { path } => write!(f, "entry {path} is missing"), + Self::EntryUnreadable { path, detail } => { + write!(f, "entry {path} is unreadable: {detail}") + } + Self::UnitManifestUnreadable { detail } => { + write!(f, "the unit manifest is unreadable: {detail}") + } + Self::UnitManifestDisagrees { member } => { + write!( + f, + "the unit manifest disagrees with the archive on {member}" + ) + } + Self::ContentHashMismatch => { + f.write_str("the content hash does not match the record bytes") + } + Self::NotCanonical { detail } => { + write!(f, "the record bytes are not canonical: {detail}") + } + Self::EnvelopeDisagrees { member } => { + write!(f, "the record's own {member} disagrees with the manifest") + } + Self::ChainHashMismatch => { + f.write_str("the chain hash does not match the predecessor hash and record bytes") + } + Self::LineageShape => { + f.write_str("a predecessor hash is present exactly for versions after the first") + } + Self::PredecessorMismatch => { + f.write_str("the predecessor in this archive has a different content hash") + } + } + } +} + +/// Whether a unit's predecessor was checked against the archive. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PredecessorLink { + /// A first version. + None, + /// The predecessor is in the archive and its content hash matches. + Linked, + /// The archive does not carry the predecessor; the chain hash was + /// still recomputed from the carried predecessor hash. + NotInExport, +} + +impl fmt::Display for PredecessorLink { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::None => "none (first version)", + Self::Linked => "linked", + Self::NotInExport => "not in export", + }) + } +} + +/// One unit's verification. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct UnitReport { + pub path: String, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub predecessor: PredecessorLink, + pub findings: Vec, +} + +impl UnitReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() + } +} + +/// The whole archive's verification. `verified` when nothing was found +/// wrong anywhere: internally consistent with its stated fingerprints, +/// which is what the format can prove. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ArchiveReport { + pub installation_id: Option, + pub exported_at: Option, + pub scope: Option, + pub units: Vec, + pub findings: Vec, +} + +impl ArchiveReport { + #[must_use] + pub fn verified(&self) -> bool { + self.findings.is_empty() && self.units.iter().all(UnitReport::verified) + } + + /// The export instant as RFC 3339, for presentation. + #[must_use] + pub fn exported_at_rfc3339(&self) -> Option { + self.exported_at.and_then(|at| { + OffsetDateTime::from_unix_timestamp(at) + .ok()? + .format(&Rfc3339) + .ok() + }) + } +} + +type Archive<'a> = zip::ZipArchive>; + +/// Verifies an archive from its bytes alone, per the normative checks +/// in docs/formats/record-export.md. +#[must_use] +pub fn verify_archive(bytes: &[u8]) -> ArchiveReport { + let mut report = ArchiveReport { + installation_id: None, + exported_at: None, + scope: None, + units: Vec::new(), + findings: Vec::new(), + }; + let mut archive = match zip::ZipArchive::new(Cursor::new(bytes)) { + Ok(archive) => archive, + Err(err) => { + report.findings.push(Finding::NotAnArchive { + detail: err.to_string(), + }); + return report; + } + }; + let names: Vec = archive.file_names().map(str::to_owned).collect(); + report.findings.extend(duplicate_entry_findings(bytes)); + let manifest_bytes = match read_entry(&mut archive, ARCHIVE_MANIFEST_PATH) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + report.findings.push(Finding::ArchiveManifestMissing); + return report; + } + Err(detail) => { + report.findings.push(Finding::EntryUnreadable { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + detail, + }); + return report; + } + }; + let manifest: ArchiveManifest = match serde_json::from_slice(&manifest_bytes) { + Ok(manifest) => manifest, + Err(err) => { + report.findings.push(Finding::ArchiveManifestUnreadable { + detail: err.to_string(), + }); + return report; + } + }; + if manifest.format != ARCHIVE_FORMAT || manifest.format_version != FORMAT_VERSION { + report.findings.push(Finding::UnsupportedFormat { + format: manifest.format.clone(), + format_version: manifest.format_version, + }); + return report; + } + if canonical_json(&manifest).ok().as_deref() != Some(manifest_bytes.as_slice()) { + report.findings.push(Finding::ManifestNotCanonical { + path: ARCHIVE_MANIFEST_PATH.to_owned(), + }); + } + report.installation_id = Some(manifest.installation_id.clone()); + report.exported_at = Some(manifest.exported_at); + report.scope = Some(manifest.scope); + + let ordered = manifest.units.windows(2).all(|pair| { + (pair[0].record_id, pair[0].version_number) < (pair[1].record_id, pair[1].version_number) + }); + if !ordered { + report.findings.push(Finding::UnitsOutOfOrder); + } + report.findings.extend(scope_findings(&manifest)); + let mut listed: BTreeSet = BTreeSet::new(); + listed.insert(ARCHIVE_MANIFEST_PATH.to_owned()); + for entry in &manifest.units { + let expected = unit_path(entry.record_id, entry.version_number); + if entry.path != expected { + report.findings.push(Finding::UnitPathUnexpected { + path: entry.path.clone(), + expected, + }); + } + listed.insert(format!("{}/{RECORD_FILE}", entry.path)); + listed.insert(format!("{}/{UNIT_MANIFEST_FILE}", entry.path)); + } + for name in &names { + if !listed.contains(name) { + report + .findings + .push(Finding::UnlistedEntry { path: name.clone() }); + } + } + let by_identity: BTreeMap<(i64, i64), &UnitEntry> = manifest + .units + .iter() + .map(|entry| ((entry.record_id, entry.version_number), entry)) + .collect(); + for entry in &manifest.units { + report + .units + .push(verify_unit(&mut archive, &manifest, entry, &by_identity)); + } + report +} + +#[allow(clippy::too_many_lines)] +fn verify_unit( + archive: &mut Archive<'_>, + manifest: &ArchiveManifest, + entry: &UnitEntry, + by_identity: &BTreeMap<(i64, i64), &UnitEntry>, +) -> UnitReport { + let mut findings = Vec::new(); + let record_path = format!("{}/{RECORD_FILE}", entry.path); + let manifest_path = format!("{}/{UNIT_MANIFEST_FILE}", entry.path); + + match read_entry(archive, &manifest_path) { + Ok(Some(bytes)) => match serde_json::from_slice::(&bytes) { + Ok(unit) => { + if unit.format != UNIT_FORMAT || unit.format_version != FORMAT_VERSION { + findings.push(Finding::UnsupportedFormat { + format: unit.format.clone(), + format_version: unit.format_version, + }); + } + if canonical_json(&unit).ok().as_deref() != Some(bytes.as_slice()) { + findings.push(Finding::ManifestNotCanonical { + path: manifest_path.clone(), + }); + } + let disagreements: [(&'static str, bool); 8] = [ + ( + "installation_id", + unit.installation_id != manifest.installation_id, + ), + ("exported_at", unit.exported_at != manifest.exported_at), + ("record_id", unit.record_id != entry.record_id), + ( + "version_number", + unit.version_number != entry.version_number, + ), + ("record_schema", unit.record_schema != entry.record_schema), + ("content_hash", unit.content_hash != entry.content_hash), + ("chain_hash", unit.chain_hash != entry.chain_hash), + ( + "predecessor_content_hash", + unit.predecessor_content_hash != entry.predecessor_content_hash, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::UnitManifestDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::UnitManifestUnreadable { + detail: err.to_string(), + }), + }, + Ok(None) => findings.push(Finding::MissingEntry { + path: manifest_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: manifest_path.clone(), + detail, + }), + } + + match read_entry(archive, &record_path) { + Ok(Some(bytes)) => { + if canonical::content_hash_hex(&bytes) != entry.content_hash { + findings.push(Finding::ContentHashMismatch); + } + match serde_json::from_slice::(&bytes) { + Ok(document) => match canonical::canonical_bytes(&document) { + Ok(again) if again == bytes => {} + Ok(_) => findings.push(Finding::NotCanonical { + detail: "re-serialization differs from the stored bytes".to_owned(), + }), + Err(err) => findings.push(Finding::NotCanonical { + detail: err.to_string(), + }), + }, + Err(err) => findings.push(Finding::NotCanonical { + detail: format!("not JSON: {err}"), + }), + } + // The bytes must be an envelope of a known record schema — + // every member the schema names, typed, and no other — before + // the identity they carry is compared with the manifest. + match record_envelope::parse(&bytes) { + Ok(envelope) => { + let disagreements: [(&'static str, bool); 6] = [ + ("record.id", envelope.record.id != entry.record_id), + ( + "record.version_number", + envelope.record.version_number != entry.version_number, + ), + ( + "record.record_schema", + envelope.record.record_schema != entry.record_schema, + ), + ( + "record.predecessor_content_hash", + envelope.record.predecessor_content_hash + != entry.predecessor_content_hash, + ), + ("instance", envelope.instance != manifest.installation_id), + ( + "canonicalization", + envelope.canonicalization != canonical::CANONICALIZATION, + ), + ]; + for (member, disagrees) in disagreements { + if disagrees { + findings.push(Finding::EnvelopeDisagrees { member }); + } + } + } + Err(err) => findings.push(Finding::EnvelopeInvalid { + detail: err.to_string(), + }), + } + match canonical::chain_hash_hex(entry.predecessor_content_hash.as_deref(), &bytes) { + Ok(chain) if chain == entry.chain_hash => {} + _ => findings.push(Finding::ChainHashMismatch), + } + } + Ok(None) => findings.push(Finding::MissingEntry { + path: record_path.clone(), + }), + Err(detail) => findings.push(Finding::EntryUnreadable { + path: record_path.clone(), + detail, + }), + } + + // Hashes are 64 lowercase hex characters by the format. The chain + // recomputation decodes hex case-insensitively, so without this an + // uppercase predecessor hash would pass for a lone successor and + // then fail to link once its lowercase predecessor joined it. + let hash_members: [(&'static str, Option<&str>); 3] = [ + ("content_hash", Some(entry.content_hash.as_str())), + ("chain_hash", Some(entry.chain_hash.as_str())), + ( + "predecessor_content_hash", + entry.predecessor_content_hash.as_deref(), + ), + ]; + for (member, value) in hash_members { + if let Some(hex) = value + && !is_lowercase_hex_hash(hex) + { + findings.push(Finding::HashNotCanonical { member }); + } + } + // Identity is positive by the format: the database assigns record + // ids from 1 and version numbers start at 1, so a zero or negative + // number is not an identity the lineage rule below can reason about. + if entry.record_id < 1 { + findings.push(Finding::IdentityOutOfRange { + member: "record_id", + }); + } + if entry.version_number < 1 { + findings.push(Finding::IdentityOutOfRange { + member: "version_number", + }); + } + if (entry.version_number == 1) != entry.predecessor_content_hash.is_none() { + findings.push(Finding::LineageShape); + } + let predecessor = match &entry.predecessor_content_hash { + None => PredecessorLink::None, + Some(hash) => match entry + .version_number + .checked_sub(1) + .and_then(|number| by_identity.get(&(entry.record_id, number))) + { + Some(previous) => { + if previous.content_hash != *hash { + findings.push(Finding::PredecessorMismatch); + } + PredecessorLink::Linked + } + None => PredecessorLink::NotInExport, + }, + }; + + UnitReport { + path: entry.path.clone(), + record_id: entry.record_id, + version_number: entry.version_number, + record_schema: entry.record_schema, + predecessor, + findings, + } +} + +/// The declared scope checked as far as the archive itself allows: no +/// scope is empty, a version scope is exactly its one unit, and a +/// record scope holds only that record's versions. Enrollment and +/// installation scopes state nothing the bytes can confirm. +fn scope_findings(manifest: &ArchiveManifest) -> Vec { + let mut findings = Vec::new(); + if manifest.units.is_empty() { + findings.push(Finding::NoUnits); + } + match manifest.scope { + Scope::Version { + record_id, + version_number, + } => { + if manifest.units.len() != 1 { + findings.push(Finding::ScopeCardinality { + expected: 1, + listed: manifest.units.len(), + }); + } + for entry in &manifest.units { + if (entry.record_id, entry.version_number) != (record_id, version_number) { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Record { record_id } => { + for entry in &manifest.units { + if entry.record_id != record_id { + findings.push(Finding::UnitOutsideScope { + path: entry.path.clone(), + }); + } + } + } + Scope::Enrollment { .. } | Scope::Installation => {} + } + findings +} + +/// The reader keeps one entry per name; only the central directory +/// itself says whether a name was written twice. +fn duplicate_entry_findings(bytes: &[u8]) -> Vec { + match central_directory_names(bytes) { + Ok(directory) => { + let mut occurrences: BTreeMap<&str, usize> = BTreeMap::new(); + for name in &directory { + *occurrences.entry(name.as_str()).or_default() += 1; + } + occurrences + .into_iter() + .filter(|(_, count)| *count > 1) + .map(|(name, _)| Finding::DuplicateEntry { + path: name.to_owned(), + }) + .collect() + } + Err(detail) => vec![Finding::CentralDirectoryUnreadable { detail }], + } +} + +/// Every entry name in the central directory, duplicates included, in +/// directory order. The `zip` reader indexes entries by name and keeps +/// one per name, so a name written twice — which extraction tools +/// resolve differently — is visible only here. The walk follows +/// APPNOTE 6.3: the end-of-central-directory record (the last record, +/// followed by at most a 65535-byte comment), the ZIP64 locator and +/// record when the classic fields overflow, then the fixed 46-byte +/// central headers with their variable name, extra, and comment parts. +fn central_directory_names(bytes: &[u8]) -> std::result::Result, String> { + const EOCD: [u8; 4] = [0x50, 0x4b, 0x05, 0x06]; + const ZIP64_LOCATOR: [u8; 4] = [0x50, 0x4b, 0x06, 0x07]; + const ZIP64_EOCD: [u8; 4] = [0x50, 0x4b, 0x06, 0x06]; + const CENTRAL_HEADER: [u8; 4] = [0x50, 0x4b, 0x01, 0x02]; + let eocd = (0..=bytes.len().saturating_sub(22)) + .rev() + .take(usize::from(u16::MAX) + 1) + .find(|&at| bytes.get(at..at + 4) == Some(&EOCD[..])) + .ok_or("no end-of-central-directory record")?; + let mut count = u64::from(le_u16(bytes, eocd + 10)?); + let mut start = u64::from(le_u32(bytes, eocd + 16)?); + if count == u64::from(u16::MAX) || start == u64::from(u32::MAX) { + let locator = eocd + .checked_sub(20) + .filter(|&at| bytes.get(at..at + 4) == Some(&ZIP64_LOCATOR[..])) + .ok_or("ZIP64 fields without a ZIP64 locator")?; + let zip64 = usize::try_from(le_u64(bytes, locator + 8)?) + .map_err(|_| "ZIP64 record offset out of range".to_owned())?; + if bytes.get(zip64..zip64 + 4) != Some(&ZIP64_EOCD[..]) { + return Err("ZIP64 locator points at no ZIP64 record".to_owned()); + } + count = le_u64(bytes, zip64 + 32)?; + start = le_u64(bytes, zip64 + 48)?; + } + let mut at = usize::try_from(start).map_err(|_| "central directory offset out of range")?; + let mut names = Vec::new(); + for _ in 0..count { + if bytes.get(at..at + 4) != Some(&CENTRAL_HEADER[..]) { + return Err(format!("no central directory header at offset {at}")); + } + let name_len = usize::from(le_u16(bytes, at + 28)?); + let extra_len = usize::from(le_u16(bytes, at + 30)?); + let comment_len = usize::from(le_u16(bytes, at + 32)?); + let name = bytes + .get(at + 46..at + 46 + name_len) + .ok_or("truncated central directory header")?; + names.push(String::from_utf8_lossy(name).into_owned()); + at += 46 + name_len + extra_len + comment_len; + } + Ok(names) +} + +fn le_u16(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u32(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 4) + .map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn le_u64(bytes: &[u8], at: usize) -> std::result::Result { + bytes + .get(at..at + 8) + .map(|b| u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])) + .ok_or_else(|| format!("truncated record at offset {at}")) +} + +fn is_lowercase_hex_hash(hex: &str) -> bool { + hex.len() == 64 + && hex + .bytes() + .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f')) +} + +/// Reads one entry: `Ok(None)` when absent, `Err(detail)` when the +/// container cannot deliver it (a CRC mismatch included). +fn read_entry( + archive: &mut Archive<'_>, + name: &str, +) -> std::result::Result>, String> { + match archive.by_name(name) { + Ok(mut file) => { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes) + .map_err(|err| err.to_string())?; + Ok(Some(bytes)) + } + Err(zip::result::ZipError::FileNotFound) => Ok(None), + Err(err) => Err(err.to_string()), + } +} diff --git a/crates/consolebook-server/src/exports_http.rs b/crates/consolebook-server/src/exports_http.rs new file mode 100644 index 0000000..485e1f1 --- /dev/null +++ b/crates/consolebook-server/src/exports_http.rs @@ -0,0 +1,122 @@ +//! Record export HTTP handlers (Milestone 5 slice 1; ADR 0014). +//! +//! `http.rs` remains the hub; this module owns the export downloads and +//! the installation-export summary. Scope rules live in +//! `record_export`; handlers translate refusals into stable error codes +//! and deliver the documented archive bytes as attachments. + +use axum::Router; +use axum::extract::{Path, State}; +use axum::http::{StatusCode, header}; +use axum::response::{IntoResponse, Json, Response}; +use axum::routing::get; + +use crate::http::{ApiError, AppState, CurrentUser}; +use crate::record_export::{self, ExportRefusal, Scope}; + +pub(crate) fn routes() -> Router { + Router::new() + .route("/api/drafts/{id}/export", get(export_record)) + .route( + "/api/drafts/{id}/versions/{number}/export", + get(export_version), + ) + .route("/api/enrollments/{id}/export", get(export_enrollment)) + .route("/api/exports/records", get(export_installation)) + .route("/api/exports/summary", get(export_summary)) +} + +fn export_refusal(refusal: ExportRefusal) -> ApiError { + match refusal { + ExportRefusal::NoSuchRecord => { + ApiError::new(StatusCode::NOT_FOUND, "no_such_record", "no such record") + } + ExportRefusal::NoSuchVersion => ApiError::new( + StatusCode::NOT_FOUND, + "no_such_version", + "this record has no finalized version with that number", + ), + ExportRefusal::NoSuchEnrollment => ApiError::new( + StatusCode::NOT_FOUND, + "no_such_enrollment", + "no such enrollment", + ), + ExportRefusal::CapabilityRequired => ApiError::new( + StatusCode::FORBIDDEN, + "capability_required", + "exporting takes the scope's read authority; the whole installation takes export_records", + ), + ExportRefusal::NothingToExport => ApiError::new( + StatusCode::CONFLICT, + "nothing_to_export", + "this scope holds no finalized version; an export never claims completeness it lacks", + ), + } +} + +/// The archive as a download: exactly the documented bytes. +async fn deliver(state: &AppState, actor_user_id: i64, scope: Scope) -> Result { + match record_export::export(&state.pool, actor_user_id, scope).await? { + Ok(export) => { + let disposition = format!("attachment; filename=\"{}\"", export.file_name); + Ok(( + [ + (header::CONTENT_TYPE, "application/zip".to_owned()), + (header::CONTENT_DISPOSITION, disposition), + ], + export.bytes, + ) + .into_response()) + } + Err(refusal) => Err(export_refusal(refusal)), + } +} + +async fn export_version( + State(state): State, + current: CurrentUser, + Path((record_id, version_number)): Path<(i64, i64)>, +) -> Result { + deliver( + &state, + current.user.id, + Scope::Version { + record_id, + version_number, + }, + ) + .await +} + +async fn export_record( + State(state): State, + current: CurrentUser, + Path(record_id): Path, +) -> Result { + deliver(&state, current.user.id, Scope::Record { record_id }).await +} + +async fn export_enrollment( + State(state): State, + current: CurrentUser, + Path(enrollment_id): Path, +) -> Result { + deliver(&state, current.user.id, Scope::Enrollment { enrollment_id }).await +} + +async fn export_installation( + State(state): State, + current: CurrentUser, +) -> Result { + deliver(&state, current.user.id, Scope::Installation).await +} + +async fn export_summary( + State(state): State, + current: CurrentUser, +) -> Result { + match record_export::summary(&state.pool, current.user.id).await? { + Ok(summary) => Ok(Json(summary).into_response()), + Err(refusal) => Err(export_refusal(refusal)), + } +} diff --git a/crates/consolebook-server/src/http.rs b/crates/consolebook-server/src/http.rs index d3d54b5..66da178 100644 --- a/crates/consolebook-server/src/http.rs +++ b/crates/consolebook-server/src/http.rs @@ -55,6 +55,7 @@ pub fn router(state: AppState) -> Router { .merge(crate::programs_http::routes()) .merge(crate::training_http::routes()) .merge(crate::drafts_http::routes()) + .merge(crate::exports_http::routes()) .fallback(crate::web_assets::serve) .with_state(state) } diff --git a/crates/consolebook-server/src/lib.rs b/crates/consolebook-server/src/lib.rs index 7777dd0..91983bd 100644 --- a/crates/consolebook-server/src/lib.rs +++ b/crates/consolebook-server/src/lib.rs @@ -18,6 +18,8 @@ pub mod draft_review; pub mod drafts_http; pub mod enrollments; pub mod evaluation_drafts; +pub mod export_verify; +pub mod exports_http; pub mod finalization; pub mod http; pub mod lifecycle; @@ -25,6 +27,8 @@ pub mod notices; pub mod program_export; pub mod programs; pub mod programs_http; +pub mod record_envelope; +pub mod record_export; pub mod restore; pub mod scheduler; pub mod secrets; diff --git a/crates/consolebook-server/src/main.rs b/crates/consolebook-server/src/main.rs index a9c97cf..d9378f1 100644 --- a/crates/consolebook-server/src/main.rs +++ b/crates/consolebook-server/src/main.rs @@ -12,7 +12,7 @@ use consolebook_server::doctor::Verdict; use consolebook_server::serve_lock::ServeLock; use consolebook_server::users::{IssueRefusal, ResetOrigin}; use consolebook_server::{ - VERSION, backup, doctor, http, restore, scheduler, setup, storage, users, + VERSION, backup, doctor, export_verify, http, restore, scheduler, setup, storage, users, }; #[derive(Parser)] @@ -77,6 +77,11 @@ enum Command { /// Path to the snapshot to restore, usually in backups/. snapshot: PathBuf, }, + /// Work with record exports (docs/formats/record-export.md). + Export { + #[command(subcommand)] + action: ExportAction, + }, /// Print a fresh first-run setup code for an uninitialized installation. SetupCode, /// Issue a password reset code for a locked-out administrator. @@ -90,6 +95,19 @@ enum Command { }, } +#[derive(Subcommand)] +enum ExportAction { + /// Verify a record export archive from the file alone. + /// + /// Opens no data directory: the archive carries everything the + /// checks need, and the verdict is consistency with its stated + /// fingerprints. + Verify { + /// Path to the export archive (.zip). + archive: PathBuf, + }, +} + fn init_logging() { use tracing_subscriber::EnvFilter; // Structured logs, no record content: log events describe operations and @@ -114,6 +132,9 @@ async fn main() -> ExitCode { Command::Doctor => run_doctor(&cli.data_dir).await, Command::Backup { keep } => run_backup(&cli.data_dir, keep).await, Command::Restore { snapshot } => run_restore(&cli.data_dir, &snapshot).await, + Command::Export { + action: ExportAction::Verify { archive }, + } => run_export_verify(&archive), Command::SetupCode => run_setup_code(&cli.data_dir).await, Command::Recover { username } => run_recover(&cli.data_dir, &username).await, }; @@ -274,3 +295,45 @@ async fn run_restore(data_dir: &std::path::Path, snapshot: &std::path::Path) -> ); Ok(ExitCode::SUCCESS) } + +fn run_export_verify(archive: &std::path::Path) -> Result { + let bytes = std::fs::read(archive).with_context(|| format!("reading {}", archive.display()))?; + let report = export_verify::verify_archive(&bytes); + if let Some(id) = &report.installation_id { + println!("installation {id}"); + } + if let Some(at) = report.exported_at_rfc3339() { + println!("exported at {at}"); + } + if let Some(scope) = report.scope { + println!("scope {scope}"); + } + for finding in &report.findings { + println!("FAIL archive {finding}"); + } + for unit in &report.units { + let mark = if unit.verified() { "ok " } else { "FAIL" }; + println!( + "{mark} {:<20} record {} version {} (schema {}); predecessor {}", + unit.path, unit.record_id, unit.version_number, unit.record_schema, unit.predecessor + ); + for finding in &unit.findings { + println!(" {finding}"); + } + } + let consistent = report.units.iter().filter(|unit| unit.verified()).count(); + if report.verified() { + println!( + "verified {consistent} of {} units: the export is consistent with its stated fingerprints", + report.units.len() + ); + Ok(ExitCode::SUCCESS) + } else { + println!( + "NOT VERIFIED: {consistent} of {} units consistent, {} archive finding(s)", + report.units.len(), + report.findings.len() + ); + Ok(ExitCode::FAILURE) + } +} diff --git a/crates/consolebook-server/src/record_envelope.rs b/crates/consolebook-server/src/record_envelope.rs new file mode 100644 index 0000000..f736c30 --- /dev/null +++ b/crates/consolebook-server/src/record_envelope.rs @@ -0,0 +1,306 @@ +//! The typed shape of a finalized record envelope, as readers see it +//! (ADR 0011 for record schema 1, ADR 0013 for schema 2). +//! +//! `finalization::envelope` is the producing side and the one owner of +//! what goes into a record. This is the reading side: the same member +//! set, typed, with every object refusing members the schema does not +//! name and every nullable member required to be present, so "these +//! bytes are a schema-1 or schema-2 envelope" is a typed contract +//! rather than a spot check of a few members. Export verification reads +//! through it; packets and rendering (Milestone 5) will too. A schema +//! bump extends this shape and never reinterprets stored bytes. + +use std::fmt; + +use serde::{Deserialize, Deserializer}; + +use crate::canonical; + +/// A user as the envelope presents them: identity plus the names shown +/// at finalization. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct User { + pub id: i64, + pub username: String, + pub display_name: String, +} + +/// Attachments exist in no schema yet: the array is always empty. The +/// element type is uninhabited, so no value of any shape deserializes +/// into it and a non-empty array is not a schema-1 or schema-2 record. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Attachment {} + +impl<'de> Deserialize<'de> for Attachment { + fn deserialize(_: D) -> Result + where + D: Deserializer<'de>, + { + Err(::custom( + "attachments exist in no known record schema", + )) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttributionEvent { + pub kind: String, + pub actor: User, + #[serde(deserialize_with = "nullable")] + pub to: Option, + pub recorded_at: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Content { + pub narratives: Vec, + pub ratings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Narrative { + pub prompt: String, + pub required: bool, + #[serde(deserialize_with = "nullable")] + pub text: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rating { + pub competency: Competency, + pub scale: Scale, + #[serde(deserialize_with = "nullable")] + pub value: Option, + pub not_observed: bool, + pub modifiers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Competency { + pub category: String, + pub name: String, + pub description: String, + pub tasks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scale { + pub name: String, + pub kind: String, + #[serde(deserialize_with = "nullable")] + pub min_value: Option, + #[serde(deserialize_with = "nullable")] + pub max_value: Option, + pub anchors: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Anchor { + pub value: i64, + pub label: String, + pub definition: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Modifier { + pub code: String, + pub label: String, + pub description: String, +} + +/// One pinned daily-report version a weekly summary covered (schema 2). +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DailyReport { + pub content_hash: String, + pub record_id: i64, + pub version_number: i64, +} + +/// The `daily_reports` member: absent in schema 1, an array in schema +/// 2. Three states are told apart — absent, `null` (never valid), and +/// present — because schema 1 requires the member to be missing. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum DailyReports { + #[default] + Absent, + Present(Vec), +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Finalization { + pub finalized_at: i64, + pub finalized_by: User, + pub policy: Policy, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Policy { + pub review_approved: bool, + pub required_narratives: bool, + pub ratings_complete: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Form { + pub name: String, + pub instructions: String, + pub record_type: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Program { + pub name: String, + pub version_number: i64, + pub label: String, +} + +/// The record's own statement of identity and lineage. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecordIdentity { + pub id: i64, + pub version_number: i64, + pub record_schema: i64, + #[serde(deserialize_with = "nullable")] + pub predecessor_content_hash: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReviewDecision { + pub reviewer: User, + pub decision: String, + pub comment: String, + pub decided_at: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Session { + pub business_date: String, + pub timezone: String, + pub local_start: String, + #[serde(deserialize_with = "nullable")] + pub local_end: Option, + pub utc_start: i64, + #[serde(deserialize_with = "nullable")] + pub utc_end: Option, + #[serde(deserialize_with = "nullable")] + pub disposition: Option, + #[serde(deserialize_with = "nullable")] + pub phase: Option, + pub trainers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Phase { + pub name: String, + pub presentation_number: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Trainee { + pub id: i64, + pub username: String, + pub display_name: String, + pub employee_id: String, + pub title: String, +} + +/// The envelope's top level: every member ADR 0011 names, plus +/// `daily_reports` for schema 2 (ADR 0013), and nothing else. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub attachments: Vec, + pub attribution: Vec, + pub canonicalization: String, + pub content: Content, + #[serde(default, deserialize_with = "daily_reports")] + pub daily_reports: DailyReports, + pub finalization: Finalization, + pub form: Form, + pub instance: String, + pub program: Program, + pub record: RecordIdentity, + pub review: Vec, + pub sessions: Vec, + pub trainee: Trainee, +} + +/// A member that may be `null` but must be present: the producer always +/// writes it, so a document without it is not an envelope. +fn nullable<'de, T, D>(deserializer: D) -> Result, D::Error> +where + T: Deserialize<'de>, + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) +} + +fn daily_reports<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Vec::::deserialize(deserializer).map(DailyReports::Present) +} + +/// Why bytes are not an envelope of any known schema. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EnvelopeError { + /// Not the member set and types of any schema this build knows. + Malformed(String), + /// A `record.record_schema` this build does not know. + UnsupportedSchema(i64), + /// `daily_reports` is present exactly in schema 2. + SchemaShape(i64), +} + +impl fmt::Display for EnvelopeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Malformed(detail) => write!(f, "not a record envelope: {detail}"), + Self::UnsupportedSchema(schema) => { + write!(f, "unsupported record schema {schema}") + } + Self::SchemaShape(schema) => write!( + f, + "record schema {schema} and the daily_reports member disagree" + ), + } + } +} + +impl std::error::Error for EnvelopeError {} + +/// Reads bytes as an envelope of the schema they declare, refusing any +/// document that is not exactly a schema-1 or schema-2 record. +pub fn parse(bytes: &[u8]) -> Result { + let envelope: Envelope = + serde_json::from_slice(bytes).map_err(|err| EnvelopeError::Malformed(err.to_string()))?; + let schema = envelope.record.record_schema; + if !(1..=canonical::RECORD_SCHEMA).contains(&schema) { + return Err(EnvelopeError::UnsupportedSchema(schema)); + } + let has_daily_reports = matches!(envelope.daily_reports, DailyReports::Present(_)); + if has_daily_reports != (schema == 2) { + return Err(EnvelopeError::SchemaShape(schema)); + } + Ok(envelope) +} diff --git a/crates/consolebook-server/src/record_export.rs b/crates/consolebook-server/src/record_export.rs new file mode 100644 index 0000000..4171eed --- /dev/null +++ b/crates/consolebook-server/src/record_export.rs @@ -0,0 +1,504 @@ +//! Structured record exports: the format vocabulary and the producing +//! side (ADR 0014; docs/formats/record-export.md; #45). +//! +//! An export unit is one finalized version's stored canonical bytes, +//! copied verbatim, beside a canonical-JSON unit manifest; an archive +//! is a ZIP container with stored entries in a fixed order holding an +//! archive manifest and its units. The archive is a pure function of +//! the scope's stored rows and the export instant, so the same scope +//! exported at the same instant is byte-identical. Export follows the +//! read rules that already exist — a unit contains exactly what its +//! reader may already read — and the installation scope takes +//! `export_records`. Verification from the archive alone is +//! `export_verify`'s, which reads the manifests defined here. + +use std::fmt; +use std::io::{Cursor, Write}; + +use anyhow::{Context, Result, anyhow}; +use serde::{Deserialize, Serialize}; +use sqlx::{Row, SqlitePool}; +use time::OffsetDateTime; +use zip::CompressionMethod; +use zip::write::{SimpleFileOptions, ZipWriter}; + +use crate::audit::{self, EventKind, Subject}; +use crate::canonical; +use crate::capabilities::{self, Capability}; +use crate::evaluation_drafts; +use crate::lifecycle; +use crate::storage; + +/// Archive-manifest discriminator; never changes. +pub const ARCHIVE_FORMAT: &str = "consolebook-record-export"; +/// Unit-manifest discriminator; never changes. +pub const UNIT_FORMAT: &str = "consolebook-record-unit"; +/// Shared by both manifests; bumped by any change to either shape. +pub const FORMAT_VERSION: i64 = 1; +/// The archive manifest's entry name. +pub const ARCHIVE_MANIFEST_PATH: &str = "manifest.json"; +/// The canonical record bytes within a unit directory. +pub const RECORD_FILE: &str = "record.json"; +/// The unit manifest within a unit directory. +pub const UNIT_MANIFEST_FILE: &str = "manifest.json"; + +/// What an archive claims to contain. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum Scope { + /// Exactly one finalized version. + Version { record_id: i64, version_number: i64 }, + /// Every retained version of one record, superseded originals + /// included. + Record { record_id: i64 }, + /// Every finalized version of every record of one enrollment. + Enrollment { enrollment_id: i64 }, + /// Every finalized version the installation holds. + Installation, +} + +impl fmt::Display for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Version { + record_id, + version_number, + } => write!(f, "record {record_id}, version {version_number}"), + Self::Record { record_id } => { + write!(f, "record {record_id}, every retained version") + } + Self::Enrollment { enrollment_id } => write!(f, "enrollment {enrollment_id}"), + Self::Installation => f.write_str("the whole installation"), + } + } +} + +/// Typed refusals for the export act. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ExportRefusal { + NoSuchRecord, + NoSuchVersion, + NoSuchEnrollment, + CapabilityRequired, + /// The scope exists but holds no finalized version; an empty + /// archive is never presented as a complete export. + NothingToExport, +} + +/// One unit as the archive manifest lists it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnitEntry { + pub path: String, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub content_hash: String, + pub chain_hash: String, + pub predecessor_content_hash: Option, +} + +/// The archive manifest (`manifest.json` at the root). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArchiveManifest { + pub format: String, + pub format_version: i64, + pub installation_id: String, + pub exported_at: i64, + pub scope: Scope, + pub units: Vec, +} + +/// The unit manifest beside each unit's record bytes. It repeats what +/// the archive manifest says so a unit directory stands on its own. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UnitManifest { + pub format: String, + pub format_version: i64, + pub installation_id: String, + pub exported_at: i64, + pub record_id: i64, + pub version_number: i64, + pub record_schema: i64, + pub content_hash: String, + pub chain_hash: String, + pub predecessor_content_hash: Option, +} + +/// A produced archive, ready to deliver. +#[derive(Debug)] +pub struct Export { + /// The documented download name, `consolebook--.zip`. + pub file_name: String, + pub bytes: Vec, + pub exported_at: i64, + pub unit_count: usize, +} + +/// The unit directory for one version: `records/{record_id}/v{n}`. +#[must_use] +pub fn unit_path(record_id: i64, version_number: i64) -> String { + format!("records/{record_id}/v{version_number}") +} + +// ------------------------------------------------------------ producing + +/// Exports `scope` now, for an actor the scope's read rule admits. +pub async fn export( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, +) -> Result> { + export_at( + pool, + actor_user_id, + scope, + OffsetDateTime::now_utc().unix_timestamp(), + ) + .await +} + +/// Exports `scope` stamped with `exported_at` (UTC unix seconds). The +/// archive is a pure function of the scope's rows and this instant. +pub async fn export_at( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, + exported_at: i64, +) -> Result> { + let audited = match authorize(pool, actor_user_id, scope).await? { + Ok(audited) => audited, + Err(refusal) => return Ok(Err(refusal)), + }; + let rows = collect(pool, scope).await?; + if rows.is_empty() { + return Ok(Err(match scope { + Scope::Version { .. } => ExportRefusal::NoSuchVersion, + Scope::Record { .. } | Scope::Enrollment { .. } | Scope::Installation => { + ExportRefusal::NothingToExport + } + })); + } + let installation_id = storage::installation_id(pool).await?; + let unit_count = rows.len(); + let bytes = build_archive(&installation_id, exported_at, scope, rows)?; + // The export is audited once it exists: actor and subject, never + // content (docs/records-integrity.md). + match audited.subject { + Some(subject) => { + audit::record_for_subject( + pool, + EventKind::RecordExported, + Some(actor_user_id), + audited.trainee, + subject, + ) + .await?; + } + None => audit::record(pool, EventKind::RecordExported, Some(actor_user_id), None).await?, + } + Ok(Ok(Export { + file_name: file_name(scope, exported_at)?, + bytes, + exported_at, + unit_count, + })) +} + +/// Counts for the installation-export interface, for `export_records` +/// holders. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExportSummary { + pub installation_id: String, + pub record_count: i64, + pub version_count: i64, +} + +/// How many finalized records and versions an installation export +/// would carry. +pub async fn summary( + pool: &SqlitePool, + actor_user_id: i64, +) -> Result> { + if !capabilities::user_has(pool, actor_user_id, Capability::ExportRecords).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + let (record_count, version_count): (i64, i64) = sqlx::query_as( + "SELECT COUNT(DISTINCT evaluation_record_id), COUNT(*) FROM evaluation_version", + ) + .fetch_one(pool) + .await + .context("counting finalized versions")?; + Ok(Ok(ExportSummary { + installation_id: storage::installation_id(pool).await?, + record_count, + version_count, + })) +} + +/// What the audit event names once the export exists. +struct Audited { + subject: Option, + trainee: Option, +} + +/// The scope's read rule, as the typed contract it already is elsewhere +/// (ADR 0010): the record read rule for a version or record, the +/// training-history read rule for an enrollment, and the explicit +/// `export_records` authority for the installation. +async fn authorize( + pool: &SqlitePool, + actor_user_id: i64, + scope: Scope, +) -> Result> { + match scope { + Scope::Version { record_id, .. } | Scope::Record { record_id } => { + let mut conn = pool.acquire().await.context("acquiring connection")?; + let Some(record) = evaluation_drafts::load_record(&mut conn, record_id).await? else { + return Ok(Err(ExportRefusal::NoSuchRecord)); + }; + drop(conn); + if !crate::draft_access::may_read(pool, actor_user_id, &record).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + let trainee: i64 = sqlx::query_scalar("SELECT user_id FROM enrollment WHERE id = ?1") + .bind(record.enrollment_id) + .fetch_one(pool) + .await + .context("reading enrollment")?; + Ok(Ok(Audited { + subject: Some(Subject::Record(record_id)), + trainee: Some(trainee), + })) + } + Scope::Enrollment { enrollment_id } => { + let trainee: Option = + sqlx::query_scalar("SELECT user_id FROM enrollment WHERE id = ?1") + .bind(enrollment_id) + .fetch_optional(pool) + .await + .context("reading enrollment")?; + let Some(trainee) = trainee else { + return Ok(Err(ExportRefusal::NoSuchEnrollment)); + }; + if !lifecycle::may_read(pool, actor_user_id, enrollment_id).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + Ok(Ok(Audited { + subject: Some(Subject::Enrollment(enrollment_id)), + trainee: Some(trainee), + })) + } + Scope::Installation => { + if !capabilities::user_has(pool, actor_user_id, Capability::ExportRecords).await? { + return Ok(Err(ExportRefusal::CapabilityRequired)); + } + Ok(Ok(Audited { + subject: None, + trainee: None, + })) + } + } +} + +/// One stored version, exactly as the archive carries it. +struct VersionRow { + record_id: i64, + version_number: i64, + record_schema: i64, + bytes: Vec, + content_hash: String, + chain_hash: String, + predecessor_content_hash: Option, +} + +/// The stored rows of a scope in archive order: ascending record id, +/// then version number. The predecessor's content hash is read from +/// its own row, so the manifest states what the chain hash was +/// computed over (ADR 0011). +macro_rules! unit_query { + ($where:literal) => { + concat!( + "SELECT v.evaluation_record_id AS record_id, v.version_number, + v.record_schema, v.canonical_bytes, v.content_hash, + v.chain_hash, p.content_hash AS predecessor_content_hash + FROM evaluation_version v + LEFT JOIN evaluation_version p ON p.id = v.predecessor_id + JOIN evaluation_record r ON r.id = v.evaluation_record_id ", + $where, + " ORDER BY v.evaluation_record_id, v.version_number" + ) + }; +} + +async fn collect(pool: &SqlitePool, scope: Scope) -> Result> { + let rows = match scope { + Scope::Version { + record_id, + version_number, + } => { + sqlx::query(unit_query!( + "WHERE v.evaluation_record_id = ?1 AND v.version_number = ?2" + )) + .bind(record_id) + .bind(version_number) + .fetch_all(pool) + .await + } + Scope::Record { record_id } => { + sqlx::query(unit_query!("WHERE v.evaluation_record_id = ?1")) + .bind(record_id) + .fetch_all(pool) + .await + } + Scope::Enrollment { enrollment_id } => { + sqlx::query(unit_query!("WHERE r.enrollment_id = ?1")) + .bind(enrollment_id) + .fetch_all(pool) + .await + } + Scope::Installation => sqlx::query(unit_query!("")).fetch_all(pool).await, + } + .context("reading finalized versions")?; + Ok(rows + .iter() + .map(|row| VersionRow { + record_id: row.get("record_id"), + version_number: row.get("version_number"), + record_schema: row.get("record_schema"), + bytes: row.get("canonical_bytes"), + content_hash: row.get("content_hash"), + chain_hash: row.get("chain_hash"), + predecessor_content_hash: row.get("predecessor_content_hash"), + }) + .collect()) +} + +/// Writes the container exactly as docs/formats/record-export.md lays +/// it out: manifest first, then units in order, stored entries, the +/// export instant as every entry's modification time, `0644`. Rows are +/// consumed so each version's bytes are released once written; the +/// archive is the one copy held to the end (#47 tracks streaming it). +fn build_archive( + installation_id: &str, + exported_at: i64, + scope: Scope, + rows: Vec, +) -> Result> { + let units: Vec = rows + .iter() + .map(|row| UnitEntry { + path: unit_path(row.record_id, row.version_number), + record_id: row.record_id, + version_number: row.version_number, + record_schema: row.record_schema, + content_hash: row.content_hash.clone(), + chain_hash: row.chain_hash.clone(), + predecessor_content_hash: row.predecessor_content_hash.clone(), + }) + .collect(); + let manifest = ArchiveManifest { + format: ARCHIVE_FORMAT.to_owned(), + format_version: FORMAT_VERSION, + installation_id: installation_id.to_owned(), + exported_at, + scope, + units, + }; + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .last_modified_time(dos_time(exported_at)?) + .unix_permissions(0o644); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + add_entry( + &mut writer, + ARCHIVE_MANIFEST_PATH, + &canonical_json(&manifest)?, + options, + )?; + for (row, entry) in rows.into_iter().zip(&manifest.units) { + add_entry( + &mut writer, + &format!("{}/{RECORD_FILE}", entry.path), + &row.bytes, + options, + )?; + let unit = UnitManifest { + format: UNIT_FORMAT.to_owned(), + format_version: FORMAT_VERSION, + installation_id: installation_id.to_owned(), + exported_at, + record_id: entry.record_id, + version_number: entry.version_number, + record_schema: entry.record_schema, + content_hash: entry.content_hash.clone(), + chain_hash: entry.chain_hash.clone(), + predecessor_content_hash: entry.predecessor_content_hash.clone(), + }; + add_entry( + &mut writer, + &format!("{}/{UNIT_MANIFEST_FILE}", entry.path), + &canonical_json(&unit)?, + options, + )?; + } + let cursor = writer.finish().context("finishing the export archive")?; + Ok(cursor.into_inner()) +} + +fn add_entry( + writer: &mut ZipWriter>>, + name: &str, + bytes: &[u8], + options: SimpleFileOptions, +) -> Result<()> { + writer + .start_file(name, options) + .with_context(|| format!("starting archive entry {name}"))?; + writer + .write_all(bytes) + .with_context(|| format!("writing archive entry {name}"))?; + Ok(()) +} + +/// Manifests are canonical JSON under the record format's subset, so +/// the archive is deterministic and a manifest is itself checkable. +pub(crate) fn canonical_json(value: &T) -> Result> { + let value = serde_json::to_value(value).context("serializing manifest")?; + canonical::canonical_bytes(&value) +} + +fn dos_time(exported_at: i64) -> Result { + let at = OffsetDateTime::from_unix_timestamp(exported_at).context("export instant")?; + zip::DateTime::from_date_and_time( + u16::try_from(at.year()).context("export year")?, + u8::from(at.month()), + at.day(), + at.hour(), + at.minute(), + at.second(), + ) + .map_err(|_| anyhow!("export instant {exported_at} is outside the ZIP date range")) +} + +fn file_name(scope: Scope, exported_at: i64) -> Result { + let stamp = OffsetDateTime::from_unix_timestamp(exported_at) + .context("export instant")? + .format(&time::macros::format_description!( + "[year][month][day]T[hour][minute][second]Z" + )) + .context("formatting export instant")?; + let scope_part = match scope { + Scope::Version { + record_id, + version_number, + } => format!("record-{record_id}-v{version_number}"), + Scope::Record { record_id } => format!("record-{record_id}"), + Scope::Enrollment { enrollment_id } => format!("enrollment-{enrollment_id}"), + Scope::Installation => "installation".to_owned(), + }; + Ok(format!("consolebook-{scope_part}-{stamp}.zip")) +} diff --git a/crates/consolebook-server/tests/record_export.rs b/crates/consolebook-server/tests/record_export.rs new file mode 100644 index 0000000..1f5e4c0 --- /dev/null +++ b/crates/consolebook-server/tests/record_export.rs @@ -0,0 +1,1576 @@ +//! Milestone 5 slice 1: structured record exports — the stored canonical +//! bytes travel verbatim beside manifests, archives are deterministic, +//! verification needs only the archive and names every finding, scopes +//! follow the read rules that already exist, the API delivers the +//! documented bytes, and the CLI verifier works from the file alone. +//! Every fixture is invented. + +use std::io::{Cursor, Read, Write}; + +use axum::body::Body; +use axum::http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, COOKIE, SET_COOKIE}; +use axum::http::{HeaderMap, Request, StatusCode}; +use consolebook_server::capabilities::RoleBundle; +use consolebook_server::draft_content::{self, DraftContent, NarrativeEntry, RatingEntry}; +use consolebook_server::export_verify::{self, Finding, PredecessorLink, UnitReport}; +use consolebook_server::programs::{ + self, AnchorDef, CompetencyDef, FormCompetencyDef, FormDef, NarrativeDef, PolicyDef, + RecordType, ScaleDef, ScaleKind, VersionContent, +}; +use consolebook_server::record_export::{ + self, ARCHIVE_FORMAT, ARCHIVE_MANIFEST_PATH, ArchiveManifest, ExportRefusal, FORMAT_VERSION, + Scope, UNIT_FORMAT, UnitManifest, +}; +use consolebook_server::training_sessions::{self, Disposition, SessionInput}; +use consolebook_server::{ + amendments, assignments, canonical, data_dir::DataDir, enrollments, evaluation_drafts, + finalization, setup, storage, users, +}; +use http_body_util::BodyExt; +use tower::ServiceExt; +use zip::write::{SimpleFileOptions, ZipWriter}; + +const PASSWORD: &str = "invented-passphrase-1"; + +/// 2026-09-01T19:00:00Z, the instant every deterministic export below +/// is stamped with. +const EXPORTED_AT: i64 = 1_788_289_200; + +const OPEN_POLICY: PolicyDef = PolicyDef { + review_approved: false, + required_narratives: false, + ratings_complete: false, +}; + +struct Fixture { + _tmp: tempfile::TempDir, + pool: sqlx::SqlitePool, + admin_id: i64, +} + +impl Fixture { + async fn new() -> Self { + let tmp = tempfile::tempdir().expect("create temp dir"); + let data_dir = DataDir::new(tmp.path().join("data")); + data_dir.ensure_layout().expect("create layout"); + let pool = storage::open(&data_dir.database()).await.expect("open"); + let code = setup::issue_setup_code(&pool) + .await + .expect("issue") + .expect("uninitialized") + .0; + let admin_id = setup::initialize( + &pool, + &code.raw, + "Example County Communications", + "avery.admin", + "Avery Admin", + PASSWORD, + ) + .await + .expect("initialize") + .expect("accepted"); + Self { + _tmp: tmp, + pool, + admin_id, + } + } + + fn app(&self) -> axum::Router { + consolebook_server::http::router(consolebook_server::http::AppState { + pool: self.pool.clone(), + }) + } + + async fn login(&self, username: &str) -> String { + let response = self + .app() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/auth/login") + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::json!({ "username": username, "password": PASSWORD }) + .to_string(), + )) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(response.status(), StatusCode::OK, "login {username}"); + let cookie = response + .headers() + .get(SET_COOKIE) + .expect("cookie") + .to_str() + .expect("ascii"); + let (pair, _) = cookie.split_once(';').expect("attrs"); + pair.split_once('=').expect("pair").1.to_string() + } + + async fn user_with_role(&self, username: &str, display_name: &str, role: RoleBundle) -> i64 { + let created = users::create_with_reset_code( + &self.pool, + self.admin_id, + username, + display_name, + "", + "", + role, + ) + .await + .expect("create") + .expect("accepted"); + assert_eq!( + users::use_reset_code(&self.pool, username, &created.reset_code.raw, PASSWORD) + .await + .expect("reset"), + users::ResetOutcome::Done + ); + created.id + } + + /// The stored row of one version: bytes and both hashes. + async fn version_row(&self, record_id: i64, number: i64) -> (Vec, String, String) { + sqlx::query_as( + "SELECT canonical_bytes, content_hash, chain_hash FROM evaluation_version + WHERE evaluation_record_id = ?1 AND version_number = ?2", + ) + .bind(record_id) + .bind(number) + .fetch_one(&self.pool) + .await + .expect("version row") + } + + async fn audit_count(&self, subject_kind: Option<&str>) -> i64 { + sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_event + WHERE kind = 'record_exported' AND subject_kind IS ?1", + ) + .bind(subject_kind) + .fetch_one(&self.pool) + .await + .expect("count") + } +} + +/// A GET whose body is delivered raw, for downloads. +async fn raw_get(app: axum::Router, uri: &str, cookie: &str) -> (StatusCode, HeaderMap, Vec) { + let response = app + .oneshot( + Request::builder() + .uri(uri) + .header( + COOKIE, + format!("{}={}", consolebook_server::http::SESSION_COOKIE, cookie), + ) + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + let status = response.status(); + let headers = response.headers().clone(); + let bytes = response + .into_body() + .collect() + .await + .expect("body") + .to_bytes() + .to_vec(); + (status, headers, bytes) +} + +/// Invented single-form program with every completion rule off, so +/// records seal from the working copy as soon as they are authored. +fn program(name: &str) -> VersionContent { + VersionContent { + name: name.to_owned(), + label: "2026 rev A".to_owned(), + description: "Invented program for export tests.".to_owned(), + phases: Vec::new(), + phase_transitions: Vec::new(), + competencies: vec![CompetencyDef { + category: "Call processing".to_owned(), + name: "Emergency Call Interrogation".to_owned(), + description: "Obtains and verifies location, callback, and nature.".to_owned(), + tasks: Vec::new(), + citations: Vec::new(), + }], + rating_scales: vec![ScaleDef { + name: "Standard 1-7".to_owned(), + kind: ScaleKind::AnchoredNumeric, + min_value: Some(1), + max_value: Some(7), + anchors: vec![AnchorDef { + value: 4, + label: "Meets standards".to_owned(), + definition: "To the invented standard.".to_owned(), + }], + }], + rating_modifiers: Vec::new(), + evaluation_forms: vec![FormDef { + record_type: RecordType::DailyReport, + name: "Daily Observation Report".to_owned(), + instructions: "Rate observed performance.".to_owned(), + competencies: vec![FormCompetencyDef { + competency: "Emergency Call Interrogation".to_owned(), + rating_scale: "Standard 1-7".to_owned(), + }], + narratives: vec![NarrativeDef { + prompt: "Most acceptable performance.".to_owned(), + required: false, + }], + }], + citations: Vec::new(), + finalization_policy: OPEN_POLICY, + } +} + +#[allow(clippy::struct_field_names)] +struct Seeded { + version_id: i64, + enrollment_id: i64, + record_id: i64, + taylor_id: i64, + jordan_id: i64, + casey_id: i64, +} + +/// A published program, an enrolled trainee, an assigned trainer, a +/// coordinator, and one daily record sealed twice: version 1, then an +/// amendment sealed as version 2 chained to it. +async fn seed(fx: &Fixture, suffix: &str) -> Seeded { + let content = program(&format!("Example County Program {suffix}")); + let program_id = programs::create_program(&fx.pool, fx.admin_id, &content.name) + .await + .expect("create program") + .expect("accepted"); + let version_id = programs::create_version(&fx.pool, fx.admin_id, program_id, &content) + .await + .expect("create version") + .expect("accepted"); + programs::publish_version(&fx.pool, fx.admin_id, version_id) + .await + .expect("publish") + .expect("accepted"); + let taylor_id = fx + .user_with_role( + &format!("taylor.{suffix}"), + "Taylor Trainee", + RoleBundle::Trainee, + ) + .await; + let jordan_id = fx + .user_with_role( + &format!("jordan.{suffix}"), + "Jordan Trainer", + RoleBundle::Trainer, + ) + .await; + let casey_id = fx + .user_with_role( + &format!("casey.{suffix}"), + "Casey Coordinator", + RoleBundle::Coordinator, + ) + .await; + let enrollment_id = enrollments::enroll(&fx.pool, fx.admin_id, version_id, taylor_id) + .await + .expect("call") + .expect("enrolled"); + assignments::create(&fx.pool, fx.admin_id, enrollment_id, jordan_id) + .await + .expect("call") + .expect("assigned"); + let record_id = draft_for(fx, jordan_id, enrollment_id, "2026-06-02").await; + let s = Seeded { + version_id, + enrollment_id, + record_id, + taylor_id, + jordan_id, + casey_id, + }; + seal_twice(fx, &s).await; + s +} + +async fn draft_for(fx: &Fixture, trainer_id: i64, enrollment_id: i64, date: &str) -> i64 { + let session_id = training_sessions::create( + &fx.pool, + trainer_id, + enrollment_id, + &SessionInput { + business_date: date.to_owned(), + timezone: "America/Chicago".to_owned(), + local_start: format!("{date}T07:00"), + local_end: Some(format!("{date}T15:00")), + disposition: Some(Disposition::Completed), + phase_id: None, + trainer_user_ids: Vec::new(), + }, + ) + .await + .expect("call") + .expect("created"); + evaluation_drafts::create(&fx.pool, trainer_id, session_id, None) + .await + .expect("call") + .expect("created") +} + +async fn author_and_seal(fx: &Fixture, s: &Seeded, value: i64, text: &str) { + let workspace = evaluation_drafts::workspace(&fx.pool, s.jordan_id, s.record_id) + .await + .expect("call") + .expect("readable"); + let revision = draft_content::save( + &fx.pool, + s.jordan_id, + s.record_id, + workspace.detail.revision, + &DraftContent { + ratings: vec![RatingEntry { + form_competency_id: workspace.form.competencies[0].form_competency_id, + value: Some(value), + not_observed: false, + modifier_ids: Vec::new(), + }], + narratives: vec![NarrativeEntry { + form_narrative_id: workspace.form.narratives[0].form_narrative_id, + text: text.to_owned(), + }], + }, + ) + .await + .expect("call") + .expect("saved"); + finalization::finalize(&fx.pool, s.casey_id, s.record_id, revision) + .await + .expect("call") + .expect("sealed"); +} + +async fn seal_twice(fx: &Fixture, s: &Seeded) { + author_and_seal(fx, s, 3, "The invented initial entry.").await; + amendments::open( + &fx.pool, + s.casey_id, + s.record_id, + "The invented rating was entered one point low.", + ) + .await + .expect("call") + .expect("opened"); + author_and_seal(fx, s, 4, "Corrected the invented rating with context.").await; +} + +async fn export(fx: &Fixture, actor: i64, scope: Scope) -> Vec { + record_export::export_at(&fx.pool, actor, scope, EXPORTED_AT) + .await + .expect("call") + .expect("exported") + .bytes +} + +async fn refusal(fx: &Fixture, actor: i64, scope: Scope) -> ExportRefusal { + record_export::export_at(&fx.pool, actor, scope, EXPORTED_AT) + .await + .expect("call") + .expect_err("refused") +} + +/// Every entry of an archive in container order. +fn entries(bytes: &[u8]) -> Vec<(String, Vec)> { + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).expect("zip"); + let mut out = Vec::new(); + for index in 0..archive.len() { + let mut file = archive.by_index(index).expect("entry"); + let mut content = Vec::new(); + file.read_to_end(&mut content).expect("read"); + out.push((file.name().to_owned(), content)); + } + out +} + +/// A well-formed container holding exactly `entries`: the way a +/// tamperer who understands ZIP would repack an export. +fn repack(entries: &[(String, Vec)]) -> Vec { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (name, content) in entries { + writer.start_file(name.as_str(), options).expect("start"); + writer.write_all(content).expect("write"); + } + writer.finish().expect("finish").into_inner() +} + +/// CRC-32 (IEEE), bitwise, for the hand-assembled container below. +fn crc32(bytes: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFF_u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc +} + +fn le16(out: &mut Vec, value: usize) { + out.extend_from_slice(&u16::try_from(value).expect("fits u16").to_le_bytes()); +} + +fn le32(out: &mut Vec, value: usize) { + out.extend_from_slice(&u32::try_from(value).expect("fits u32").to_le_bytes()); +} + +/// A ZIP assembled by hand (APPNOTE 6.3, stored entries) so a name can +/// be written twice — the `zip` writer refuses to, and the `zip` reader +/// keeps one entry per name, which is exactly what the verifier must +/// see through. +fn handmade_zip(entries: &[(&str, &[u8])]) -> Vec { + let mut out = Vec::new(); + let mut central = Vec::new(); + for (name, data) in entries { + let offset = out.len(); + let crc = crc32(data); + // Local file header. + out.extend_from_slice(&[0x50, 0x4b, 0x03, 0x04]); + le16(&mut out, 20); // version needed + le16(&mut out, 0); // flags + le16(&mut out, 0); // stored + le16(&mut out, 0); // time + le16(&mut out, 0x21); // date: 1980-01-01 + out.extend_from_slice(&crc.to_le_bytes()); + le32(&mut out, data.len()); + le32(&mut out, data.len()); + le16(&mut out, name.len()); + le16(&mut out, 0); // extra + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(data); + // Central directory header. + central.extend_from_slice(&[0x50, 0x4b, 0x01, 0x02]); + le16(&mut central, 0x031E); // made by: Unix, 3.0 + le16(&mut central, 20); + le16(&mut central, 0); + le16(&mut central, 0); + le16(&mut central, 0); + le16(&mut central, 0x21); + central.extend_from_slice(&crc.to_le_bytes()); + le32(&mut central, data.len()); + le32(&mut central, data.len()); + le16(&mut central, name.len()); + le16(&mut central, 0); // extra + le16(&mut central, 0); // comment + le16(&mut central, 0); // disk + le16(&mut central, 0); // internal attributes + central.extend_from_slice(&(0o100_644_u32 << 16).to_le_bytes()); + le32(&mut central, offset); + central.extend_from_slice(name.as_bytes()); + } + let directory_offset = out.len(); + out.extend_from_slice(¢ral); + // End of central directory. + out.extend_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + le16(&mut out, 0); + le16(&mut out, 0); + le16(&mut out, entries.len()); + le16(&mut out, entries.len()); + le32(&mut out, central.len()); + le32(&mut out, directory_offset); + le16(&mut out, 0); + out +} + +/// A single-unit archive whose record bytes are replaced and whose +/// manifests are rehashed and relabeled to match: the honest way to +/// forge a unit, leaving only the envelope's own shape to object. +fn rehashed_unit( + listed: &[(String, Vec)], + unit: &str, + bytes: &[u8], + predecessor: Option<&str>, + record_schema: i64, +) -> Vec { + let content_hash = canonical::content_hash_hex(bytes); + let chain_hash = canonical::chain_hash_hex(predecessor, bytes).expect("chain"); + let relabel = |manifest: &mut serde_json::Value| { + manifest["content_hash"] = serde_json::Value::String(content_hash.clone()); + manifest["chain_hash"] = serde_json::Value::String(chain_hash.clone()); + manifest["record_schema"] = serde_json::Value::from(record_schema); + manifest["predecessor_content_hash"] = predecessor + .map_or(serde_json::Value::Null, |hash| { + serde_json::Value::String(hash.to_owned()) + }); + }; + let entries: Vec<(String, Vec)> = listed + .iter() + .map(|(name, content)| { + let content = if *name == format!("{unit}/record.json") { + bytes.to_vec() + } else if name == ARCHIVE_MANIFEST_PATH { + edit_json(content, |manifest| relabel(&mut manifest["units"][0])) + } else if *name == format!("{unit}/manifest.json") { + edit_json(content, relabel) + } else { + content.clone() + }; + (name.clone(), content) + }) + .collect(); + repack(&entries) +} + +fn edit_json(bytes: &[u8], edit: impl FnOnce(&mut serde_json::Value)) -> Vec { + let mut value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); + edit(&mut value); + canonical::canonical_bytes(&value).expect("canonical") +} + +fn entry<'a>(entries: &'a [(String, Vec)], name: &str) -> &'a [u8] { + &entries + .iter() + .find(|(entry_name, _)| entry_name == name) + .unwrap_or_else(|| panic!("entry {name}")) + .1 +} + +fn with_entry( + entries: &[(String, Vec)], + name: &str, + replace: impl FnOnce(&[u8]) -> Option>, +) -> Vec<(String, Vec)> { + let mut replace = Some(replace); + entries + .iter() + .filter_map(|(entry_name, content)| { + if entry_name == name { + replace.take().expect("one edit")(content) + } else { + Some(content.clone()) + } + .map(|content| (entry_name.clone(), content)) + }) + .collect() +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn exports_carry_stored_bytes_verbatim_and_verify() { + let fx = Fixture::new().await; + let s = seed(&fx, "verbatim").await; + let installation_id = storage::installation_id(&fx.pool).await.expect("id"); + let (v1_bytes, v1_content, v1_chain) = fx.version_row(s.record_id, 1).await; + let (v2_bytes, v2_content, v2_chain) = fx.version_row(s.record_id, 2).await; + + // One version: the archive lays out exactly the documented entries, + // the record bytes are the stored bytes, and both manifests carry + // the stored identity and fingerprints. + let scope = Scope::Version { + record_id: s.record_id, + version_number: 2, + }; + let exported = record_export::export_at(&fx.pool, s.casey_id, scope, EXPORTED_AT) + .await + .expect("call") + .expect("exported"); + assert_eq!( + exported.file_name, + format!("consolebook-record-{}-v2-20260901T190000Z.zip", s.record_id) + ); + assert_eq!(exported.unit_count, 1); + let unit_dir = format!("records/{}/v2", s.record_id); + let listed = entries(&exported.bytes); + let names: Vec<&str> = listed.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!( + names, + [ + ARCHIVE_MANIFEST_PATH.to_owned(), + format!("{unit_dir}/record.json"), + format!("{unit_dir}/manifest.json"), + ] + ); + assert_eq!( + entry(&listed, &format!("{unit_dir}/record.json")), + v2_bytes.as_slice(), + "the record bytes are the stored bytes, not a re-serialization" + ); + let manifest_bytes = entry(&listed, ARCHIVE_MANIFEST_PATH); + let manifest: ArchiveManifest = serde_json::from_slice(manifest_bytes).expect("manifest"); + assert_eq!(manifest.format, ARCHIVE_FORMAT); + assert_eq!(manifest.format_version, FORMAT_VERSION); + assert_eq!(manifest.installation_id, installation_id); + assert_eq!(manifest.exported_at, EXPORTED_AT); + assert_eq!(manifest.scope, scope); + assert_eq!(manifest.units.len(), 1); + let unit = &manifest.units[0]; + assert_eq!(unit.path, unit_dir); + assert_eq!(unit.record_id, s.record_id); + assert_eq!(unit.version_number, 2); + assert_eq!(unit.record_schema, canonical::RECORD_SCHEMA); + assert_eq!(unit.content_hash, v2_content); + assert_eq!(unit.chain_hash, v2_chain); + assert_eq!( + unit.predecessor_content_hash.as_deref(), + Some(v1_content.as_str()) + ); + // Manifests are canonical JSON: members sorted, compact. + let manifest_value: serde_json::Value = serde_json::from_slice(manifest_bytes).expect("json"); + assert_eq!( + canonical::canonical_bytes(&manifest_value).expect("canonical"), + manifest_bytes + ); + let unit_manifest: UnitManifest = + serde_json::from_slice(entry(&listed, &format!("{unit_dir}/manifest.json"))) + .expect("unit manifest"); + assert_eq!(unit_manifest.format, UNIT_FORMAT); + assert_eq!(unit_manifest.format_version, FORMAT_VERSION); + assert_eq!(unit_manifest.installation_id, installation_id); + assert_eq!(unit_manifest.exported_at, EXPORTED_AT); + assert_eq!(unit_manifest.record_id, s.record_id); + assert_eq!(unit_manifest.version_number, 2); + assert_eq!(unit_manifest.content_hash, v2_content); + assert_eq!(unit_manifest.chain_hash, v2_chain); + assert_eq!( + unit_manifest.predecessor_content_hash.as_deref(), + Some(v1_content.as_str()) + ); + + // Verification from the archive alone: the chain hash recomputes + // from the carried predecessor hash, and the predecessor itself is + // honestly reported as not in this export. + let report = export_verify::verify_archive(&exported.bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!( + report.installation_id.as_deref(), + Some(installation_id.as_str()) + ); + assert_eq!(report.exported_at, Some(EXPORTED_AT)); + assert_eq!(report.scope, Some(scope)); + assert_eq!(report.units.len(), 1); + assert_eq!(report.units[0].predecessor, PredecessorLink::NotInExport); + + // Deterministic: the same scope at the same instant is byte-identical. + let again = export(&fx, s.casey_id, scope).await; + assert_eq!(again, exported.bytes); + let later = export(&fx, s.casey_id, scope).await; + assert_eq!(later, exported.bytes); + + // The whole record: both versions in order, the successor linked to + // its predecessor within the archive. + let record_scope = Scope::Record { + record_id: s.record_id, + }; + let bytes = export(&fx, s.casey_id, record_scope).await; + let listed = entries(&bytes); + assert_eq!(listed.len(), 5); + assert_eq!( + entry(&listed, &format!("records/{}/v1/record.json", s.record_id)), + v1_bytes.as_slice() + ); + assert_eq!( + entry(&listed, &format!("records/{}/v2/record.json", s.record_id)), + v2_bytes.as_slice() + ); + let manifest: ArchiveManifest = + serde_json::from_slice(entry(&listed, ARCHIVE_MANIFEST_PATH)).expect("manifest"); + assert_eq!(manifest.scope, record_scope); + assert_eq!(manifest.units.len(), 2); + assert_eq!(manifest.units[0].version_number, 1); + assert_eq!(manifest.units[0].chain_hash, v1_chain); + assert_eq!(manifest.units[0].predecessor_content_hash, None); + assert_eq!(manifest.units[1].version_number, 2); + let report = export_verify::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units[0].predecessor, PredecessorLink::None); + assert_eq!(report.units[1].predecessor, PredecessorLink::Linked); + + // Every export is audited with its subject and never with content. + assert_eq!(fx.audit_count(Some("record")).await, 4); + let bytes = export( + &fx, + s.jordan_id, + Scope::Enrollment { + enrollment_id: s.enrollment_id, + }, + ) + .await; + assert!(export_verify::verify_archive(&bytes).verified()); + assert_eq!(fx.audit_count(Some("enrollment")).await, 1); + let bytes = export(&fx, fx.admin_id, Scope::Installation).await; + let report = export_verify::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.scope, Some(Scope::Installation)); + assert_eq!(report.units.len(), 2); + assert_eq!(fx.audit_count(None).await, 1); + let with_content: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM audit_event WHERE kind = 'record_exported' + AND (subject_user_id IS NULL) != (subject_kind IS NULL)", + ) + .fetch_one(&fx.pool) + .await + .expect("count"); + assert_eq!(with_content, 0, "subject and trainee travel together"); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn verification_names_every_finding() { + let fx = Fixture::new().await; + let s = seed(&fx, "findings").await; + let original = export( + &fx, + s.casey_id, + Scope::Record { + record_id: s.record_id, + }, + ) + .await; + let listed = entries(&original); + let v1 = format!("records/{}/v1", s.record_id); + let v2 = format!("records/{}/v2", s.record_id); + let (v1_bytes, _, _) = fx.version_row(s.record_id, 1).await; + + // Not an archive at all. + let report = export_verify::verify_archive(b"not a zip archive"); + assert!(!report.verified()); + assert!(matches!(report.findings[0], Finding::NotAnArchive { .. })); + + // Repacked with altered record content: the content hash no longer + // covers the bytes, and neither does the chain hash. + let altered = repack(&with_entry( + &listed, + &format!("{v2}/record.json"), + |bytes| { + let text = String::from_utf8(bytes.to_vec()).expect("utf-8"); + Some( + text.replace("Corrected the invented", "Corrected the inveNted") + .into_bytes(), + ) + }, + )); + let report = export_verify::verify_archive(&altered); + assert!(!report.verified()); + assert!(report.findings.is_empty(), "{report:?}"); + assert!(report.units[0].verified()); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::ChainHashMismatch), + "{findings:?}" + ); + assert!( + !findings + .iter() + .any(|f| matches!(f, Finding::NotCanonical { .. })), + "the altered text is still canonical JSON: {findings:?}" + ); + + // A byte flipped inside the container without repacking fails too: + // the container's own checksum or the content hash catches it. + let mut flipped = original.clone(); + let needle = b"Corrected the invented"; + let at = flipped + .windows(needle.len()) + .position(|window| window == needle) + .expect("record text in the archive"); + flipped[at] ^= 0x20; + assert!(!export_verify::verify_archive(&flipped).verified()); + + // The archive manifest claims a different content hash for version + // 1: the bytes disagree, the unit manifest disagrees, and version + // 2's predecessor no longer matches the version 1 in the archive. + let other_hash = "0".repeat(64); + let swapped = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][0]["content_hash"] = serde_json::Value::String(other_hash.clone()); + })) + })); + let report = export_verify::verify_archive(&swapped); + assert!(!report.verified()); + assert!(report.findings.is_empty(), "{report:?}"); + let findings = &report.units[0].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::UnitManifestDisagrees { + member: "content_hash" + }), + "{findings:?}" + ); + assert!( + !findings.contains(&Finding::ChainHashMismatch), + "the chain hash still covers the bytes: {findings:?}" + ); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::PredecessorMismatch), + "{findings:?}" + ); + assert_eq!(report.units[1].predecessor, PredecessorLink::Linked); + + // A unit's record bytes replaced by another version's: every + // identity member the envelope itself carries disagrees. + let replaced = repack(&with_entry(&listed, &format!("{v2}/record.json"), |_| { + Some(v1_bytes.clone()) + })); + let report = export_verify::verify_archive(&replaced); + let findings = &report.units[1].findings; + assert!( + findings.contains(&Finding::ContentHashMismatch), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::EnvelopeDisagrees { + member: "record.version_number" + }), + "{findings:?}" + ); + assert!( + findings.contains(&Finding::EnvelopeDisagrees { + member: "record.predecessor_content_hash" + }), + "{findings:?}" + ); + + // Missing and unlisted entries. + let missing = repack(&with_entry(&listed, &format!("{v1}/record.json"), |_| None)); + let report = export_verify::verify_archive(&missing); + assert!( + report.units[0].findings.contains(&Finding::MissingEntry { + path: format!("{v1}/record.json") + }), + "{report:?}" + ); + let mut extra = listed.clone(); + extra.push(("README.txt".to_owned(), b"nothing to see here".to_vec())); + let report = export_verify::verify_archive(&repack(&extra)); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::UnlistedEntry { + path: "README.txt".to_owned() + }), + "{report:?}" + ); + assert!(report.units.iter().all(UnitReport::verified)); + + // A manifest that parses but is not canonical (pretty-printed) is a + // finding: the format fixes the bytes, not only the members. + let pretty = repack(&with_entry( + &listed, + &format!("{v2}/manifest.json"), + |bytes| { + let value: serde_json::Value = serde_json::from_slice(bytes).expect("json"); + Some(serde_json::to_vec_pretty(&value).expect("pretty")) + }, + )); + let report = export_verify::verify_archive(&pretty); + assert!( + report.units[1] + .findings + .contains(&Finding::ManifestNotCanonical { + path: format!("{v2}/manifest.json") + }), + "{report:?}" + ); + + // An unknown format version is refused by name, not guessed at. + let future = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["format_version"] = serde_json::Value::from(2); + })) + })); + let report = export_verify::verify_archive(&future); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::UnsupportedFormat { + format: ARCHIVE_FORMAT.to_owned(), + format_version: 2 + }), + "{report:?}" + ); + assert!(report.units.is_empty()); + + // Units out of order, with paths that do not derive from identity. + let reordered = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + let units = manifest["units"].as_array_mut().expect("units"); + units.swap(0, 1); + })) + })); + let report = export_verify::verify_archive(&reordered); + assert!( + report.findings.contains(&Finding::UnitsOutOfOrder), + "{report:?}" + ); + let renamed = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][1]["path"] = serde_json::Value::String("records/elsewhere".into()); + })) + })); + let report = export_verify::verify_archive(&renamed); + assert!( + report.findings.contains(&Finding::UnitPathUnexpected { + path: "records/elsewhere".to_owned(), + expected: v2.clone() + }), + "{report:?}" + ); + + // Lineage shape: a version 2 claiming no predecessor. + let orphan = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"][1]["predecessor_content_hash"] = serde_json::Value::Null; + })) + })); + let report = export_verify::verify_archive(&orphan); + let findings = &report.units[1].findings; + assert!(findings.contains(&Finding::LineageShape), "{findings:?}"); + assert!( + findings.contains(&Finding::ChainHashMismatch), + "{findings:?}" + ); + assert_eq!(report.units[1].predecessor, PredecessorLink::None); + + // The declared scope is checked against the listed units as far as + // the archive allows: no units at all, a version scope with two + // units (one of them outside it), a record scope naming another + // record. + let hollow = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["units"] = serde_json::Value::Array(Vec::new()); + })) + })); + let report = export_verify::verify_archive(&hollow); + assert!(!report.verified()); + assert!(report.findings.contains(&Finding::NoUnits), "{report:?}"); + let mislabeled = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["scope"] = serde_json::json!({ + "kind": "version", + "record_id": s.record_id, + "version_number": 2, + }); + })) + })); + let report = export_verify::verify_archive(&mislabeled); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::ScopeCardinality { + expected: 1, + listed: 2 + }), + "{report:?}" + ); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v1.clone() }), + "{report:?}" + ); + assert!( + !report + .findings + .contains(&Finding::UnitOutsideScope { path: v2.clone() }), + "{report:?}" + ); + let foreign = repack(&with_entry(&listed, ARCHIVE_MANIFEST_PATH, |bytes| { + Some(edit_json(bytes, |manifest| { + manifest["scope"]["record_id"] = serde_json::Value::from(9999); + })) + })); + let report = export_verify::verify_archive(&foreign); + assert!(!report.verified()); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v1.clone() }), + "{report:?}" + ); + assert!( + report + .findings + .contains(&Finding::UnitOutsideScope { path: v2.clone() }), + "{report:?}" + ); + + // A name written twice: the reader resolves the genuine copy, so + // every per-unit check passes, and only the central directory shows + // the alternate record bytes another tool might extract instead. + let plain: Vec<(&str, &[u8])> = listed + .iter() + .map(|(name, content)| (name.as_str(), content.as_slice())) + .collect(); + let handmade = handmade_zip(&plain); + assert!( + export_verify::verify_archive(&handmade).verified(), + "a hand-assembled container of the same entries verifies" + ); + let alternate = String::from_utf8(entry(&listed, &format!("{v2}/record.json")).to_vec()) + .expect("utf-8") + .replace("Corrected the invented", "Corrected the inveNted") + .into_bytes(); + let mut doubled: Vec<(&str, &[u8])> = Vec::new(); + for (name, content) in &plain { + if *name == format!("{v2}/record.json") { + doubled.push((name, alternate.as_slice())); + } + doubled.push((name, content)); + } + let report = export_verify::verify_archive(&handmade_zip(&doubled)); + assert!(!report.verified()); + assert!( + report.findings.contains(&Finding::DuplicateEntry { + path: format!("{v2}/record.json") + }), + "{report:?}" + ); + assert!( + report.units.iter().all(UnitReport::verified), + "the reader saw the genuine copy: {report:?}" + ); + + // The untouched original still verifies after all that. + assert!(export_verify::verify_archive(&original).verified()); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn verification_reads_envelopes_as_typed_records() { + let fx = Fixture::new().await; + let s = seed(&fx, "typed").await; + let installation_id = storage::installation_id(&fx.pool).await.expect("id"); + let (_, v1_content, _) = fx.version_row(s.record_id, 1).await; + let original = export( + &fx, + s.casey_id, + Scope::Version { + record_id: s.record_id, + version_number: 2, + }, + ) + .await; + let listed = entries(&original); + let unit = format!("records/{}/v2", s.record_id); + let genuine: serde_json::Value = + serde_json::from_slice(entry(&listed, &format!("{unit}/record.json"))).expect("json"); + let invalid = |bytes: &[u8], record_schema: i64| { + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + bytes, + Some(&v1_content), + record_schema, + )); + assert!(!report.verified(), "{report:?}"); + let findings = &report.units[0].findings; + assert!( + findings + .iter() + .any(|f| matches!(f, Finding::EnvelopeInvalid { .. })), + "{findings:?}" + ); + assert!( + !findings.contains(&Finding::ContentHashMismatch) + && !findings.contains(&Finding::ChainHashMismatch) + && !findings + .iter() + .any(|f| matches!(f, Finding::NotCanonical { .. })), + "the forgery is otherwise self-consistent: {findings:?}" + ); + }; + + // A canonical, correctly hashed document carrying only the members + // the identity cross-check reads is not a record. + let counterfeit = canonical::canonical_bytes(&serde_json::json!({ + "canonicalization": canonical::CANONICALIZATION, + "instance": installation_id, + "record": { + "id": s.record_id, + "version_number": 2, + "record_schema": 2, + "predecessor_content_hash": v1_content, + }, + })) + .expect("canonical"); + invalid(&counterfeit, 2); + + // Nor is a schema-2 envelope missing its daily_reports member, one + // carrying a member no schema names, one with a mistyped member, one + // whose nullable member is absent instead of null, or one declaring + // a schema this build does not know. + let mut without = genuine.clone(); + without + .as_object_mut() + .expect("object") + .remove("daily_reports"); + invalid(&canonical::canonical_bytes(&without).expect("canonical"), 2); + let mut extra = genuine.clone(); + extra["annotations"] = serde_json::json!([]); + invalid(&canonical::canonical_bytes(&extra).expect("canonical"), 2); + let mut retyped = genuine.clone(); + retyped["finalization"]["finalized_at"] = serde_json::json!("yesterday"); + invalid(&canonical::canonical_bytes(&retyped).expect("canonical"), 2); + let mut absent = genuine.clone(); + absent["content"]["narratives"][0] + .as_object_mut() + .expect("object") + .remove("text"); + invalid(&canonical::canonical_bytes(&absent).expect("canonical"), 2); + let mut unknown_schema = genuine.clone(); + unknown_schema["record"]["record_schema"] = serde_json::json!(7); + invalid( + &canonical::canonical_bytes(&unknown_schema).expect("canonical"), + 7, + ); + // Hashes are lowercase hex by the format. An uppercase predecessor + // hash, consistent across both manifests and the envelope, decodes + // to the same bytes, so the chain recomputes — and nothing else + // objects for a lone successor whose predecessor is not present. + let shouted = v1_content.to_uppercase(); + let mut loud = genuine.clone(); + loud["record"]["predecessor_content_hash"] = serde_json::Value::String(shouted.clone()); + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + &canonical::canonical_bytes(&loud).expect("canonical"), + Some(&shouted), + 2, + )); + assert!(!report.verified()); + assert_eq!( + report.units[0].findings, + vec![Finding::HashNotCanonical { + member: "predecessor_content_hash" + }], + "{report:?}" + ); + + // Attachments exist in no known schema: an element of any shape, + // even an empty object, is not a record. + let mut attached = genuine.clone(); + attached["attachments"] = serde_json::json!([{}]); + invalid( + &canonical::canonical_bytes(&attached).expect("canonical"), + 2, + ); + + // Identity is positive: a version 0 with a predecessor satisfies the + // lineage rule's two sides trivially, so the range check has to be + // its own finding — and it is the only objection. + let v0 = format!("records/{}/v0", s.record_id); + let mut zeroed = genuine.clone(); + zeroed["record"]["version_number"] = serde_json::json!(0); + let zeroed_bytes = canonical::canonical_bytes(&zeroed).expect("canonical"); + let zeroed_content = canonical::content_hash_hex(&zeroed_bytes); + let zeroed_chain = canonical::chain_hash_hex(Some(&v1_content), &zeroed_bytes).expect("chain"); + let relabel_zero = |manifest: &mut serde_json::Value| { + manifest["version_number"] = serde_json::json!(0); + manifest["content_hash"] = serde_json::Value::String(zeroed_content.clone()); + manifest["chain_hash"] = serde_json::Value::String(zeroed_chain.clone()); + }; + let renumbered: Vec<(String, Vec)> = listed + .iter() + .map(|(name, content)| { + if name == ARCHIVE_MANIFEST_PATH { + let edited = edit_json(content, |manifest| { + manifest["scope"]["version_number"] = serde_json::json!(0); + relabel_zero(&mut manifest["units"][0]); + manifest["units"][0]["path"] = serde_json::Value::String(v0.clone()); + }); + (name.clone(), edited) + } else if name.ends_with("/record.json") { + (format!("{v0}/record.json"), zeroed_bytes.clone()) + } else { + ( + format!("{v0}/manifest.json"), + edit_json(content, relabel_zero), + ) + } + }) + .collect(); + let report = export_verify::verify_archive(&repack(&renumbered)); + assert!(!report.verified()); + assert_eq!( + report.units[0].findings, + vec![Finding::IdentityOutOfRange { + member: "version_number" + }], + "{report:?}" + ); + assert!(report.findings.is_empty(), "{report:?}"); + + // The same content shaped as schema 1 — no daily_reports, schema 1 + // declared throughout — is a valid record of its own schema. + let mut schema1 = without; + schema1["record"]["record_schema"] = serde_json::json!(1); + let report = export_verify::verify_archive(&rehashed_unit( + &listed, + &unit, + &canonical::canonical_bytes(&schema1).expect("canonical"), + Some(&v1_content), + 1, + )); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units[0].record_schema, 1); + + // And the genuine bytes read as the typed envelope they are. + let envelope = + consolebook_server::record_envelope::parse(entry(&listed, &format!("{unit}/record.json"))) + .expect("typed envelope"); + assert_eq!(envelope.record.version_number, 2); + assert_eq!(envelope.trainee.display_name, "Taylor Trainee"); + assert_eq!( + envelope.content.narratives[0].text.as_deref(), + Some("Corrected the invented rating with context.") + ); + assert!(matches!( + envelope.daily_reports, + consolebook_server::record_envelope::DailyReports::Present(ref links) if links.is_empty() + )); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn scopes_follow_the_read_rules() { + let fx = Fixture::new().await; + let s = seed(&fx, "scopes").await; + let outsider = fx + .user_with_role("robin.scopes", "Robin Outsider", RoleBundle::Trainer) + .await; + let version = Scope::Version { + record_id: s.record_id, + version_number: 1, + }; + let record = Scope::Record { + record_id: s.record_id, + }; + let enrollment = Scope::Enrollment { + enrollment_id: s.enrollment_id, + }; + + // A version or record exports for whoever may read the record: the + // assigned trainer, the coordinator, and the trainee on their own + // finalized record — every retained version included. A trainer + // outside the scope is refused. + for actor in [s.jordan_id, s.casey_id, s.taylor_id] { + let bytes = export(&fx, actor, record).await; + let report = export_verify::verify_archive(&bytes); + assert!(report.verified()); + assert_eq!(report.units.len(), 2, "actor {actor} sees both versions"); + export(&fx, actor, version).await; + } + assert_eq!( + refusal(&fx, outsider, version).await, + ExportRefusal::CapabilityRequired + ); + assert_eq!( + refusal(&fx, outsider, record).await, + ExportRefusal::CapabilityRequired + ); + + // An enrollment exports for whoever may read its training history; + // the trainee's own-record grant is not that. + export(&fx, s.jordan_id, enrollment).await; + export(&fx, s.casey_id, enrollment).await; + export(&fx, fx.admin_id, enrollment).await; + assert_eq!( + refusal(&fx, outsider, enrollment).await, + ExportRefusal::CapabilityRequired + ); + assert_eq!( + refusal(&fx, s.taylor_id, enrollment).await, + ExportRefusal::CapabilityRequired + ); + + // The whole installation takes export_records: the administrator + // bundle carries it, the coordinator bundle does not. + export(&fx, fx.admin_id, Scope::Installation).await; + assert_eq!( + refusal(&fx, s.casey_id, Scope::Installation).await, + ExportRefusal::CapabilityRequired + ); + let summary = record_export::summary(&fx.pool, fx.admin_id) + .await + .expect("call") + .expect("summarized"); + assert_eq!((summary.record_count, summary.version_count), (1, 2)); + assert_eq!( + record_export::summary(&fx.pool, s.casey_id) + .await + .expect("call"), + Err(ExportRefusal::CapabilityRequired) + ); + + // Unknown identities and empty scopes are typed, never an empty + // archive presented as complete. + assert_eq!( + refusal(&fx, s.casey_id, Scope::Record { record_id: 9999 }).await, + ExportRefusal::NoSuchRecord + ); + assert_eq!( + refusal( + &fx, + s.casey_id, + Scope::Version { + record_id: s.record_id, + version_number: 7, + } + ) + .await, + ExportRefusal::NoSuchVersion + ); + assert_eq!( + refusal( + &fx, + s.casey_id, + Scope::Enrollment { + enrollment_id: 9999 + } + ) + .await, + ExportRefusal::NoSuchEnrollment + ); + let unfinalized = draft_for(&fx, s.jordan_id, s.enrollment_id, "2026-06-03").await; + assert_eq!( + refusal( + &fx, + s.jordan_id, + Scope::Record { + record_id: unfinalized + } + ) + .await, + ExportRefusal::NothingToExport + ); + let riley = fx + .user_with_role("riley.scopes", "Riley Trainee", RoleBundle::Trainee) + .await; + let empty_enrollment = enrollments::enroll(&fx.pool, fx.admin_id, s.version_id, riley) + .await + .expect("call") + .expect("enrolled"); + assert_eq!( + refusal( + &fx, + fx.admin_id, + Scope::Enrollment { + enrollment_id: empty_enrollment + } + ) + .await, + ExportRefusal::NothingToExport + ); + // Refusals leave no export audit behind. + let audited: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM audit_event WHERE kind = 'record_exported'") + .fetch_one(&fx.pool) + .await + .expect("count"); + assert_eq!(audited, 10); +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn export_api_delivers_the_documented_bytes() { + let fx = Fixture::new().await; + let s = seed(&fx, "api").await; + let casey = fx.login("casey.api").await; + let admin = fx.login("avery.admin").await; + + let (status, headers, bytes) = raw_get( + fx.app(), + &format!("/api/drafts/{}/versions/2/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + headers + .get(CONTENT_TYPE) + .expect("type") + .to_str() + .expect("ascii"), + "application/zip" + ); + let disposition = headers + .get(CONTENT_DISPOSITION) + .expect("disposition") + .to_str() + .expect("ascii"); + assert!( + disposition.starts_with(&format!( + "attachment; filename=\"consolebook-record-{}-v2-", + s.record_id + )) && disposition.ends_with("Z.zip\""), + "got: {disposition}" + ); + let report = export_verify::verify_archive(&bytes); + assert!(report.verified(), "{report:?}"); + assert_eq!(report.units.len(), 1); + + let (status, _, bytes) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(export_verify::verify_archive(&bytes).units.len(), 2); + let (status, _, bytes) = raw_get( + fx.app(), + &format!("/api/enrollments/{}/export", s.enrollment_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(export_verify::verify_archive(&bytes).verified()); + + // The installation scope answers to export_records only. + let (status, _, body) = raw_get(fx.app(), "/api/exports/records", &casey).await; + assert_eq!(status, StatusCode::FORBIDDEN); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "capability_required"); + let (status, _, bytes) = raw_get(fx.app(), "/api/exports/records", &admin).await; + assert_eq!(status, StatusCode::OK); + let report = export_verify::verify_archive(&bytes); + assert!(report.verified()); + assert_eq!(report.scope, Some(Scope::Installation)); + let (status, _, body) = raw_get(fx.app(), "/api/exports/summary", &admin).await; + assert_eq!(status, StatusCode::OK); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["record_count"], 1); + assert_eq!(body["version_count"], 2); + let (status, _, _) = raw_get(fx.app(), "/api/exports/summary", &casey).await; + assert_eq!(status, StatusCode::FORBIDDEN); + + // Typed refusals over the wire. + let (status, _, body) = raw_get(fx.app(), "/api/enrollments/9999/export", &casey).await; + assert_eq!(status, StatusCode::NOT_FOUND); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "no_such_enrollment"); + let (status, _, body) = raw_get( + fx.app(), + &format!("/api/drafts/{}/versions/9/export", s.record_id), + &casey, + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "no_such_version"); + let unfinalized = draft_for(&fx, s.jordan_id, s.enrollment_id, "2026-06-03").await; + let (status, _, body) = raw_get( + fx.app(), + &format!("/api/drafts/{unfinalized}/export"), + &casey, + ) + .await; + assert_eq!(status, StatusCode::CONFLICT); + let body: serde_json::Value = serde_json::from_slice(&body).expect("json"); + assert_eq!(body["error"], "nothing_to_export"); + let outsider_cookie = { + fx.user_with_role("robin.api", "Robin Outsider", RoleBundle::Trainer) + .await; + fx.login("robin.api").await + }; + let (status, _, _) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + &outsider_cookie, + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN); + let (status, _, _) = raw_get( + fx.app(), + &format!("/api/drafts/{}/export", s.record_id), + "not-a-session", + ) + .await; + assert_eq!(status, StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn cli_verifies_from_the_file_alone() { + let fx = Fixture::new().await; + let s = seed(&fx, "cli").await; + let bytes = export( + &fx, + s.casey_id, + Scope::Record { + record_id: s.record_id, + }, + ) + .await; + let scratch = tempfile::tempdir().expect("scratch"); + let archive = scratch.path().join("export.zip"); + std::fs::write(&archive, &bytes).expect("write"); + // The data directory named here must never be touched: the archive + // carries everything the checks need. + let untouched = scratch.path().join("never-created"); + + let output = std::process::Command::new(env!("CARGO_BIN_EXE_consolebook-server")) + .args(["--data-dir"]) + .arg(&untouched) + .args(["export", "verify"]) + .arg(&archive) + .output() + .expect("run verifier"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(output.status.success(), "stdout: {stdout}"); + assert!(stdout.contains("verified 2 of 2 units"), "stdout: {stdout}"); + assert!(stdout.contains("predecessor linked"), "stdout: {stdout}"); + assert!(!untouched.exists(), "the verifier opened no data directory"); + + // A tampered file fails with a named finding and a failing exit code. + let listed = entries(&bytes); + let tampered = repack(&with_entry( + &listed, + &format!("records/{}/v1/record.json", s.record_id), + |content| { + let text = String::from_utf8(content.to_vec()).expect("utf-8"); + Some( + text.replace("initial entry", "initial entry, revised") + .into_bytes(), + ) + }, + )); + std::fs::write(&archive, &tampered).expect("write"); + let output = std::process::Command::new(env!("CARGO_BIN_EXE_consolebook-server")) + .args(["export", "verify"]) + .arg(&archive) + .output() + .expect("run verifier"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(!output.status.success(), "stdout: {stdout}"); + assert!(stdout.contains("NOT VERIFIED"), "stdout: {stdout}"); + assert!( + stdout.contains("the content hash does not match the record bytes"), + "stdout: {stdout}" + ); + // Version 2 still verifies on its own: its chain covers the hash the + // manifest states for version 1, and that statement is unchanged; + // the altered bytes fail where they sit. + assert!( + stdout.contains("NOT VERIFIED: 1 of 2 units consistent, 0 archive finding(s)"), + "stdout: {stdout}" + ); + assert!( + stdout.contains(&format!("FAIL records/{}/v1", s.record_id)), + "stdout: {stdout}" + ); + assert!( + stdout.contains(&format!("ok records/{}/v2", s.record_id)), + "stdout: {stdout}" + ); +} diff --git a/docs/decisions/0014-record-export-format.md b/docs/decisions/0014-record-export-format.md new file mode 100644 index 0000000..f3aa8ce --- /dev/null +++ b/docs/decisions/0014-record-export-format.md @@ -0,0 +1,148 @@ +# ADR 0014: Record export format and verification + +- **Status:** Accepted +- **Date:** 2026-09-01 + +## Context + +Milestone 5 makes an installation leave-able: "a center can leave with +all of its data and can prove recovery from a clean installation" +(`docs/roadmap.md`). Milestone 4 finished the record substrate — +canonical bytes with content and chain hashes (ADR 0011), successor +versions (ADR 0012), record schema 2 (ADR 0013) — but a finalized +version is readable only through the installation that holds it. +`docs/records-integrity.md` requires export round-trip tests and +honest verification wording, and #44 decision 1 settled the owner +choice: the stored canonical bytes travel verbatim beside a manifest, +multi-record exports are an archive of the same units plus an archive +manifest, and verification needs nothing but the export. This ADR +fixes the format (`docs/formats/record-export.md`), its scopes, who +may export, and what verification claims (#45; Milestone 5 slice 1). + +## Decision + +### Units carry stored bytes, never a re-serialization + +- A unit is one finalized version's `canonical_bytes` copied from + storage, beside a unit manifest naming the installation, the record + and version identity, the stored `record_schema`, both stored hashes, + the predecessor's content hash, the export instant, and the format + version. Schema-1 and schema-2 bytes travel as stored; nothing + upgrades, rewraps, or reformats a record on the way out. +- Manifests are themselves canonical JSON under the record format's + JCS subset, so an export is deterministic: the same scope exported + at the same instant is byte-identical. + +### Archives are ZIP containers, one format for every scope + +- Every export, a single version included, is a ZIP archive with + stored entries, a fixed entry order, fixed entry metadata, and an + archive manifest listing every unit with its identity and hashes. One + container means one reader, one verifier, and one document for all + four scopes: version, record (every retained version), enrollment + (every finalized version of its records), and installation. +- The archive holds nothing this format does not name; extra entries + are verification findings, so a tampered or repacked archive cannot + hide content beside the records. +- A scope with no finalized version is a typed refusal + (`nothing_to_export`), never an empty archive presented as a + complete export. + +### Verification from the export alone + +- The verifier recomputes the content hash over the bytes, proves the + bytes are canonical by re-serialization, reads them as a typed + envelope of their declared record schema (`record_envelope`, the + reading side of ADR 0011's shape: every named member, typed, and no + other), cross-checks the envelope's own `record` and `instance` + members against the manifests, + recomputes the chain hash from the carried predecessor hash, links + predecessors present in the same archive, checks the listed units + against the declared scope as far as the archive allows, and walks + the container's central directory itself so an entry name written + twice is a finding rather than whichever copy one reader happens to + pick. It reports per unit and per archive with typed findings; the + verdict is `verified` only when every check passes. +- It ships as a library function and as + `consolebook-server export verify `, which opens no data + directory — the artifact is checked wherever it landed. +- Honest limits (ADR 0010, ADR 0011): a verified export is internally + consistent with its stated fingerprints. It does not prove which + installation produced it, nor completeness against what that + installation held; the scope member is the exporter's statement of + intent. Signatures (the future signed mode) attach beside the hashes + without changing a record byte. + +### Export is a read in a portable shape + +- Authorization reuses the read contracts that exist rather than + adding parallel ones: a version or record exports for whoever may + read the record (workflow readers, and the trainee on their own + finalized record — every retained version, per ADR 0012); an + enrollment exports for whoever may read its training history; the + whole installation exports for `export_records` holders, the + explicit administrative authority PRINCIPLES.md 10 demands for + breadth. A unit contains exactly what its reader can already read + through the API; the capability gates breadth, not format. +- Every export is audited (`record_exported`) with actor and subject + and never with content, in the append-only audit trail + (`docs/records-integrity.md`). + +### Nothing is retained on disk + +- Exports are produced on request and delivered as downloads; this + slice writes nothing under `data/exports/`. A future persisted export + (scheduled exports, large archives) must register each file so + lawful disposition can reach derived exports in scope + (`docs/records-integrity.md` step 4); until then there are none. + +## Consequences + +### Positive + +- a record leaves the installation exactly as it was sealed, and any + reader of the format document can check it without Consolebook; +- one container and one verifier cover single versions, records, + enrollments, and the whole installation, and the trainee packet + (slice 2) can be built from the same units; +- determinism makes exports comparable and testable byte for byte, + the property `docs/records-integrity.md` asks of every record + representation; and +- verification wording carries the same honesty as in-database + verification — consistency, not tamper-proofing. + +### Costs + +- a new dependency (`zip`, without default features: stored entries + only) enters the one executable; +- the format version is shared by both manifests, so any manifest + change bumps it for every export; +- an archive pins the predecessor by content hash only, so an + exported successor without its predecessor verifies its own chain + hash but cannot prove the predecessor's bytes — reported as *not in + export*, never inferred; and +- on-demand exports mean nothing on disk to dispose of and nothing to + resume: an installation export is one response, assembled in memory + while it is produced, so a very large history costs memory in + proportion until exports stream (tracked separately). + +## Rejected alternatives + +- **JSON re-serialization of the envelope (a "pretty" export):** the + hash is over the specified bytes; any re-serialization is either a + no-op (pointless) or a change (a record that no longer verifies). +- **A single JSON document embedding the bytes as strings or + base64:** embedding re-encodes the bytes, and base64 hides the + record from a human reader; the decision was units *beside* a + manifest, unchanged. +- **Tar or a bespoke container:** ZIP opens on every operator + platform without tooling; a bespoke container would need its own + reader in every verifier. +- **A separate `export_records` gate on every scope:** a trainee may + already read every retained version of their own record and a + reviewer every record in their scope; forcing an administrator to + produce those exports adds a gatekeeper without adding protection, + which is not what PRINCIPLES.md 10's explicit-authority rule is for. +- **Exporter identity in the manifest:** provenance of a file is the + audit trail's job; putting an operator's identity in every artifact + that leaves the system is personal data the record does not need. diff --git a/docs/domain-model.md b/docs/domain-model.md index feab28f..682ee0d 100644 --- a/docs/domain-model.md +++ b/docs/domain-model.md @@ -138,6 +138,12 @@ Acknowledgment means receipt, not agreement. A successor version requires a new An Amendment links an original finalized version to its successor and records the reason, authority, author, and timestamps. The original remains readable and exportable while retained. +## Exports + +### RecordExport + +A RecordExport is an archive of finalized EvaluationVersions as stored: each version's canonical bytes, unchanged, beside a manifest carrying the installation identity, record and version identity, record schema, both hashes, the predecessor's content hash, and the export instant (`docs/formats/record-export.md`, ADR 0014). Scopes are one version, one record, one enrollment, or the installation. Verifying an export needs nothing but the export; the verdict reports consistency with the stated fingerprints, never tamper-proofing. Every export is audited without record content. + ## Retention and disposition ### RetentionPolicy diff --git a/docs/formats/record-export.md b/docs/formats/record-export.md new file mode 100644 index 0000000..7ba7b47 --- /dev/null +++ b/docs/formats/record-export.md @@ -0,0 +1,270 @@ +# Record Export Format + +Portable, verifiable exports of finalized evaluation versions (ADR 0014; +`docs/records-integrity.md`; #44 decision 1). +`consolebook-server/src/record_export.rs` produces exports and +`consolebook-server/src/export_verify.rs` verifies them; +`tests/record_export.rs` proves the round trip, determinism, and every +verification finding. This document is normative: the implementation +follows it, not the other way around. + +An export never re-serializes a record. The stored canonical bytes of +each version (ADR 0011, ADR 0013) travel byte for byte, beside a +manifest that names them. Verifying an export needs nothing but the +export. + +## Vocabulary + +- **Unit** — one finalized version as exported: its canonical record + bytes and its unit manifest. +- **Archive** — the container holding one archive manifest and one or + more units. Every export, a single version included, is an archive, + so one reader and one verifier cover every scope. +- **Scope** — what the archive claims to contain: one version, one + record (every retained version, superseded originals included), one + enrollment (every finalized version of its records), or the whole + installation. + +## Container + +The container is a ZIP archive (APPNOTE 6.3): + +- every entry uses compression method 0 (stored); nothing is encrypted; + ZIP64 structures appear only when the container's size or entry count + requires them (a record is kilobytes, so no single entry ever does); +- entries are files only — no directory entries — named with forward + slashes and ASCII characters; +- entry order is the archive manifest first, then units in ascending + (`record_id`, `version_number`) order, `record.json` before + `manifest.json` within a unit; +- every entry's modification time is the export instant (DOS time, UTC + as written, so seconds round down to even), and every entry carries + Unix permissions `0644`; +- the archive holds nothing but what this document names; anything else + is a verification finding. + +Entry names: + +```text +manifest.json archive manifest +records/{record_id}/v{version_number}/record.json canonical record bytes +records/{record_id}/v{version_number}/manifest.json unit manifest +``` + +`{record_id}` and `{version_number}` are the decimal integers of the +unit's identity with no padding. The archive manifest states each +unit's `path` as the directory prefix (`records/12/v2`) and the path +must equal the derived form. + +## Determinism + +An archive is a pure function of its scope's stored rows and the +export instant: the same scope exported at the same instant produces +identical bytes. Manifests are canonical JSON under the record format's +JCS subset (ADR 0011): UTF-8, members sorted by code point, no +insignificant whitespace, integers only. Record bytes are copied from +storage unchanged. + +## `record.json` + +The version's `canonical_bytes` exactly as stored — a schema-1 or +schema-2 envelope under `jcs-v1`, presented under its own stored +`record_schema`. The export never upgrades, rewraps, or reformats them. +The envelope's own `record` and `instance` members are what the +verifier cross-checks against the manifests: the bytes commit to their +identity and lineage on their own (ADR 0011). + +## Unit manifest (`records/…/manifest.json`) + +```json +{ + "chain_hash": "…64 lowercase hex…", + "content_hash": "…64 lowercase hex…", + "exported_at": 1756753200, + "format": "consolebook-record-unit", + "format_version": 1, + "installation_id": "…", + "predecessor_content_hash": null, + "record_id": 12, + "record_schema": 2, + "version_number": 1 +} +``` + +| Member | Type | Meaning | +| --- | --- | --- | +| `format` | string | Always `consolebook-record-unit` | +| `format_version` | integer | `1`; bumped by any change to either manifest's shape | +| `installation_id` | string | The exporting installation's identity (`instance.installation_id`) | +| `record_id` | integer | The record's instance-local identity (positive) | +| `version_number` | integer | This version's number within the record (from 1) | +| `record_schema` | integer | The stored envelope schema of `record.json` | +| `content_hash` | string | The stored SHA-256 of `record.json`, lowercase hex | +| `chain_hash` | string | The stored integrity-chain hash (ADR 0011), lowercase hex | +| `predecessor_content_hash` | string or `null` | The prior version's content hash; `null` exactly when `version_number` is 1 | +| `exported_at` | integer | The export instant, UTC unix seconds; identical across the archive | + +A unit manifest repeats what the archive manifest says about the unit +so that a unit directory stands on its own; the two must agree. + +## Archive manifest (`manifest.json`) + +```json +{ + "exported_at": 1756753200, + "format": "consolebook-record-export", + "format_version": 1, + "installation_id": "…", + "scope": { "kind": "record", "record_id": 12 }, + "units": [ + { + "chain_hash": "…", + "content_hash": "…", + "path": "records/12/v1", + "predecessor_content_hash": null, + "record_id": 12, + "record_schema": 2, + "version_number": 1 + }, + { + "chain_hash": "…", + "content_hash": "…", + "path": "records/12/v2", + "predecessor_content_hash": "…", + "record_id": 12, + "record_schema": 2, + "version_number": 2 + } + ] +} +``` + +| Member | Type | Meaning | +| --- | --- | --- | +| `format` | string | Always `consolebook-record-export` | +| `format_version` | integer | `1` | +| `installation_id` | string | As in the unit manifest | +| `exported_at` | integer | The export instant, UTC unix seconds | +| `scope` | object | What the archive claims to contain (below) | +| `units` | array | Every unit, ascending by (`record_id`, `version_number`), no duplicates | + +`units[]` members are `path`, `record_id`, `version_number`, +`record_schema`, `content_hash`, `chain_hash`, and +`predecessor_content_hash`, with the unit-manifest meanings. + +### `scope` + +`kind` is one of: + +- `version` — with `record_id` and `version_number`; exactly one unit. +- `record` — with `record_id`; every retained version of that record. +- `enrollment` — with `enrollment_id`; every finalized version of every + record of that enrollment. +- `installation` — no further members; every finalized version the + installation holds. + +The scope is the exporter's statement of intent. Verification checks +the archive against itself — a `version` scope must be exactly its one +unit, a `record` scope must hold only that record's versions — but it +cannot know whether an installation held versions the archive omits. + +## Verification + +A verifier reads only the archive. It reports, per unit and for the +archive as a whole, and its verdict is `verified` only when every check +below passes. Wording stays honest (ADR 0010, ADR 0011, +`docs/records-integrity.md`): a verified export is internally +consistent with its stated fingerprints. Without the future signed +mode nothing in the archive proves which installation produced it, and +the checks are not tamper-proofing against whoever produced the file. + +Archive checks: + +1. the container is a readable ZIP archive whose central directory names + each entry once — a name written twice is a finding, because + extraction tools disagree on which copy they take; +2. `manifest.json` exists, parses, and carries a known `format` and + `format_version`; +3. `units` lists at least one unit, ascending by (`record_id`, + `version_number`) with no duplicate identity, and each `path` equals + the derived form; +4. the listed units fit the declared scope as far as the archive can + tell: a `version` scope lists exactly one unit with that identity, a + `record` scope lists only units of that record; an `enrollment` or + `installation` scope states nothing the archive can confirm on its + own; +5. every entry in the container is `manifest.json` or one of a listed + unit's two files — an unlisted entry is a finding; and +6. both files of every listed unit exist. + +Unit checks, for every listed unit: + +1. `manifest.json` parses, carries the known `format` and + `format_version`, and agrees with the archive entry and the archive + manifest on every shared member (`installation_id`, `exported_at`, + identity, schema, hashes); +2. `content_hash`, `chain_hash`, and a non-null + `predecessor_content_hash` are each 64 lowercase hex characters, and + `content_hash` equals SHA-256 over the bytes of `record.json`; +3. `record.json` parses as JSON and re-serializing it under the + canonical subset reproduces the identical bytes — the bytes are + canonical, so the hash is over the specified representation; +4. `record.json` is an envelope of a known record schema: every member + ADR 0011 (schema 1) or ADR 0013 (schema 2) names, with its type, + every nullable member present, no member the schema does not name, + `daily_reports` present exactly for schema 2, and `attachments` + empty (no known schema carries attachments); +5. the envelope agrees with the manifest: `record.id`, + `record.version_number`, `record.record_schema`, + `record.predecessor_content_hash`, `instance`, and + `canonicalization` (`jcs-v1`); +6. `chain_hash` equals + `SHA-256("consolebook-version-v1" || 0x00 || predecessor || bytes)` + with `predecessor` the raw 32 bytes of `predecessor_content_hash`, + or 32 zero bytes when it is `null`; +7. `record_id` and `version_number` are positive integers, and + `predecessor_content_hash` is `null` exactly when `version_number` + is 1; and +8. when the archive also lists (`record_id`, `version_number - 1`), + that unit's `content_hash` equals this unit's + `predecessor_content_hash` — reported as *linked*; a predecessor the + archive does not carry is reported as *not in export*, which is not + a failure (a single-version scope is legitimate), and a first + version reports *none*. + +`consolebook-server export verify ` runs exactly these checks, +prints one line per unit and every finding, and exits non-zero unless +the verdict is `verified`. It opens no data directory. + +Verification deliberately does not check the container's entry order, +modification times, or permissions. Those are production rules: they +make a fresh export deterministic, and they carry no record content. +An archive whose entries and manifests are intact is still a verified +export after a tool has repacked it, which is what an operator asking +"are these records intact?" needs to hear. Proving an archive is +byte-identical to a fresh export is a byte comparison, not a +verification finding. + +## What the archive does not carry + +- **Drafts.** An unfinalized record is not a record; scopes contain + finalized versions only, and a scope with none is refused, never + exported empty. +- **Acknowledgments, amendments, and signoff history.** They are + separate records bound to versions; the trainee packet (#44 decision + 3) is the artifact that gathers them. An amendment's existence is + visible here only as a successor version's `predecessor_content_hash`. +- **Exporter identity.** Who exported what is the installation's + audit trail (`record_exported`), not the artifact's business. +- **Signatures.** The future signed mode adds them beside the hashes + without changing a record byte (`docs/records-integrity.md`). + +## Authorization (behavior of the producing installation) + +Export is a read in a portable shape, so it follows the read rules that +already exist rather than inventing parallel ones: a version or record +exports for whoever may read the record (workflow readers and the +trainee's own finalized record, ADR 0012); an enrollment exports for +whoever may read its training history; the whole installation exports +for holders of `export_records`. Every export is audited with actor and +subject and never with content. diff --git a/web/e2e/drafts.spec.ts b/web/e2e/drafts.spec.ts index 3a8741f..1bbe944 100644 --- a/web/e2e/drafts.spec.ts +++ b/web/e2e/drafts.spec.ts @@ -340,6 +340,19 @@ test('draft, collaborate, transfer, and submit a daily evaluation', async ({ pag await expect( page.getByText('Recomputed from the stored record: both fingerprints match.') ).toBeVisible(); + // The sealed record leaves as it was stored: the browser download is + // an archive whose record bytes are the canonical bytes verbatim, and + // the CLI verifies it from the file alone (ADR 0014). + const downloadPromise = page.waitForEvent('download'); + await page.getByRole('button', { name: 'Export this version' }).click(); + const download = await downloadPromise; + expect(download.suggestedFilename()).toMatch( + /^consolebook-record-\d+-v1-\d{8}T\d{6}Z\.zip$/ + ); + const archivePath = await download.path(); + const verified = await execFileAsync(BINARY, ['export', 'verify', archivePath]); + expect(verified.stdout).toContain('verified 1 of 1 units'); + await expect(page.getByText(/^Downloaded consolebook-record-/)).toBeVisible(); // The sealed record takes no further decisions or edits. await expect(page.getByRole('button', { name: 'Finalize record' })).toHaveCount(0); await expect(page.getByLabel('Most acceptable performance.')).toHaveCount(0); diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 3ed6f53..e4cdcd9 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1182,3 +1182,68 @@ export function blankContent(name: string): VersionContent { } }; } + +// Record exports (Milestone 5 slice 1; docs/formats/record-export.md; +// ADR 0014): the stored canonical bytes travel verbatim beside +// manifests, and an archive verifies from its own contents alone. + +export interface ExportSummary { + installation_id: string; + record_count: number; + version_count: number; +} + +export function exportSummary(): Promise { + return request('/api/exports/summary'); +} + +/** Download path for one finalized version. */ +export function recordVersionExportPath(recordId: number, versionNumber: number): string { + return `/api/drafts/${recordId}/versions/${versionNumber}/export`; +} + +/** Download path for every retained version of a record. */ +export function recordExportPath(recordId: number): string { + return `/api/drafts/${recordId}/export`; +} + +/** Download path for every finalized version of an enrollment's records. */ +export function enrollmentExportPath(enrollmentId: number): string { + return `/api/enrollments/${enrollmentId}/export`; +} + +/** Download path for every finalized version the installation holds. */ +export function installationExportPath(): string { + return '/api/exports/records'; +} + +/** + * Fetches an export archive and hands it to the browser as a download, so + * a refusal surfaces as an error instead of a saved error document. + * Returns the server's file name. + */ +export async function downloadExport(path: string): Promise { + const response = await fetch(path); + if (!response.ok) { + let body: ApiErrorBody; + try { + body = (await response.json()) as ApiErrorBody; + } catch { + body = { error: 'unreachable', message: `server returned ${response.status}` }; + } + throw new ApiError(response.status, body); + } + const disposition = response.headers.get('Content-Disposition') ?? ''; + const named = /filename="([^"]+)"/.exec(disposition); + const fileName = named?.[1] ?? 'consolebook-export.zip'; + const url = URL.createObjectURL(await response.blob()); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = fileName; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + // Revoke once the browser has had time to start the download. + setTimeout(() => URL.revokeObjectURL(url), 60_000); + return fileName; +} diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index c9585fe..26480f6 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -6,8 +6,11 @@ createDraft, createUser, dailyForms, + downloadExport, + exportSummary, getHealth, getNotices, + installationExportPath, issueResetCode, logout, markNoticeRead, @@ -16,6 +19,7 @@ reviewQueue, type AssignedTrainee, type CreatedUser, + type ExportSummary, type Health, type MySession, type Notice, @@ -43,12 +47,16 @@ let canReview = $derived( data.session?.capabilities.includes('review_evaluation') ?? false ); + let canExport = $derived( + data.session?.capabilities.includes('export_records') ?? false + ); let health: Health | null = $state(null); let notices: Notice[] = $state([]); let assigned: AssignedTrainee[] = $state([]); let sessions: MySession[] = $state([]); let queue: ReviewQueueRow[] = $state([]); + let summary: ExportSummary | null = $state(null); $effect(() => { getHealth().then( (h) => (health = h), @@ -76,6 +84,12 @@ () => (queue = []) ); } + if (canExport) { + exportSummary().then( + (body) => (summary = body), + () => (summary = null) + ); + } }); async function acknowledge(notice: Notice) { @@ -194,6 +208,25 @@ busy = false; } } + + // The whole installation leaves as one archive: every finalized + // version as its stored bytes, verifiable from the archive alone + // (ADR 0014). This is the explicit export_records authority. + let exportError = $state(''); + let exported = $state(''); + async function exportInstallation() { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(installationExportPath()); + } catch (err) { + exportError = + err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + }

Installation status

@@ -239,6 +272,35 @@ {/if} +{#if canExport} +
+

Record exports

+ {#if summary === null} +

Checking finalized records…

+ {:else if summary.version_count === 0} +

No finalized records yet; there is nothing to export.

+ {:else} +

+ This installation holds {summary.version_count} finalized + {summary.version_count === 1 ? 'version' : 'versions'} across + {summary.record_count} + {summary.record_count === 1 ? 'record' : 'records'}. An export carries + every one as its stored canonical bytes beside a manifest and verifies + from the archive alone with consolebook-server export verify. +

+ + {#if exported} +

Downloaded {exported}.

+ {/if} + {/if} + {#if exportError} + + {/if} +
+{/if} + {#if canViewAssigned}

My trainees

diff --git a/web/src/routes/drafts/[id]/+page.svelte b/web/src/routes/drafts/[id]/+page.svelte index 824df5b..3f96228 100644 --- a/web/src/routes/drafts/[id]/+page.svelte +++ b/web/src/routes/drafts/[id]/+page.svelte @@ -5,12 +5,15 @@ acknowledgeRecord, amendRecord, attestRecord, + downloadExport, finalizeDraft, finalizedVersion, finalizedVersionAt, getAcknowledgment, getDraft, linkableDailies, + recordExportPath, + recordVersionExportPath, addSummaryLink, removeSummaryLink, reviewDraft, @@ -78,6 +81,30 @@ let verification: Verification | null = $state(null); let ack: Acknowledgment | null = $state(null); let versions: VersionHistoryRow[] = $state([]); + + // An export is the sealed record leaving as it was stored: the + // canonical bytes verbatim beside a manifest, verifiable anywhere + // (ADR 0014). + let exportError = $state(''); + let exported = $state(''); + async function exportArchive(path: string) { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(path); + } catch (err) { + exportError = err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + } + function exportSealedVersion() { + if (sealed === null) { + return; + } + void exportArchive(recordVersionExportPath(draftId, sealed.meta.version_number)); + } // The link picker for weekly summaries: the enrollment's finalized // dailies not yet linked. let linkable: SummaryLink[] = $state([]); @@ -915,6 +942,34 @@ stored; it is not by itself proof against a writer with direct database access.

+
+ + + {#if exported} + Downloaded {exported}. + {/if} + {#if exportError} + {exportError} + {/if} +
+

+ An export carries the stored record bytes verbatim beside a manifest + and verifies anywhere with consolebook-server export verify. +

{#if !viewingSuperseded} diff --git a/web/src/routes/enrollments/[id]/+page.svelte b/web/src/routes/enrollments/[id]/+page.svelte index a6444ae..46d802e 100644 --- a/web/src/routes/enrollments/[id]/+page.svelte +++ b/web/src/routes/enrollments/[id]/+page.svelte @@ -9,7 +9,9 @@ createDraft, createSession, dailyForms, + downloadExport, endAssignment, + enrollmentExportPath, getEnrollment, getProgramVersions, listSessions, @@ -50,6 +52,23 @@ let error = $state(''); let busy = $state(false); + // Every finalized version of this enrollment's records leaves as one + // archive of the stored record bytes with manifests (ADR 0014). + let exportError = $state(''); + let exported = $state(''); + async function exportEnrollment() { + exportError = ''; + exported = ''; + busy = true; + try { + exported = await downloadExport(enrollmentExportPath(enrollmentId)); + } catch (err) { + exportError = err instanceof ApiError ? err.message : 'the server could not be reached'; + } finally { + busy = false; + } + } + async function reload() { try { detail = await getEnrollment(enrollmentId); @@ -825,6 +844,27 @@ {/if} +
+

Export

+

+ Every finalized version of this enrollment's records — superseded + originals included — leaves as one archive of the stored record bytes + with manifests. Verify it anywhere with + consolebook-server export verify. +

+
+ + {#if exported} + Downloaded {exported}. + {/if} +
+ {#if exportError} + + {/if} +
+ {#if canAssign || canAuthor}

Weekly summaries