test(spec): a rendering corpus, for the shapes the fleet cannot cover

conformance/Cargo.toml

@@ -28,3 +28,7 @@ usage-derive = { workspace = true }
 [[bin]]
 name = "oracle"
 path = "src/bin/oracle.rs"
+
+[[bin]]
+name = "render-oracle"
+path = "src/bin/render-oracle.rs"

conformance/src/argv.rs

@@ -1,13 +1,8 @@
 //! Running the corpus against [`usage_argv`], the compiled parser.
 //!
 //! usage-argv reads `static` tables that a derive macro is meant to emit. Nothing
-//! emits them yet, so this module builds them from a [`Spec`] instead, which also
-//! makes the corpus usable as usage-argv's test suite from the first commit.
-//!
-//! The tables are leaked. They must outlive the parse and be `'static`-shaped,
-//! and a test process that builds a handful of small tables and exits is the one
-//! place where leaking is the simplest correct answer. Generated code has no such
-//! problem: its tables really are `static`.
+//! emits them here, so [`crate::tables`] builds them from a [`Spec`] instead, which
+//! also makes the corpus usable as usage-argv's test suite from the first commit.
 //!
 //! # Scope
 //!
@@ -21,10 +16,9 @@ use std::collections::BTreeMap;
 use std::ffi::OsStr;
 
 use usage::{Spec, SpecCommand};
-use usage_argv::{
-    Arg, Command, DoubleDash, Error, Event, Flag, Parser, UnknownFlags as ArgvUnknownFlags,
-};
+use usage_argv::{Command, Error, Event, Parser};
 
+use crate::tables::{self, convert_unknown_flags, leak};
 use crate::{ErrorCode, Expect, Layer, Parsed, Value, Vector};
 
 /// What usage-argv did with a vector.
@@ -64,7 +58,15 @@ pub fn run(vector: &Vector) -> Outcome {
     // hold it. Everything below inherits it, which the parser now does itself rather than
     // this flattening it on the way in — a second implementation of the same rule, and the
     // one that hid the parser not having it.
-    let root = build(&spec.cmd, spec.unknown_flags.map(convert_unknown_flags));
+    //
+    // No completers, because this harness binds a line and no completion metadata changes what
+    // binds. `build_spec` resolves a spec's own, for the rendering corpus.
+    let root = tables::build(
+        &spec.cmd,
+        spec.unknown_flags.map(convert_unknown_flags),
+        &[],
+    )
+    .cmd;
     // `default_subcommand` is a property of the spec rather than of a command, so it is
     // resolved once, here, against the root's own subcommands. A name that answers to
     // nothing is left as None: the spec is what it is, and a vector that expects routing
@@ -177,14 +179,6 @@ fn code(err: Error<'_, '_>) -> ErrorCode {
     }
 }
 
-/// The spec's spelling of the setting, in the parser's terms.
-fn convert_unknown_flags(mode: usage::UnknownFlags) -> ArgvUnknownFlags {
-    match mode {
-        usage::UnknownFlags::Value => ArgvUnknownFlags::Value,
-        usage::UnknownFlags::Error => ArgvUnknownFlags::Error,
-    }
-}
-
 /// Which flags accumulate rather than replace.
 enum Multi {
     Count,
@@ -224,108 +218,3 @@ fn out_of_scope(vector: &Vector) -> Option<&'static str> {
         ),
     }
 }
-
-/// Build leaked tables mirroring a spec command.
-///
-/// `unknown_flags` is carried through as the spec states it — `None` where a command says
-/// nothing — because the parser inherits it. The root takes the spec-level setting, since
-/// that is the command a spec's own property describes.
-fn build(
-    cmd: &SpecCommand,
-    root_unknown_flags: Option<ArgvUnknownFlags>,
-) -> &'static Command<'static> {
-    let unknown_flags = cmd
-        .unknown_flags
-        .map(convert_unknown_flags)
-        .or(root_unknown_flags);
-    let flags: Vec<&'static Flag<'static>> = cmd
-        .flags
-        .iter()
-        .map(|f| -> &'static Flag<'static> {
-            let longs: Vec<&'static str> = f.long.iter().map(|l| leak(l)).collect();
-            let shorts: Vec<u8> = f.short.iter().map(|c| *c as u8).collect();
-            Box::leak(Box::new(Flag {
-                key: 0,
-                name: leak(&f.name),
-                longs: Box::leak(longs.into_boxed_slice()),
-                shorts: Box::leak(shorts.into_boxed_slice()),
-                // usage-lib stores the negation with its dashes; the table wants
-                // the bare name.
-                negate: f.negate.as_ref().map(|n| leak(n.trim_start_matches('-'))),
-                takes_value: f.arg.is_some(),
-                // Only a variadic *argument* is greedy. A `var` flag with a
-                // single-value argument is repeatable instead: one value per
-                // occurrence, which the parser gets by not collecting.
-                variadic: f.arg.as_ref().is_some_and(|a| a.var),
-                // The bound on one occurrence's values, which is the argument's. A
-                // repeatable flag's own `var_max` counts occurrences and is checked after
-                // the parse, so it does not belong in this table.
-                var_max: f
-                    .arg
-                    .as_ref()
-                    .filter(|a| a.var)
-                    .and_then(|a| a.var_max)
-                    // Saturating rather than truncating: `4294967296 as u32` is zero,
-                    // which would read as "stop at once" rather than "no real limit".
-                    .map(|max| u32::try_from(max).unwrap_or(u32::MAX)),
-                global: f.global,
-            }))
-        })
-        .collect();
-
-    let args: Vec<&'static Arg<'static>> = cmd
-        .args
-        .iter()
-        .map(|a| -> &'static Arg<'static> {
-            Box::leak(Box::new(Arg {
-                key: 0,
-                name: leak(&a.name),
-                var: a.var,
-                var_max: a
-                    .var_max
-                    .filter(|_| a.var)
-                    .map(|max| u32::try_from(max).unwrap_or(u32::MAX)),
-                double_dash: match a.double_dash {
-                    usage::SpecDoubleDashChoices::Required => DoubleDash::Required,
-                    usage::SpecDoubleDashChoices::Preserve => DoubleDash::Preserve,
-                    usage::SpecDoubleDashChoices::Automatic => DoubleDash::Automatic,
-                    _ => DoubleDash::Optional,
-                },
-            }))
-        })
-        .collect();
-
-    let subcommands: Vec<&'static Command<'static>> = cmd
-        .subcommands
-        .values()
-        // A subcommand states its own or says nothing; there is no spec-level setting to
-        // hand it, since the root has already taken that.
-        .map(|sub| build(sub, None))
-        .collect();
-
-    let aliases: Vec<&'static str> = cmd
-        .aliases
-        .iter()
-        .chain(cmd.hidden_aliases.iter())
-        .map(|a| leak(a))
-        .collect();
-
-    Box::leak(Box::new(Command {
-        name: leak(&cmd.name),
-        aliases: Box::leak(aliases.into_boxed_slice()),
-        flags: Box::leak(flags.into_boxed_slice()),
-        args: Box::leak(args.into_boxed_slice()),
-        subcommands: Box::leak(subcommands.into_boxed_slice()),
-        // Filled in by the caller for the root, which is the only place a spec declares one.
-        default_subcommand: None,
-        unknown_flags,
-        // The corpus describes argv parsing; `--version` is a question the *caller* answers,
-        // so no vector turns on it and the harness leaves it off.
-        version: false,
-        key: 0,
-    }))
-}
-
-fn leak(s: &str) -> &'static str {
-    Box::leak(s.to_string().into_boxed_str())
-}

conformance/src/bin/render-oracle.rs

@@ -0,0 +1,99 @@
+//! Report what each implementation renders for every vector in the rendering corpus.
+//!
+//! Authoring aid, not a test. `cargo run -p usage-conformance --bin render-oracle` prints what
+//! usage-lib and usage-argv produce beside what the vector expects, which is how an
+//! expectation gets filled in with a measurement rather than a guess — and how the `reference`
+//! label gets set honestly when the two disagree.
+//!
+//! `--json` emits the same thing machine-readably, which is what makes it usable for filling in
+//! a new vector: an array of `{"id", "usage-lib", "usage-argv"}`, where each of the two is a
+//! vector's `expect` object exactly as written. So it is not the file's own shape — pick the
+//! renderer you have decided is right and copy that object, which `corpus/render/README.md`
+//! shows with `jq`. Copying a measurement beats transcribing a page by hand either way.
+//!
+//! The test suite (`conformance/tests/render.rs`) is what actually enforces agreement in CI.
+
+use usage_conformance::render::{self, Outcome, Rendered};
+
+fn main() -> Result<(), String> {
+    let json = std::env::args().any(|a| a == "--json");
+    let filter = std::env::args()
+        .skip(1)
+        .find(|a| !a.starts_with("--"))
+        .unwrap_or_default();
+
+    let files = render::load(render::corpus_dir())?;
+    let mut rows = Vec::new();
+
+    for file in &files {
+        for vector in &file.vectors {
+            if !filter.is_empty() && !vector.id.contains(&filter) {
+                continue;
+            }
+            rows.push((vector, render::reference(vector), render::argv(vector)));
+        }
+    }
+
+    if json {
+        // One row per vector, each renderer's result in the shape a vector's `expect` takes, so
+        // a new one can be filled in by copying the object of whichever renderer is right.
+        let out: Vec<serde_json::Value> = rows
+            .iter()
+            .map(|(vector, lib, argv)| {
+                serde_json::json!({
+                    "id": vector.id,
+                    "usage-lib": value(lib),
+                    "usage-argv": value(argv),
+                })
+            })
+            .collect();
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&out).map_err(|e| e.to_string())?
+        );
+        return Ok(());
+    }
+
+    for (vector, lib, argv) in &rows {
+        let lib_diff = lib.difference(&vector.expect);
+        let argv_diff = argv.difference(&vector.expect);
+        let mark = match (&lib_diff, &argv_diff) {
+            // A vector usage-argv is not asked to answer reports as agreeing, so it is called
+            // what it is rather than being counted as a pass.
+            _ if matches!(argv, Outcome::OutOfScope(_)) => "skip",
+            (None, None) => "ok  ",
+            (Some(_), Some(_)) => "BOTH",
+            (Some(_), None) => "LIB ",
+            (None, Some(_)) => "ARGV",
+        };
+        println!("{mark}  {}", vector.id);
+        if let Some(diff) = lib_diff {
+            println!("        usage-lib:  {}", indent(&diff));
+        }
+        if let Some(diff) = argv_diff {
+            println!("        usage-argv: {}", indent(&diff));
+        }
+    }
+    Ok(())
+}
+
+/// What an implementation produced, in the vector's own shape.
+fn value(outcome: &Outcome) -> serde_json::Value {
+    match outcome {
+        Outcome::Bad(why) => serde_json::json!({ "error": why }),
+        Outcome::OutOfScope(why) => serde_json::json!({ "out_of_scope": why }),
+        Outcome::Rendered(Rendered {
+            usage,
+            short_help,
+            long_help,
+        }) => serde_json::json!({
+            "usage": usage,
+            "short_help": short_help,
+            "long_help": long_help,
+        }),
+    }
+}
+
+fn indent(text: &str) -> String {
+    text.replace('\n', "\n        ")
+}

conformance/src/lib.rs

@@ -16,6 +16,8 @@ use serde::{Deserialize, Serialize};
 pub mod argv;
 pub mod config;
 pub mod reference;
+pub mod render;
+pub mod tables;
 
 /// One `corpus/*.json` file: a themed group of vectors.
 ///