Add the XSD schema model to the core crate

README.md

@@ -20,6 +20,27 @@ Website: <https://matdata-eu.github.io/schema-explorer/>
 - A command-line tool (`schema-explorer serve` / `index`), a Docker image and
   an embeddable `<schema-explorer>` web component (`@matdata/schema-explorer`)
 
+## Indexing a schema set
+
+The core reads an XSD set (XSD 1.0, tolerating 1.1 constructs) into one JSON
+bundle for the viewer: every global and local component keyed by qualified
+name and source file and line, with documentation, references resolved to
+ids, "where used" back-references, derived types, substitution group
+members, effective content models and a search index.
+
+```sh
+schema-explorer index path/to/NeTEx_publication.xsd -o bundle.json
+schema-explorer index schemas/ --compact -o bundle.json   # directories are read recursively
+```
+
+Imports and includes are followed from disk; a location that cannot be
+found falls back to an input file with the imported namespace or the same
+file name, and otherwise becomes an `unresolved-import` diagnostic in the
+bundle. The bundle format is versioned (`format`, see
+`crates/core/src/bundle.rs`); `--compact` leaves out the parts the viewer
+can recompute. In the browser, the WebAssembly build exposes
+`index(names, contents, compact)` with the same output.
+
 ## Layout
 
 - `crates/core`: Rust library that reads schema sets into an indexed model
@@ -41,7 +62,9 @@ cd web && npm ci && npm run build
 Tests on real-world schemas read them from the directory in `SCHEMA_FIXTURES`
 (default `tests/fixtures`, gitignored) and are skipped when absent.
 `scripts/fetch-fixtures.sh` fetches the public [NeTEx](https://github.com/NeTEx-CEN/NeTEx)
-schemas there. Schemas whose licence forbids redistribution (such as railML)
+schemas there; the tests also look for `railml/<version>/{schema,codelists}`
+and `telematics/{netex,taf,taf-v3.5.1,b5,b10}`. For timings, run
+`cargo test --release --test fixtures -- --nocapture`. Schemas whose licence forbids redistribution (such as railML)
 must never be committed.
 
 ## Licence

crates/cli/src/main.rs

@@ -3,11 +3,14 @@
 //!
 //! Early development: the commands are stubs.
 
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
 
-use anyhow::{Result, bail};
+use anyhow::{Context, Result, bail};
 use clap::{Parser, Subcommand};
 use include_dir::{Dir, include_dir};
+use schema_explorer_core::bundle::Level;
+use schema_explorer_core::{Options, Source};
 
 /// The web viewer, embedded at build time (see build.rs).
 static WEB: Dir<'static> = include_dir!("$OUT_DIR/web");
@@ -38,12 +41,23 @@ enum Command {
         no_open: bool,
     },
     /// Index schema files into a bundle for the viewer.
+    ///
+    /// Directories are read recursively (every .xsd file); imports and
+    /// includes are followed from disk.
     Index {
         /// Schema files or directories.
+        #[arg(required = true)]
         files: Vec<PathBuf>,
         /// Output file.
         #[arg(short, long, default_value = "bundle.json")]
         output: PathBuf,
+        /// Write the compact form (no effective content models or search
+        /// index; the viewer recomputes them).
+        #[arg(long)]
+        compact: bool,
+        /// Indent the JSON.
+        #[arg(long)]
+        pretty: bool,
     },
 }
 
@@ -65,16 +79,93 @@ fn main() -> Result<()> {
                 WEB.entries().len()
             )
         }
-        Some(Command::Index { files, output }) => {
-            let _ = files;
-            bail!(
-                "index is not implemented yet (would write {})",
-                output.display()
-            )
-        }
+        Some(Command::Index {
+            files,
+            output,
+            compact,
+            pretty,
+        }) => index(&files, &output, compact, pretty),
         None => {
             println!("schema-explorer {}", schema_explorer_core::version());
             Ok(())
         }
     }
 }
+
+/// Every .xsd file under `path` (or `path` itself when it is a file).
+fn collect(path: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
+    if path.is_dir() {
+        let mut entries: Vec<PathBuf> = std::fs::read_dir(path)
+            .with_context(|| format!("reading {}", path.display()))?
+            .map(|e| e.map(|e| e.path()))
+            .collect::<std::io::Result<_>>()?;
+        entries.sort();
+        for p in entries {
+            if p.is_dir() || p.extension().is_some_and(|x| x.eq_ignore_ascii_case("xsd")) {
+                collect(&p, out)?;
+            }
+        }
+    } else if path.is_file() {
+        out.push(path.to_path_buf());
+    } else {
+        bail!("{} not found", path.display());
+    }
+    Ok(())
+}
+
+fn index(inputs: &[PathBuf], output: &Path, compact: bool, pretty: bool) -> Result<()> {
+    let start = Instant::now();
+    let mut paths = Vec::new();
+    for p in inputs {
+        collect(p, &mut paths)?;
+    }
+    let sources = paths
+        .iter()
+        .map(|p| {
+            let bytes = std::fs::read(p).with_context(|| format!("reading {}", p.display()))?;
+            Ok(Source::new(p.to_string_lossy(), bytes))
+        })
+        .collect::<Result<Vec<_>>>()?;
+    let mut resolver = |path: &str| -> Option<Vec<u8>> {
+        if path.contains("://") {
+            return None;
+        }
+        std::fs::read(path).ok()
+    };
+    let opts = Options {
+        generator: format!("schema-explorer {}", env!("CARGO_PKG_VERSION")),
+        compact,
+    };
+    let bundle = schema_explorer_core::index(sources, Some(&mut resolver), &opts);
+    let json = if pretty {
+        schema_explorer_core::to_json_pretty(&bundle)?
+    } else {
+        schema_explorer_core::to_json(&bundle)?
+    };
+    std::fs::write(output, &json).with_context(|| format!("writing {}", output.display()))?;
+    for d in &bundle.diagnostics {
+        if d.level == Level::Info {
+            continue;
+        }
+        let file = d
+            .file
+            .map_or("", |f| bundle.files[f as usize].name.as_str());
+        eprintln!(
+            "{:?}: {file}:{}: {} [{}]",
+            d.level, d.line, d.message, d.code
+        );
+    }
+    let s = &bundle.stats;
+    eprintln!(
+        "{} files, {} global and {} local components, {} unresolved imports, {} unresolved references; wrote {} ({} KB) in {:.2?}",
+        s.files,
+        s.global.values().sum::<u32>(),
+        s.local.values().sum::<u32>(),
+        s.unresolved_imports,
+        s.unresolved_refs,
+        output.display(),
+        json.len() / 1024,
+        start.elapsed()
+    );
+    Ok(())
+}

crates/core/Cargo.toml

@@ -11,3 +11,6 @@ keywords = ["xsd", "xml-schema", "json-schema", "schema", "uml"]
 categories = ["parser-implementations", "visualization"]
 
 [dependencies]
+roxmltree = "0.21.1"
+serde.workspace = true
+serde_json.workspace = true