From b51defc9fb5d6d280cf1a618d9649a604d19e5a6 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 21 Oct 2026 16:00:58 +0000 Subject: [PATCH 1/2] Add the XSD schema model to the core crate Read XSD sets from in-memory files into a serde JSON bundle (format 1): - roxmltree parser; imports, includes, redefines and overrides resolved relative to the including file, through an optional resolver callback, then by namespace or file name; unresolved ones become diagnostics - global and local elements, complex and simple types, attributes, attribute groups, model groups and identity constraints, keyed by qualified name with source file and line, documentation (xml:lang) and appinfo; built-in types as components; chameleon includes - derived indexes: where used, derived types, substitution group members, effective content models and a search index; a compact form - wasm `index(names, contents, compact)` and CLI `index -o` - tests on hand-written schemas and on real-world fixtures (NeTEx, railML 3.1-3.3, TAF, TAP B.5/B.10) from SCHEMA_FIXTURES --- Cargo.lock | 153 +++ README.md | 25 +- crates/cli/src/main.rs | 109 +- crates/core/Cargo.toml | 3 + crates/core/src/bundle.rs | 710 ++++++++++++ crates/core/src/index.rs | 381 +++++++ crates/core/src/input.rs | 444 ++++++++ crates/core/src/lib.rs | 118 +- crates/core/src/xsd.rs | 1078 +++++++++++++++++++ crates/core/tests/data/basic/main.xsd | 145 +++ crates/core/tests/data/basic/other.xsd | 10 + crates/core/tests/data/basic/sub/common.xsd | 14 + crates/core/tests/data/broken/broken.xsd | 12 + crates/core/tests/data/broken/notxml.xsd | 1 + crates/core/tests/data/redefine/base.xsd | 6 + crates/core/tests/data/redefine/main.xsd | 13 + crates/core/tests/fixtures.rs | 277 ++++- crates/core/tests/model.rs | 501 +++++++++ crates/wasm/Cargo.toml | 1 + crates/wasm/src/lib.rs | 35 + 20 files changed, 3996 insertions(+), 40 deletions(-) create mode 100644 crates/core/src/bundle.rs create mode 100644 crates/core/src/index.rs create mode 100644 crates/core/src/input.rs create mode 100644 crates/core/src/xsd.rs create mode 100644 crates/core/tests/data/basic/main.xsd create mode 100644 crates/core/tests/data/basic/other.xsd create mode 100644 crates/core/tests/data/basic/sub/common.xsd create mode 100644 crates/core/tests/data/broken/broken.xsd create mode 100644 crates/core/tests/data/broken/notxml.xsd create mode 100644 crates/core/tests/data/redefine/base.xsd create mode 100644 crates/core/tests/data/redefine/main.xsd create mode 100644 crates/core/tests/model.rs diff --git a/Cargo.lock b/Cargo.lock index cf53090..8a281b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,48 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -141,12 +183,45 @@ dependencies = [ "quote", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + [[package]] name = "once_cell" version = "1.21.4" @@ -159,6 +234,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -177,6 +258,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -196,15 +286,72 @@ dependencies = [ [[package]] name = "schema-explorer-core" version = "0.1.0" +dependencies = [ + "roxmltree", + "serde", + "serde_json", +] [[package]] name = "schema-explorer-wasm" version = "0.1.0" dependencies = [ + "js-sys", "schema-explorer-core", "wasm-bindgen", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "foldhash", + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "strsim" version = "0.11.1" @@ -293,3 +440,9 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/README.md b/README.md index c58c788..301ef85 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,27 @@ Website: - A command-line tool (`schema-explorer serve` / `index`), a Docker image and an embeddable `` web component (`@matdata/schema-explorer`) +## Indexing a schema set + +The core reads an XSD set (XSD 1.0, tolerating 1.1 constructs) into one JSON +bundle for the viewer: every global and local component keyed by qualified +name and source file and line, with documentation, references resolved to +ids, "where used" back-references, derived types, substitution group +members, effective content models and a search index. + +```sh +schema-explorer index path/to/NeTEx_publication.xsd -o bundle.json +schema-explorer index schemas/ --compact -o bundle.json # directories are read recursively +``` + +Imports and includes are followed from disk; a location that cannot be +found falls back to an input file with the imported namespace or the same +file name, and otherwise becomes an `unresolved-import` diagnostic in the +bundle. The bundle format is versioned (`format`, see +`crates/core/src/bundle.rs`); `--compact` leaves out the parts the viewer +can recompute. In the browser, the WebAssembly build exposes +`index(names, contents, compact)` with the same output. + ## Layout - `crates/core`: Rust library that reads schema sets into an indexed model @@ -41,7 +62,9 @@ cd web && npm ci && npm run build Tests on real-world schemas read them from the directory in `SCHEMA_FIXTURES` (default `tests/fixtures`, gitignored) and are skipped when absent. `scripts/fetch-fixtures.sh` fetches the public [NeTEx](https://github.com/NeTEx-CEN/NeTEx) -schemas there. Schemas whose licence forbids redistribution (such as railML) +schemas there; the tests also look for `railml//{schema,codelists}` +and `telematics/{netex,taf,taf-v3.5.1,b5,b10}`. For timings, run +`cargo test --release --test fixtures -- --nocapture`. Schemas whose licence forbids redistribution (such as railML) must never be committed. ## Licence diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d77bf0f..3636171 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -3,11 +3,14 @@ //! //! Early development: the commands are stubs. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::time::Instant; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use include_dir::{Dir, include_dir}; +use schema_explorer_core::bundle::Level; +use schema_explorer_core::{Options, Source}; /// The web viewer, embedded at build time (see build.rs). static WEB: Dir<'static> = include_dir!("$OUT_DIR/web"); @@ -38,12 +41,23 @@ enum Command { no_open: bool, }, /// Index schema files into a bundle for the viewer. + /// + /// Directories are read recursively (every .xsd file); imports and + /// includes are followed from disk. Index { /// Schema files or directories. + #[arg(required = true)] files: Vec, /// Output file. #[arg(short, long, default_value = "bundle.json")] output: PathBuf, + /// Write the compact form (no effective content models or search + /// index; the viewer recomputes them). + #[arg(long)] + compact: bool, + /// Indent the JSON. + #[arg(long)] + pretty: bool, }, } @@ -65,16 +79,93 @@ fn main() -> Result<()> { WEB.entries().len() ) } - Some(Command::Index { files, output }) => { - let _ = files; - bail!( - "index is not implemented yet (would write {})", - output.display() - ) - } + Some(Command::Index { + files, + output, + compact, + pretty, + }) => index(&files, &output, compact, pretty), None => { println!("schema-explorer {}", schema_explorer_core::version()); Ok(()) } } } + +/// Every .xsd file under `path` (or `path` itself when it is a file). +fn collect(path: &Path, out: &mut Vec) -> Result<()> { + if path.is_dir() { + let mut entries: Vec = std::fs::read_dir(path) + .with_context(|| format!("reading {}", path.display()))? + .map(|e| e.map(|e| e.path())) + .collect::>()?; + entries.sort(); + for p in entries { + if p.is_dir() || p.extension().is_some_and(|x| x.eq_ignore_ascii_case("xsd")) { + collect(&p, out)?; + } + } + } else if path.is_file() { + out.push(path.to_path_buf()); + } else { + bail!("{} not found", path.display()); + } + Ok(()) +} + +fn index(inputs: &[PathBuf], output: &Path, compact: bool, pretty: bool) -> Result<()> { + let start = Instant::now(); + let mut paths = Vec::new(); + for p in inputs { + collect(p, &mut paths)?; + } + let sources = paths + .iter() + .map(|p| { + let bytes = std::fs::read(p).with_context(|| format!("reading {}", p.display()))?; + Ok(Source::new(p.to_string_lossy(), bytes)) + }) + .collect::>>()?; + let mut resolver = |path: &str| -> Option> { + if path.contains("://") { + return None; + } + std::fs::read(path).ok() + }; + let opts = Options { + generator: format!("schema-explorer {}", env!("CARGO_PKG_VERSION")), + compact, + }; + let bundle = schema_explorer_core::index(sources, Some(&mut resolver), &opts); + let json = if pretty { + schema_explorer_core::to_json_pretty(&bundle)? + } else { + schema_explorer_core::to_json(&bundle)? + }; + std::fs::write(output, &json).with_context(|| format!("writing {}", output.display()))?; + for d in &bundle.diagnostics { + if d.level == Level::Info { + continue; + } + let file = d + .file + .map_or("", |f| bundle.files[f as usize].name.as_str()); + eprintln!( + "{:?}: {file}:{}: {} [{}]", + d.level, d.line, d.message, d.code + ); + } + let s = &bundle.stats; + eprintln!( + "{} files, {} global and {} local components, {} unresolved imports, {} unresolved references; wrote {} ({} KB) in {:.2?}", + s.files, + s.global.values().sum::(), + s.local.values().sum::(), + s.unresolved_imports, + s.unresolved_refs, + output.display(), + json.len() / 1024, + start.elapsed() + ); + Ok(()) +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 80a72d0..1c47249 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -11,3 +11,6 @@ keywords = ["xsd", "xml-schema", "json-schema", "schema", "uml"] categories = ["parser-implementations", "visualization"] [dependencies] +roxmltree = "0.21.1" +serde.workspace = true +serde_json.workspace = true diff --git a/crates/core/src/bundle.rs b/crates/core/src/bundle.rs new file mode 100644 index 0000000..bf22096 --- /dev/null +++ b/crates/core/src/bundle.rs @@ -0,0 +1,710 @@ +//! The bundle: one XSD set read into components with small integer ids, plus +//! the indexes the viewer needs ("where used", derived types, substitution +//! group members, effective content models, search), as serde JSON. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Bundle format version. Bump when the viewer must change to read it. +pub const FORMAT: u32 = 1; + +/// Index into [`Bundle::components`]. +pub type Id = u32; +/// Index into [`Bundle::files`]. +pub type FileId = u32; +/// Index into [`Bundle::namespaces`]. +pub type NsId = u32; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Bundle { + pub format: u32, + pub generator: String, + /// True when the bundle was written in the compact form: no + /// `effective` content models and no `search` index (the viewer + /// recomputes them). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub compact: bool, + pub files: Vec, + pub namespaces: Vec, + /// Component ids are indexes into this vector. + pub components: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub diagnostics: Vec, + pub stats: Stats, + /// Lower-case search terms (names, name parts, words of the + /// documentation and enumeration values) to the components they occur in. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub search: BTreeMap>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FileInfo { + /// Path of the file, relative to the common directory of all files. + pub name: String, + /// The `targetNamespace` attribute. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target_namespace: Option, + /// The namespace the components of the file are in: the target + /// namespace, or that of the including schema for a chameleon include. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + /// Prefixes declared on the `xs:schema` element. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub prefixes: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub element_form_qualified: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub attribute_form_qualified: bool, + /// True when the file was given as input, false when it was pulled in + /// through an import or include. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub input: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub directives: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub docs: Vec, + /// Number of global components declared in the file. + pub components: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum DirectiveKind { + Import, + Include, + Redefine, + Override, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Directive { + pub kind: DirectiveKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub location: Option, + /// The file it resolved to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file: Option, + /// How it was resolved when not by its location: `namespace` (another + /// input file has the imported namespace), `filename` (an input file has + /// the same file name) or `builtin` (the XML namespace). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub via: Option, + pub line: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Namespace { + pub uri: String, + /// Prefix for display, unique in the bundle. + pub prefix: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Kind { + Element, + ComplexType, + SimpleType, + Attribute, + AttributeGroup, + Group, + Key, + Keyref, + Unique, +} + +impl Kind { + pub fn is_type(self) -> bool { + matches!(self, Kind::ComplexType | Kind::SimpleType) + } + pub fn is_identity(self) -> bool { + matches!(self, Kind::Key | Kind::Keyref | Kind::Unique) + } +} + +/// A reference to a component: its id, or `{ns}local` when it was not +/// found (the diagnostics say why). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Ref { + Id(Id), + Missing(String), +} + +impl Ref { + pub fn id(&self) -> Option { + match self { + Ref::Id(id) => Some(*id), + Ref::Missing(_) => None, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Component { + pub kind: Kind, + /// Local name; empty for an anonymous type. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ns: Option, + /// The enclosing component of a local declaration or anonymous type; + /// none for a global one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parent: Option, + /// None for the built-in types of XML Schema and the XML namespace. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file: Option, + #[serde(default, skip_serializing_if = "is_zero")] + pub line: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub docs: Vec, + /// Content of `xs:appinfo`, as XML text. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub appinfo: Vec, + /// The global component this one redefines (`xs:redefine` / `xs:override`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub redefines: Option, + /// Who references this component. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub used_by: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub element: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub complex_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub simple_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attribute: Option, + /// Model group definition: its particle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attribute_group: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity: Option, + /// A built-in type or attribute (no source file). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub builtin: bool, +} + +impl Component { + pub fn new(kind: Kind, name: String) -> Component { + Component { + kind, + name, + ns: None, + parent: None, + file: None, + line: 0, + docs: Vec::new(), + appinfo: Vec::new(), + redefines: None, + used_by: Vec::new(), + element: None, + complex_type: None, + simple_type: None, + attribute: None, + group: None, + attribute_group: None, + identity: None, + builtin: false, + } + } + pub fn is_global(&self) -> bool { + self.parent.is_none() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Doc { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lang: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + pub text: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum UseKind { + /// `type=` of an element or attribute. + Type, + /// `base=` of an extension or restriction. + Base, + /// `itemType=` of a list. + Item, + /// A member type of a union. + Member, + /// `ref=` to a global element. + Element, + /// `ref=` to a global attribute. + Attribute, + /// `ref=` to a model group. + Group, + /// `ref=` to an attribute group. + AttributeGroup, + /// `substitutionGroup=` naming the head element. + Substitution, + /// `refer=` of a keyref. + Refer, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Usage { + /// The referencing component (a local one has a `parent` chain up to + /// the global component). + pub by: Id, + pub how: UseKind, +} + +fn is_zero(n: &u32) -> bool { + *n == 0 +} +fn one() -> u32 { + 1 +} +fn is_one(n: &u32) -> bool { + *n == 1 +} +fn some_one() -> Option { + Some(1) +} +fn is_some_one(n: &Option) -> bool { + *n == Some(1) +} + +/// `minOccurs`/`maxOccurs`; `max: null` means unbounded, absent means 1. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct Occurs { + #[serde(default = "one", skip_serializing_if = "is_one")] + pub min: u32, + #[serde(default = "some_one", skip_serializing_if = "is_some_one")] + pub max: Option, +} + +impl Default for Occurs { + fn default() -> Self { + Occurs { + min: 1, + max: Some(1), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "camelCase")] +pub enum Particle { + /// A local element declaration or a reference to a global one. + Element { + el: Ref, + #[serde(flatten)] + occurs: Occurs, + }, + /// A reference to a model group. + Group { + group: Ref, + #[serde(flatten)] + occurs: Occurs, + }, + Sequence { + items: Vec, + #[serde(flatten)] + occurs: Occurs, + }, + Choice { + items: Vec, + #[serde(flatten)] + occurs: Occurs, + }, + All { + items: Vec, + #[serde(flatten)] + occurs: Occurs, + }, + Any { + #[serde(flatten)] + wildcard: Wildcard, + #[serde(flatten)] + occurs: Occurs, + }, +} + +impl Particle { + pub fn occurs(&self) -> Occurs { + match self { + Particle::Element { occurs, .. } + | Particle::Group { occurs, .. } + | Particle::Sequence { occurs, .. } + | Particle::Choice { occurs, .. } + | Particle::All { occurs, .. } + | Particle::Any { occurs, .. } => *occurs, + } + } + + /// Calls `f` on this particle and every nested one. + pub fn walk<'a>(&'a self, f: &mut impl FnMut(&'a Particle)) { + f(self); + if let Particle::Sequence { items, .. } + | Particle::Choice { items, .. } + | Particle::All { items, .. } = self + { + for p in items { + p.walk(f); + } + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Wildcard { + /// The `namespace` attribute (`##any` when absent). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub not_namespace: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub process_contents: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Constraint { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ElementInfo { + /// The declared type (named, built-in or anonymous). None when the + /// declaration has neither: the type is that of the substitution group + /// head, or `xs:anyType`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#type: Option, + /// Substitution group heads (one in XSD 1.0). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub substitution_group: Vec, + /// Elements whose substitution group names this one (direct members). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub members: Vec, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub r#abstract: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub nillable: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub block: Option, + /// Identity constraints (key, keyref, unique) declared on the element. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub identity: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Method { + Extension, + Restriction, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Derivation { + pub method: Method, + pub base: Ref, + /// True for `xs:simpleContent`. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub simple: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComplexInfo { + /// None for a type that restricts `xs:anyType` implicitly. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub derivation: Option, + /// The particle declared by this type itself (for an extension: only + /// what it adds to the base). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any_attribute: Option, + /// Facets of a `xs:simpleContent` restriction. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub facets: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub enumerations: Vec, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub mixed: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub r#abstract: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub block: Option, + /// XSD 1.1 `xs:assert` tests. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub asserts: Vec, + /// Types that name this one as their base (both complex and simple). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub derived: Vec, + /// The effective content model, inheritance included. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effective: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum AttrUse { + #[default] + Optional, + Required, + Prohibited, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "camelCase")] +pub enum AttrItem { + /// A local attribute declaration or a reference to a global one. + Attribute { + attr: Ref, + #[serde(default, skip_serializing_if = "is_optional")] + r#use: AttrUse, + #[serde(default, skip_serializing_if = "Option::is_none")] + default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + fixed: Option, + }, + /// A reference to an attribute group. + Group { group: Ref }, +} + +fn is_optional(u: &AttrUse) -> bool { + *u == AttrUse::Optional +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Effective { + /// The types whose own `content` particles, in this order, make up the + /// content model (a sequence of them). Empty for empty or simple content. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub content: Vec, + /// All attributes, inherited and from attribute groups (expanded). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any_attribute: Option, + /// The simple type of the text content, for simple content. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub simple: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub mixed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EffectiveAttr { + pub attr: Ref, + #[serde(default, skip_serializing_if = "is_optional")] + pub r#use: AttrUse, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed: Option, + /// The type that declares it. + pub from: Id, + /// The attribute group it comes through, if any (the outermost one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub via: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Variety { + Atomic, + List, + Union, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SimpleInfo { + pub variety: Variety, + /// Restriction base. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base: Option, + /// List item type. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub item: Option, + /// Union member types. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub members: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub facets: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub enumerations: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#final: Option, + /// Types that name this one as their base. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub derived: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Facet { + /// Facet element name: `pattern`, `maxLength`, `minInclusive`, ... + pub kind: String, + pub value: String, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub fixed: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Enumeration { + pub value: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub docs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub appinfo: Vec, + #[serde(default, skip_serializing_if = "is_zero")] + pub line: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AttributeInfo { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub r#type: Option, + /// Value constraint on the declaration (global ones; local uses carry + /// theirs in [`AttrItem`]). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub inheritable: bool, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AttributeGroupInfo { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub attributes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub any_attribute: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IdentityInfo { + pub selector: String, + pub fields: Vec, + /// For a keyref: the key or unique constraint it refers to. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub refer: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Level { + Info, + Warning, + Error, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Diagnostic { + pub level: Level, + /// Stable machine-readable code: `parse-error`, `unresolved-import`, + /// `unresolved-ref`, `duplicate`, `cycle`, ... + pub code: String, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file: Option, + #[serde(default, skip_serializing_if = "is_zero")] + pub line: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Stats { + pub files: u32, + /// Global components per kind. + pub global: BTreeMap, + /// Local declarations and anonymous types per kind. + pub local: BTreeMap, + pub enumerations: u32, + pub unresolved_imports: u32, + pub unresolved_refs: u32, +} + +impl Bundle { + /// The display name `prefix:name` of a component. + pub fn qname(&self, id: Id) -> String { + let c = &self.components[id as usize]; + match c.ns { + Some(ns) => format!("{}:{}", self.namespaces[ns as usize].prefix, c.name), + None => c.name.clone(), + } + } + + /// The global component of `kind` with display name `qname` (`prefix:name`, + /// or a bare name for no namespace). + pub fn find(&self, kind: Kind, qname: &str) -> Option { + let (prefix, name) = match qname.split_once(':') { + Some((p, n)) => (Some(p), n), + None => (None, qname), + }; + let ns = match prefix { + Some(p) => Some(self.namespaces.iter().position(|n| n.prefix == p)? as NsId), + None => None, + }; + self.components + .iter() + .position(|c| c.kind == kind && c.parent.is_none() && c.ns == ns && c.name == name) + .map(|i| i as Id) + } + + /// Strip the parts the viewer can recompute (effective content models + /// and the search index) to make big bundles smaller. + pub fn make_compact(&mut self) { + self.compact = true; + self.search.clear(); + for c in &mut self.components { + if let Some(ct) = &mut c.complex_type { + ct.effective = None; + } + } + } + + pub fn unresolved_imports(&self) -> impl Iterator { + self.diagnostics + .iter() + .filter(|d| d.code == "unresolved-import") + } +} diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs new file mode 100644 index 0000000..b60b284 --- /dev/null +++ b/crates/core/src/index.rs @@ -0,0 +1,381 @@ +//! Derived indexes: where used, derived types, substitution group members, +//! effective content models and the search index. + +use crate::bundle::*; +use std::collections::{BTreeMap, HashSet}; + +/// Fill `used_by`, `derived` and `members`. +pub(crate) fn links(components: &mut [Component], usages: Vec<(Id, Usage)>) { + for (target, u) in usages { + let list = &mut components[target as usize].used_by; + if list.last() != Some(&u) { + list.push(u); + } + } + let mut derived: Vec<(Id, Id)> = Vec::new(); + let mut members: Vec<(Id, Id)> = Vec::new(); + for (i, c) in components.iter().enumerate() { + let i = i as Id; + let base = c + .complex_type + .as_ref() + .and_then(|ct| ct.derivation.as_ref()) + .and_then(|d| d.base.id()) + .or_else(|| { + c.simple_type + .as_ref() + .and_then(|st| st.base.as_ref()) + .and_then(Ref::id) + }); + if let Some(b) = base { + derived.push((b, i)); + } + if let Some(e) = &c.element { + for h in e.substitution_group.iter().filter_map(Ref::id) { + members.push((h, i)); + } + } + } + for (b, d) in derived { + let c = &mut components[b as usize]; + if let Some(ct) = &mut c.complex_type { + ct.derived.push(d); + } else if let Some(st) = &mut c.simple_type { + st.derived.push(d); + } + } + for (h, m) in members { + if let Some(e) = &mut components[h as usize].element { + e.members.push(m); + } + } +} + +#[derive(Clone)] +enum State { + Todo, + Busy, + Done(Box), +} + +/// Compute the effective content model of every complex type. +pub(crate) fn effective(components: &mut [Component], diags: &mut Vec) { + let mut memo: Vec = vec![State::Todo; components.len()]; + for i in 0..components.len() { + if components[i].complex_type.is_some() { + compute(components, i as Id, &mut memo, diags); + } + } + for (i, s) in memo.into_iter().enumerate() { + if let (State::Done(e), Some(ct)) = (s, &mut components[i].complex_type) { + ct.effective = Some(*e); + } + } +} + +fn attr_key(components: &[Component], r: &Ref) -> (Option, String) { + match r { + Ref::Id(a) => { + let c = &components[*a as usize]; + (c.ns, c.name.clone()) + } + Ref::Missing(s) => (None, s.clone()), + } +} + +/// Expand attribute items (attribute groups inlined) declared by `from`. +fn expand( + components: &[Component], + items: &[AttrItem], + from: Id, + via: Option, + seen: &mut HashSet, + out: &mut Vec, + any: &mut Option, +) { + for it in items { + match it { + AttrItem::Attribute { + attr, + r#use, + default, + fixed, + } => { + let (default, fixed) = match (default, fixed, attr.id()) { + (None, None, Some(a)) => components[a as usize] + .attribute + .as_ref() + .map_or((None, None), |i| (i.default.clone(), i.fixed.clone())), + _ => (default.clone(), fixed.clone()), + }; + out.push(EffectiveAttr { + attr: attr.clone(), + r#use: *r#use, + default, + fixed, + from, + via, + }); + } + AttrItem::Group { group } => { + let Some(g) = group.id() else { continue }; + if !seen.insert(g) { + continue; + } + if let Some(info) = &components[g as usize].attribute_group { + expand( + components, + &info.attributes, + from, + via.or(Some(g)), + seen, + out, + any, + ); + if any.is_none() { + any.clone_from(&info.any_attribute); + } + } + seen.remove(&g); + } + } + } +} + +fn compute( + components: &[Component], + id: Id, + memo: &mut Vec, + diags: &mut Vec, +) -> Option { + match &memo[id as usize] { + State::Done(e) => return Some((**e).clone()), + State::Busy => { + let c = &components[id as usize]; + diags.push(Diagnostic { + level: Level::Warning, + code: "cycle".into(), + message: format!("type {} derives from itself", c.name), + file: c.file, + line: c.line, + }); + return None; + } + State::Todo => {} + } + let ct = components[id as usize].complex_type.as_ref()?; + memo[id as usize] = State::Busy; + let mut own = Vec::new(); + let mut own_any = None; + expand( + components, + &ct.attributes, + id, + None, + &mut HashSet::new(), + &mut own, + &mut own_any, + ); + if ct.any_attribute.is_some() { + own_any.clone_from(&ct.any_attribute); + } + let base = ct.derivation.as_ref().and_then(|d| d.base.id()); + let base_eff = base + .filter(|b| components[*b as usize].complex_type.is_some()) + .and_then(|b| compute(components, b, memo, diags)) + .unwrap_or(Effective { + content: Vec::new(), + attributes: Vec::new(), + any_attribute: None, + simple: None, + mixed: false, + }); + let own_content = ct.content.is_some().then_some(id); + let mut e = Effective { + content: Vec::new(), + attributes: base_eff.attributes, + any_attribute: None, + simple: None, + mixed: ct.mixed, + }; + match &ct.derivation { + None => { + e.content.extend(own_content); + } + Some(d) if d.simple => { + e.simple = match d.base.id() { + Some(b) if components[b as usize].simple_type.is_some() => Some(d.base.clone()), + _ => base_eff.simple, + }; + if d.method == Method::Restriction + && !(ct.facets.is_empty() && ct.enumerations.is_empty()) + { + e.simple = Some(Ref::Id(id)); + } + e.any_attribute = base_eff.any_attribute.clone(); + } + Some(d) if d.method == Method::Extension => { + e.content = base_eff.content; + e.content.extend(own_content); + e.mixed |= base_eff.mixed; + e.any_attribute = base_eff.any_attribute.clone(); + } + Some(_) => { + e.content.extend(own_content); + } + } + for a in own { + let key = attr_key(components, &a.attr); + let pos = e + .attributes + .iter() + .position(|x| attr_key(components, &x.attr) == key); + match (pos, a.r#use) { + (Some(p), AttrUse::Prohibited) => { + e.attributes.remove(p); + } + (None, AttrUse::Prohibited) => {} + (Some(p), _) => e.attributes[p] = a, + (None, _) => e.attributes.push(a), + } + } + if own_any.is_some() { + e.any_attribute = own_any; + } + memo[id as usize] = State::Done(Box::new(e.clone())); + Some(e) +} + +const STOP: &[&str] = &[ + "the", "and", "for", "are", "this", "that", "with", "from", "which", "not", "can", "may", + "has", "have", "its", "was", "were", "will", "shall", "into", "all", "any", "but", "one", + "used", "use", "type", "element", "value", "values", "see", "also", "other", "only", "when", + "where", "should", "must", "than", "then", "there", "these", "those", "such", "their", "been", + "being", "each", "does", "per", +]; + +/// Lower-case parts of a name: the name itself and its camelCase, `_`, +/// `-` and `.` separated words. +pub fn name_terms(name: &str) -> Vec { + let mut out = vec![name.to_lowercase()]; + let mut word = String::new(); + let chars: Vec = name.chars().collect(); + for (i, &ch) in chars.iter().enumerate() { + let boundary = !ch.is_alphanumeric() + || (ch.is_uppercase() + && i > 0 + && (chars[i - 1].is_lowercase() + || chars.get(i + 1).is_some_and(|n| n.is_lowercase()))) + || (ch.is_ascii_digit() + != chars + .get(i.wrapping_sub(1)) + .is_some_and(|p| p.is_ascii_digit()) + && i > 0); + if boundary && !word.is_empty() { + out.push(std::mem::take(&mut word).to_lowercase()); + } + if ch.is_alphanumeric() { + word.push(ch); + } + } + if !word.is_empty() { + out.push(word.to_lowercase()); + } + out.retain(|w| w.chars().count() >= 2); + out.sort(); + out.dedup(); + out +} + +fn doc_terms(text: &str, out: &mut Vec) { + for w in text.split(|c: char| !c.is_alphanumeric()) { + if w.chars().count() < 3 || w.chars().all(|c| c.is_ascii_digit()) { + continue; + } + let w = w.to_lowercase(); + if !STOP.contains(&w.as_str()) { + out.push(w); + } + } +} + +/// Build the search index over names, documentation and enumerations. +pub(crate) fn search(components: &[Component]) -> BTreeMap> { + let mut index: BTreeMap> = BTreeMap::new(); + let mut terms = Vec::new(); + for (i, c) in components.iter().enumerate() { + if c.builtin { + continue; + } + terms.clear(); + if !c.name.is_empty() { + terms.extend(name_terms(&c.name)); + } + for d in &c.docs { + doc_terms(&d.text, &mut terms); + } + let enums = c + .simple_type + .as_ref() + .map(|s| &s.enumerations) + .or(c.complex_type.as_ref().map(|s| &s.enumerations)); + for e in enums.into_iter().flatten() { + terms.extend(name_terms(&e.value)); + for d in &e.docs { + doc_terms(&d.text, &mut terms); + } + } + terms.sort(); + terms.dedup(); + for t in &terms { + index.entry(t.clone()).or_default().push(i as Id); + } + } + index +} + +pub(crate) fn stats(b: &Bundle, unresolved_refs: u32) -> Stats { + let mut s = Stats { + files: b.files.len() as u32, + unresolved_refs, + unresolved_imports: b.unresolved_imports().count() as u32, + ..Default::default() + }; + for c in &b.components { + if c.builtin { + continue; + } + let map = if c.is_global() && !c.kind.is_identity() { + &mut s.global + } else { + &mut s.local + }; + *map.entry(c.kind).or_default() += 1; + s.enumerations += c + .simple_type + .as_ref() + .map_or(0, |st| st.enumerations.len() as u32); + } + s +} + +#[cfg(test)] +mod tests { + use super::name_terms; + + #[test] + fn terms() { + assert_eq!( + name_terms("StopPlaceRef_VersionStructure"), + vec![ + "place", + "ref", + "stop", + "stopplaceref_versionstructure", + "structure", + "version" + ] + ); + assert_eq!(name_terms("XMLSchema"), vec!["schema", "xml", "xmlschema"]); + } +} diff --git a/crates/core/src/input.rs b/crates/core/src/input.rs new file mode 100644 index 0000000..51025b8 --- /dev/null +++ b/crates/core/src/input.rs @@ -0,0 +1,444 @@ +//! Loading an XSD set from in-memory files: decoding, path handling and +//! resolving `xs:import` / `xs:include` / `xs:redefine` / `xs:override` +//! locations, with an optional callback for files that were not given. + +use crate::bundle::{Diagnostic, DirectiveKind, Level}; +use std::collections::{HashMap, VecDeque}; + +pub(crate) const XS: &str = "http://www.w3.org/2001/XMLSchema"; +pub(crate) const XML: &str = "http://www.w3.org/XML/1998/namespace"; + +/// One input file: its name (a path, used to resolve relative schema +/// locations) and its bytes. +#[derive(Debug, Clone)] +pub struct Source { + pub name: String, + pub bytes: Vec, +} + +impl Source { + pub fn new(name: impl Into, bytes: impl Into>) -> Source { + Source { + name: name.into(), + bytes: bytes.into(), + } + } +} + +/// Supplies the bytes of a file that an import or include names but that +/// was not among the inputs; gets the normalized path (or URL). +pub type Resolver<'r> = dyn FnMut(&str) -> Option> + 'r; + +pub(crate) struct RawDirective { + pub kind: DirectiveKind, + pub namespace: Option, + pub location: Option, + pub line: u32, + pub file: Option, + pub via: Option<&'static str>, +} + +pub(crate) struct LoadedFile { + /// Normalized path. + pub name: String, + /// Path relative to the common directory of all files. + pub display: String, + pub text: String, + pub input: bool, + pub ok: bool, + pub target_namespace: Option, + pub directives: Vec, + pub lines: LineIndex, +} + +/// Byte offset to line number. +pub(crate) struct LineIndex(Vec); + +impl LineIndex { + pub fn new(text: &str) -> LineIndex { + LineIndex( + text.bytes() + .enumerate() + .filter(|(_, b)| *b == b'\n') + .map(|(i, _)| i) + .collect(), + ) + } + /// 1-based line of byte offset `pos`. + pub fn line(&self, pos: usize) -> u32 { + (self.0.partition_point(|&nl| nl < pos) + 1) as u32 + } +} + +pub(crate) fn parse_options<'a>() -> roxmltree::ParsingOptions<'a> { + roxmltree::ParsingOptions { + allow_dtd: true, + ..Default::default() + } +} + +/// Decode XML bytes: UTF-8 (with or without BOM), UTF-16 with a BOM, else +/// Latin-1. +pub(crate) fn decode(bytes: &[u8]) -> (String, bool) { + if let Some(b) = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]) { + return (String::from_utf8_lossy(b).into_owned(), true); + } + let utf16 = |b: &[u8], le: bool| { + let units: Vec = b + .chunks_exact(2) + .map(|c| { + if le { + u16::from_le_bytes([c[0], c[1]]) + } else { + u16::from_be_bytes([c[0], c[1]]) + } + }) + .collect(); + String::from_utf16_lossy(&units) + }; + if let Some(b) = bytes.strip_prefix(&[0xFF, 0xFE]) { + return (utf16(b, true), true); + } + if let Some(b) = bytes.strip_prefix(&[0xFE, 0xFF]) { + return (utf16(b, false), true); + } + match std::str::from_utf8(bytes) { + Ok(s) => (s.to_string(), true), + Err(_) => (bytes.iter().map(|&b| b as char).collect(), false), + } +} + +pub(crate) fn is_url(s: &str) -> bool { + s.split_once("://") + .is_some_and(|(scheme, _)| scheme.chars().all(|c| c.is_ascii_alphanumeric())) +} + +/// Normalize a path: `\` to `/`, drop `.` segments and fold `..`. +pub fn normalize(path: &str) -> String { + if is_url(path) { + return path.to_string(); + } + let path = path.replace('\\', "/"); + let absolute = path.starts_with('/'); + let mut out: Vec<&str> = Vec::new(); + for seg in path.split('/') { + match seg { + "" | "." => {} + ".." => { + if out.last().is_some_and(|s| *s != "..") { + out.pop(); + } else if !absolute { + out.push(".."); + } + } + s => out.push(s), + } + } + let joined = out.join("/"); + if absolute { + format!("/{joined}") + } else { + joined + } +} + +/// Resolve `location` against the file `base`. +pub fn join(base: &str, location: &str) -> String { + let location = location.trim(); + if is_url(location) || location.starts_with('/') { + return normalize(location); + } + if is_url(base) { + let dir = &base[..base.rfind('/').map_or(base.len(), |i| i + 1)]; + return format!("{dir}{location}"); + } + let dir = match base.rfind(['/', '\\']) { + Some(i) => &base[..=i], + None => "", + }; + normalize(&format!("{dir}{location}")) +} + +fn basename(path: &str) -> &str { + path.rsplit(['/', '\\']).next().unwrap_or(path) +} + +fn file_diag(level: Level, code: &str, message: String, file: usize, line: u32) -> Diagnostic { + Diagnostic { + level, + code: code.into(), + message, + file: Some(file as u32), + line, + } +} + +/// Read the root element of one file: target namespace and directives. +fn scan(f: &mut LoadedFile, idx: usize, diags: &mut Vec) { + let doc = match roxmltree::Document::parse_with_options(&f.text, parse_options()) { + Ok(d) => d, + Err(e) => { + let line = e.pos().row; + diags.push(file_diag( + Level::Error, + "parse-error", + format!("not well-formed XML: {e}"), + idx, + line, + )); + return; + } + }; + let root = doc.root_element(); + if root.tag_name().namespace() != Some(XS) || root.tag_name().name() != "schema" { + diags.push(file_diag( + Level::Warning, + "not-a-schema", + "the root element is not xs:schema".to_string(), + idx, + 1, + )); + return; + } + f.ok = true; + f.target_namespace = root + .attribute("targetNamespace") + .filter(|s| !s.is_empty()) + .map(str::to_string); + for c in root + .children() + .filter(|c| c.tag_name().namespace() == Some(XS)) + { + let kind = match c.tag_name().name() { + "import" => DirectiveKind::Import, + "include" => DirectiveKind::Include, + "redefine" => DirectiveKind::Redefine, + "override" => DirectiveKind::Override, + _ => continue, + }; + f.directives.push(RawDirective { + kind, + namespace: c.attribute("namespace").map(str::to_string), + location: c + .attribute("schemaLocation") + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string), + line: f.lines.line(c.range().start), + file: None, + via: None, + }); + } +} + +/// Load the sources and everything they import or include. +pub(crate) fn load( + sources: Vec, + mut resolver: Option<&mut Resolver<'_>>, + diags: &mut Vec, +) -> Vec { + let mut files: Vec = Vec::new(); + let mut by_name: HashMap = HashMap::new(); + let add = |files: &mut Vec, + by_name: &mut HashMap, + diags: &mut Vec, + name: String, + bytes: &[u8], + input: bool| { + let (text, utf8) = decode(bytes); + let idx = files.len(); + if !utf8 { + diags.push(file_diag( + Level::Info, + "encoding", + "not UTF-8, read as Latin-1".to_string(), + idx, + 0, + )); + } + by_name.insert(name.clone(), idx); + files.push(LoadedFile { + display: String::new(), + name, + lines: LineIndex::new(&text), + text, + input, + ok: false, + target_namespace: None, + directives: Vec::new(), + }); + idx + }; + for s in sources { + let name = normalize(&s.name); + if by_name.contains_key(&name) { + continue; + } + add(&mut files, &mut by_name, diags, name, &s.bytes, true); + } + let mut queue: VecDeque = (0..files.len()).collect(); + while let Some(i) = queue.pop_front() { + let mut f = std::mem::replace( + &mut files[i], + LoadedFile { + display: String::new(), + name: String::new(), + text: String::new(), + input: false, + ok: false, + target_namespace: None, + directives: Vec::new(), + lines: LineIndex(Vec::new()), + }, + ); + scan(&mut f, i, diags); + for d in &mut f.directives { + let Some(loc) = &d.location else { continue }; + let target = join(&f.name, loc); + if let Some(&j) = by_name.get(&target) { + d.file = Some(j); + continue; + } + if let Some(r) = resolver.as_deref_mut() + && let Some(bytes) = r(&target) + { + let j = add(&mut files, &mut by_name, diags, target, &bytes, false); + queue.push_back(j); + d.file = Some(j); + } + } + files[i] = f; + } + fallback(&mut files, diags); + let names: Vec<&str> = files.iter().map(|f| f.name.as_str()).collect(); + let root = common_dir(&names); + for f in &mut files { + f.display = f.name.strip_prefix(&root).unwrap_or(&f.name).to_string(); + } + files +} + +/// Resolve the directives whose location was not found: by namespace, by +/// file name, or as the built-in XML namespace; report the rest. +fn fallback(files: &mut [LoadedFile], diags: &mut Vec) { + let mut by_ns: HashMap<&str, Vec> = HashMap::new(); + let mut by_base: HashMap<&str, Vec> = HashMap::new(); + for (i, f) in files.iter().enumerate() { + if let Some(ns) = &f.target_namespace { + by_ns.entry(ns.as_str()).or_default().push(i); + } + by_base.entry(basename(&f.name)).or_default().push(i); + } + let mut found: Vec<(usize, usize, Option, Option<&'static str>)> = Vec::new(); + for (i, f) in files.iter().enumerate() { + for (k, d) in f.directives.iter().enumerate() { + if d.file.is_some() { + continue; + } + let ns = d.namespace.as_deref(); + let by_namespace = (d.kind == DirectiveKind::Import) + .then(|| ns.and_then(|ns| by_ns.get(ns))) + .flatten() + .and_then(|v| v.iter().copied().find(|&j| j != i)); + let by_file = d + .location + .as_deref() + .and_then(|l| by_base.get(basename(l))) + .filter(|v| v.len() == 1) + .map(|v| v[0]) + .filter(|&j| j != i) + .filter(|&j| { + d.kind != DirectiveKind::Import || files[j].target_namespace.as_deref() == ns + }); + let r = if let Some(j) = by_namespace { + (Some(j), Some("namespace")) + } else if let Some(j) = by_file { + (Some(j), Some("filename")) + } else if d.kind == DirectiveKind::Import && ns == Some(XML) { + (None, Some("builtin")) + } else if d.kind == DirectiveKind::Import && d.location.is_none() { + // An import without a location only declares the namespace; + // components from it show up as unresolved references. + (None, None) + } else { + diags.push(file_diag( + Level::Warning, + "unresolved-import", + format!( + "cannot find the {} {}", + match d.kind { + DirectiveKind::Import => "import", + DirectiveKind::Include => "include", + DirectiveKind::Redefine => "redefine", + DirectiveKind::Override => "override", + }, + d.location.as_deref().or(ns).unwrap_or("?") + ), + i, + d.line, + )); + continue; + }; + found.push((i, k, r.0, r.1)); + } + } + for (i, k, file, via) in found { + let d = &mut files[i].directives[k]; + d.file = file; + d.via = via; + } +} + +/// The longest common directory of the file names, with a trailing `/`. +pub(crate) fn common_dir(names: &[&str]) -> String { + let local: Vec<&str> = names.iter().copied().filter(|n| !is_url(n)).collect(); + let Some(first) = local.first() else { + return String::new(); + }; + let mut prefix: &str = match first.rfind('/') { + Some(i) => &first[..=i], + None => return String::new(), + }; + for n in &local[1..] { + while !n.starts_with(prefix) { + prefix = match prefix[..prefix.len() - 1].rfind('/') { + Some(i) => &prefix[..=i], + None => return String::new(), + }; + } + } + prefix.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn paths() { + assert_eq!(normalize("a/./b/../c.xsd"), "a/c.xsd"); + assert_eq!(normalize("../x.xsd"), "../x.xsd"); + assert_eq!(normalize("/r/a/../../b.xsd"), "/b.xsd"); + assert_eq!(join("a/b/c.xsd", "../../d/e.xsd"), "d/e.xsd"); + assert_eq!(join("c.xsd", "e.xsd"), "e.xsd"); + assert_eq!(join("a\\c.xsd", "e.xsd"), "a/e.xsd"); + assert_eq!(join("a/c.xsd", "http://x.org/y.xsd"), "http://x.org/y.xsd"); + assert_eq!(common_dir(&["/a/b/c.xsd", "/a/b/d/e.xsd"]), "/a/b/"); + assert_eq!(common_dir(&["a.xsd"]), ""); + } + + #[test] + fn lines() { + let l = LineIndex::new("a\nb\nc"); + assert_eq!(l.line(0), 1); + assert_eq!(l.line(2), 2); + assert_eq!(l.line(4), 3); + } + + #[test] + fn decoding() { + assert_eq!(decode(b"\xEF\xBB\xBFab").0, "ab"); + assert_eq!(decode(b"\xFF\xFEa\0").0, "a"); + assert_eq!(decode(b"\xE9").0, "\u{e9}"); + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index ead12b5..bc1d457 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,13 +1,129 @@ //! Core of schema-explorer: reads XML Schema (XSD) sets, and later JSON Schema, //! into one indexed model for the viewer. //! -//! Early development: the XSD reader is not implemented yet. +//! The input is a set of in-memory files (name and bytes), so the same code +//! runs in the command line tool and, compiled to WebAssembly, in the +//! browser. Imports and includes are resolved relative to the including +//! file, among the given files or through an optional [`Resolver`]. +//! Problems (unparsable files, unresolved imports or references) become +//! [diagnostics](bundle::Diagnostic) in the [`Bundle`]; indexing never fails. + +pub mod bundle; +pub mod index; +pub mod input; +mod xsd; + +pub use bundle::{Bundle, FORMAT, Id, Kind}; +pub use input::{Resolver, Source}; + +/// Indexing options. +#[derive(Debug, Clone, Default)] +pub struct Options { + /// Written to [`Bundle::generator`]. + pub generator: String, + /// Write the compact form (see [`Bundle::make_compact`]). + pub compact: bool, +} /// The version of this crate. pub fn version() -> &'static str { env!("CARGO_PKG_VERSION") } +/// Read an XSD set into a bundle. +pub fn index(sources: Vec, resolver: Option<&mut Resolver<'_>>, opts: &Options) -> Bundle { + let mut diagnostics = Vec::new(); + let files = input::load(sources, resolver, &mut diagnostics); + let docs: Vec> = files + .iter() + .map(|f| { + f.ok.then(|| { + roxmltree::Document::parse_with_options(&f.text, input::parse_options()).ok() + }) + .flatten() + }) + .collect(); + let mut b = xsd::Builder::new(&files); + b.run(&docs); + let file_infos: Vec = files + .iter() + .enumerate() + .map(|(i, f)| { + let root_el = docs[i].as_ref().map(|d| d.root_element()); + bundle::FileInfo { + name: f.display.clone(), + target_namespace: f.target_namespace.clone(), + namespace: b.file_ns[i], + prefixes: docs[i] + .as_ref() + .map(xsd::Builder::file_prefixes) + .unwrap_or_default(), + version: root_el + .and_then(|r| r.attribute("version")) + .map(str::to_string), + element_form_qualified: root_el + .is_some_and(|r| r.attribute("elementFormDefault") == Some("qualified")), + attribute_form_qualified: root_el + .is_some_and(|r| r.attribute("attributeFormDefault") == Some("qualified")), + input: f.input, + directives: f + .directives + .iter() + .map(|d| bundle::Directive { + kind: d.kind, + namespace: d.namespace.clone(), + location: d.location.clone(), + file: d.file.map(|j| j as bundle::FileId), + via: d.via.map(str::to_string), + line: d.line, + }) + .collect(), + docs: root_el.map(|r| xsd::annotations(&r).0).unwrap_or_default(), + components: b.file_globals[i], + } + }) + .collect(); + diagnostics.append(&mut b.diagnostics); + let mut components = std::mem::take(&mut b.components); + index::links(&mut components, std::mem::take(&mut b.usages)); + index::effective(&mut components, &mut diagnostics); + let search = if opts.compact { + Default::default() + } else { + index::search(&components) + }; + let mut bundle = Bundle { + format: FORMAT, + generator: if opts.generator.is_empty() { + format!("schema-explorer-core {}", version()) + } else { + opts.generator.clone() + }, + compact: false, + files: file_infos, + namespaces: std::mem::take(&mut b.namespaces), + components, + diagnostics, + stats: Default::default(), + search, + }; + bundle.stats = index::stats(&bundle, b.unresolved_refs); + if opts.compact { + bundle.make_compact(); + } + bundle +} + +/// Serialize a bundle as JSON (no whitespace). +pub fn to_json(bundle: &Bundle) -> serde_json::Result { + serde_json::to_string(bundle) +} + +/// Serialize a bundle as indented JSON. +pub fn to_json_pretty(bundle: &Bundle) -> serde_json::Result { + serde_json::to_string_pretty(bundle) +} + #[cfg(test)] mod tests { #[test] diff --git a/crates/core/src/xsd.rs b/crates/core/src/xsd.rs new file mode 100644 index 0000000..4caee8f --- /dev/null +++ b/crates/core/src/xsd.rs @@ -0,0 +1,1078 @@ +//! Reading the components of the loaded schema documents into the bundle. +//! +//! Pass 1 registers every global component (and every identity constraint) +//! under its qualified name; pass 2 reads their content and the local +//! declarations, resolving references by qualified name. + +use crate::bundle::*; +use crate::input::{LoadedFile, XML, XS}; +use roxmltree::{Document, Node, NodeId}; +use std::collections::{BTreeMap, HashMap, HashSet}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum Space { + Type, + Element, + Attribute, + AttributeGroup, + Group, + Identity, +} + +fn space_of(kind: Kind) -> Space { + match kind { + Kind::ComplexType | Kind::SimpleType => Space::Type, + Kind::Element => Space::Element, + Kind::Attribute => Space::Attribute, + Kind::AttributeGroup => Space::AttributeGroup, + Kind::Group => Space::Group, + Kind::Key | Kind::Keyref | Kind::Unique => Space::Identity, + } +} + +fn kind_of(name: &str) -> Option { + Some(match name { + "element" => Kind::Element, + "complexType" => Kind::ComplexType, + "simpleType" => Kind::SimpleType, + "attribute" => Kind::Attribute, + "attributeGroup" => Kind::AttributeGroup, + "group" => Kind::Group, + "key" => Kind::Key, + "keyref" => Kind::Keyref, + "unique" => Kind::Unique, + _ => return None, + }) +} + +/// Built-in types: name, base, and whether it is a list type. +const BUILTINS: &[(&str, &str)] = &[ + ("anyType", ""), + ("anySimpleType", "anyType"), + ("anyAtomicType", "anySimpleType"), + ("string", "anyAtomicType"), + ("boolean", "anyAtomicType"), + ("decimal", "anyAtomicType"), + ("float", "anyAtomicType"), + ("double", "anyAtomicType"), + ("duration", "anyAtomicType"), + ("dateTime", "anyAtomicType"), + ("time", "anyAtomicType"), + ("date", "anyAtomicType"), + ("gYearMonth", "anyAtomicType"), + ("gYear", "anyAtomicType"), + ("gMonthDay", "anyAtomicType"), + ("gDay", "anyAtomicType"), + ("gMonth", "anyAtomicType"), + ("hexBinary", "anyAtomicType"), + ("base64Binary", "anyAtomicType"), + ("anyURI", "anyAtomicType"), + ("QName", "anyAtomicType"), + ("NOTATION", "anyAtomicType"), + ("normalizedString", "string"), + ("token", "normalizedString"), + ("language", "token"), + ("NMTOKEN", "token"), + ("NMTOKENS", "anySimpleType"), + ("Name", "token"), + ("NCName", "Name"), + ("ID", "NCName"), + ("IDREF", "NCName"), + ("IDREFS", "anySimpleType"), + ("ENTITY", "NCName"), + ("ENTITIES", "anySimpleType"), + ("integer", "decimal"), + ("nonPositiveInteger", "integer"), + ("negativeInteger", "nonPositiveInteger"), + ("long", "integer"), + ("int", "long"), + ("short", "int"), + ("byte", "short"), + ("nonNegativeInteger", "integer"), + ("unsignedLong", "nonNegativeInteger"), + ("unsignedInt", "unsignedLong"), + ("unsignedShort", "unsignedInt"), + ("unsignedByte", "unsignedShort"), + ("positiveInteger", "nonNegativeInteger"), + ("dateTimeStamp", "dateTime"), + ("dayTimeDuration", "duration"), + ("yearMonthDuration", "duration"), + ("error", "anySimpleType"), +]; + +/// Attributes of the XML namespace, available without importing it. +const XML_ATTRS: &[(&str, &str)] = &[ + ("lang", "language"), + ("space", "NCName"), + ("base", "anyURI"), + ("id", "ID"), +]; + +const FACETS: &[&str] = &[ + "length", + "minLength", + "maxLength", + "pattern", + "whiteSpace", + "maxInclusive", + "maxExclusive", + "minInclusive", + "minExclusive", + "totalDigits", + "fractionDigits", + "assertion", + "explicitTimezone", +]; + +fn is_xs(n: &Node) -> bool { + n.is_element() && n.tag_name().namespace() == Some(XS) +} + +fn xs_children<'a, 'i>(n: Node<'a, 'i>) -> impl Iterator> { + n.children().filter(is_xs) +} + +fn name_of<'a>(n: &Node<'a, '_>) -> &'a str { + n.tag_name().name() +} + +fn flag(n: &Node, attr: &str) -> bool { + matches!(n.attribute(attr).map(str::trim), Some("true" | "1")) +} + +fn opt(n: &Node, attr: &str) -> Option { + n.attribute(attr).map(str::to_string) +} + +fn occurs(n: &Node) -> Occurs { + let num = |s: &str| -> u32 { + let s = s.trim(); + s.parse::().unwrap_or_else(|_| { + if !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) { + u32::MAX + } else { + 1 + } + }) + }; + Occurs { + min: n.attribute("minOccurs").map_or(1, num), + max: match n.attribute("maxOccurs").map(str::trim) { + None => Some(1), + Some("unbounded") => None, + Some(s) => Some(num(s)), + }, + } +} + +fn wildcard(n: &Node) -> Wildcard { + Wildcard { + namespace: opt(n, "namespace"), + not_namespace: opt(n, "notNamespace"), + process_contents: opt(n, "processContents"), + } +} + +/// Text of a documentation element: descendant text with lines trimmed and +/// runs of blank lines folded. +fn doc_text(n: &Node) -> String { + let mut raw = String::new(); + for d in n.descendants().filter(|d| d.is_text()) { + raw.push_str(d.text().unwrap_or("")); + } + let mut out = String::new(); + let mut blank = false; + for line in raw.lines().map(str::trim) { + if line.is_empty() { + blank = !out.is_empty(); + continue; + } + if !out.is_empty() { + out.push_str(if blank { "\n\n" } else { "\n" }); + } + blank = false; + out.push_str(line); + } + out +} + +/// The documentation and appinfo of the `xs:annotation` children of `n`. +pub(crate) fn annotations(n: &Node) -> (Vec, Vec) { + let mut docs = Vec::new(); + let mut appinfo = Vec::new(); + for a in xs_children(*n).filter(|a| name_of(a) == "annotation") { + for c in xs_children(a) { + match name_of(&c) { + "documentation" => { + let text = doc_text(&c); + if !text.is_empty() { + docs.push(Doc { + lang: c.attribute((XML, "lang")).map(str::to_string), + source: opt(&c, "source"), + text, + }); + } + } + "appinfo" => { + let input = c.document().input_text(); + let r = c.range(); + let whole = &input[r.clone()]; + let inner = match (whole.find('>'), whole.rfind('<')) { + (Some(s), Some(e)) if s < e && !whole[..=s].ends_with("/>") => { + &whole[s + 1..e] + } + _ => "", + }; + let inner = inner.trim(); + if !inner.is_empty() { + appinfo.push(inner.to_string()); + } + } + _ => {} + } + } + } + (docs, appinfo) +} + +#[derive(Clone, Copy)] +struct Ctx { + file: usize, + /// Namespace of the components of the file. + ns: Option, + /// The file has no targetNamespace and takes the including one. + chameleon: bool, + qualified_el: bool, + qualified_attr: bool, + /// The global component being read. + global: Id, +} + +pub(crate) struct Builder<'f> { + files: &'f [LoadedFile], + pub components: Vec, + pub namespaces: Vec, + ns_ids: HashMap, + prefixes: HashMap, + table: HashMap<(Space, Option, String), Id>, + /// Pass 1 ids of global components and identity constraints. + pre: HashMap<(usize, NodeId), Id>, + builtins: HashMap<(bool, &'static str), Id>, + pub usages: Vec<(Id, Usage)>, + pub diagnostics: Vec, + missing: HashSet, + pub unresolved_refs: u32, + pub file_ns: Vec>, + pub file_globals: Vec, +} + +impl<'f> Builder<'f> { + pub fn new(files: &'f [LoadedFile]) -> Builder<'f> { + Builder { + files, + components: Vec::new(), + namespaces: Vec::new(), + ns_ids: HashMap::new(), + prefixes: HashMap::new(), + table: HashMap::new(), + pre: HashMap::new(), + builtins: HashMap::new(), + usages: Vec::new(), + diagnostics: Vec::new(), + missing: HashSet::new(), + unresolved_refs: 0, + file_ns: vec![None; files.len()], + file_globals: vec![0; files.len()], + } + } + + /// Read all documents. + pub fn run(&mut self, docs: &[Option>]) { + self.collect_prefixes(docs); + self.assign_file_namespaces(); + let ctxs: Vec> = docs + .iter() + .enumerate() + .map(|(i, d)| d.as_ref().map(|d| self.ctx(i, d))) + .collect(); + // Pass 1: register global components, then redefinitions. + for (i, d) in docs.iter().enumerate() { + let (Some(d), Some(ctx)) = (d, ctxs[i]) else { + continue; + }; + for c in xs_children(d.root_element()) { + self.register(ctx, c, false); + } + } + for (i, d) in docs.iter().enumerate() { + let (Some(d), Some(ctx)) = (d, ctxs[i]) else { + continue; + }; + for r in xs_children(d.root_element()) + .filter(|r| matches!(name_of(r), "redefine" | "override")) + { + for c in xs_children(r) { + self.register(ctx, c, true); + } + } + } + // Pass 2: content. + for (i, d) in docs.iter().enumerate() { + let (Some(d), Some(ctx)) = (d, ctxs[i]) else { + continue; + }; + let root = d.root_element(); + let tops = xs_children(root).flat_map(|c| { + if matches!(name_of(&c), "redefine" | "override") { + xs_children(c).collect::>() + } else { + vec![c] + } + }); + for c in tops { + if let Some(&id) = self.pre.get(&(i, c.id())) { + let ctx = Ctx { global: id, ..ctx }; + self.global(ctx, c, id); + } + } + } + } + + fn collect_prefixes(&mut self, docs: &[Option>]) { + self.prefixes.insert(XS.into(), "xs".into()); + self.prefixes.insert(XML.into(), "xml".into()); + // Prefer the prefix a schema uses for its own target namespace. + for own in [true, false] { + for (i, d) in docs.iter().enumerate() { + let Some(d) = d else { continue }; + let tns = self.files[i].target_namespace.as_deref(); + for ns in d.root_element().namespaces() { + let Some(p) = ns.name() else { continue }; + if own != (Some(ns.uri()) == tns) { + continue; + } + self.prefixes + .entry(ns.uri().to_string()) + .or_insert_with(|| p.to_string()); + } + } + } + } + + fn assign_file_namespaces(&mut self) { + for (i, f) in self.files.iter().enumerate() { + if let Some(tns) = &f.target_namespace { + self.file_ns[i] = Some(self.ns_id(tns)); + } + } + // Chameleon includes take the namespace of the including schema. + loop { + let mut changed = false; + for (i, f) in self.files.iter().enumerate() { + for d in &f.directives { + if d.kind == DirectiveKind::Import { + continue; + } + if let Some(j) = d.file + && self.files[j].target_namespace.is_none() + && self.file_ns[j].is_none() + && self.file_ns[i].is_some() + { + self.file_ns[j] = self.file_ns[i]; + changed = true; + } + } + } + if !changed { + break; + } + } + } + + fn ctx(&self, i: usize, d: &Document) -> Ctx { + let root = d.root_element(); + Ctx { + file: i, + ns: self.file_ns[i], + chameleon: self.files[i].target_namespace.is_none() && self.file_ns[i].is_some(), + qualified_el: root.attribute("elementFormDefault") == Some("qualified"), + qualified_attr: root.attribute("attributeFormDefault") == Some("qualified"), + global: 0, + } + } + + pub fn ns_id(&mut self, uri: &str) -> NsId { + if let Some(&id) = self.ns_ids.get(uri) { + return id; + } + let id = self.namespaces.len() as NsId; + let base = self + .prefixes + .get(uri) + .cloned() + .or_else(|| prefix_from_uri(uri)) + .unwrap_or_else(|| format!("ns{}", id + 1)); + let mut prefix = base.clone(); + let mut n = 2; + while self.namespaces.iter().any(|ns| ns.prefix == prefix) { + prefix = format!("{base}{n}"); + n += 1; + } + self.namespaces.push(Namespace { + uri: uri.to_string(), + prefix, + }); + self.ns_ids.insert(uri.to_string(), id); + id + } + + fn line(&self, ctx: Ctx, n: &Node) -> u32 { + self.files[ctx.file].lines.line(n.range().start) + } + + fn diag(&mut self, level: Level, code: &str, message: String, ctx: Ctx, n: &Node) { + let line = self.line(ctx, n); + self.diagnostics.push(Diagnostic { + level, + code: code.into(), + message, + file: Some(ctx.file as FileId), + line, + }); + } + + fn push(&mut self, c: Component) -> Id { + self.components.push(c); + (self.components.len() - 1) as Id + } + + /// A new component for the declaration `n`, with its location and + /// annotations. + fn create(&mut self, ctx: Ctx, n: &Node, kind: Kind, name: &str, ns: Option) -> Id { + let (docs, appinfo) = annotations(n); + let mut c = Component::new(kind, name.to_string()); + c.ns = ns; + c.file = Some(ctx.file as FileId); + c.line = self.line(ctx, n); + c.docs = docs; + c.appinfo = appinfo; + self.push(c) + } + + /// Pass 1: register a top-level declaration (and its identity + /// constraints) under its name. + fn register(&mut self, ctx: Ctx, n: Node, redefinition: bool) { + let Some(kind) = kind_of(name_of(&n)) else { + return; + }; + if kind.is_identity() { + return; + } + let Some(name) = n.attribute("name").map(str::trim) else { + self.diag( + Level::Warning, + "no-name", + format!("global xs:{} without a name", name_of(&n)), + ctx, + &n, + ); + return; + }; + let key = (space_of(kind), ctx.ns, name.to_string()); + let existing = self.table.get(&key).copied(); + if let Some(prev) = existing + && !redefinition + { + let p = &self.components[prev as usize]; + let msg = format!( + "{} {name} is declared again (first in {}:{})", + name_of(&n), + self.files[p.file.unwrap_or(0) as usize].display, + p.line + ); + self.diag(Level::Warning, "duplicate", msg, ctx, &n); + return; + } + let id = self.create(ctx, &n, kind, name, ctx.ns); + self.components[id as usize].redefines = existing; + self.table.insert(key, id); + self.pre.insert((ctx.file, n.id()), id); + self.file_globals[ctx.file] += 1; + for d in n.descendants().filter(is_xs) { + let Some(k) = kind_of(name_of(&d)).filter(|k| k.is_identity()) else { + continue; + }; + let Some(iname) = d.attribute("name").map(str::trim) else { + continue; + }; + let key = (Space::Identity, ctx.ns, iname.to_string()); + if self.table.contains_key(&key) && !redefinition { + self.diag( + Level::Warning, + "duplicate", + format!("identity constraint {iname} is declared again"), + ctx, + &d, + ); + continue; + } + let iid = self.create(ctx, &d, k, iname, ctx.ns); + self.table.insert(key, iid); + self.pre.insert((ctx.file, d.id()), iid); + } + } + + fn builtin(&mut self, name: &str, attribute: bool) -> Option { + let (table, ns): (&[(&'static str, &'static str)], &str) = if attribute { + (XML_ATTRS, XML) + } else { + (BUILTINS, XS) + }; + let &(static_name, base) = table.iter().find(|(n, _)| *n == name)?; + if let Some(&id) = self.builtins.get(&(attribute, static_name)) { + return Some(id); + } + let ns = self.ns_id(ns); + let kind = if attribute { + Kind::Attribute + } else if name == "anyType" { + Kind::ComplexType + } else { + Kind::SimpleType + }; + let mut c = Component::new(kind, name.to_string()); + c.ns = Some(ns); + c.builtin = true; + let id = self.push(c); + self.builtins.insert((attribute, static_name), id); + let base = if base.is_empty() { + None + } else { + self.builtin(base, false).map(Ref::Id) + }; + let c = &mut self.components[id as usize]; + match kind { + Kind::Attribute => { + c.attribute = Some(AttributeInfo { + r#type: base, + ..Default::default() + }) + } + Kind::ComplexType => c.complex_type = Some(ComplexInfo::default()), + _ => { + let list = matches!(name, "NMTOKENS" | "IDREFS" | "ENTITIES"); + let item = list.then(|| { + let item = &name[..name.len() - 1]; + let item = if item == "ENTITIE" { "ENTITY" } else { item }; + item.to_string() + }); + c.simple_type = Some(SimpleInfo { + variety: if list { Variety::List } else { Variety::Atomic }, + base, + item: None, + members: Vec::new(), + facets: Vec::new(), + enumerations: Vec::new(), + r#final: None, + derived: Vec::new(), + }); + if let Some(item) = item { + let item = self.builtin(&item, false).map(Ref::Id); + if let Some(st) = &mut self.components[id as usize].simple_type { + st.item = item; + } + } + } + } + Some(id) + } + + /// Resolve the QName `value` (in the scope of `n`) in `space`; records + /// the usage by `by`. + fn resolve( + &mut self, + ctx: Ctx, + n: &Node, + value: &str, + space: Space, + by: Id, + how: UseKind, + ) -> Ref { + let value = value.trim(); + let (prefix, local) = match value.split_once(':') { + Some((p, l)) => (Some(p), l), + None => (None, value), + }; + let uri = match prefix { + Some("xml") => Some(XML), + _ => n.lookup_namespace_uri(prefix), + }; + if prefix.is_some() && uri.is_none() { + self.report_missing(ctx, n, value, format!("undeclared prefix in {value}")); + return Ref::Missing(value.to_string()); + } + let ns_owned: Option = match uri.filter(|u| !u.is_empty()) { + Some(u) => Some(u.to_string()), + None if ctx.chameleon => ctx.ns.map(|i| self.namespaces[i as usize].uri.clone()), + None => None, + }; + let ns = ns_owned.as_deref(); + let ns_id = ns.and_then(|u| self.ns_ids.get(u).copied()); + let key = (space, ns_id, local.to_string()); + if let Some(&id) = self.table.get(&key) { + // A redefinition refers to the component it redefines. + let id = match self.components[id as usize].redefines { + Some(orig) if id == ctx.global => orig, + _ => id, + }; + return self.used(Ref::Id(id), by, how); + } + let builtin = match (ns, space) { + (Some(XS), Space::Type) => self.builtin(local, false), + (Some(XML), Space::Attribute) => self.builtin(local, true), + _ => None, + }; + if let Some(id) = builtin { + return self.used(Ref::Id(id), by, how); + } + let clark = match ns { + Some(u) => format!("{{{u}}}{local}"), + None => local.to_string(), + }; + let what = match space { + Space::Type => "type", + Space::Element => "element", + Space::Attribute => "attribute", + Space::AttributeGroup => "attribute group", + Space::Group => "group", + Space::Identity => "identity constraint", + }; + self.report_missing( + ctx, + n, + &clark, + format!("cannot find {what} {value} ({clark})"), + ); + Ref::Missing(clark) + } + + fn report_missing(&mut self, ctx: Ctx, n: &Node, key: &str, message: String) { + if self.missing.insert(key.to_string()) { + self.unresolved_refs += 1; + self.diag(Level::Warning, "unresolved-ref", message, ctx, n); + } + } + + fn used(&mut self, r: Ref, by: Id, how: UseKind) -> Ref { + if let Ref::Id(t) = r { + self.usages.push((t, Usage { by, how })); + } + r + } + + /// Pass 2 for a global component. + fn global(&mut self, ctx: Ctx, n: Node, id: Id) { + match self.components[id as usize].kind { + Kind::Element => self.element(ctx, n, id), + Kind::ComplexType => self.complex(ctx, n, id), + Kind::SimpleType => self.simple(ctx, n, id), + Kind::Attribute => self.attribute(ctx, n, id, true), + Kind::AttributeGroup => { + let mut info = AttributeGroupInfo::default(); + self.attr_items(ctx, n, id, &mut info.attributes, &mut info.any_attribute); + self.components[id as usize].attribute_group = Some(info); + } + Kind::Group => { + let p = xs_children(n).find_map(|c| self.particle(ctx, c, id)); + self.components[id as usize].group = p; + } + _ => {} + } + } + + fn local( + &mut self, + ctx: Ctx, + n: &Node, + kind: Kind, + name: &str, + ns: Option, + parent: Id, + ) -> Id { + let id = self.create(ctx, n, kind, name, ns); + self.components[id as usize].parent = Some(parent); + id + } + + fn anonymous_type(&mut self, ctx: Ctx, c: Node, parent: Id) -> Option { + let id = match name_of(&c) { + "complexType" => { + let id = self.local(ctx, &c, Kind::ComplexType, "", None, parent); + self.complex(ctx, c, id); + id + } + "simpleType" => { + let id = self.local(ctx, &c, Kind::SimpleType, "", None, parent); + self.simple(ctx, c, id); + id + } + _ => return None, + }; + Some(Ref::Id(id)) + } + + fn element(&mut self, ctx: Ctx, n: Node, id: Id) { + let mut info = ElementInfo { + r#abstract: flag(&n, "abstract"), + nillable: flag(&n, "nillable"), + default: opt(&n, "default"), + fixed: opt(&n, "fixed"), + r#final: opt(&n, "final"), + block: opt(&n, "block"), + ..Default::default() + }; + if let Some(t) = n.attribute("type") { + info.r#type = Some(self.resolve(ctx, &n, t, Space::Type, id, UseKind::Type)); + } + if let Some(sg) = n.attribute("substitutionGroup") { + for h in sg.split_whitespace() { + let r = self.resolve(ctx, &n, h, Space::Element, id, UseKind::Substitution); + info.substitution_group.push(r); + } + } + for c in xs_children(n) { + match name_of(&c) { + "complexType" | "simpleType" => { + if info.r#type.is_none() { + info.r#type = self.anonymous_type(ctx, c, id); + } + } + "key" | "keyref" | "unique" => { + let iid = match self.pre.get(&(ctx.file, c.id())) { + Some(&iid) => iid, + None => { + let kind = kind_of(name_of(&c)).unwrap_or(Kind::Key); + let name = c.attribute("name").unwrap_or(""); + self.create(ctx, &c, kind, name, ctx.ns) + } + }; + self.components[iid as usize].parent = Some(id); + self.identity(ctx, c, iid); + info.identity.push(iid); + } + _ => {} + } + } + self.components[id as usize].element = Some(info); + } + + fn identity(&mut self, ctx: Ctx, n: Node, id: Id) { + let mut info = IdentityInfo::default(); + for c in xs_children(n) { + match name_of(&c) { + "selector" => info.selector = c.attribute("xpath").unwrap_or("").to_string(), + "field" => info + .fields + .push(c.attribute("xpath").unwrap_or("").to_string()), + _ => {} + } + } + if let Some(r) = n.attribute("refer") { + info.refer = Some(self.resolve(ctx, &n, r, Space::Identity, id, UseKind::Refer)); + } + self.components[id as usize].identity = Some(info); + } + + fn complex(&mut self, ctx: Ctx, n: Node, id: Id) { + let mut ct = ComplexInfo { + mixed: flag(&n, "mixed"), + r#abstract: flag(&n, "abstract"), + r#final: opt(&n, "final"), + block: opt(&n, "block"), + ..Default::default() + }; + for c in xs_children(n) { + let simple = match name_of(&c) { + "simpleContent" => true, + "complexContent" => false, + _ => continue, + }; + if !simple && c.has_attribute("mixed") { + ct.mixed = flag(&c, "mixed"); + } + for d in xs_children(c) { + let method = match name_of(&d) { + "extension" => Method::Extension, + "restriction" => Method::Restriction, + _ => continue, + }; + let base = match d.attribute("base") { + Some(b) => self.resolve(ctx, &d, b, Space::Type, id, UseKind::Base), + None => match xs_children(d).find(|x| name_of(x) == "simpleType") { + Some(st) => self + .anonymous_type(ctx, st, id) + .unwrap_or(Ref::Missing(String::new())), + None => Ref::Missing(String::new()), + }, + }; + ct.derivation = Some(Derivation { + method, + base, + simple, + }); + self.content(ctx, d, id, &mut ct); + } + } + self.content(ctx, n, id, &mut ct); + self.components[id as usize].complex_type = Some(ct); + } + + fn content(&mut self, ctx: Ctx, n: Node, id: Id, ct: &mut ComplexInfo) { + for c in xs_children(n) { + match name_of(&c) { + "sequence" | "choice" | "all" | "group" => ct.content = self.particle(ctx, c, id), + "assert" => ct + .asserts + .push(c.attribute("test").unwrap_or("").to_string()), + "enumeration" => ct.enumerations.push(self.enumeration(ctx, c)), + f if FACETS.contains(&f) => ct.facets.push(facet(c)), + _ => {} + } + } + self.attr_items(ctx, n, id, &mut ct.attributes, &mut ct.any_attribute); + } + + fn particle(&mut self, ctx: Ctx, n: Node, owner: Id) -> Option { + let occurs = occurs(&n); + Some(match name_of(&n) { + "element" => { + let el = match n.attribute("ref") { + Some(r) => self.resolve(ctx, &n, r, Space::Element, owner, UseKind::Element), + None => Ref::Id(self.local_element(ctx, n, owner)), + }; + Particle::Element { el, occurs } + } + "group" => { + let r = n.attribute("ref").unwrap_or(""); + let group = self.resolve(ctx, &n, r, Space::Group, owner, UseKind::Group); + Particle::Group { group, occurs } + } + "sequence" | "choice" | "all" => { + let items = xs_children(n) + .filter_map(|c| self.particle(ctx, c, owner)) + .collect(); + match name_of(&n) { + "sequence" => Particle::Sequence { items, occurs }, + "choice" => Particle::Choice { items, occurs }, + _ => Particle::All { items, occurs }, + } + } + "any" => Particle::Any { + wildcard: wildcard(&n), + occurs, + }, + _ => return None, + }) + } + + fn local_ns(&mut self, ctx: Ctx, n: &Node, qualified_default: bool) -> Option { + if let Some(tns) = n.attribute("targetNamespace") { + return (!tns.is_empty()).then(|| self.ns_id(tns)); + } + let qualified = match n.attribute("form") { + Some(f) => f == "qualified", + None => qualified_default, + }; + if qualified { ctx.ns } else { None } + } + + fn local_element(&mut self, ctx: Ctx, n: Node, owner: Id) -> Id { + let name = n.attribute("name").unwrap_or("").trim(); + let ns = self.local_ns(ctx, &n, ctx.qualified_el); + let id = self.local(ctx, &n, Kind::Element, name, ns, owner); + self.element(ctx, n, id); + id + } + + fn attr_items( + &mut self, + ctx: Ctx, + n: Node, + owner: Id, + items: &mut Vec, + any: &mut Option, + ) { + for c in xs_children(n) { + match name_of(&c) { + "attribute" => { + let r#use = match c.attribute("use") { + Some("required") => AttrUse::Required, + Some("prohibited") => AttrUse::Prohibited, + _ => AttrUse::Optional, + }; + let attr = match c.attribute("ref") { + Some(r) => { + self.resolve(ctx, &c, r, Space::Attribute, owner, UseKind::Attribute) + } + None => { + let name = c.attribute("name").unwrap_or("").trim(); + let ns = self.local_ns(ctx, &c, ctx.qualified_attr); + let id = self.local(ctx, &c, Kind::Attribute, name, ns, owner); + self.attribute(ctx, c, id, false); + Ref::Id(id) + } + }; + items.push(AttrItem::Attribute { + attr, + r#use, + default: opt(&c, "default"), + fixed: opt(&c, "fixed"), + }); + } + "attributeGroup" => { + let r = c.attribute("ref").unwrap_or(""); + let group = self.resolve( + ctx, + &c, + r, + Space::AttributeGroup, + owner, + UseKind::AttributeGroup, + ); + items.push(AttrItem::Group { group }); + } + "anyAttribute" => *any = Some(wildcard(&c)), + _ => {} + } + } + } + + fn attribute(&mut self, ctx: Ctx, n: Node, id: Id, global: bool) { + let mut info = AttributeInfo { + inheritable: flag(&n, "inheritable"), + ..Default::default() + }; + if global { + info.default = opt(&n, "default"); + info.fixed = opt(&n, "fixed"); + } + if let Some(t) = n.attribute("type") { + info.r#type = Some(self.resolve(ctx, &n, t, Space::Type, id, UseKind::Type)); + } else if let Some(st) = xs_children(n).find(|c| name_of(c) == "simpleType") { + info.r#type = self.anonymous_type(ctx, st, id); + } + self.components[id as usize].attribute = Some(info); + } + + fn enumeration(&mut self, ctx: Ctx, c: Node) -> Enumeration { + let (docs, appinfo) = annotations(&c); + Enumeration { + value: c.attribute("value").unwrap_or("").to_string(), + docs, + appinfo, + line: self.line(ctx, &c), + } + } + + fn simple(&mut self, ctx: Ctx, n: Node, id: Id) { + let mut st = SimpleInfo { + variety: Variety::Atomic, + base: None, + item: None, + members: Vec::new(), + facets: Vec::new(), + enumerations: Vec::new(), + r#final: opt(&n, "final"), + derived: Vec::new(), + }; + for c in xs_children(n) { + match name_of(&c) { + "restriction" => { + if let Some(b) = c.attribute("base") { + st.base = Some(self.resolve(ctx, &c, b, Space::Type, id, UseKind::Base)); + } + for f in xs_children(c) { + match name_of(&f) { + "simpleType" if st.base.is_none() => { + st.base = self.anonymous_type(ctx, f, id); + } + "enumeration" => { + let e = self.enumeration(ctx, f); + st.enumerations.push(e); + } + k if FACETS.contains(&k) => st.facets.push(facet(f)), + _ => {} + } + } + } + "list" => { + st.variety = Variety::List; + if let Some(t) = c.attribute("itemType") { + st.item = Some(self.resolve(ctx, &c, t, Space::Type, id, UseKind::Item)); + } else if let Some(t) = xs_children(c).find(|x| name_of(x) == "simpleType") { + st.item = self.anonymous_type(ctx, t, id); + } + } + "union" => { + st.variety = Variety::Union; + if let Some(m) = c.attribute("memberTypes") { + for t in m.split_whitespace() { + let r = self.resolve(ctx, &c, t, Space::Type, id, UseKind::Member); + st.members.push(r); + } + } + for t in xs_children(c).filter(|x| name_of(x) == "simpleType") { + if let Some(r) = self.anonymous_type(ctx, t, id) { + st.members.push(r); + } + } + } + _ => {} + } + } + // A restriction of a list or union keeps its variety. + if let Some(Ref::Id(b)) = &st.base + && let Some(bs) = &self.components[*b as usize].simple_type + && bs.variety != Variety::Atomic + && !self.components[*b as usize].builtin + { + st.variety = bs.variety; + } + self.components[id as usize].simple_type = Some(st); + } + + /// Per-file prefix declarations of the schema element. + pub fn file_prefixes(d: &Document) -> BTreeMap { + d.root_element() + .namespaces() + .map(|ns| (ns.name().unwrap_or("").to_string(), ns.uri().to_string())) + .collect() + } +} + +/// A display prefix guessed from a namespace URI: its last word +/// (`http://www.netex.org.uk/netex` gives `netex`). +fn prefix_from_uri(uri: &str) -> Option { + let last = uri + .trim_end_matches(['/', '#']) + .rsplit(['/', ':', '#']) + .find(|s| !s.is_empty())?; + let p: String = last + .chars() + .take_while(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-') + .collect(); + let ok = p.len() <= 12 && p.starts_with(|c: char| c.is_ascii_alphabetic()); + ok.then(|| p.to_lowercase()) +} + +fn facet(f: Node) -> Facet { + Facet { + kind: name_of(&f).to_string(), + value: f + .attribute("value") + .or_else(|| f.attribute("test")) + .unwrap_or("") + .to_string(), + fixed: flag(&f, "fixed"), + } +} diff --git a/crates/core/tests/data/basic/main.xsd b/crates/core/tests/data/basic/main.xsd new file mode 100644 index 0000000..c5f7d66 --- /dev/null +++ b/crates/core/tests/data/basic/main.xsd @@ -0,0 +1,145 @@ + + + Main test schema. + + + + + + + The base of everything. + La base de tout. + stable + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Red as a rose. + Rood. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/crates/core/tests/data/basic/other.xsd b/crates/core/tests/data/basic/other.xsd new file mode 100644 index 0000000..6b7df6d --- /dev/null +++ b/crates/core/tests/data/basic/other.xsd @@ -0,0 +1,10 @@ + + + + + + + + + diff --git a/crates/core/tests/data/basic/sub/common.xsd b/crates/core/tests/data/basic/sub/common.xsd new file mode 100644 index 0000000..2c8aa13 --- /dev/null +++ b/crates/core/tests/data/basic/sub/common.xsd @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/crates/core/tests/data/broken/broken.xsd b/crates/core/tests/data/broken/broken.xsd new file mode 100644 index 0000000..85b379f --- /dev/null +++ b/crates/core/tests/data/broken/broken.xsd @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/crates/core/tests/data/broken/notxml.xsd b/crates/core/tests/data/broken/notxml.xsd new file mode 100644 index 0000000..d6fb8a6 --- /dev/null +++ b/crates/core/tests/data/broken/notxml.xsd @@ -0,0 +1 @@ + diff --git a/crates/core/tests/data/redefine/base.xsd b/crates/core/tests/data/redefine/base.xsd new file mode 100644 index 0000000..4c30df0 --- /dev/null +++ b/crates/core/tests/data/redefine/base.xsd @@ -0,0 +1,6 @@ + + + + + + diff --git a/crates/core/tests/data/redefine/main.xsd b/crates/core/tests/data/redefine/main.xsd new file mode 100644 index 0000000..079109d --- /dev/null +++ b/crates/core/tests/data/redefine/main.xsd @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/crates/core/tests/fixtures.rs b/crates/core/tests/fixtures.rs index 60e2ecf..f4d99d3 100644 --- a/crates/core/tests/fixtures.rs +++ b/crates/core/tests/fixtures.rs @@ -3,9 +3,18 @@ //! Real schemas are not committed. Tests read them from the directory in the //! `SCHEMA_FIXTURES` environment variable (default: `tests/fixtures` at the //! repository root) and are skipped when a schema set is absent. Fetch the -//! public ones with `scripts/fetch-fixtures.sh`. +//! public ones with `scripts/fetch-fixtures.sh`; the railML and telematics +//! sets are expected under `railml//{schema,codelists}` and +//! `telematics/{netex,taf,taf-v3.5.1,b5,b10}`. +//! +//! Run with `--nocapture` (in release mode for meaningful timings) to see +//! the counts and indexing times. +use schema_explorer_core::bundle::{Bundle, Kind}; +use schema_explorer_core::{Options, Source, index}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; fn fixtures_dir() -> PathBuf { match std::env::var_os("SCHEMA_FIXTURES") { @@ -14,39 +23,249 @@ fn fixtures_dir() -> PathBuf { } } -/// The fixture directory `name`, or `None` (with a note) when it is absent. -fn fixture(name: &str) -> Option { - let dir = fixtures_dir().join(name); - if dir.is_dir() { - Some(dir) - } else { - eprintln!("skipping: fixture {} not found", dir.display()); - None +fn collect(path: &Path, out: &mut Vec) { + if path.is_dir() { + let mut entries: Vec = std::fs::read_dir(path) + .into_iter() + .flatten() + .flatten() + .map(|e| e.path()) + .collect(); + entries.sort(); + for p in entries { + if p.is_dir() || p.extension().is_some_and(|x| x == "xsd") { + collect(&p, out); + } + } + } else if path.is_file() { + out.push(path.to_path_buf()); } } -fn count_xsd(dir: &Path) -> usize { - std::fs::read_dir(dir) - .into_iter() - .flatten() - .flatten() - .map(|e| { - let p = e.path(); - if p.is_dir() { - count_xsd(&p) - } else { - usize::from(p.extension().is_some_and(|x| x == "xsd")) - } - }) - .sum() +struct Run { + bundle: Bundle, + /// Indexing time from memory (files already read). + time: Duration, } -#[test] -fn netex_schemas_are_present() { - let Some(dir) = fixture("NeTEx") else { return }; +/// Index the given fixture paths (files or directories, relative to the +/// fixtures directory), following imports from disk. `None` when absent. +fn run(label: &str, paths: &[&str]) -> Option { + let root = fixtures_dir(); + let mut files = Vec::new(); + for p in paths { + let full = root.join(p); + if !full.exists() { + eprintln!("skipping {label}: {} not found", full.display()); + return None; + } + collect(&full, &mut files); + } + let sources: Vec = files + .iter() + .map(|f| Source::new(f.to_string_lossy(), std::fs::read(f).unwrap())) + .collect(); + // First run: from disk, remembering what the resolver read. + let mut read: HashMap> = HashMap::new(); + let mut resolver = |path: &str| { + let bytes = std::fs::read(path).ok()?; + read.insert(path.to_string(), bytes.clone()); + Some(bytes) + }; + let first = index(sources.clone(), Some(&mut resolver), &Options::default()); + // Second run: everything in memory, timed. + let mut all = sources; + all.extend(read.into_iter().map(|(n, b)| Source::new(n, b))); + let start = Instant::now(); + let bundle = index(all, None, &Options::default()); + let time = start.elapsed(); + assert_eq!(first.components.len(), bundle.components.len()); + let s = &bundle.stats; + let g = |k| s.global.get(&k).copied().unwrap_or(0); + eprintln!( + "{label}: {} files, {} complex types, {} simple types, {} elements, {} groups, {} attribute groups (global); {} local; {} enumerations; {} unresolved imports, {} unresolved refs; indexed in {time:.2?}", + s.files, + g(Kind::ComplexType), + g(Kind::SimpleType), + g(Kind::Element), + g(Kind::Group), + g(Kind::AttributeGroup), + s.local.values().sum::(), + s.enumerations, + s.unresolved_imports, + s.unresolved_refs, + ); + for d in bundle.unresolved_imports() { + eprintln!( + " {}: {}", + bundle.files[d.file.unwrap() as usize].name, + d.message + ); + } + Some(Run { bundle, time }) +} + +/// Assert the counts are at least the given minima. +fn at_least(b: &Bundle, complex: u32, simple: u32, elements: u32) { + let g = |k| b.stats.global.get(&k).copied().unwrap_or(0); + assert!( + g(Kind::ComplexType) >= complex, + "complex types: {}", + g(Kind::ComplexType) + ); + assert!( + g(Kind::SimpleType) >= simple, + "simple types: {}", + g(Kind::SimpleType) + ); assert!( - count_xsd(&dir.join("xsd")) > 0, - "no .xsd files in {}", - dir.display() + g(Kind::Element) >= elements, + "elements: {}", + g(Kind::Element) + ); +} + +fn no_unresolved_imports(b: &Bundle) { + let list: Vec<&str> = b.unresolved_imports().map(|d| d.message.as_str()).collect(); + assert!(list.is_empty(), "unresolved imports: {list:?}"); +} + +fn netex(label: &str, entry: &str) { + let Some(r) = run(label, &[entry]) else { + return; + }; + let b = &r.bundle; + no_unresolved_imports(b); + assert_eq!(b.stats.unresolved_refs, 0); + at_least(b, 2500, 1500, 1800); + let pd = b + .find(Kind::Element, "netex:PublicationDelivery") + .expect("PublicationDelivery"); + assert!( + !b.components[pd as usize] + .element + .as_ref() + .unwrap() + .identity + .is_empty() + ); + // Deep inheritance: the effective content of a stop place spans many types. + let sp = b + .find(Kind::ComplexType, "netex:StopPlace_VersionStructure") + .unwrap(); + let eff = b.components[sp as usize] + .complex_type + .as_ref() + .unwrap() + .effective + .as_ref() + .unwrap(); + assert!(eff.content.len() >= 5, "{:?}", eff.content); + assert!( + eff.attributes + .iter() + .any(|a| b.qname(a.attr.id().unwrap()) == "id") + ); + if !cfg!(debug_assertions) { + assert!(r.time < Duration::from_secs(5), "NeTEx took {:?}", r.time); + } +} + +#[test] +fn netex_upstream() { + netex("NeTEx (upstream clone)", "NeTEx/xsd/NeTEx_publication.xsd"); +} + +#[test] +fn netex_telematics() { + netex( + "NeTEx (telematics)", + "telematics/netex/NeTEx_publication.xsd", ); } + +#[test] +fn railml_3_1() { + let Some(r) = run("railML 3.1", &["railml/3.1/schema"]) else { + return; + }; + let b = &r.bundle; + // Known upstream breakage: common3.xsd imports Dublin Core from + // dublincore.org and railML 3.1 does not ship a local copy. + let list: Vec<&str> = b.unresolved_imports().map(|d| d.message.as_str()).collect(); + assert_eq!(list.len(), 1, "{list:?}"); + assert!(list[0].contains("dublincore.org")); + at_least(b, 350, 50, 1); +} + +fn railml(version: &str) { + let label = format!("railML {version}"); + let schema = format!("railml/{version}/schema"); + let codelists = format!("railml/{version}/codelists"); + let Some(r) = run(&label, &[&schema, &codelists]) else { + return; + }; + let b = &r.bundle; + no_unresolved_imports(b); + assert_eq!(b.stats.unresolved_refs, 0); + at_least(b, 600, 150, 1); + let railml = b + .find(Kind::Element, "rail3:railML") + .expect("railML root element"); + assert!(b.components[railml as usize].file.is_some()); +} + +#[test] +fn railml_3_2() { + railml("3.2"); +} + +#[test] +fn railml_3_3() { + railml("3.3"); +} + +#[test] +fn taf() { + for dir in ["telematics/taf", "telematics/taf-v3.5.1"] { + // taf_cat_complete_fixed.xsd is a corrected copy of + // taf_cat_complete.xsd: index each on its own. + for entry in ["taf_cat_complete.xsd", "taf_cat_complete_fixed.xsd"] { + let path = format!("{dir}/{entry}"); + let Some(r) = run(&path, &[&path]) else { + continue; + }; + no_unresolved_imports(&r.bundle); + at_least(&r.bundle, 10, 150, 350); + } + } +} + +#[test] +fn taf_extension() { + // Known upstream gap: EI_TAF_Extension.xsd imports TAF 4.0 + // (taf_cat_complete.xsd as a sibling) while the fixtures hold TAF 3.5, + // whose namespace differs. One unresolved import; the rest reads. + let Some(r) = run("EI_TAF_Extension", &["telematics/EI_TAF_Extension.xsd"]) else { + return; + }; + let list: Vec<&str> = r + .bundle + .unresolved_imports() + .map(|d| d.message.as_str()) + .collect(); + assert_eq!(list, ["cannot find the import taf_cat_complete.xsd"]); + at_least(&r.bundle, 20, 1, 30); +} + +#[test] +fn tap_b5_b10() { + if let Some(r) = run("TAP B.5", &["telematics/b5"]) { + no_unresolved_imports(&r.bundle); + at_least(&r.bundle, 50, 10, 20); + } + if let Some(r) = run("TAP B.10", &["telematics/b10"]) { + no_unresolved_imports(&r.bundle); + at_least(&r.bundle, 10, 10, 1); + } +} diff --git a/crates/core/tests/model.rs b/crates/core/tests/model.rs new file mode 100644 index 0000000..349ebcf --- /dev/null +++ b/crates/core/tests/model.rs @@ -0,0 +1,501 @@ +//! Tests on the small hand-written schemas in tests/data. + +use schema_explorer_core::bundle::*; +use schema_explorer_core::{Options, Source, index, to_json}; +use std::path::Path; + +fn data(files: &[&str]) -> Vec { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data"); + files + .iter() + .map(|f| Source::new(*f, std::fs::read(dir.join(f)).unwrap())) + .collect() +} + +fn basic() -> Bundle { + index( + data(&["basic/main.xsd", "basic/sub/common.xsd", "basic/other.xsd"]), + None, + &Options::default(), + ) +} + +fn get<'a>(b: &'a Bundle, kind: Kind, q: &str) -> (Id, &'a Component) { + let id = b + .find(kind, q) + .unwrap_or_else(|| panic!("{kind:?} {q} not found")); + (id, &b.components[id as usize]) +} + +fn name(b: &Bundle, r: &Ref) -> String { + match r { + Ref::Id(id) => b.qname(*id), + Ref::Missing(s) => format!("?{s}"), + } +} + +#[test] +fn files_and_namespaces() { + let b = basic(); + assert_eq!(b.format, FORMAT); + let names: Vec<&str> = b.files.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, ["main.xsd", "sub/common.xsd", "other.xsd"]); + let main = &b.files[0]; + assert_eq!(main.version.as_deref(), Some("1.2")); + assert!(main.element_form_qualified); + assert_eq!(main.docs[0].text, "Main test schema."); + assert_eq!(main.directives[0].file, Some(1)); + assert_eq!(main.directives[1].file, Some(2)); + assert_eq!(main.directives[2].via.as_deref(), Some("builtin")); + assert_eq!(b.stats.unresolved_imports, 0); + // The chameleon include takes the main namespace. + assert_eq!(b.files[1].namespace, b.files[0].namespace); + let prefixes: Vec<(&str, &str)> = b + .namespaces + .iter() + .map(|n| (n.prefix.as_str(), n.uri.as_str())) + .collect(); + assert!(prefixes.contains(&("m", "urn:test:main"))); + assert!(prefixes.contains(&("o", "urn:test:other"))); + assert!(prefixes.contains(&("xs", "http://www.w3.org/2001/XMLSchema"))); + assert_eq!( + b.files[2].prefixes.get(""), + Some(&"urn:test:other".to_string()) + ); +} + +#[test] +fn only_the_missing_type_is_unresolved() { + let b = basic(); + let refs: Vec<&str> = b + .diagnostics + .iter() + .filter(|d| d.code == "unresolved-ref") + .map(|d| d.message.as_str()) + .collect(); + assert_eq!(refs.len(), 1, "{refs:?}"); + assert!(refs[0].contains("m:Missing")); + assert_eq!(b.stats.unresolved_refs, 1); + let (_, unknown) = get(&b, Kind::Element, "m:Unknown"); + assert_eq!( + unknown.element.as_ref().unwrap().r#type, + Some(Ref::Missing("{urn:test:main}Missing".into())) + ); +} + +#[test] +fn complex_types_and_inheritance() { + let b = basic(); + let (base_id, base) = get(&b, Kind::ComplexType, "m:Base"); + assert_eq!(base.line, 10); + assert_eq!(base.file, Some(0)); + assert_eq!(base.docs.len(), 2); + assert_eq!(base.docs[1].lang.as_deref(), Some("fr")); + assert_eq!(base.appinfo, ["stable"]); + let ct = base.complex_type.as_ref().unwrap(); + assert!(ct.r#abstract); + let Some(Particle::Sequence { items, .. }) = &ct.content else { + panic!("{:?}", ct.content) + }; + let Particle::Element { el, occurs } = &items[1] else { + panic!() + }; + assert_eq!(occurs.min, 0); + assert_eq!(occurs.max, None); + let note = &b.components[el.id().unwrap() as usize]; + assert_eq!(note.name, "note"); + assert_eq!(note.parent, Some(base_id)); + assert_eq!( + b.qname(el.id().unwrap()), + "m:note", + "qualified local element" + ); + assert!(note.element.as_ref().unwrap().nillable); + + let (derived_id, derived) = get(&b, Kind::ComplexType, "m:Derived"); + let (restricted_id, _) = get(&b, Kind::ComplexType, "m:Restricted"); + let (circle_id, circle) = get(&b, Kind::Element, "m:Circle"); + let circle_type = circle + .element + .as_ref() + .unwrap() + .r#type + .as_ref() + .unwrap() + .id() + .unwrap(); + assert_eq!(b.components[circle_type as usize].parent, Some(circle_id)); + // Both directions of the base chain. + assert_eq!(ct.derived, [derived_id]); + let dct = derived.complex_type.as_ref().unwrap(); + assert_eq!(dct.derived, [restricted_id, circle_type]); + let d = dct.derivation.as_ref().unwrap(); + assert_eq!( + (d.method, d.base.clone()), + (Method::Extension, Ref::Id(base_id)) + ); + assert!( + matches!(&dct.content, Some(Particle::Choice { items, .. }) if matches!(items[2], Particle::Any { .. })) + ); + assert_eq!( + dct.any_attribute.as_ref().unwrap().namespace.as_deref(), + Some("##other") + ); + + // Effective content: inherited particles and attributes. + let eff = dct.effective.as_ref().unwrap(); + assert_eq!(eff.content, [base_id, derived_id]); + let attrs: Vec = eff.attributes.iter().map(|a| name(&b, &a.attr)).collect(); + assert_eq!(attrs, ["version", "obsolete", "created", "unit", "o:flag"]); + assert_eq!(eff.attributes[0].default.as_deref(), Some("1")); + assert_eq!( + eff.attributes[2].via, + b.find(Kind::AttributeGroup, "m:Common") + ); + assert_eq!(eff.attributes[4].r#use, AttrUse::Required); + assert_eq!(eff.attributes[4].from, derived_id); + + let circle_eff = b.components[circle_type as usize] + .complex_type + .as_ref() + .unwrap() + .effective + .as_ref() + .unwrap(); + assert_eq!(circle_eff.content, [base_id, derived_id, circle_type]); + + let restricted = b.components[restricted_id as usize] + .complex_type + .as_ref() + .unwrap(); + let eff = restricted.effective.as_ref().unwrap(); + assert_eq!(eff.content, [restricted_id]); + let attrs: Vec = eff.attributes.iter().map(|a| name(&b, &a.attr)).collect(); + assert_eq!( + attrs, + ["version", "created", "unit", "o:flag"], + "obsolete is prohibited" + ); + + let (_, measure) = get(&b, Kind::ComplexType, "m:Measure"); + let eff = measure + .complex_type + .as_ref() + .unwrap() + .effective + .as_ref() + .unwrap(); + assert_eq!( + eff.simple.as_ref().map(|r| name(&b, r)).as_deref(), + Some("xs:decimal") + ); + assert!(eff.content.is_empty()); + + let (_, tag) = get(&b, Kind::ComplexType, "o:Tag"); + let tag = tag.complex_type.as_ref().unwrap(); + assert!(tag.mixed); + let Some(Particle::All { items, .. }) = &tag.content else { + panic!() + }; + let Particle::Element { el, .. } = &items[0] else { + panic!() + }; + assert_eq!( + b.qname(el.id().unwrap()), + "label", + "unqualified local element" + ); + + let (_, range) = get(&b, Kind::ComplexType, "m:Range"); + assert_eq!(range.complex_type.as_ref().unwrap().asserts, ["min le max"]); +} + +#[test] +fn simple_types() { + let b = basic(); + let (colour_id, colour) = get(&b, Kind::SimpleType, "m:Colour"); + let st = colour.simple_type.as_ref().unwrap(); + assert_eq!(st.variety, Variety::Atomic); + assert_eq!(name(&b, st.base.as_ref().unwrap()), "xs:token"); + let values: Vec<&str> = st.enumerations.iter().map(|e| e.value.as_str()).collect(); + assert_eq!(values, ["red", "green", "blue"]); + assert_eq!(st.enumerations[0].docs[1].lang.as_deref(), Some("nl")); + assert_eq!(st.enumerations[0].docs[0].text, "Red as a rose."); + + let (_, list) = get(&b, Kind::SimpleType, "m:Colours"); + let list = list.simple_type.as_ref().unwrap(); + assert_eq!(list.variety, Variety::List); + assert_eq!(list.item, Some(Ref::Id(colour_id))); + + let (union_id, union) = get(&b, Kind::SimpleType, "m:ColourOrCode"); + let union = union.simple_type.as_ref().unwrap(); + assert_eq!(union.variety, Variety::Union); + assert_eq!(union.members[0], Ref::Id(colour_id)); + let anon = &b.components[union.members[1].id().unwrap() as usize]; + assert!(anon.name.is_empty()); + assert_eq!(anon.parent, Some(union_id)); + let facets = &anon.simple_type.as_ref().unwrap().facets; + assert_eq!(facets[0].kind, "pattern"); + assert_eq!(facets[1].kind, "maxLength"); + assert!(facets[1].fixed); + + // Built-in types are components with their own base chain. + let token = st.base.as_ref().unwrap().id().unwrap(); + assert!(b.components[token as usize].builtin); + let token_base = b.components[token as usize] + .simple_type + .as_ref() + .unwrap() + .base + .clone(); + assert_eq!(name(&b, &token_base.unwrap()), "xs:normalizedString"); + + // The chameleon type is in the including namespace. + let (_, unit) = get(&b, Kind::SimpleType, "m:Unit"); + assert_eq!(unit.file, Some(1)); +} + +#[test] +fn elements_substitution_and_identity() { + let b = basic(); + let (shape_id, shape) = get(&b, Kind::Element, "m:Shape"); + let (circle_id, _) = get(&b, Kind::Element, "m:Circle"); + let (square_id, square) = get(&b, Kind::Element, "m:Square"); + let e = shape.element.as_ref().unwrap(); + assert!(e.r#abstract); + assert_eq!(e.members, [circle_id, square_id]); + let sq = square.element.as_ref().unwrap(); + assert_eq!(sq.substitution_group, [Ref::Id(shape_id)]); + assert_eq!(sq.r#final.as_deref(), Some("#all")); + let (_, colour) = get(&b, Kind::Element, "m:colour"); + assert_eq!( + colour.element.as_ref().unwrap().default.as_deref(), + Some("red") + ); + let (_, answer) = get(&b, Kind::Element, "m:answer"); + assert_eq!( + answer.element.as_ref().unwrap().fixed.as_deref(), + Some("42") + ); + + let (root_id, root) = get(&b, Kind::Element, "m:Root"); + let ids = &root.element.as_ref().unwrap().identity; + assert_eq!(ids.len(), 3); + let key = &b.components[ids[0] as usize]; + assert_eq!( + (key.kind, key.name.as_str(), key.parent), + (Kind::Key, "ShapeKey", Some(root_id)) + ); + assert_eq!(key.identity.as_ref().unwrap().fields, ["m:id"]); + let keyref = &b.components[ids[1] as usize]; + assert_eq!(keyref.kind, Kind::Keyref); + assert_eq!( + keyref.identity.as_ref().unwrap().refer, + Some(Ref::Id(ids[0])) + ); + assert_eq!(b.components[ids[2] as usize].kind, Kind::Unique); + assert_eq!( + key.used_by, + [Usage { + by: ids[1], + how: UseKind::Refer + }] + ); + + // xml:lang without the XML namespace schema. + let lang = b.find(Kind::Attribute, "xml:lang").unwrap(); + assert!(b.components[lang as usize].builtin); +} + +#[test] +fn where_used() { + let b = basic(); + let users = |kind, q| -> Vec<(String, UseKind)> { + let (_, c) = get(&b, kind, q); + c.used_by + .iter() + .map(|u| { + let mut top = u.by; + while let Some(p) = b.components[top as usize].parent { + top = p; + } + (b.qname(top), u.how) + }) + .collect() + }; + assert_eq!( + users(Kind::ComplexType, "m:Base"), + [ + ("m:Derived".into(), UseKind::Base), + ("m:Shape".into(), UseKind::Type) + ] + ); + assert_eq!( + users(Kind::Element, "m:Shape"), + [ + ("m:Derived".into(), UseKind::Element), + ("m:Restricted".into(), UseKind::Element), + ("m:Circle".into(), UseKind::Substitution), + ("m:Square".into(), UseKind::Substitution), + ("m:Root".into(), UseKind::Element), + ] + ); + assert_eq!( + users(Kind::Group, "m:Extra"), + [("m:Derived".into(), UseKind::Group)] + ); + assert_eq!( + users(Kind::AttributeGroup, "m:Common"), + [("m:Base".into(), UseKind::AttributeGroup)] + ); + assert_eq!( + users(Kind::Attribute, "o:flag"), + [("m:Derived".into(), UseKind::Attribute)] + ); + assert_eq!( + users(Kind::SimpleType, "m:Colour"), + [ + ("m:Colours".into(), UseKind::Item), + ("m:ColourOrCode".into(), UseKind::Member), + ("m:colour".into(), UseKind::Type), + ] + ); + assert_eq!(users(Kind::SimpleType, "m:Unit").len(), 2); +} + +#[test] +fn search_index() { + let b = basic(); + let shape = b.find(Kind::Element, "m:Shape").unwrap(); + assert!(b.search["shape"].contains(&shape)); + let colour = b.find(Kind::SimpleType, "m:Colour").unwrap(); + assert!( + b.search["rose"].contains(&colour), + "enumeration documentation" + ); + assert!(b.search["green"].contains(&colour), "enumeration value"); + let base = b.find(Kind::ComplexType, "m:Base").unwrap(); + assert!(b.search["everything"].contains(&base)); + let derived = b.find(Kind::ComplexType, "m:Derived").unwrap(); + assert!(b.search["derived"].contains(&derived)); +} + +#[test] +fn json_round_trip_and_compact() { + let b = basic(); + let json = to_json(&b).unwrap(); + let back: Bundle = serde_json::from_str(&json).unwrap(); + assert_eq!(to_json(&back).unwrap(), json); + let compact = index( + data(&["basic/main.xsd", "basic/sub/common.xsd", "basic/other.xsd"]), + None, + &Options { + compact: true, + ..Default::default() + }, + ); + assert!(compact.compact); + assert!(compact.search.is_empty()); + let cjson = to_json(&compact).unwrap(); + assert!(cjson.len() < json.len()); + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(v["format"], FORMAT); +} + +#[test] +fn imports_are_followed_through_the_resolver() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data"); + let mut asked = Vec::new(); + let mut resolver = |path: &str| { + asked.push(path.to_string()); + std::fs::read(dir.join(path)).ok() + }; + let b = index( + data(&["basic/main.xsd"]), + Some(&mut resolver), + &Options::default(), + ); + assert_eq!( + asked, + [ + "basic/sub/common.xsd", + "basic/other.xsd", + "http://www.w3.org/2001/xml.xsd" + ] + ); + assert_eq!(b.files.len(), 3); + assert!(b.files[0].input && !b.files[1].input); + assert_eq!(b.stats.unresolved_imports, 0); + assert!(b.find(Kind::ComplexType, "o:Tag").is_some()); +} + +#[test] +fn flat_upload_resolves_by_file_name() { + // As when files are dropped in the browser without their folders. + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/data/basic"); + let sources = ["main.xsd", "sub/common.xsd", "other.xsd"] + .iter() + .map(|f| { + Source::new( + f.rsplit('/').next().unwrap(), + std::fs::read(dir.join(f)).unwrap(), + ) + }) + .collect(); + let b = index(sources, None, &Options::default()); + assert_eq!(b.stats.unresolved_imports, 0); + assert_eq!(b.files[0].directives[0].via.as_deref(), Some("filename")); + assert_eq!(b.files[0].directives[1].file, Some(2)); +} + +#[test] +fn broken_input_gives_diagnostics() { + let b = index( + data(&["broken/broken.xsd", "broken/notxml.xsd"]), + None, + &Options::default(), + ); + let codes: Vec<&str> = b.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert_eq!( + codes.iter().filter(|c| **c == "unresolved-import").count(), + 2, + "{:?}", + b.diagnostics + ); + assert!(codes.contains(&"parse-error")); + assert!(codes.contains(&"duplicate")); + assert!(codes.contains(&"cycle")); + let dup = b + .diagnostics + .iter() + .find(|d| d.code == "duplicate") + .unwrap(); + assert_eq!((dup.file, dup.line), (Some(0), 11)); + assert_eq!(b.stats.global[&Kind::Element], 1); +} + +#[test] +fn redefine() { + let b = index( + data(&["redefine/main.xsd", "redefine/base.xsd"]), + None, + &Options::default(), + ); + assert_eq!(b.stats.unresolved_imports, 0); + let (_, person) = get(&b, Kind::Element, "r:person"); + let ty = person + .element + .as_ref() + .unwrap() + .r#type + .as_ref() + .unwrap() + .id() + .unwrap(); + let redefined = &b.components[ty as usize]; + let original = redefined.redefines.expect("redefines the original"); + assert_eq!(b.components[original as usize].file, Some(1)); + let ct = redefined.complex_type.as_ref().unwrap(); + assert_eq!(ct.derivation.as_ref().unwrap().base, Ref::Id(original)); + assert_eq!(ct.effective.as_ref().unwrap().content, [original, ty]); +} diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index 0e83dd5..3b19261 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -13,5 +13,6 @@ publish = false crate-type = ["cdylib", "rlib"] [dependencies] +js-sys = "0.3.106" schema-explorer-core = { path = "../core", version = "0.1.0" } wasm-bindgen = "0.2" diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs index 33b4d45..c4ca565 100644 --- a/crates/wasm/src/lib.rs +++ b/crates/wasm/src/lib.rs @@ -1,9 +1,44 @@ //! WebAssembly bindings of schema-explorer-core for the web viewer. +use schema_explorer_core::{Options, Source}; use wasm_bindgen::prelude::*; +/// Index an XSD set and return the bundle as JSON. +/// +/// `names` and `contents` are parallel arrays: file paths (relative paths +/// let imports and includes resolve; bare file names resolve by name) and +/// their bytes as `Uint8Array`s. `compact` writes the compact form. Problems +/// in the schemas are reported in the bundle's `diagnostics`. +#[wasm_bindgen] +pub fn index( + names: Vec, + contents: Vec, + compact: bool, +) -> Result { + if names.len() != contents.len() { + return Err(JsValue::from_str("names and contents differ in length")); + } + let sources = names + .into_iter() + .zip(contents) + .map(|(name, bytes)| Source::new(name, bytes.to_vec())) + .collect(); + let opts = Options { + generator: format!("schema-explorer-wasm {}", env!("CARGO_PKG_VERSION")), + compact, + }; + let bundle = schema_explorer_core::index(sources, None, &opts); + schema_explorer_core::to_json(&bundle).map_err(|e| JsValue::from_str(&e.to_string())) +} + /// The version of the core library. #[wasm_bindgen] pub fn version() -> String { schema_explorer_core::version().to_string() } + +/// The bundle format version this build writes. +#[wasm_bindgen] +pub fn format() -> u32 { + schema_explorer_core::FORMAT +} From da41df5e02ef99629170e1c6c11e0bd9de70a068 Mon Sep 17 00:00:00 2001 From: Mathias Vanden Auweele Date: Wed, 21 Oct 2026 16:20:02 +0000 Subject: [PATCH 2/2] Use as_chunks for UTF-16 decoding (clippy chunks_exact_to_as_chunks) --- crates/core/src/input.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/core/src/input.rs b/crates/core/src/input.rs index 51025b8..a6262b1 100644 --- a/crates/core/src/input.rs +++ b/crates/core/src/input.rs @@ -85,12 +85,14 @@ pub(crate) fn decode(bytes: &[u8]) -> (String, bool) { } let utf16 = |b: &[u8], le: bool| { let units: Vec = b - .chunks_exact(2) - .map(|c| { + .as_chunks::<2>() + .0 + .iter() + .map(|&c| { if le { - u16::from_le_bytes([c[0], c[1]]) + u16::from_le_bytes(c) } else { - u16::from_be_bytes([c[0], c[1]]) + u16::from_be_bytes(c) } }) .collect();