From fdb56ec21462c3c0b3fd9915f8aa1c76011bed20 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 03:24:32 +0000 Subject: [PATCH 01/10] fix(bundle): Rust loader follow-ups: lexer offsets, dead items, paths through re-exports and macros, include!, review fixes, --cargo-target, --cargo-manifests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups to the Rust loader from #173, squashed from the review rounds of this PR: - the lexer's masked view stays aligned after astral characters; - a dead item is skipped whole whatever commas its generics hold; - paths resolve through imports, re-exports, globs and exported macros, with a memoized per-crate import table, textual macro scope and the extern prelude, cfg-aware (`any(…)` leaves, gate macros, custom cfgs presumed off); - include!, include_str!/include_bytes! and #[doc = include_str!] are carried; #[path] on an inline module is honoured; - --cargo-target decides target cfgs from rustc (run from the home dir, $RUSTC honoured, no auto-install); - --cargo-manifests carries manifests, the lockfile, the cargo config and build scripts, each as written (as #182 carries --manifests files): a file that isn't UTF-8 text is refused, not altered; - vendored crates are held to their package through symlinks and #[path]. On the strict TOML reader of #179: Cargo.toml, Cargo.lock and the cargo config are read by table path (`[package] build`, `edition.workspace`, target-specific dependency tables, every `[patch]` spelling). A Cargo or Foundry file that exists but isn't TOML stops the build, naming the file and line; a dependency's foundry.toml the loader can't read is no longer skipped with a warning but stops the build too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 246 +- stasis/bin/stasis.js | 25 +- stasis/src/cmd/bundle.js | 118 +- stasis/src/loaders/cargo.js | 299 ++- stasis/src/loaders/rust.js | 2213 +++++++++++++++-- tests/bundle-cmd.test.js | 50 + .../fixtures/rust-bundle/includes/Cargo.lock | 7 + .../fixtures/rust-bundle/includes/Cargo.toml | 4 + tests/fixtures/rust-bundle/includes/README.md | 3 + tests/fixtures/rust-bundle/includes/build.rs | 8 + .../rust-bundle/includes/build/helper.rs | 1 + .../rust-bundle/includes/data/blob.bin | Bin 0 -> 4 bytes .../rust-bundle/includes/data/table.txt | 2 + .../rust-bundle/includes/src/gated.rs | 4 + .../includes/src/generated/consts.rs | 1 + .../fixtures/rust-bundle/includes/src/lib.rs | 18 + .../rust-bundle/includes/src/macros.rs | 5 + tests/rust-features.test.js | 650 ++++- tests/rust-loader.test.js | 1385 ++++++++++- 19 files changed, 4701 insertions(+), 338 deletions(-) create mode 100644 tests/fixtures/rust-bundle/includes/Cargo.lock create mode 100644 tests/fixtures/rust-bundle/includes/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/includes/README.md create mode 100644 tests/fixtures/rust-bundle/includes/build.rs create mode 100644 tests/fixtures/rust-bundle/includes/build/helper.rs create mode 100644 tests/fixtures/rust-bundle/includes/data/blob.bin create mode 100644 tests/fixtures/rust-bundle/includes/data/table.txt create mode 100644 tests/fixtures/rust-bundle/includes/src/gated.rs create mode 100644 tests/fixtures/rust-bundle/includes/src/generated/consts.rs create mode 100644 tests/fixtures/rust-bundle/includes/src/lib.rs create mode 100644 tests/fixtures/rust-bundle/includes/src/macros.rs diff --git a/doc/file-formats.md b/doc/file-formats.md index 1baeba92..b694f686 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -65,7 +65,7 @@ if the run/scan never reached it — so a `bundle = load` of the artifact, or a *writing* a bundle (`bundle = add | replace`); its effect is the extra bundled files (attested like any other), so the flag itself is not serialized. JS bundles only on the `stasis bundle` side (`.sol`/`.php`/`.sh`/`.rs` bundles have no npm -`package.json`). +`package.json`; Rust bundles have `--cargo-manifests`, see below). Unknown keys are rejected. A key set by both file and env var must match. Only `scope` is persisted into the lockfile/bundle `config` block; `debug`, @@ -493,7 +493,18 @@ the bundle couldn't reproduce the resolution. Rust entries are crate roots (`src/main.rs`, `src/lib.rs`, `src/bin/*.rs`, `tests/*.rs`, …): their `mod` declarations resolve as siblings, as rustc does, -and so do those of a file a `#[path = …]` loaded. Each root gets its own module +and so do those of a file a `#[path = …]` loaded. A `mod` declared inside +inline modules resolves under their directories, each an inline module's name +or its own `#[path = "…"]` (solana-program's `#[path = ""] mod non_bpf_modules +{ mod account_keys; }` finds `account_keys.rs` beside the file). A `mod` inside +an `include!`d file is the including module's child, as rustc splices it, but +its file is looked up beside the included file: `include!("gen/list.rs")` with +`mod bar;` in it finds `gen/bar.rs`, whose own submodules sit under `gen/bar/` +(checked against rustc). When several +`#[path]` / `#[cfg_attr(, path = …)]` attributes sit on one `mod`, the +first whose predicate holds decides the file and ends the list, and the default +`name.rs` / `name/mod.rs` lookup is off; variants whose predicate is undecided +stay as cfg-keyed candidates. Each root gets its own module tree, so a lib and its bin bundled together don't collide on `crate::`. A `tests/*.rs` or `benches/*.rs` entry is compiled the way `cargo test` does: `cfg(test)` holds, its `#[test]` fns and `#[cfg(test)]` modules are live, and @@ -501,11 +512,96 @@ the package's dev-dependencies take part in the feature resolution. A `use`/`ext that crate's root in: the package's own lib target (`use my_app::…` from `main.rs`), a Cargo `path` dependency (incl. `workspace = true` ones and `package = …` renames, honouring `[lib] path`), or a `cargo vendor`ed crate under -`vendor/`. Registry dependencies live in `~/.cargo/registry`, outside the bundle +`vendor/` (or the directory `.cargo/config.toml` names under +`[source.vendored-sources]`). Registry dependencies live in `~/.cargo/registry`, outside the bundle root, so they're never read: vendor them first (`cargo vendor`). When a bundle references crates it can't find and there is no `vendor/` dir, `stasis bundle` says so and suggests it. +A path edge (`crate::a::b::Item`, `super::x`, a relative `child::y`) resolves +module by module along the crate's tree, inline `mod x { … }` blocks included +(a `mod imp;` file wins over an inline `mod imp { … }` of the same name under +another cfg). A segment that names no module may be bound by an import of the +module reached, which is followed: a `use` naming it outright (`pub use +serde_core as serde`, `extern crate serde_core as s`, `pub(crate) use helper` +of a `macro_rules!` macro), as far as its visibility reaches -- a plain `use` +only for a path written in that module or below, `pub(super)` / `pub(in …)` +within that scope -- or a glob (`pub use external::*`), when the module it +names is one of this crate and that module has the name: a `pub` child module +(a private `mod` is not glob-importable), an item it defines (a `struct`, +`fn`, `const`, … at module level, an `extern "C"` block's too -- not an +`impl`'s associated items or a fn's local ones; libc's `crate::sigset_t` lands +on the platform file whose `s! { … }` defines it, not on a later platform's +re-export of the name), or a name a `use` of its own binds. A segment with more +path after it names a module or a type-namespace item: a `fn log` or `const +libc` beside `use log::info` / `*const libc::c_char` never stands for the +crate, nor does a `use crate::util::log;` of such a fn -- `log::info!` names +the crate `log`, whose root the bundle loads on the tree pass's request when +the walk had taken the name for the fn. Where a module has several files or imports under cfgs that can't hold +together -- libc's platform modules, each glob re-exported from a `cfg_if!` +branch; mio's two `pub use … Waker` in two branches of one file; tokio's +macro-keyed `mod imp` variants -- a path means the candidate compatible with +the cfgs its own file was mounted under, checked along the whole chain of +imports followed: a file under `#[cfg(unix)] mod unix;` never resolves through +the fuchsia or windows branch (exclusive: a cfg and its negation, two values of +a single-valued key such as `target_os`, two variants of one `mod`, `windows` +against `unix`, an `any(…)` none of whose alternatives can hold -- libc's +`any(target_os = "linux", target_os = "l4re")` branch against a file under +`target_os = "aix"`), and of the compatible ones the first whose cfgs its own +entail (a candidate under the same `target_os` leaf it was mounted under, over +one under an `any(…)` nothing rules out), else the first in written order +under no custom cfg, else the first in written order at all: a positive cfg +that neither rustc nor cargo sets (`loom`, `docsrs`, `tokio_unstable`, mio's +`mio_unsupported_force_poll_poll`, an `any(…)` of such and of alternatives +the file rules out) is a `--cfg` a default build lacks, so a candidate under +one is taken only when nothing else fits, and one under its negation +(`not(loom)`) counts as certain. With `--cargo-target`, every file compiled +for the target is under the target's cfgs too, so a `windows` candidate is +out for every file and a `target_os = "linux"` file takes the `any(android, +linux)` branch over the `any(aix, solaris)` one (mio's `sys::Waker` is the +eventfd one under a Linux target; without one, written order decides). An +item a module defines in several of its files follows the same ranking, the +module's own tree file first within a rank. A `cfg_if!` branch is +under its own predicate and the negation of every branch before it, nested +chains each on their own; a `#[cfg(…)]` on a macro invocation gates everything +its body declares (mio's `#[cfg(unix)] cfg_os_poll! { mod unix; … }`); the body +of a `cfg_!` macro (tokio's `cfg_io_uring!`, mio's `cfg_os_poll!`) is a gate +the loader can't see into and `cfg_not_!` its negation, so what the two +declare is never taken together and neither counts as certainly compiled; a +file mounted by several declarations is under any of their cfgs; a module +reached by several glob paths is under either's, and so is everything its own +globs reach. A glob +into the sysroot or into another crate brings in nothing the loader can see, so +it claims nothing -- and since a glob into a crate that isn't in-tree may well +provide a name, an unresolved lead in a file with such a glob (`use syn::*; … +punctuated::Punctuated`) is not reported as an unresolved crate; a glob into an +in-tree crate explains only the names the module its path names there has, or +its own globs bring in (tokio's `use tokio::sync::*;` explains `mpsc`; +tracing's `use tracing_core::*;` does not hide its `tracing_attributes`), and a +glob into the sysroot or an enum explains none. An `extern crate x as y;` at +the crate root puts `y` in the extern prelude, so `y::…` and `::y::…` resolve +from every file of the crate. At the crate root the final segment may be a +`#[macro_export]` macro (`$crate::name!`), which lives in the file defining it: +an invocation (`crate::helper!()`, recorded as `crate::helper!`) names the +macro before an import or module of that name, an ordinary path +(`crate::helper()`) the other way round, and a one-segment path written at the +crate root names it when no crate has the name (anyhow's `pub use anyhow as +format_err;` in lib.rs; a child module's `use x;` does not reach it). The +answers are the same whatever order the files are listed in: each module's +files count in tree order, the variants of a `mod` as declared. Anything +else is an item of the module reached, whose file is the edge's target (a plain +item behind a glob re-export can't be placed more precisely). An import that +leads to another in-tree crate records that crate +(`use serde_core`); one that leads out of the bundle, to the sysroot (`pub use +core::result::Result`) or to a crate that isn't vendored, binds the name to +its own file as far as the bundle knows (serde's `$crate::__private::Result` +records the file of the `lib` module that re-exports `core::result`). The +module an import was first followed from is a dependency too: when nothing +else in the file points at its file, the path's prefix naming it is recorded +(`crate::__private` → private/mod.rs) -- unless that file is the crate root, +which every file of the crate hangs off anyway. An edge from a file to itself +is never recorded. + Dependency buckets carry an `ecosystem`, attributed by the install layout each file resolves out of: @@ -529,7 +625,7 @@ What counts as a fatal unresolved reference differs by language: | Solidity | every `import` | — | | PHP | every literal `require`/`include` path | Composer-autoloaded class refs (unresolved ones usually built-in/extension classes); a dynamic include with a static dir prefix pulls in that dir's `.php` files as candidates | | Bash | every in-root `.sh`/`.bash` reference | PATH commands, `$VAR`/absolute/system paths, `../`-escaping sources (external); dynamic `source "${VAR}/x.sh"` followed via `# shellcheck source=` when present | -| Rust | every `mod foo;` not gated on an undecidable cfg (see the cfg rules below), incl. one whose `#[path]` names no file, or escapes the bundle root | a `mod` gated on a cfg the loader can't decide (`unix`, a feature of a package outside the resolved build, …); a `mod` inside a macro invocation body (`cfg_if! { … }` emits real ones, other macros may not — followed when the file exists); every path edge (`crate::`/`self::`/`super::`/relative `use`s, recorded best-effort and never widening the walk); crates not in-tree (see above); `include_str!`/`include_bytes!` assets | +| Rust | every `mod foo;` not gated on an undecidable cfg (see the cfg rules below), incl. one whose `#[path]` names no file, or escapes the bundle root | a `mod` gated on a cfg the loader can't decide (`unix`, a feature of a package outside the resolved build, …); a `mod` inside a macro invocation body (`cfg_if! { … }` emits real ones, other macros may not — followed when the file exists); every path edge (`crate::`/`self::`/`super::`/relative `use`s, recorded best-effort and never widening the walk); crates not in-tree (see above); an `include!`/`include_str!`/`include_bytes!` whose literal path names no file, or whose argument is no literal the loader can read (a `concat!` of anything but `env!("CARGO_MANIFEST_DIR")` and literals, a macro variable) -- both warned; a vendored crate's include, `#[path]` or build script that reaches outside its own package (refused and warned; a `mod` so refused is missing) | A missing entry is always fatal. @@ -552,19 +648,53 @@ test code reaches for. Two kinds of cfg are decided: doesn't turn on digest's `dev`); the entries' packages' dev-dependencies count under resolver 1 only, since resolver 2 (edition 2021+, or `resolver = "2"`) keeps them out of a normal build. Target-specific - dependency tables always count (an over-approximation: it only keeps files). + dependency tables count unless `--cargo-target` says they are for another + target (without it, an over-approximation: it only keeps files). The crate a versioned dependency resolves to comes from `Cargo.lock`, so two vendored versions of one crate each get their own features and edges. A package the resolved build doesn't pull in has unknown features, and its gated - code is kept. The manifests and the lock are read with `@preventive/lockfile`'s - strict TOML parser (so is a `foundry.toml`): a `Cargo.toml` or `Cargo.lock` - that exists but isn't TOML stops the build, naming the file and line, and so - does TOML those files are never written in (a local date, a byte order mark, - U+FFFD where bytes weren't UTF-8). + code is kept. The manifests, the lock and the cargo config are read with + `@preventive/lockfile`'s strict TOML parser (so is a `foundry.toml`): a + `Cargo.toml`, `Cargo.lock` or cargo config the loader reads that exists but + isn't TOML stops the build, naming the file and line, and so does TOML those + files are never written in (a local date, a byte order mark, U+FFFD where + bytes weren't UTF-8). + +- With `--cargo-target=`, **target cfgs** are decided too: the + loader asks `rustc --print cfg --target ` (`host`: the running + rustc's own target, from `rustc -vV`) for the target's cfg set — `unix`, + `windows`, `target_os`, `target_family`, `target_arch`, `target_env`, + `target_vendor`, `target_abi`, `target_pointer_width`, `target_endian`, + `target_has_atomic` — and `#[cfg(windows)]` code, a + `#[cfg_attr(windows, path = …)]` variant or a `[target.'cfg(windows)'.dependencies]` + table stays out of a Linux bundle. `target_feature` and `target_thread_local` + stay undecided: a build may add features (`-C target-cpu=native`), and + both differ with the toolchain and its flags. Profile cfgs + (`debug_assertions`, `panic = …`) and custom ones stay undecided too. Build + scripts (see `--cargo-manifests`) compile for the host, so no target cfg is + decided in them and every `[target.'cfg(…)'.build-dependencies]` table counts. + This runs rustc — `$RUSTC` when set, as cargo honours it, else `rustc` from + `PATH` — from the user's home directory (the filesystem root when there is + none, or when the home directory is the bundle root or lies inside it, real + paths compared), never + from the project being bundled nor from a temp dir: a rustup proxy picks its + toolchain from the `rust-toolchain(.toml)` files of the working directory + and its parents, and such a file may name a `path` to any binary, so the + project's own (untrusted input) and one planted in a world-writable directory + must not choose what runs. To use a project's pinned toolchain, set + `RUSTUP_TOOLCHAIN` or `RUSTC` yourself. It runs with a 60-second limit and + `RUSTUP_AUTO_INSTALL=0`, so an uninstalled toolchain is an error, never a + download on the loader's behalf, and needs only rustc's built-in knowledge + of the target, not its standard library. A target only another + toolchain knows needs that toolchain's rustc: Solana's `sbf-solana-solana` is + known to the rustc in its platform-tools + (`RUSTC=~/.cache/solana//platform-tools/rust/bin/rustc`), not to a + rustup one. With `--cargo`, the same triple goes to `cargo metadata + --filter-platform`. `all(…)`/`any(…)`/`not(…)` compose; a predicate that reduces to true (`not(test)`, an enabled feature) is as firm as no cfg, so a missing module behind it is fatal. -Target cfgs (`unix`, `target_os = …`) stay undecided and their code is kept. A +Without a target, target cfgs stay undecided and their code is kept. A `cfg_attr` that applies a non-cfg attribute (`#[cfg_attr(docsrs, doc(cfg(…)))]`) gates nothing. @@ -586,6 +716,39 @@ packages leaves off; the manifest replay describes that build. Set `EXODUS_STASIS_DEBUG=1` to have `stasis bundle` print the resolved features per package, in either mode. +`stasis bundle --cargo-manifests` also carries what describes each bundled +package's build, the way `--package-json` carries npm manifests: every bundled +package's `Cargo.toml` and the workspace `Cargo.toml` above it, and that +workspace's -- or, for a package outside any, its own -- `Cargo.lock` and cargo +config (`.cargo/config` before `.cargo/config.toml`, the order cargo reads +them; a workspace nested below the bundle root keeps its own pair), when they +lie inside the bundle root (a workspace's may sit above), all as `resource` +files in their package's bucket, or the workspace bucket for the root-level +ones. They are carried as written, as `--package-json` carries `package.json`: +stasis doesn't edit them, so whatever they hold -- a `[registries]` token or +`[http]` proxy in the cargo config, the credentials in a `git = …` URL or a +lockfile's `source = "git+https://…"` -- is in the bundle too. Keep registry +tokens in `~/.cargo/credentials.toml` or the environment +(`CARGO_REGISTRIES__TOKEN`) rather than in these files, or don't pass +`--cargo-manifests`. A file that isn't UTF-8 text is refused, not altered. +Each package's build script +(`[package] build = "…"`, else `build.rs` beside the manifest; `build = false` +means none) is walked like a crate root of its own, compiled for the host, so +its modules and the in-tree `[build-dependencies]` it reaches (and their build +scripts, in turn) come along as Rust code. Nothing runs. A vendored crate (one +under the `vendor/` directory) is held to its own package directory: an +`include!`/`include_str!`/`include_bytes!`, a `#[path = …]` or a `build = "…"` +in it that reaches outside (`include_str!("../../../.git/config")`, +`concat!(env!("CARGO_MANIFEST_DIR"), "/../../.env")`, an inline module's +`#[path = "../../.."]`) is refused with a warning, so a crate from the registry +can't carry the project's secrets into a bundle; the project's own code may +include anything under the bundle root. The check follows symlinks: a file of +a vendored crate that really lies outside its package (a link to the project's +`.env`, a linked directory of modules, a linked manifest) is refused too, and +so is a `[lib] path` outside the package. Credentials in the URLs of the +carried config tables (`registry = "sparse+https://user:token@…"`, a `[patch]` +`git = "https://token@…"`) are stripped. + The root packages' features follow the same flags as `cargo build`, in either mode: `--cargo-features=a,b` (repeatable; `x/feat` is a feature of the entries' package named `x`, else of their dependency `x`, cargo's `dep/feat` form; a name @@ -598,12 +761,61 @@ disabled them too, as cargo warns. Rust edge specs are the path as written (`crate::net::client::Client`, `super::config::Config`, a `use crate::{a::B, c::D}` group flattened to one edge per path); `mod ` for a module file (`mod outer::inner` when declared inside -inline module `outer`); `use ` for a crate root. A `mod` whose files vary -by cfg — `#[cfg_attr(, path = …)]` variants, or same-name declarations -under exclusive `#[cfg()]`s (`#[cfg(unix)] #[path = "u.rs"] mod sys;` -beside `#[cfg(windows)] #[path = "w.rs"] mod sys;`) — records a -`{ : file, …, "*": }` map, the same shape as a JS edge that -diverges per Metro platform. +inline module `outer`); `use ` for a crate root; `include `, +`include_str ` and `include_bytes ` for the file an include macro +names with a literal — a plain string with its escapes read (`"a \"b\".txt"`), +a raw string (`r#"…"#`), or `concat!(env!("CARGO_MANIFEST_DIR"), "/…")`, the +one `concat!` form the loader can read, relative to the package root, inside a +`#[doc = …]` attribute as much as in an expression; anything else, build output +under `concat!(env!("OUT_DIR"), …)` included, can't be followed and is warned +about unless it is `OUT_DIR`. `include!` splices Rust source, carried and +scanned as a file of its own; the other two carry an asset as `resource` / +`resource:base64`, each relative to the including file. A file one place names +with `mod` or `include!` and another with `include_str!` is Rust: scanned and +followed, never a resource. An include written inside a `macro_rules!` body +resolves relative to each file that invokes the macro, as rustc expands it (the +edge is that file's, at the invocation). `!` marks a `macro_rules!` +invoked by bare name and defined in another file the invocation can see -- in +textual scope, as rustc has it: the files above it in the module tree, each up +to the `mod` that mounts the next (a macro defined after `mod early;` is not +early.rs's), what a `#[macro_use] mod` hands the file mounting it, from that +`mod` on (tokio's `ready!` through `#[macro_use] mod macros;`; a nested +`#[macro_use]` reaches its own parent, not the crate), the later definition of +a name shadowing the earlier in source order whichever kind each is (serde's +`tri!`, defined by its root's `crate_root!()` after `#[macro_use] mod +crate_root;`, is the root's) as far as each call (a `macro_rules!` written +after a call doesn't reach it), a `mod` or `macro_rules!` a macro's body +declares standing where that macro is first invoked -- in the file, else in +the first file of the crate that invokes it (serde's `crate_root!`, declared +in core/crate_root.rs and invoked in lib.rs after `#[macro_use] mod macros;`, +so `de` sees `forward_to_deserialize_any!`), and, in the crate +root's file, whose items they are, the `#[macro_export]`ed ones -- or by path, +through the module's imports (`use crate::combinator::dispatch;`, a `use +super::*` from the root); a `macro_rules!` in a sibling file is out of scope +otherwise. A file whose inner +`#![cfg(…)]` can never hold is carried but compiled empty, so nothing in it is +followed (a `#![cfg]` inside a macro invocation's body gates nothing). A `mod` +whose files vary by cfg — `#[cfg_attr(, path = …)]` variants, or +same-name declarations under exclusive `#[cfg()]`s (`#[cfg(unix)] #[path += "u.rs"] mod sys;` beside `#[cfg(windows)] #[path = "w.rs"] mod sys;`, the +`if #[cfg(unix)] { … }` branches of a `cfg_if!`, each later branch keyed by +the earlier cfgs not holding: `all(not(unix), windows)`, `all(not(unix), +not(windows))`) — records a `{ : file, …, "*": }` map, +the same shape as a JS edge that diverges per Metro platform; when every +predicate names one file (libc's `mod primitives` under a dozen) the edge is +that file. A +declaration a macro emits under a cfg the loader can't see (tokio's +`cfg_has_atomic_u64! { mod imp; }` beside `cfg_not_has_atomic_u64! { mod imp; }`) +is keyed by the macro (`cfg_has_atomic_u64!`); a second file under an already +used key gets it numbered (`*#2`) rather than dropped. A `use` is an import only +when a use tree follows it: the word in macro input (syn's `Token![use]`) or a +tree interpolating a metavariable (`use #path as _serde;` in a `quote!`) is not +one. The body of a `quote!`, `quote_spanned!`, `parse_quote!` or +`parse_quote_spanned!` is a token template for another crate's code and is +skipped whole -- unless the package defines a `macro_rules!` of that name +itself, in whichever of its files (syn's `parse_quote!`), when it is that +macro's input like any other -- while reading the files from disk too, so a +`mod` such a body declares is found whichever file the walk meets first. ## Resources in the bundle diff --git a/stasis/bin/stasis.js b/stasis/bin/stasis.js index 509b5d64..35358c26 100755 --- a/stasis/bin/stasis.js +++ b/stasis/bin/stasis.js @@ -41,11 +41,15 @@ function usage(prefix = '') { stasis bundle [--scope=(node_modules|full)] [--conditions=cond1,cond2] [--mainFields=field1,field2] [--jsx] [--flow] [--typescript [--tsconfig=path/to/tsconfig.json]] [--resources=ext,ext] [--package-json] [--lockfile=path/to/stasis.lock.json] [--add] [--output=(path|-)] path/to/file.(js|ts) ... stasis bundle --metro [--metro-resolver] --platforms=ios,android [--platforms=web] [--jsx] [--flow] [--typescript [--tsconfig=path/to/tsconfig.json]] [--resources=ext,ext] [--package-json] [--lockfile=path/to/stasis.lock.json] [--add] [--output=(path|-)] path/to/file.(js|ts) ... stasis bundle [--add] [--output=(path|-)] path/to/file.(sh|bash) ... - stasis bundle [--cargo] [--cargo-features=a,b,pkg/c] [--cargo-no-default-features] [--cargo-all-features] [--add] [--output=(path|-)] path/to/file.rs ... + stasis bundle [--cargo] [--cargo-features=a,b,pkg/c] [--cargo-no-default-features] [--cargo-all-features] [--cargo-target=(triple|host)] [--cargo-manifests] [--add] [--output=(path|-)] path/to/file.rs ... (Rust: each crate's Cargo features are resolved from Cargo.toml/Cargo.lock like "cargo build" of the entries' packages, so #[cfg(feature = ...)] code that is off stays out; --cargo takes the resolution from "cargo metadata" instead -- it runs cargo, so only on a project you trust; the --cargo-* - flags are cargo's --features / --no-default-features / --all-features for those packages) + feature flags are cargo's --features / --no-default-features / --all-features for those packages; + --cargo-target asks rustc for the target's cfgs so #[cfg(unix)]-style code for other targets stays + out too, otherwise it is all kept; --cargo-manifests also bundles each bundled package's Cargo.toml + and build script, the workspace Cargo.toml, Cargo.lock and .cargo/config.toml, as written: tokens + and URL credentials in them included) (writes to stasis.code.br by default; --output=- streams to stdout; --add merges into an existing bundle instead of replacing it (not with --output=-); --brotli-quality=0..11, default 9; --jsx parses JSX in .js/.cjs/.mjs files, e.g. React Native source (put JSX-in-TS in a .tsx file); @@ -252,11 +256,13 @@ if (command === '-v' || command === '--version') { 'cargo-features': { type: 'string', multiple: true }, 'cargo-no-default-features': { type: 'boolean' }, 'cargo-all-features': { type: 'boolean' }, + 'cargo-target': { type: 'string' }, + 'cargo-manifests': { type: 'boolean' }, 'brotli-quality': { type: 'string' }, add: { type: 'boolean' }, } const values = parseLeadingOptions(argv, options, { - valueFlags: ['--mapping', '--output', '--scope', '--lockfile', '--conditions', '--mainFields', '--platforms', '--resources', '--tsconfig', '--cargo-features', '--brotli-quality', '-o'], + valueFlags: ['--mapping', '--output', '--scope', '--lockfile', '--conditions', '--mainFields', '--platforms', '--resources', '--tsconfig', '--cargo-features', '--cargo-target', '--brotli-quality', '-o'], onError: usage, }) if (argv.length === 0) usage('Nothing to bundle: no entry file given') @@ -290,10 +296,21 @@ if (command === '-v' || command === '--version') { } const cargoNoDefaultFeatures = Boolean(values['cargo-no-default-features']) const cargoAllFeatures = Boolean(values['cargo-all-features']) + // --cargo-target=: the build's target, whose cfgs (from `rustc --print cfg`) decide + // `#[cfg(unix)]`-style code. A triple is letters, digits, `-`, `_` and `.` (`x86_64-unknown-linux-gnu`). + const cargoTarget = values['cargo-target'] ?? null + if (cargoTarget !== null && !/^[\w.-]+$/u.test(cargoTarget)) { + usage('Error: --cargo-target must be a target triple or "host" (e.g. --cargo-target=aarch64-apple-darwin)') + } + // --cargo-manifests: carry each bundled package's Cargo.toml and build script, the workspace + // Cargo.toml, Cargo.lock and .cargo/config.toml too (the Rust counterpart of --package-json). + const cargoManifests = Boolean(values['cargo-manifests']) if (!allRust) { if (cargoFeatures.length > 0) usage('Error: --cargo-features is only valid for Rust bundles') if (cargoNoDefaultFeatures) usage('Error: --cargo-no-default-features is only valid for Rust bundles') if (cargoAllFeatures) usage('Error: --cargo-all-features is only valid for Rust bundles') + if (cargoTarget !== null) usage('Error: --cargo-target is only valid for Rust bundles') + if (cargoManifests) usage('Error: --cargo-manifests is only valid for Rust bundles') } if (values.scope && !allJs) usage('Error: --scope is only valid for JS bundles') if (values.scope && !['node_modules', 'full'].includes(values.scope)) { @@ -413,6 +430,8 @@ if (command === '-v' || command === '--version') { cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, + cargoTarget, + cargoManifests, brotliQuality, add, }) diff --git a/stasis/src/cmd/bundle.js b/stasis/src/cmd/bundle.js index 7b6a02b6..199207fd 100644 --- a/stasis/src/cmd/bundle.js +++ b/stasis/src/cmd/bundle.js @@ -1,5 +1,5 @@ import { isUtf8 } from 'node:buffer' -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs' import { dirname, extname, isAbsolute, join, posix, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { brotliCompressSync, brotliDecompressSync } from 'node:zlib' @@ -27,7 +27,7 @@ import { import { decodeUtf8 } from '../loaders/solidity-ownership.js' import { buildBashTree, collectBashFilesFromDisk } from '../loaders/bash.js' import { buildRustTree, collectRustFilesFromDisk } from '../loaders/rust.js' -import { VENDOR_DIR as CARGO_VENDOR_DIR, createCargoContext } from '../loaders/cargo.js' +import { createCargoContext } from '../loaders/cargo.js' import { bucketizePhpSources, buildPhpTree, @@ -123,15 +123,16 @@ function makeSolidityClassifier(baseDir, ownership, host) { } } -// Classify a Rust file by the nearest Cargo.toml `[package]`: a `cargo vendor`ed crate under -// `vendor//` is a dependency (tagged `cargo`); any other package (the crate itself, a -// workspace member reached through a `path` dependency) is first-party, so no ecosystem. Null -// (no manifest above the file) defers to the package.json/placeholder logic. +// Classify a Rust file by the nearest Cargo.toml `[package]`: a `cargo vendor`ed crate under the +// vendor dir (`vendor//`, or where .cargo/config.toml points) is a dependency (tagged +// `cargo`); any other package (the crate itself, a workspace member reached through a `path` +// dependency) is first-party, so no ecosystem. Null (no manifest above the file) defers to the +// package.json/placeholder logic. function makeRustClassifier(cargo) { return (path) => { const pkg = cargo.packageInfo(path) if (!pkg) return null - const vendored = pkg.dir.startsWith(`${CARGO_VENDOR_DIR}/`) + const vendored = pkg.dir.startsWith(`${cargo.vendorDir}/`) return { bucketDir: pkg.dir, name: pkg.name, version: pkg.version, ecosystem: vendored ? 'cargo' : undefined } } } @@ -139,6 +140,28 @@ function makeRustClassifier(cargo) { // A file's key inside its bucket: the inverse of moduleFileKey. const fileInBucket = (bucketDir, path) => (bucketDir === '.' ? path : path.slice(bucketDir.length + 1)) +// A file of the bundle root, its bytes, by project-relative path, or null when it isn't a regular +// file there (a manifest the context saw may have gone since; a FIFO would hang a read) or, with +// `within` a directory, doesn't really lie in it (a vendored crate's manifest linked to a project +// file). +function readFileWithinBase(baseDir, rel, within = '.') { + try { + assertRealPathWithinBase(realpathSync(baseDir), baseDir, rel) + if (within !== '.') { + const real = relative(realpathSync(join(baseDir, within)), realpathSync(join(baseDir, rel))).split(/[\\/]/u).join('/') + if (real.startsWith('..') || isAbsolute(real)) { + console.warn(`[loader.cargo] Refusing file outside its package: ${rel} (a link out of ${within})`) + return null + } + } + if (!statSync(join(baseDir, rel)).isFile()) return null + return readFileSync(join(baseDir, rel)) + } catch (err) { + if (err.code === 'ENOENT' || err.code === 'EISDIR') return null + throw err + } +} + // Project-relative paths in `sources` whose on-disk file carries a POSIX execute bit -- the // `executable` list both artifacts record. The State-driven path derives this in addFile; the // static builders never touch a State, so they stat here. @@ -414,7 +437,12 @@ export async function buildBashBundle({ cwd = process.cwd(), entries } = {}) { // that is off stays out; `cargo` takes them from `cargo metadata` instead (opt-in: it runs cargo). // `cargoFeatures` / `cargoNoDefaultFeatures` / `cargoAllFeatures` are cargo's `--features` / // `--no-default-features` / `--all-features` for the entries' packages, in either mode. -export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false } = {}) { +// `cargoTarget` (a triple, or `host`) names the build's target: its cfgs, asked of rustc, decide +// `#[cfg(unix)]`-style code and target-specific dependency tables, which are otherwise all kept. +// `cargoManifests` also carries what describes each bundled package's build: its Cargo.toml (and +// the workspace's), its build script -- walked like a crate root, so its modules and the in-tree +// build-dependencies it reaches come along -- plus the root's Cargo.lock and .cargo/config.toml. +export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false, cargoTarget = null, cargoManifests = false } = {}) { if (!Array.isArray(entries) || entries.length === 0) { throw new Error('buildRustBundle: at least one entry .rs file is required') } @@ -433,9 +461,58 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa features: cargoFeatures, noDefaultFeatures: cargoNoDefaultFeatures, allFeatures: cargoAllFeatures, + target: cargoTarget, }) - const sources = await collectRustFilesFromDisk(baseDir, normalized, { cargo: cargoCtx }) - const { resolutions, missing, unresolvedCrates } = buildRustTree(sources, { roots: normalized, baseDir, cargo: cargoCtx }) + // `include_str!` / `include_bytes!` assets ride along as resources: the walk records their format. + const formats = new Map() + const sources = new Map() + // The crate roots: the entries, then -- with `cargoManifests` -- the build script of every + // package the walk reaches, each a crate root of its own, until no package is new. Build scripts + // and what they reach compile for the host, not the target (`hostFiles`). + const roots = [...normalized] + const hostFiles = new Set() + const walk = async (wave) => { + await collectRustFilesFromDisk(baseDir, wave, { cargo: cargoCtx, formats, sources, hostFiles }) + if (!cargoManifests) return + const scripts = new Set() + for (const path of sources.keys()) { + const script = formats.has(path) ? null : cargoCtx.buildScriptOf(path) + if (script !== null && !roots.includes(script)) scripts.add(script) + } + if (scripts.size === 0) return + roots.push(...scripts) + for (const s of scripts) hostFiles.add(s) + await walk([...scripts]) + } + await walk(normalized) + // The tree pass may name in-tree crate roots the walk left out (a crate whose name a file also + // binds as a value, `use crate::util::log;` beside `log::info!`: buildRustTree's wantedRoots): + // load them and build again, until none is new. + const complete = async () => { + const built = buildRustTree(sources, { roots, baseDir, cargo: cargoCtx, formats, hostFiles }) + const wanted = built.wantedRoots.filter((r) => !roots.includes(r)) + if (wanted.length === 0) return built + roots.push(...wanted) + await walk(wanted) + return complete() + } + const tree = await complete() + if (cargoManifests) { + // Each bundled package's manifests, and its workspace's lockfile and cargo config, carried as + // written: whatever they hold (a registry token, a `git` URL's credentials) is in the bundle + // too, as with --package-json. A file that isn't UTF-8 text is refused, not altered. + const files = new Set() + for (const path of sources.keys()) for (const f of cargoCtx.buildFilesFor(path)) files.add(f.path) + for (const rel of files) { + if (sources.has(rel)) continue + const buf = readFileWithinBase(baseDir, rel, cargoCtx.isVendored(rel) ? cargoCtx.packageInfo(rel).dir : '.') + if (buf === null) continue + if (!isUtf8(buf)) throw new Error(`Rust manifest is not valid UTF-8: ${rel}`) + sources.set(rel, buf.toString('utf8')) + formats.set(rel, 'resource') + } + } + const { resolutions, missing, unresolvedCrates } = tree const issues = [] for (const entry of normalized) { @@ -460,8 +537,8 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa } } - // Deps live outside the bundle root unless vendored; with no `vendor/` dir the fix is one command. - if (unresolvedCrates.size > 0 && !existsSync(join(baseDir, CARGO_VENDOR_DIR))) { + // Deps live outside the bundle root unless vendored; with no vendor dir the fix is one command. + if (unresolvedCrates.size > 0 && !existsSync(join(baseDir, cargoCtx.vendorDir))) { const names = [...unresolvedCrates].toSorted() const shown = names.slice(0, 10).join(', ') + (names.length > 10 ? `, ... and ${names.length - 10} more` : '') console.warn(`[stasis] ${names.length} crate${names.length === 1 ? '' : 's'} referenced but not found in the bundle root: ${shown}`) @@ -476,6 +553,7 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa workspaceName: RUST_WORKSPACE_NAME, workspaceVersion: RUST_WORKSPACE_VERSION, format: RUST_FORMAT, + formats, conditionKey: 'rust', classifyDep: makeRustClassifier(cargoCtx), }) @@ -1037,7 +1115,7 @@ async function buildResolvedJsBundle({ cwd = process.cwd(), entries, mainFields, // Classify entries into their single shared language and check option applicability; `name` prefixes // errors. A directory entry (resolved against `cwd`, on `host`) stands for the .sol files under it: Solidity only. -function classifyEntries(name, { cwd = process.cwd(), entries, mappingFile, manifests, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, host = diskHost, fetched }) { +function classifyEntries(name, { cwd = process.cwd(), entries, mappingFile, manifests, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, cargoTarget, cargoManifests, host = diskHost, fetched }) { if (!Array.isArray(entries) || entries.length === 0) { throw new Error(`${name}: at least one entry file is required`) } @@ -1066,6 +1144,8 @@ function classifyEntries(name, { cwd = process.cwd(), entries, mappingFile, mani if (Array.isArray(cargoFeatures) && cargoFeatures.length > 0) throw new Error(`${name}: --cargo-features is only valid for Rust bundles`) if (cargoNoDefaultFeatures) throw new Error(`${name}: --cargo-no-default-features is only valid for Rust bundles`) if (cargoAllFeatures) throw new Error(`${name}: --cargo-all-features is only valid for Rust bundles`) + if (cargoTarget) throw new Error(`${name}: --cargo-target is only valid for Rust bundles`) + if (cargoManifests) throw new Error(`${name}: --cargo-manifests is only valid for Rust bundles`) } if (scope !== undefined && kind !== 'js') { throw new Error(`${name}: --scope is only valid for JS bundles`) @@ -1209,12 +1289,12 @@ export async function buildVfsBundle({ vfs, packageManager, cwd = '/', packageMa // Programmatic equivalent of `stasis bundle`: build and return an in-memory Bundle without // writing to disk. Files are attributed to the `bundle` consumer. Option applicability // (--mapping|--manifests/.sol, --scope|--conditions|--mainFields|--metro|--jsx|--flow|--typescript/JS) is enforced by classifyEntries. -export async function buildBundle({ cwd = process.cwd(), env = process.env, entries, mappingFile, manifests = false, scope, conditions, mainFields, platforms, metro, metroResolver, jsx = false, flow = false, typescript = false, tsconfig, resources = [], packageJSON = false, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false } = {}) { - const kind = classifyEntries('buildBundle', { cwd, entries, mappingFile, manifests, scope, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures }) +export async function buildBundle({ cwd = process.cwd(), env = process.env, entries, mappingFile, manifests = false, scope, conditions, mainFields, platforms, metro, metroResolver, jsx = false, flow = false, typescript = false, tsconfig, resources = [], packageJSON = false, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false, cargoTarget = null, cargoManifests = false } = {}) { + const kind = classifyEntries('buildBundle', { cwd, entries, mappingFile, manifests, scope, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, cargoTarget, cargoManifests }) if (kind === 'sol') return buildSolidityBundle({ cwd, env, entries, mappingFile, manifests }) if (kind === 'php') return buildPhpBundle({ cwd, entries }) if (kind === 'bash') return buildBashBundle({ cwd, entries }) - if (kind === 'rust') return buildRustBundle({ cwd, entries, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures }) + if (kind === 'rust') return buildRustBundle({ cwd, entries, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, cargoTarget, cargoManifests }) return (await buildJs({ cwd, env, entries, scope, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON })).bundle } @@ -1228,8 +1308,8 @@ const DEFAULT_BUNDLE_FILE = 'stasis.code.br' // `stasis run --lock=frozen` (which doesn't replay them) fails closed -- pair it with // `--bundle=load` or replay the conditions. `add` unions the fresh build into the bundle // already on disk (strict; a conflicting file throws) and can't target stdout. -export async function bundleCommand({ cwd = process.cwd(), env = process.env, entries, mappingFile, manifests = false, output, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx = false, flow = false, typescript = false, tsconfig, resources = [], packageJSON = false, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false, brotliQuality, add = false } = {}) { - const kind = classifyEntries('bundleCommand', { cwd, entries, mappingFile, manifests, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures }) +export async function bundleCommand({ cwd = process.cwd(), env = process.env, entries, mappingFile, manifests = false, output, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx = false, flow = false, typescript = false, tsconfig, resources = [], packageJSON = false, cargo = false, cargoFeatures = [], cargoNoDefaultFeatures = false, cargoAllFeatures = false, cargoTarget = null, cargoManifests = false, brotliQuality, add = false } = {}) { + const kind = classifyEntries('bundleCommand', { cwd, entries, mappingFile, manifests, scope, lockfile, conditions, mainFields, platforms, metro, metroResolver, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, cargoTarget, cargoManifests }) const target = output ?? DEFAULT_BUNDLE_FILE // --add has nothing to merge into on stdout (write-only). @@ -1245,7 +1325,7 @@ export async function bundleCommand({ cwd = process.cwd(), env = process.env, en ;({ bundle, stateBuilt } = built) if (lockfile) lockData = built.lockfile().serialize() } else { - bundle = await buildBundle({ cwd, env, entries, mappingFile, manifests, scope, conditions, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures }) + bundle = await buildBundle({ cwd, env, entries, mappingFile, manifests, scope, conditions, jsx, flow, typescript, tsconfig, resources, packageJSON, cargo, cargoFeatures, cargoNoDefaultFeatures, cargoAllFeatures, cargoTarget, cargoManifests }) } // State-built bundles keep the State root's repo (no cwd fallback: their paths are relative to that root). diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index 38791fa0..32ecbc8c 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -5,11 +5,12 @@ import { spawnSync } from 'node:child_process' import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' -import { dirname, isAbsolute, join, posix, relative } from 'node:path' +import { homedir } from 'node:os' +import { dirname, isAbsolute, join, parse, posix, relative, resolve, sep } from 'node:path' import { toPosix } from '@exodus/stasis-core/util' -import { isTomlTable, readToml } from './toml.js' +import { isTomlTable, readToml, splitTopLevel } from './toml.js' // `cargo vendor` copies registry crates in-tree under this dir. export const VENDOR_DIR = 'vendor' @@ -45,7 +46,10 @@ export function normalizeRel(dir, sub) { // --- Cargo.toml ----------------------------------------------------------------------- -const DEP_KINDS = { __proto__: null, dependencies: 'normal', 'dev-dependencies': 'dev', 'build-dependencies': 'build' } +const DEP_KINDS = new Map([['dependencies', 'normal'], ['dev-dependencies', 'dev'], ['build-dependencies', 'build']]) +// A request's kind (`normal`/`dev`/`build`), a target-specific table's `@` suffix dropped. +const kindOf = (request) => request.split('@')[0] + // Entries of a `[features]` list beyond a plain feature name: `dep:key` and `key/feat` / `key?/feat`. const DEP_IMPLICATION_RE = /^dep:(.+)$/u const DEP_FEATURE_RE = /^([^/?]+)(\?)?\/(.+)$/u @@ -77,17 +81,20 @@ export function isTestTargetPath(pkgDir, fileRel) { // `file` on text that isn't TOML. export function parseCargoManifest(text, file = null) { const manifest = { - package: null, // { name, version, versionFromWorkspace, edition } + package: null, // { name, version, versionFromWorkspace, edition, editionFromWorkspace, build } resolver: null, // "1" | "2" | "3" from [workspace] or [package] lib: { name: null, path: null }, features: new Map(), // name -> implied entries (`other`, `dep:key`, `key/feat`, `key?/feat`) - // key -> { key, name, version, path, package, workspace, kinds: Map }: + // key -> { key, name, version, path, package, workspace, kinds: Map }: // what identifies the crate is shared, what is asked of it is per dependency table -- sha2's // `[dependencies] digest = "0.10"` and `[dev-dependencies] digest = { features = ["dev"] }` are - // two requests, and only the first is part of a build of sha2's dependents. + // two requests, and only the first is part of a build of sha2's dependents. A request is the + // table's kind (`normal`, `dev`, `build`), with `@` appended for a target-specific table: + // `[target.'cfg(unix)'.dependencies] foo = { optional = true }` beside `[dependencies] foo = "1"` + // is a second request of `foo`, not a change to the first. deps: new Map(), workspaceDeps: new Map(), // key -> { key, name, version, path, package, optional, defaultFeatures, features } - workspacePackage: { version: null }, + workspacePackage: { version: null, edition: null }, patches: new Map(), // crate -> path isWorkspace: false, } @@ -121,20 +128,26 @@ export function parseCargoManifest(text, file = null) { const doc = readToml(text, file) const str = (v) => (typeof v === 'string' ? v : null) const table = (v) => (isTomlTable(v) ? v : null) + // `version = { workspace = true }` or `version.workspace = true`: inherited, as `edition` may be. + const inherited = (v) => table(v)?.workspace === true const pkg = table(doc.package) if (pkg !== null && typeof pkg.name === 'string') { manifest.package = { name: pkg.name, version: str(pkg.version), - versionFromWorkspace: table(pkg.version)?.workspace === true, + versionFromWorkspace: inherited(pkg.version), edition: str(pkg.edition), + editionFromWorkspace: inherited(pkg.edition), + // the build script: a path, `false` for none, null to look for `build.rs` + build: typeof pkg.build === 'string' || pkg.build === false ? pkg.build : null, } } const ws = table(doc.workspace) // A `[workspace]` table, however it is spelled out, makes this a workspace root. manifest.isWorkspace = 'workspace' in doc manifest.resolver = str(ws?.resolver) ?? str(pkg?.resolver) - manifest.workspacePackage.version = str(table(ws?.package)?.version) + const wsPackage = table(ws?.package) + manifest.workspacePackage = { version: str(wsPackage?.version), edition: str(wsPackage?.edition) } const lib = table(doc.lib) manifest.lib = { name: str(lib?.name), path: str(lib?.path) } for (const [name, list] of Object.entries(table(doc.features) ?? {})) { @@ -147,17 +160,19 @@ export function parseCargoManifest(text, file = null) { if (patch !== null) manifest.patches.set(normName(crate), patch) } } - // `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`, under `target.` or - // `workspace`, in the order the manifest has them: each dependency a version string or a table. - const depTables = (scope, into, flat) => { + // `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`, under `target.` (a + // request of its own, `@`) or `workspace`, in the order the manifest has them: each + // dependency a version string or a table. + const depTables = (scope, into, flat, cfg = null) => { for (const [key, value] of Object.entries(scope)) { - if (key === 'target') { - for (const cfg of Object.values(table(value) ?? {})) depTables(table(cfg) ?? {}, into, flat) + if (key === 'target' && cfg === null) { + for (const [platform, tables] of Object.entries(table(value) ?? {})) depTables(table(tables) ?? {}, into, flat, platform) continue } - const kind = DEP_KINDS[key] + const kind = DEP_KINDS.get(key) if (kind === undefined) continue - for (const [depName, spec] of Object.entries(table(value) ?? {})) setDepFields(depOf(into, depName, { flat }), spec, flat ? null : kind) + const request = flat ? null : (cfg === null ? kind : `${kind}@${cfg}`) + for (const [depName, spec] of Object.entries(table(value) ?? {})) setDepFields(depOf(into, depName, { flat }), spec, request) } } depTables(doc, manifest.deps, false) @@ -243,7 +258,8 @@ function satisfiesComparator(v, comparator) { let text = m[2] const wild = /^(\d+)(?:\.(\d+))?\.[*xX]$/u.exec(text) if (wild) { - op = '~' + // A bare wildcard is a tilde range; with an operator the `*` just omits the component (`>=1.*` is `>=1`). + if (m[1] === undefined) op = '~' text = wild[2] === undefined ? wild[1] : `${wild[1]}.${wild[2]}` } const r = parseVersion(text) @@ -281,29 +297,137 @@ export function satisfiesCargoReq(version, req) { // --- cargo metadata ------------------------------------------------------------------- -// The Cargo.lock governing `baseDir`: in it or in an ancestor (a member dir's lock is its workspace root's). +// The Cargo.lock governing `baseDir`: beside the workspace root's Cargo.toml -- the nearest manifest +// at or above `baseDir` with a `[workspace]` table, as cargo finds it -- else beside `baseDir`'s +// own. A lock further up belongs to some other project: passing `--locked` for it would make +// cargo refuse to create the one it needs. export function findCargoLock(baseDir) { - let dir = baseDir - for (;;) { - const candidate = join(dir, 'Cargo.lock') - if (existsSync(candidate)) return candidate + let root = baseDir + for (let dir = baseDir; ;) { + const file = join(dir, 'Cargo.toml') + const text = readFileOrNull(file) + if (text !== null && parseCargoManifest(text, file).isWorkspace) { + root = dir + break + } const parent = dirname(dir) - if (parent === dir) return null + if (parent === dir) break dir = parent } + const candidate = join(root, 'Cargo.lock') + return existsSync(candidate) ? candidate : null +} + +// --- cfg predicates --------------------------------------------------------------------- + +export const normalizeCfg = (pred) => pred.replaceAll(/\s+/gu, ' ').trim() + +const FEATURE_CFG_RE = /^feature\s*=\s*"([^"]*)"$/u +// Any other leaf: `unix`, `target_os = "linux"`. +const CFG_LEAF_RE = /^([A-Za-z_]\w*)\s*(?:=\s*"([^"]*)")?$/u + +// The cfg keys rustc sets from the target alone (`rustc --print cfg --target `), decided by +// membership in that set when the target is known. Not among them: `debug_assertions`, `panic`, +// `overflow_checks` and the like, which the build profile sets; `target_feature`, which the build +// may add to or take from (`-C target-feature=-crt-static`, `-C target-cpu=native`); and +// `target_thread_local`, which a stable rustc never prints. +export const TARGET_CFG_KEYS = new Set(['unix', 'windows', 'target_abi', 'target_arch', 'target_endian', 'target_env', 'target_family', 'target_has_atomic', 'target_os', 'target_pointer_width', 'target_vendor']) + +// Three-valued evaluation of a cfg predicate: `false` when it can never hold in the build (so the +// item it gates is dead code for the bundle), `true` when it always does, `null` when the loader +// can't tell. `all`/`any`/`not` compose. `test` holds only in a test/bench target (`env.test`); +// `doctest` and `doc` never do when a program is built; `feature = "x"` is decided against +// `env.features`, the crate's resolved feature set (see createCargoContext), and unknown without +// one; a target leaf (`unix`, `target_os = …`) against `env.target`, the target's cfg set as rustc +// prints it (rustcTargetCfgs), and unknown without one; every other leaf stays unknown. +export function evalCfg(pred, env = {}) { + const p = pred.trim() + const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) + if (!m) { + if (p === 'test') return env.test === true + if (p === 'doctest' || p === 'doc') return false + const feature = FEATURE_CFG_RE.exec(p) + if (feature) return env.features ? env.features.has(feature[1]) : null + const leaf = CFG_LEAF_RE.exec(p) + if (leaf && env.target && TARGET_CFG_KEYS.has(leaf[1])) return env.target.has(leaf[2] === undefined ? leaf[1] : `${leaf[1]}="${leaf[2]}"`) + return null + } + const args = splitTopLevel(m[2]).map((a) => a.trim()).filter(Boolean).map((a) => evalCfg(a, env)) + if (m[1] === 'not') return args.length === 1 && args[0] !== null ? !args[0] : null + if (m[1] === 'all') return args.includes(false) ? false : (args.every((a) => a === true) ? true : null) + return args.includes(true) ? true : (args.every((a) => a === false) ? false : null) +} + +// `rustc --print cfg` output → the set of cfg leaves as printed, one per line (`unix`, +// `target_os="linux"`). +export function parseRustcCfg(text) { + return new Set(text.split('\n').map((l) => l.trim()).filter(Boolean)) +} + +// How long a toolchain query may take before it counts as hung. +const TOOL_TIMEOUT_MS = 60_000 +// Never let rustup install a toolchain on the loader's behalf, whatever a `rust-toolchain` file says. +const TOOL_ENV = { ...process.env, RUSTUP_AUTO_INSTALL: '0' } +// Where rustc is run from: the user's home directory, else the filesystem root. A rustup proxy +// picks its toolchain from the `rust-toolchain(.toml)` files of the working directory and its +// parents, and such a file may name a `path` to any binary -- so never the project being bundled +// (untrusted input: its toolchain file must not choose what runs), and never a temp dir (anyone +// can plant a file in a world-writable one). The home directory is the user's -- unless it is the +// bundle root (`baseDir`) or lies inside it (a project bundled from a home directory, a CI job +// whose HOME is its checkout): then the root, whose toolchain file, if any, is the machine's. +function toolCwd(baseDir) { + const home = homedir() + const root = parse(process.cwd()).root + if (!home) return root + if (baseDir === null) return home + // Real paths: a home directory or bundle root reached through a link is where it really is. + const real = (p) => { + try { + return realpathSync(p) + } catch { + return resolve(p) + } + } + const base = real(baseDir) + const homeAbs = real(home) + return homeAbs === base || homeAbs.startsWith(base.endsWith(sep) ? base : base + sep) ? root : home +} + +// Run rustc: `$RUSTC` when set, as cargo honours it (a target only another toolchain knows, such +// as Solana's `sbf-solana-solana` in its platform-tools), else `rustc` from PATH -- a rustup proxy +// resolving to the user's default toolchain (or `RUSTUP_TOOLCHAIN`), from toolCwd. With +// RUSTUP_AUTO_INSTALL=0 an uninstalled toolchain is an error, not a download. +function rustc(args, what, baseDir) { + const bin = process.env.RUSTC || 'rustc' + const r = spawnSync(bin, args, { cwd: toolCwd(baseDir), encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: TOOL_TIMEOUT_MS, env: TOOL_ENV }) + if (r.error) throw new Error(`${bin} could not run (${r.error.message}); ${what} needs rustc on PATH (or $RUSTC)`, { cause: r.error }) + if (r.status !== 0) throw new Error(`${bin} ${args.join(' ')} failed (exit ${r.status}):\n${(r.stderr ?? '').trim()}`) + return r.stdout +} + +// The cfg set of a target: `{ triple, cfgs }` from `rustc --print cfg --target `, which +// needs only rustc's built-in knowledge of the target, not its standard library. `host` is the +// running rustc's host triple (`rustc -vV`). `baseDir` is the bundle root, which rustc is never +// run from (toolCwd). +export function rustcTargetCfgs(target, baseDir = null) { + const triple = target === 'host' ? /^host: (\S+)$/mu.exec(rustc(['-vV'], '--cargo-target=host', baseDir))?.[1] ?? null : target + if (triple === null) throw new Error('rustc -vV printed no host triple') + return { triple, cfgs: parseRustcCfg(rustc(['--print', 'cfg', '--target', triple], '--cargo-target', baseDir)) } } // Run `cargo metadata` in the bundle root and return its JSON. Opt-in (`--cargo`) because it runs // cargo, which reads the project's `.cargo/config.toml` and may touch the registry and Cargo.lock // (doc/file-formats.md has the full caveat); never run on a bundle root unasked. With a lockfile -// present, `--locked` keeps the resolution the one the build uses. The feature flags pass through. -export function runCargoMetadata(baseDir, { features = [], noDefaultFeatures = false, allFeatures = false } = {}) { +// present, `--locked` keeps the resolution the one the build uses. The feature flags pass through; +// `platform`, a target triple, restricts the graph to that target's dependencies. +export function runCargoMetadata(baseDir, { features = [], noDefaultFeatures = false, allFeatures = false, platform = null } = {}) { const args = ['metadata', '--format-version', '1'] - // A lock anywhere above governs too: never let metadata rewrite a workspace lock outside the bundle root. + // The workspace's lock may sit above the bundle root: never let metadata rewrite it. if (findCargoLock(baseDir) !== null) args.push('--locked') if (allFeatures) args.push('--all-features') if (noDefaultFeatures) args.push('--no-default-features') for (const f of features) args.push('--features', f) + if (platform !== null) args.push('--filter-platform', platform) const r = spawnSync('cargo', args, { cwd: baseDir, encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }) if (r.error) throw new Error(`cargo metadata could not run (${r.error.message}); is cargo on PATH?`, { cause: r.error }) if (r.status !== 0) throw new Error(`cargo metadata failed (exit ${r.status}):\n${(r.stderr ?? '').trim()}`) @@ -353,6 +477,24 @@ export function resolutionFromMetadata(metadata, baseDir, { locate = null } = {} return { enabled, deps } } +// The cargo config in directory `dir` (project-relative; `.` for the bundle root), project-relative: +// `.cargo/config` when it exists -- cargo prefers the extensionless file over `.cargo/config.toml` +// when both are there -- else `.cargo/config.toml`; null when there is neither. +function cargoConfigIn(baseDir, dir) { + return ['.cargo/config', '.cargo/config.toml'].map((f) => (dir === '.' ? f : `${dir}/${f}`)).find((rel) => isFile(join(baseDir, rel))) ?? null +} +const cargoConfigOf = (baseDir) => cargoConfigIn(baseDir, '.') + +// The directory `cargo vendor` filled: what `[source.vendored-sources] directory = "…"` in the +// project's cargo config says (however the table is spelled out), else the default `vendor`. +// Throws a TomlError when the config isn't TOML. +function vendorDirOf(baseDir) { + const rel = cargoConfigOf(baseDir) + const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) + const dir = text === null ? undefined : readToml(text, rel).source?.['vendored-sources']?.directory + return (typeof dir === 'string' ? normalizeRel('.', dir)?.replace(/\/+$/u, '') : null) || VENDOR_DIR +} + // --- Context ---------------------------------------------------------------------------- // Per-bundle Cargo state: manifest lookup (memoized per directory), crate-name resolution against @@ -360,8 +502,12 @@ export function resolutionFromMetadata(metadata, baseDir, { locate = null } = {} // bundled, like `cargo build -p …`). `features` / `noDefaultFeatures` / `allFeatures` mirror // cargo's flags for those root packages (`pkg/feat` targets one of them). `cargo: true` takes the // dependency graph and features from `cargo metadata` (see runCargoMetadata) instead of replaying -// the manifests. `baseDir` is the bundle root; every path in and out is project-relative POSIX. -export function createCargoContext(baseDir, { entries = [], features = [], noDefaultFeatures = false, allFeatures = false, cargo = false } = {}) { +// the manifests. `target` names the build's target -- a triple or `host`, asked of rustc +// (rustcTargetCfgs), or its `{ triple, cfgs }` outright -- so target-specific dependency tables +// and `#[cfg(unix)]`-style code are decided; without one they are kept. `baseDir` is the bundle +// root; every path in and out is project-relative POSIX. +export function createCargoContext(baseDir, { entries = [], features = [], noDefaultFeatures = false, allFeatures = false, cargo = false, target = null } = {}) { + const targetInfo = target === null ? null : (typeof target === 'string' ? rustcTargetCfgs(target, baseDir) : target) const manifests = new Map() const readManifest = (dir) => { if (!manifests.has(dir)) { @@ -403,10 +549,14 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } // The manifest's lib target root when it is on disk (`[lib] path`, default `src/lib.rs`); // memoized on the manifest, since it is asked for once per crate reference. + // A vendored package's `[lib] path` may not leave its directory (a published crate's never + // does; one that tries is reaching for the project's files). const libPath = (m) => { if (m.libRoot === undefined) { const rel = normalizeRel(m.dir, m.lib.path ?? 'src/lib.rs') - m.libRoot = rel !== null && isFile(join(baseDir, rel)) ? rel : null + const inside = rel !== null && (!isVendoredDir(m.dir) || rel.startsWith(`${m.dir}/`)) + if (rel !== null && !inside) console.warn(`[loader.cargo] Refusing lib path outside its package: ${m.lib.path} in ${m.dir}`) + m.libRoot = inside && isFile(join(baseDir, rel)) ? rel : null } return m.libRoot } @@ -420,20 +570,23 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // --- dependency resolution - // `vendor//` crates as `[{ version, dir }]`, indexed `byName` (normalized package name; the - // dir may hyphenate a snake_case name, and an older duplicate version lives in + // The vendored crates, `//`, as `[{ version, dir }]`, indexed `byName` (normalized + // package name; the dir may hyphenate a snake_case name, and an older duplicate version lives in // `-/`) and `byLib` (lib name, for the crates whose `[lib] name` differs: // `md-5` → `md5`). + const vendorDir = vendorDirOf(baseDir) + // Whether a package directory is a vendored crate's (a registry crate `cargo vendor` copied in). + const isVendoredDir = (dir) => dir !== vendorDir && dir.startsWith(`${vendorDir}/`) let vendorIndex = null const vendored = () => { if (vendorIndex === null) { vendorIndex = { byName: new Map(), byLib: new Map() } let dirs = [] try { - dirs = readdirSync(join(baseDir, VENDOR_DIR)) + dirs = readdirSync(join(baseDir, vendorDir)) } catch { /* no vendor dir */ } for (const d of dirs) { - const m = readManifest(`${VENDOR_DIR}/${d}`) + const m = readManifest(`${vendorDir}/${d}`) if (!m?.package) continue const entry = { version: version(m), dir: m.dir } for (const [index, key] of [[vendorIndex.byName, normName(m.package.name)], [vendorIndex.byLib, libName(m)]]) { @@ -452,7 +605,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // `--cargo`: the graph and features as cargo resolved them, with registry packages it read from // the registry cache matched to their `vendor/` copy by name + version. const metadata = cargo - ? resolutionFromMetadata(runCargoMetadata(baseDir, { features, noDefaultFeatures, allFeatures }), baseDir, { + ? resolutionFromMetadata(runCargoMetadata(baseDir, { features, noDefaultFeatures, allFeatures, platform: targetInfo?.triple ?? null }), baseDir, { locate: (name, ver) => (typeof name === 'string' ? vendored().byName.get(normName(name))?.find((c) => c.version === ver)?.dir ?? null : null), }) : null @@ -539,7 +692,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const root = readManifest('.') const explicit = Number(root?.resolver) if (Number.isInteger(explicit) && explicit > 0) return explicit - return Number(root?.package?.edition ?? 0) >= 2021 ? 2 : 1 + const edition = root?.package?.editionFromWorkspace ? workspaceFor('.')?.workspacePackage.edition : root?.package?.edition + return Number(edition ?? 0) >= 2021 ? 2 : 1 } // An optional dependency no `dep:` entry names gets an implicit feature of its own name -- as // the manifest spells it (`proc-macro-crate`), which is what `std = ["proc-macro-crate"]` and @@ -563,9 +717,10 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // the roots start from `default` (or the flags), then features imply features, activate // optional deps and request dependency features, and active deps get `default` plus what the // dependent asks for, to a fixed point -- cargo's unification, over-approximating where the - // loader can't tell (target-specific dependency tables always count). Dev-dependencies: a - // dependency's own are never built by anyone, so they never count; the root packages' count - // under resolver 1 only (resolver 2 keeps them out of a normal build). + // loader can't tell (a target-specific dependency table counts unless the target is known and + // it isn't for it). Dev-dependencies: a dependency's own are never built by anyone, so they + // never count; the root packages' count under resolver 1 only (resolver 2 keeps them out of a + // normal build). const ensureResolved = () => { if (enabled !== null) return enabled if (metadata) { @@ -602,10 +757,21 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // Dev-dependencies count for a root package under resolver 1, and for one whose entries include a // test/bench target (that build is `cargo test`'s, which links them). const testRootDirs = new Set(entries.filter((e) => isTestTarget(e)).map((e) => packageFor(e)?.dir)) - const kindApplies = (m, kind) => kind !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) - // The tables of `d` that take part in the build, with an optional one only once activated. + const kindApplies = (m, request) => kindOf(request) !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) + // Whether a table's `[target..…]` is for the build's target: a `cfg(…)` decided against + // its cfg set (kept when undecidable), a triple compared; any table when no target is known, + // and any build-dependency table, which is about the host the build script runs on. + const targetApplies = (request) => { + const at = request.indexOf('@') + if (at === -1 || targetInfo === null || kindOf(request) === 'build') return true + const spec = request.slice(at + 1) + const cfg = /^cfg\((.*)\)$/u.exec(spec) + return cfg ? evalCfg(cfg[1], { target: targetInfo.cfgs }) !== false : spec === targetInfo.triple + } + // The tables of `d` that take part in the build (a target-specific one when it is for the + // target), with an optional one only once activated. const activeRequests = (m, d) => [...d.kinds] - .filter(([kind, r]) => kindApplies(m, kind) && (!r.optional || active.get(m.dir)?.has(d.key) === true)) + .filter(([request, r]) => kindApplies(m, request) && targetApplies(request) && (!r.optional || active.get(m.dir)?.has(d.key) === true)) .map(([, r]) => r) // One entry of a feature's list: `other`, `dep:key`, `key/feat`, `key?/feat`. Returns whether it named anything. const applyImplication = (m, imp) => { @@ -646,19 +812,25 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // `--features a,b,pkg/c`: a bare name is a feature of every root package; `x/c` is a feature of // the root package named `x`, else of the dependency `x` of each root (cargo's `dep/feat` form). const rootNames = new Set(roots.map((r) => normName(r.package.name))) + const requested = [] for (const flag of parseFeatureList(features)) { const slash = flag.indexOf('/') const pkg = slash === -1 ? null : normName(flag.slice(0, slash)) const scoped = pkg !== null && rootNames.has(pkg) const targets = scoped ? roots.filter((m) => normName(m.package.name) === pkg) : roots const imp = scoped ? flag.slice(slash + 1) : flag - if (!targets.some((m) => applyImplication(m, imp))) { + // Whether a name is known doesn't depend on what is active yet, so the check happens once. + if (targets.filter((m) => applyImplication(m, imp)).length === 0) { console.warn(`[stasis] --cargo-features: '${flag}' names no feature of the entries' packages, nor a dependency of theirs`) } + requested.push({ targets, imp }) } do { changed = false + // The requested features re-apply on every pass like the manifest's entries do: a weak + // `dep?/x` asked for on the command line takes effect once `default` has activated `dep`. + for (const { targets, imp } of requested) for (const m of targets) applyImplication(m, imp) // Map/Set iteration is live: packages and features added mid-pass are visited in this pass. for (const dir of enabled.keys()) { const m = readManifest(dir) @@ -720,6 +892,9 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return { + // Where `cargo vendor` put the registry crates (project-relative): `vendor`, or what + // .cargo/config.toml names. A package under it is a vendored dependency. + vendorDir, // Identity of the package owning `fileRel` -- `{ dir, name, version }` from the nearest // Cargo.toml with a [package] -- or null when no manifest claims it. packageInfo(fileRel) { @@ -746,6 +921,46 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return depCrate(m, norm) }, isTestTarget, + // The build's target: its triple and cfg set (`unix`, `target_os="linux"`, … as rustc prints + // them), or null for both when none was given. + targetTriple: targetInfo?.triple ?? null, + targetCfgs: targetInfo?.cfgs ?? null, + // The files that describe the build of `fileRel`'s package, project-relative, as `{ path, + // kind }`: its own Cargo.toml and, inside the bundle root, the workspace's above it + // (`manifest`); that workspace's -- or, without one, the package's own -- Cargo.lock (`lock`) + // and cargo config (`config`) when on disk. Empty for a file no manifest claims. + buildFilesFor(fileRel) { + const m = packageFor(fileRel) + if (!m) return [] + const out = [{ path: posix.join(m.dir, 'Cargo.toml'), kind: 'manifest' }] + const ws = workspaceFor(m.dir) + if (ws && ws.dir !== m.dir) out.push({ path: posix.join(ws.dir, 'Cargo.toml'), kind: 'manifest' }) + const top = ws?.dir ?? m.dir + const lockPath = posix.join(top, 'Cargo.lock') + if (isFile(join(baseDir, lockPath))) out.push({ path: lockPath, kind: 'lock' }) + const config = cargoConfigIn(baseDir, top) + if (config !== null) out.push({ path: config, kind: 'config' }) + return out + }, + // Whether `fileRel` belongs to a vendored package (a registry crate `cargo vendor` copied in). + isVendored(fileRel) { + const m = packageFor(fileRel) + return m !== null && isVendoredDir(m.dir) + }, + // The build script of `fileRel`'s package, project-relative, when there is one on disk: + // `[package] build = "…"`, else `build.rs` beside the manifest; `build = false` means none. A + // vendored package's may not name a file outside the package (a published crate never does). + buildScriptOf(fileRel) { + const m = packageFor(fileRel) + if (!m || m.package.build === false) return null + const rel = normalizeRel(m.dir, m.package.build ?? 'build.rs') + if (rel === null || !isFile(join(baseDir, rel))) return null + if (this.isVendored(fileRel) && !rel.startsWith(`${m.dir}/`)) { + console.warn(`[loader.cargo] Refusing build script outside its package: ${m.package.build} in ${m.dir}`) + return null + } + return rel + }, // The features enabled for the package owning `fileRel` in the build of the root packages, or // null when that is unknown: no owning manifest, no root package to resolve from, or a package // the resolved build doesn't pull in (its gated code is then kept, not dropped). diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index ecd73744..bb33a66e 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -8,12 +8,13 @@ // file to a crate, only `mod` can. Registry dependencies that aren't vendored live outside the // bundle root and are dropped. +import { isUtf8 } from 'node:buffer' import { realpathSync } from 'node:fs' import { readFile } from 'node:fs/promises' import { dirname, extname, isAbsolute, join, posix, relative, resolve } from 'node:path' import { assertRealPathWithinBase } from '@exodus/stasis-core/util' -import { VENDOR_DIR, createCargoContext, isFile, isTestTargetPath, normName, normalizeRel } from './cargo.js' +import { TARGET_CFG_KEYS, VENDOR_DIR, createCargoContext, evalCfg, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' import { matchClose, splitTopLevel } from './toml.js' // Leads of the expression-position paths anchored on the module tree rather than on a name. @@ -47,13 +48,14 @@ const isWordChar = (ch) => ch !== undefined && /\w/u.test(ch) // wrong in both directions (a commented-out `mod` taken for a real one, or real ones swallowed). export function lexRust(content) { const n = content.length - // Each view is assembled from chunks: the source up to a blanked range, then the range with its - // non-newline chars replaced by spaces (indexes stay those of `content`). + // Each view is assembled from chunks: the source up to a blanked range, then the range with + // every line replaced by as many spaces as it has UTF-16 units (a char may be two), so indexes + // stay those of `content`. const code = [] const masked = [] let codeAt = 0 let maskedAt = 0 - const blanks = (start, end) => content.slice(start, end).replaceAll(/[^\n]/gu, ' ') + const blanks = (start, end) => content.slice(start, end).split('\n').map((line) => ' '.repeat(line.length)).join('\n') const blankMasked = (start, end) => { masked.push(content.slice(maskedAt, start), blanks(start, end)) maskedAt = end @@ -164,39 +166,60 @@ function charLiteralEnd(content, i) { const WORD_RE = /[A-Za-z_]\w*/uy const WS_RE = /\s/u +// First word of an item or statement that runs to its `;` or `{ … }` body, whatever commas its +// generics, arguments or `where` clause hold; anything else (a field, a variant, a match arm) +// ends at a top-level comma. Weak keywords a field may be named after (`default`, `union`) are +// left out on purpose. +const ITEM_KEYWORDS = new Set(['fn', 'impl', 'struct', 'enum', 'trait', 'type', 'const', 'static', 'mod', 'use', 'extern', 'macro_rules', 'unsafe', 'async', 'let']) +// Macros whose literal string argument names a file relative to the invoking one: Rust source +// spliced in (`include!`), or an asset embedded as text / bytes. Only a literal can be followed; +// `concat!(env!("OUT_DIR"), …)` is build output. +const INCLUDE_MACROS = new Set(['include', 'include_str', 'include_bytes']) +// Macros whose body is a template of code for some other crate (a proc macro's output): nothing +// in it is an item, import or path of the crate holding it. +const TEMPLATE_MACROS = new Set(['quote', 'quote_spanned', 'parse_quote', 'parse_quote_spanned']) +// An include macro's argument: a string literal (plain, with escapes, or raw with any number of +// `#`s), or `concat!(env!("CARGO_MANIFEST_DIR"), "")`, a path from the package root. +const STRING_LITERAL_RE = /^\s*(?:"((?:[^"\\]|\\[\s\S])*)"|r(#*)"([\s\S]*?)"\2)\s*$/u +const MANIFEST_DIR_CONCAT_RE = /^\s*concat!\s*\(\s*env!\s*\(\s*"CARGO_MANIFEST_DIR"\s*\)\s*,\s*([\s\S]*?),?\s*\)\s*$/u +// An include macro opening inside an attribute's text: `#![doc = include_str!("../README.md")]`. +const INCLUDE_IN_ATTR_RE = /\b(include|include_str|include_bytes)!\s*(?=[([{])/gu +// Items a module defines by name, for `pub struct X` behind a glob; `mod`, `use`, `extern crate` +// and `macro_rules!` have their own records. +const DEFINING_KEYWORDS = new Set(['struct', 'enum', 'union', 'trait', 'type', 'fn', 'const', 'static']) +// Of those, the items in the value namespace: never a path's lead or prefix (a `fn log` beside +// `use log::info` hides no crate). +const VALUE_KEYWORDS = new Set(['fn', 'const', 'static']) + +// The path an include macro's argument names: `{ path, base }` (`base`: `file` for a path relative +// to the invoking file, `manifest` for one from the package root), or null when it isn't one the +// loader can follow (`concat!(env!("OUT_DIR"), …)` is build output). +function includeArg(text) { + const manifest = MANIFEST_DIR_CONCAT_RE.exec(text) + const lit = STRING_LITERAL_RE.exec(manifest ? manifest[1] : text) + if (!lit) return null + const path = lit[1] === undefined ? lit[3] : unescapeString(lit[1]) + return manifest ? { path: path.replace(/^\/+/u, ''), base: 'manifest' } : { path, base: 'file' } +} + +// The value of a plain string literal's body: `\\`, `\"`, `\'`, `\n`, `\r`, `\t`, `\0`, +// `\x41`, `\u{1F600}` and a backslash-newline continuation. +function unescapeString(body) { + return body.replaceAll(/\\(?:u\{([0-9a-fA-F]{1,6})\}|x([0-9a-fA-F]{2})|\n\s*|(.))/gsu, (m, u, x, c) => { + if (u !== undefined) return String.fromCodePoint(Number.parseInt(u, 16)) + if (x !== undefined) return String.fromCharCode(Number.parseInt(x, 16)) + if (c === undefined) return '' + return { n: '\n', r: '\r', t: '\t', 0: '\0' }[c] ?? c + }) +} const EXTERN_CRATE_RE = /extern\s+crate\s+(?:r#)?(\w+)(?:\s+as\s+(?:r#)?(\w+))?\s*;/uy +const MACRO_EXPORT_RE = /^\s*macro_export\s*(?:\([\s\S]*\))?\s*$/u // Expression-position paths: any `lead::…` path whose lead is lowercase (skipping `Type::assoc` // associated-item paths) -- a `crate`/`self`/`super` keyword path, or one led by a module or // crate name. Runs over the masked view, so string contents can't fake one. const LEAD_PATH_RE = /\b([a-z_]\w*)(?:::\w+)+/gu const ATTR_PATH_RE = /^\s*path\s*=\s*"([^"]*)"\s*$/u -const normalizeCfg = (pred) => pred.replaceAll(/\s+/gu, ' ').trim() - -const FEATURE_CFG_RE = /^feature\s*=\s*"([^"]*)"$/u - -// Three-valued evaluation of a cfg predicate: `false` when it can never hold in the build (so the -// item it gates is dead code for the bundle), `true` when it always does, `null` when the loader -// can't tell. `all`/`any`/`not` compose. `test` holds only in a test/bench target (`env.test`); -// `doctest` and `doc` never do when a program is built; `feature = "x"` is decided against -// `env.features`, the crate's resolved feature set (see cargo.js), and unknown without one; every -// other leaf (`unix`, `target_os = …`) depends on the target and stays unknown. -export function evalCfg(pred, env = {}) { - const p = pred.trim() - const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) - if (!m) { - if (p === 'test') return env.test === true - if (p === 'doctest' || p === 'doc') return false - const feature = FEATURE_CFG_RE.exec(p) - if (feature && env.features) return env.features.has(feature[1]) - return null - } - const args = splitTopLevel(m[2]).map((a) => a.trim()).filter(Boolean).map((a) => evalCfg(a, env)) - if (m[1] === 'not') return args.length === 1 && args[0] !== null ? !args[0] : null - if (m[1] === 'all') return args.includes(false) ? false : (args.every((a) => a === true) ? true : null) - return args.includes(true) ? true : (args.every((a) => a === false) ? false : null) -} - // Combine an item's cfg predicates (several `#[cfg]` attributes all apply) into one; null when ungated. const joinCfgs = (preds) => (preds.length === 0 ? null : (preds.length === 1 ? preds[0] : `all(${preds.join(', ')})`)) @@ -205,37 +228,67 @@ const CFG_ATTR_RE = /^\s*cfg\s*\(([\s\S]*)\)\s*$/u // One outer attribute's text (inside `#[…]`) → `cfg`: the predicate gating the item (`#[cfg()]`; // `test` for a `#[test]` fn; for `#[cfg_attr(, cfg())]` the item is compiled unless // pred holds and inner doesn't, i.e. `any(not(pred), inner)`; null when the item is unconditional -// -- a `cfg_attr` applying any other attribute, `doc(cfg(…))` included, gates nothing) and +// -- a `cfg_attr` applying any other attribute, `doc(cfg(…))` included, gates nothing); // `paths`: the module file paths it names, `#[path = "…"]` outright (cfg null) or each -// `#[cfg_attr(, path = "…")]` under its predicate. -function parseAttr(text) { +// `#[cfg_attr(, path = "…")]` under its predicate; `pred`: a `cfg_attr`'s own predicate +// (null for any other attribute) -- when it can't hold, the whole attribute applies nothing; +// `macroExport`: true for `#[macro_export]` / `#[macro_export(local_inner_macros)]`, outright or +// under a `cfg_attr` (the macro then lives at the crate root, as `$crate::name!`); `includes`: the +// files an include macro in the text names (`#[doc = include_str!("../README.md")]`, or from the +// package root with `concat!(env!("CARGO_MANIFEST_DIR"), …)`; see includeArg). `masked` is the same +// text with string contents blanked (lexRust), for matching the macro's brackets. +function parseAttr(text, masked = text) { + const includes = [] + for (const m of masked.matchAll(INCLUDE_IN_ATTR_RE)) { + const open = m.index + m[0].length + const arg = includeArg(text.slice(open + 1, matchClose(masked, open))) + if (arg !== null) includes.push({ kind: m[1], ...arg }) + } + if (MACRO_EXPORT_RE.test(text)) return { cfg: null, paths: [], pred: null, includes, macroExport: true } + if (/^\s*macro_use\s*$/u.test(text)) return { cfg: null, paths: [], pred: null, includes, macroUse: true } const path = ATTR_PATH_RE.exec(text) - if (path) return { cfg: null, paths: [{ path: path[1], cfg: null }] } - if (/^\s*test\s*$/u.test(text)) return { cfg: 'test', paths: [] } + if (path) return { cfg: null, paths: [{ path: path[1], cfg: null }], pred: null, includes } + if (/^\s*test\s*$/u.test(text)) return { cfg: 'test', paths: [], pred: null, includes } const cfg = CFG_ATTR_RE.exec(text) - if (cfg) return { cfg: normalizeCfg(cfg[1]), paths: [] } + if (cfg) return { cfg: normalizeCfg(cfg[1]), paths: [], pred: null, includes } const cfgAttr = /^\s*cfg_attr\s*\(([\s\S]*)\)\s*$/u.exec(text) if (cfgAttr) { const parts = splitTopLevel(cfgAttr[1]) const pred = normalizeCfg(parts.shift() ?? '') const paths = [] const inner = [] + let macroExport = false for (const part of parts) { const m = ATTR_PATH_RE.exec(part) const c = CFG_ATTR_RE.exec(part) if (m) paths.push({ path: m[1], cfg: pred }) else if (c) inner.push(normalizeCfg(c[1])) + else if (MACRO_EXPORT_RE.test(part)) macroExport = true } - return { cfg: inner.length === 0 ? null : `any(not(${pred}), ${joinCfgs(inner)})`, paths } + return { cfg: inner.length === 0 ? null : `any(not(${pred}), ${joinCfgs(inner)})`, paths, pred, includes, macroExport } } - return { cfg: null, paths: [] } + return { cfg: null, paths: [], pred: null, includes } +} + +// An attribute that applies nothing in this build (a `cfg_attr` whose predicate can't hold). +const INERT_ATTR = { cfg: null, paths: [], pred: null, includes: [] } + +// The visibility a `pub(…)` restriction spells, from the text between its parentheses: `crate`, +// `self`, `super`, or `in ` (normalized to `in a::b`); a bare `pub` is `crate` -- the loader +// never follows a path into another crate's tree, so `pub` and `pub(crate)` are one to it. +function parseVisibility(restriction) { + const r = restriction.trim().replaceAll(/\s+/gu, ' ') + if (r === 'crate' || r === 'self' || r === 'super') return r + if (r.startsWith('in ')) return `in ${r.slice(3).replaceAll(' ', '')}` + return 'crate' } // Flatten a `use` tree body (the text between `use` and `;`) into the paths it imports, as -// `{ segments, absolute, spec, binding }`: `a::{b::C, d::{E, F}}` → `a::b::C`, `a::d::E`, `a::d::F`; -// a glob or `self` inside a group names the group's own module; a leading `::` marks the path -// absolute (an external crate). `spec` is the flat path as it would be written; `binding` the name -// the import brings into scope (its last segment, or the `as` alias; null for a glob or `_`). +// `{ segments, absolute, spec, binding, glob }`: `a::{b::C, d::{E, F}}` → `a::b::C`, `a::d::E`, +// `a::d::F`; a glob (`glob` true) or `self` inside a group names the group's own module; a leading +// `::` marks the path absolute (an external crate). `spec` is the flat path as it would be +// written; `binding` the name the import brings into scope (its last segment, or the `as` alias; +// null for a glob or `_`). export function parseUseTree(body) { const tokens = [...body.matchAll(/::|[{},*]|\bas\b|(?:r#)?\w+/gu)].map((m) => m[0]) const out = [] @@ -243,9 +296,9 @@ export function parseUseTree(body) { const peek = () => tokens[pos] const next = () => tokens[pos++] const isIdent = (tok) => tok !== undefined && tok !== 'as' && /^(?:r#)?\w+$/u.test(tok) - const emit = (segments, absolute, binding) => { + const emit = (segments, absolute, binding, glob) => { if (segments.length > 0) { - out.push({ segments, absolute, spec: `${absolute ? '::' : ''}${segments.join('::')}`, binding: binding === '_' ? null : binding }) + out.push({ segments, absolute, spec: `${absolute ? '::' : ''}${segments.join('::')}`, binding: binding === '_' ? null : binding, glob }) } } const parse = (prefix, absolute) => { @@ -268,7 +321,7 @@ export function parseUseTree(body) { } if (tok === '*') { next() - emit(prefix, absolute, null) + emit(prefix, absolute, null, true) return } if (!isIdent(tok)) return @@ -291,57 +344,190 @@ export function parseUseTree(body) { } // `use a::{self, b}`: `self` names the group's own module. if (prefix.length > 0 && segments.length === prefix.length + 1 && segments.at(-1) === 'self') segments.pop() - emit(segments, absolute, alias ?? segments.at(-1)) + emit(segments, absolute, alias ?? segments.at(-1), false) } parse([], false) return out } // Statically scan one file's items. An item whose cfg can never hold in the build (`#[cfg(test)]`, -// `#[test]`, `#[cfg(doc)]`, or a `#[cfg(feature = "x")]` with `x` off in the crate's resolved -// `features`) is skipped whole -- an inline `mod tests { … }` with everything in it, a `fn`'s +// `#[test]`, `#[cfg(doc)]`, a `#[cfg(feature = "x")]` with `x` off in the crate's resolved +// `features`, a `#[cfg(windows)]` when `target` -- the build target's cfg set, see evalCfg -- says +// otherwise) is skipped whole -- an inline `mod tests { … }` with everything in it, a `fn`'s // body, a `use` -- so dead modules aren't bundled and the dependencies only dead code reaches for // don't get pulled in. Returns -// mods: external `mod` declarations `{ name, inlinePath, cfg, conditional, paths }` -- -// `inlinePath` is the chain of inline `mod x { … }` blocks it sits in, `cfg` the +// mods: external `mod` declarations `{ name, inlinePath, inlineDirs, cfg, conditional, +// paths, noDefault, vis, macroUse, macro }` -- `inlinePath` is the chain of inline +// `mod x { … }` blocks it sits in, `inlineDirs` the directories those stand for +// (each its name, or its own `#[path = "…"]`, which may be empty), `vis` its +// visibility (see imports), `macroUse` a `#[macro_use]` on it, `macro` the macro +// invocation it sits in (if any), `noDefault` that a `#[cfg_attr(…, path)]` variant +// always applies so the default file lookup never happens, `cfg` the // `#[cfg(…)]` predicate gating it (several → `all(…)`; null when ungated), // `conditional` marks one that may not exist as an item (a cfg the loader can't // decide, an inline ancestor so gated, or inside a macro invocation body), `paths` // the explicit file paths its attributes name (see parseAttr), minus the -// `cfg_attr` variants whose predicate can't hold in the build; -// refs: path references `{ spec, segments, absolute, inlinePath, fromUse }` -- flattened -// `use` trees plus expression-position `crate::`/`self::`/`super::`/`lead::…` paths; -// externCrates: `extern crate x [as y];` names, with their inlinePath; +// `cfg_attr` variants whose predicate can't hold in the build, `at` where the +// declaration is written and `offset` where it stands (textual macro scope: what a +// mounted file sees) -- the same, unless it sits in a `macro_rules!` body +// (`template`, that macro's name): then the macro's first bare invocation in +// this file, Infinity for none; +// refs: path references `{ spec, segments, absolute, inlinePath, fromUse, macroCall }` -- +// flattened `use` trees plus expression-position `crate::`/`self::`/`super::`/ +// `lead::…` paths (`macroCall`: one invoked, `$crate::name!(…)`); +// externCrates: `extern crate x [as y];` as `{ name, alias, inlinePath }`; +// defined: the items the file defines by name, `{ name, inlinePath, vis, cfg }`: a `struct`, +// `enum`, `union`, `trait`, `type`, `fn`, `const` or `static` at module level (not +// an `impl`'s or trait's associated item, nor a `macro_rules!` template's) -- what +// a glob into the module brings in; `cfg` the predicate gating it (see mods); // bindings: names the file's `use` items and `extern crate … as` aliases bring into scope -- -// a path lead among them names an import, not a crate. -export function scanRustItems(content, { features = null, test = false } = {}) { - const env = { features, test } +// a path lead among them names an import, not a crate; +// imports: the `use` items, and `extern crate` declarations as the imports they amount to +// (`extern crate serde_core as s` ≡ `use ::serde_core as s`; `extern crate self as +// x` ≡ `use crate as x`): `{ segments, absolute, binding, glob, inlinePath, vis, +// cfg, macro }` -- what a path written in that module may be anchored on, and, as +// far as `vis` reaches (null: private to the module; `crate`; `super`; `self`; `in +// a::b`), what a path from another module may continue along, i.e. a re-export; +// `cfg` gates it (see mods), `macro` is the invocation body it sits in, if any (one +// in a `macro_rules!` template is the invoking module's, not this one's); +// macros: `macro_rules!` definitions `{ name, exported, includes, offset, template }` -- an +// exported one is `$crate::name!` from anywhere in the crate, and lives in this +// file; `includes` are the include macros in its body, which rustc resolves +// relative to each file that invokes the macro; one written in another +// `macro_rules!`'s body (`template`, its name) is defined where that one is invoked +// (`offset`: the first bare invocation in this file, Infinity for none), as is a +// `mod` there; +// invocations: the names of the macros invoked by bare name (`ready!(…)`) -- for an edge to the +// file of a `macro_rules!` defined elsewhere in the crate; `calls` has each name's +// invocation offsets, in order (textual scope is decided at the call); +// includes: `include!` / `include_str!` / `include_bytes!` invocations outside macro +// definitions, `{ kind, path, base, conditional }` -- a file to carry, named +// relative to this one (`base` `file`) or to the package root (`manifest`, for +// `concat!(env!("CARGO_MANIFEST_DIR"), …)`); `unfollowed` counts the ones whose +// argument names no file the loader can follow (`concat!(env!("OUT_DIR"), …)`); +// inlineModules: the paths of the inline `mod x { … }` blocks, for the module tree, and +// `inlineModuleVis`, each path (`a::b`) → its visibility; +// skipped: the template macros (see `templates` below) whose bodies were skipped. +// A file whose inner `#![cfg()]` can never hold is compiled empty: the scan is empty too. +// `templates` names the macros whose bodies are token templates for another crate's code (quote!) +// and are skipped whole: TEMPLATE_MACROS, less the ones the file's package defines itself. +export function scanRustItems(content, { features = null, test = false, target = null, templates = TEMPLATE_MACROS } = {}) { + const env = { features, test, target } const { code, masked } = lexRust(content) const n = masked.length const mods = [] const uses = [] + const imports = [] + const macros = [] + const invocations = new Set() + const calls = new Map() // macro name → the offsets of its bare invocations, in order + const includes = [] + const defined = [] + const skipped = new Set() // the template macros whose bodies were skipped + let unfollowed = 0 const useSpans = [] // [start, end) of each `use` item's body, parsed as a tree below const deadSpans = [] // [start, end) of each skipped test/doc-only item const externCrates = [] const bindings = new Set() - const spans = [] // closed inline module blocks: { start, end, path } - const stack = [] // open inline modules: { name, depth, conditional, start } + const spans = [] // closed inline module blocks: { start, end, path, vis } + const stack = [] // open inline modules: { name, dir, depth, conditional, start, vis } + // `impl … { }` and `trait … { }` bodies, `{ start, end, depth }`: a `fn`, `const` or `type` at + // their depth is an associated item, not one of the module's. `extern "C" { }` blocks the same + // way: what they declare is the module's. Both are met in source order and nest or lie apart, + // so each list is a stack: the ones the scan has left are popped off its end. + const assocBlocks = [] + const foreignBlocks = [] + const openBlocks = (blocks, at) => { + while (blocks.length > 0 && blocks.at(-1).end < at) blocks.pop() + return blocks + } + // The `macro_rules!` names this file defines: a `quote!` of its own is no template. + const localMacros = new Set([...masked.matchAll(/\bmacro_rules!\s*([A-Za-z_]\w*)/gu)].map((m) => m[1])) let depth = 0 let pending = [] // parsed outer attributes (parseAttr) waiting for their item + let attrStart = 0 // where the first pending attribute begins: a dead item is dead from there + let visNext = null // the previous token was a `pub` visibility (parseVisibility): the coming item has it + // Open blocks a live `#[cfg]` gates -- `if #[cfg(unix)] { mod imp; }` in a `cfg_if!` body, a + // `#[cfg(x)] mod m { mod a; }` -- as `{ depth, cfg }`: everything declared inside is gated too. + const cfgScopes = [] + const scopeCfgs = () => cfgScopes.map((s) => s.cfg) + // The cfg of a `#[cfg(…)] name! { … }`: it gates the invocation, so everything the body + // declares (mio's `#[cfg(unix)] cfg_os_poll! { mod unix; }`); taken up by the body's `{`. + let macroBodyCfg = null + // A chain of cfg-gated branches, `if #[cfg(a)] { … } else if #[cfg(b)] { … } else { … }` (a + // `cfg_if!` body): a branch applies only when the cfgs of those before it don't hold, so its + // block is gated on `all(not(a), b)` -- dead outright once an earlier cfg is known to hold. + // Open chains by the depth of their branch blocks (a `cfg_if!` inside a `cfg_if!` branch is a + // chain of its own): depth → the cfgs of the branches so far. + const chains = new Map() + // The word right before `pos` (skipping whitespace) and where it starts, or null. + const wordBefore = (pos) => { + let k = pos - 1 + while (k >= 0 && WS_RE.test(masked[k])) k-- + const end = k + 1 + while (k >= 0 && isWordChar(masked[k])) k-- + return k + 1 < end ? { word: masked.slice(k + 1, end), start: k + 1 } : null + } + // The cfg a `{` at `i` opens its block under, when the block is a branch of a cfg chain: its own + // (`own`, from the attributes right after the `if`) and the negation of every earlier branch's; + // null for any other block. Records the branch on its chain. + const branchCfg = (i, own) => { + const lead = wordBefore(pending.length > 0 ? attrStart : i) + if (lead === null || (lead.word !== 'if' && lead.word !== 'else')) return null + const continuing = lead.word === 'else' || wordBefore(lead.start)?.word === 'else' + const negations = continuing ? chains.get(depth + 1) ?? [] : [] + if (own === null && negations.length === 0) return null + chains.set(depth + 1, own === null ? negations : [...negations, own]) + return joinCfgs([...negations.map((c) => `not(${c})`), ...(own === null ? [] : [own])]) + } + // After a branch block at `blockDepth` closed at `end`: its chain goes on only if an `else` follows. + const endBranch = (end, blockDepth) => { + if (chains.has(blockDepth) && readWord(skipWs(end)) !== 'else') chains.delete(blockDepth) + } // End of the outermost macro invocation body being scanned (`m! { … }`, `m!( … )`, // `macro_rules! m { … }`). Its tokens are macro input: a `mod x;` there only becomes an item if // the macro emits it (cfg_if! does; serde_with's generate_guide! turns it into an inline module // documented from a .md file), so such declarations are followed when their file exists and - // tolerated when it doesn't -- the same footing as a `#[cfg]`-gated one. + // tolerated when it doesn't -- the same footing as a `#[cfg]`-gated one. `macroName` is that + // macro's name: what stands for the cfg a `mod` it emits is under (tokio's `cfg_has_atomic_u64! + // { mod imp; }` beside `cfg_not_has_atomic_u64! { mod imp; }`). let macroUntil = -1 + let macroName = null + // The `macro_rules!` being defined (its name) while inside its body: what it declares (serde's + // `crate_root! { … mod de; … }`) appears where the macro is invoked, not where it is defined, so + // such a `mod` takes the offset of the macro's first bare invocation in this file (`calls`). + let templateName = null const inlinePath = () => stack.map((s) => s.name) const closeTo = (targetDepth, at) => { while (stack.length > 0 && stack.at(-1).depth > targetDepth) { const top = stack.pop() - spans.push({ start: top.start, end: at, path: [...inlinePath(), top.name] }) + spans.push({ start: top.start, end: at, path: [...inlinePath(), top.name], vis: top.vis }) } } + // Whether an item at `at` (depth `d`) sits directly in an `impl`/`trait` body. + const isAssociated = (at, d) => openBlocks(assocBlocks, at).some((b) => b.start < at && b.depth === d) + // Whether an item at `at` is at its module's own level: the module body (a file's, an inline + // module's), a `#[cfg]`-gated block or an `extern "C" { }` block there, or a macro invocation's + // body (libc's `s! { … }`); not a `fn` body. + const atModuleLevel = (at) => { + if (at < macroUntil) return true + const base = stack.length > 0 ? stack.at(-1).depth : 0 + return depth === base || cfgScopes.some((s) => s.depth === depth) || openBlocks(foreignBlocks, at).some((b) => b.start < at && b.depth === depth) + } + // The `{` opening the body of the item starting at `from` (an `impl`/`trait` header may hold + // generics, bounds and a `where` clause), or -1 when a `;` ends it first. + const bodyOpen = (from) => { + let nest = 0 + for (let k = from; k < n; k++) { + const ch = masked[k] + if (ch === '(' || ch === '[' || ch === '<') nest++ + else if (ch === ')' || ch === ']' || (ch === '>' && masked[k - 1] !== '-' && masked[k - 1] !== '=')) nest = Math.max(0, nest - 1) + else if (nest === 0 && ch === '{') return k + else if (nest === 0 && ch === ';') return -1 + } + return -1 + } const readWord = (at) => { WORD_RE.lastIndex = at const m = WORD_RE.exec(masked) @@ -351,21 +537,46 @@ export function scanRustItems(content, { features = null, test = false } = {}) { while (at < n && WS_RE.test(masked[at])) at++ return at } + const skipWsBack = (at) => { + while (at >= 0 && WS_RE.test(masked[at])) at-- + return at + } // The end of the item starting at `from`, without leaving its enclosing block: through a - // top-level `;` or `,` (a field, variant or match arm), or its `{ … }` body; a `}` or `)` - // closing the enclosing block is left for the main loop. Parentheses/brackets nest (a tuple - // field's `(u8, u8)` comma isn't the item's end). - const skipItem = (from) => { + // top-level `;` or its `{ … }` body, or -- for a field, variant or match arm (`itemLike` + // false) -- a top-level `,`; a `}` or `)` closing the enclosing block is left for the main + // loop. Parentheses/brackets nest (a tuple field's `(u8, u8)` comma isn't the field's end), and + // so does a generic argument list -- a `<` right after a name or `::` (`HashMap`, `f::`) + // up to its `>` (the `>` of `->` / `=>` is not one) -- so the comma in `Result` isn't + // either. An item (`fn f<'a, T>(…) -> R where T: Tr, { … }`, `impl …`) never ends at a + // comma at all. A dead statement or arm whose value is an `if` (`elseChain`: anything but a bare + // `{ … }` block) takes its `else` branches with it; a dead bare block does not: in `cfg_if! { if + // #[cfg(a)] { … } else { … } }` the `else` is what applies when the cfg doesn't. + // A `let` (`toSemicolon`) runs to its `;` whatever blocks its initializer holds. + const skipItem = (from, itemLike, { elseChain = false, toSemicolon = false } = {}) => { let nest = 0 + let angle = 0 for (let k = from; k < n; k++) { const ch = masked[k] if (ch === '(' || ch === '[') nest++ else if (ch === ')' || ch === ']') { if (--nest < 0) return k } else if (nest === 0) { - if (ch === ';' || ch === ',') return k + 1 + if (ch === ';') return k + 1 if (ch === '}') return k - if (ch === '{') return matchClose(masked, k) + 1 + if (ch === '{') { + const close = matchClose(masked, k) + if (toSemicolon) { + k = close + continue + } + const after = skipWs(close + 1) + if (!elseChain || readWord(after) !== 'else') return close + 1 + k = after + 3 + continue + } + if (ch === '<' && (angle > 0 || isWordChar(masked[k - 1]) || masked[k - 1] === ':')) angle++ + else if (ch === '>' && angle > 0 && masked[k - 1] !== '-' && masked[k - 1] !== '=') angle-- + else if (ch === ',' && angle === 0 && !itemLike) return k + 1 } } return n @@ -378,18 +589,80 @@ export function scanRustItems(content, { features = null, test = false } = {}) { i++ continue } + if (i >= macroUntil) macroName = null if (ch === '#' && (masked[i + 1] === '[' || (masked[i + 1] === '!' && masked[i + 2] === '['))) { const outer = masked[i + 1] === '[' const open = outer ? i + 1 : i + 2 const close = matchClose(masked, open) - // An inner `#![…]` attribute applies to the enclosing module, not to the next item. - if (outer) pending.push(parseAttr(code.slice(open + 1, close))) - else pending = [] + if (outer) { + const attr = parseAttr(code.slice(open + 1, close), masked.slice(open + 1, close)) + // `#[cfg_attr(, derive(serde::Serialize))]` with pred never holding applies nothing: + // the paths in its text are dead code too, and so is whatever else it would apply. + const inert = attr.pred !== null && evalCfg(attr.pred, env) === false + if (inert) deadSpans.push([i, close + 1]) + if (pending.length === 0) attrStart = i + pending.push(inert ? INERT_ATTR : attr) + } else { + // An inner `#![…]` attribute applies to the enclosing module, not to the next item. A + // `#![cfg()]` that can't hold empties that module: the whole file at the top, else + // the inline module it opens. + const attr = parseAttr(code.slice(open + 1, close), masked.slice(open + 1, close)) + // (An inner attribute inside a macro invocation's body is the macro's business.) + if (attr.cfg !== null && evalCfg(attr.cfg, env) === false && i >= macroUntil) { + if (depth === 0) return { mods: [], refs: [], externCrates: [], bindings: new Set(), imports: [], macros: [], invocations: new Set(), calls: new Map(), includes: [], unfollowed: 0, defined: [], inlineModules: [], inlineModuleVis: new Map(), skipped } + const top = stack.at(-1) + if (top !== undefined && top.depth === depth) { + const end = matchClose(masked, top.start - 1) + deadSpans.push([i, end]) + pending = [] + i = end + continue + } + } + // `#![doc = include_str!("../README.md")]`: the module's own include, applied here. + const conditional = stack.some((s) => s.conditional) || i < macroUntil + for (const inc of attr.includes) includes.push({ ...inc, conditional }) + pending = [] + } i = close + 1 continue } + // The cfg gating the item the pending attributes belong to (several `#[cfg]`s all apply, + // and so does every enclosing block's). `pub` is visibility, not the item: its attributes + // stay pending for the keyword after it. + const word = readWord(i) + const attrCfg = word === 'pub' ? null : joinCfgs(pending.map((a) => a.cfg).filter((c) => c !== null)) + // A `{` opening a branch of a cfg chain is gated on the earlier branches' cfgs not holding too. + const own = ch === '{' ? branchCfg(i, attrCfg) ?? attrCfg : attrCfg + const cfg = joinCfgs(own === null ? scopeCfgs() : [...scopeCfgs(), own]) + // An item gated on a cfg that never holds in the build is dead code for the bundle: skip it + // whole -- from its first attribute (a `#[derive(serde::Serialize)]` on it names nothing + // live) through a declaration's `;`, a field's or variant's `,`, or a body's or block's `}` + // -- without recording anything in it, whatever token starts it: a keyword or a name, a + // `{ … }` block statement, a `(a, b)` / `[a, ..]` pattern or tuple type, a `&x`, `*x` or literal. + if (cfg !== null && evalCfg(cfg, env) === false) { + pending = [] + visNext = null + const end = skipItem(word === null ? i : i + word.length, word !== null && ITEM_KEYWORDS.has(word), { elseChain: ch !== '{', toSemicolon: word === 'let' }) + deadSpans.push([attrStart, end]) + if (ch === '{') endBranch(end, depth + 1) + i = end + continue + } + // The item is live, so the includes its attributes name (`#[doc = include_str!("docs/x.md")]`) + // are too; a `pub` keeps them pending for the keyword after it. + if (word !== 'pub' && pending.length > 0) { + const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil + for (const a of pending) { + for (const inc of a.includes) includes.push({ ...inc, conditional }) + a.includes = [] + } + } if (ch === '{') { depth++ + const gate = own ?? macroBodyCfg + macroBodyCfg = null + if (gate !== null) cfgScopes.push({ depth, cfg: gate }) pending = [] i++ continue @@ -397,45 +670,104 @@ export function scanRustItems(content, { features = null, test = false } = {}) { if (ch === '}') { depth-- closeTo(depth, i) + while (cfgScopes.length > 0 && cfgScopes.at(-1).depth > depth) cfgScopes.pop() + endBranch(i + 1, depth + 1) pending = [] i++ continue } - const word = readWord(i) + const vis = visNext + visNext = null if (word === null) { pending = [] i++ continue } - // The cfg gating the item the pending attributes belong to (several `#[cfg]`s all apply). - // `pub` is visibility, not the item: its attributes stay pending for the keyword after it. - const cfg = word === 'pub' ? null : joinCfgs(pending.map((a) => a.cfg).filter((c) => c !== null)) - // An item gated on a cfg that never holds in the build is dead code for the bundle: skip it - // whole -- a declaration through its `;`, a field or variant through its `,`, a body or block - // through its `}` -- without recording anything in it. - if (cfg !== null && evalCfg(cfg, env) === false) { - pending = [] - const end = skipItem(i + word.length) - deadSpans.push([i, end]) - i = end - continue - } // `name!` followed by a bracket opens a macro invocation body (`macro_rules! name` too). const bang = skipWs(i + word.length) + const pathQualified = masked[i - 1] === ':' && masked[i - 2] === ':' if (masked[bang] === '!') { let open = skipWs(bang + 1) if (word === 'macro_rules') { const name = readWord(open) - if (name !== null) open = skipWs(open + name.length) + if (name !== null) { + open = skipWs(open + name.length) + macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], offset: i, at: i, template: i < macroUntil && macroName === 'macro_rules' ? templateName : null }) + if (i >= macroUntil) templateName = name + } } if (masked[open] === '{' || masked[open] === '(' || masked[open] === '[') { - macroUntil = Math.max(macroUntil, matchClose(masked, open)) + const close = matchClose(masked, open) + if (templates.has(word) && !localMacros.has(word)) { + // serde_derive's `quote! { use #path as _serde; … _serde::__private::Ok(…) }`, bare or as + // `quote::quote! { … }`: skipped whole. (A crate's own `macro_rules! quote` is no template.) + deadSpans.push([i, close + 1]) + skipped.add(word) + pending = [] + i = close + 1 + continue + } + if (i >= macroUntil) macroName = word + // (A cfg on a `macro_rules!` definition gates the definition, not what its template + // declares wherever it is invoked -- an approximation kept from before.) + if (own !== null && masked[open] === '{' && word !== 'macro_rules') macroBodyCfg = own + if (word !== 'macro_rules') { + // Invoked by bare name; `$crate::name!` / `serde_json::json!` is a path (a ref). + if (!pathQualified) { + invocations.add(word) + ;(calls.get(word) ?? calls.set(word, []).get(word)).push(i) + } + if (INCLUDE_MACROS.has(word)) { + const text = code.slice(open + 1, close) + const arg = includeArg(text) + const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil + // Inside a `macro_rules!` body the path is relative to whichever file invokes the + // macro: recorded on the definition, resolved per invoking file. Build output + // (`concat!(env!("OUT_DIR"), …)`) is nobody's to follow; any other unreadable argument + // is counted, for a warning. + if (arg === null) unfollowed += /\bOUT_DIR\b/u.test(text) ? 0 : 1 + else if (i < macroUntil && macroName === 'macro_rules' && macros.length > 0) macros.at(-1).includes.push({ kind: word, ...arg }) + else includes.push({ kind: word, ...arg, conditional }) + } + } + macroUntil = Math.max(macroUntil, close) } } + // An `impl`/`trait` body holds associated items, an `extern "C" { }` block the module's own + // (an `extern "C" fn f() { }` is a fn, with a body of its own). + if (word === 'impl' || word === 'trait') { + const open = bodyOpen(i + word.length) + if (open !== -1) assocBlocks.push({ start: open, end: matchClose(masked, open), depth: depth + 1 }) + } else if (word === 'extern' && readWord(bang) !== 'crate') { + let k = bang + if (masked[k] === '"') k = skipWs(masked.indexOf('"', k + 1) + 1) + if (masked[k] === '{') foreignBlocks.push({ start: k, end: matchClose(masked, k), depth: depth + 1 }) + } + // An item the module defines by name (`pub struct sigset_t`, inside libc's `s! { … }` or an + // `extern "C" { }` block too), at the module's own level -- not a `fn`'s local item, nor an + // `impl`/`trait` body's associated one, nor a `macro_rules!` template's (that is some invoking + // module's item); `const fn` / `static mut` / lazy_static's `static ref` are not items named + // `fn` / `mut` / `ref`. + if (DEFINING_KEYWORDS.has(word) && atModuleLevel(i) && !isAssociated(i, depth) && !(i < macroUntil && macroName === 'macro_rules') && !(word === 'const' && masked[skipWsBack(i - 1)] === '*')) { + let at = skipWs(i + word.length) + if (masked.startsWith('mut ', at) || masked.startsWith('ref ', at)) at = skipWs(at + 3) + if (masked.startsWith('r#', at)) at += 2 + const name = readWord(at) + const next = name === null ? null : masked[skipWs(at + name.length)] + // `ns`: a `fn`, `const` or `static` lives in the value namespace and can't lead a path + // (`*const libc::c_char`, a raw pointer type, is no item at all; a `fn log` beside + // `use log::info` hides no crate). + if (name !== null && !ITEM_KEYWORDS.has(name) && (word !== 'union' || next === '{' || next === '<')) defined.push({ name, inlinePath: inlinePath(), vis, cfg, macro: i < macroUntil ? macroName : null, ns: VALUE_KEYWORDS.has(word) ? 'value' : 'type' }) + } if (word === 'pub') { // Visibility sits between an item's attributes and its keyword; `pub(crate)` etc. included. i = skipWs(i + 3) - if (masked[i] === '(') i = matchClose(masked, i) + 1 + visNext = 'crate' + if (masked[i] === '(') { + const close = matchClose(masked, i) + visNext = parseVisibility(masked.slice(i + 1, close)) + i = close + 1 + } continue } if (word === 'mod') { @@ -453,15 +785,28 @@ export function scanRustItems(content, { features = null, test = false } = {}) { // Dead cfgs were skipped above; a decidable-true one (`not(test)`) is as firm as no cfg at all. const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil if (masked[k] === ';') { - // A `#[cfg_attr(, path = …)]` whose predicate can't hold names nothing in this build. - const paths = attrs.flatMap((a) => a.paths).filter((p) => p.cfg === null || evalCfg(p.cfg, env) !== false) - mods.push({ name, inlinePath: inlinePath(), cfg, conditional, paths }) + // rustc applies the first `#[path]` / `#[cfg_attr(, path = …)]` whose predicate holds. + // A variant whose predicate can't hold names nothing in this build; one whose predicate + // holds ends the list -- outright the `#[path]` when nothing undecided precedes it -- and + // either way the default `.rs` lookup never happens. + const live = attrs.flatMap((a) => a.paths).filter((p) => p.cfg === null || evalCfg(p.cfg, env) !== false) + const applies = live.findIndex((p) => p.cfg === null || evalCfg(p.cfg, env) === true) + const paths = applies === -1 ? live : (applies === 0 ? [{ path: live[0].path, cfg: null }] : live.slice(0, applies + 1)) + mods.push({ + name, inlinePath: inlinePath(), inlineDirs: stack.map((s) => s.dir), cfg, conditional, paths, noDefault: applies !== -1, vis, + macroUse: attrs.some((a) => a.macroUse === true), macro: i < macroUntil ? macroName : null, offset: i, at: i, + template: i < macroUntil && macroName === 'macro_rules' ? templateName : null, + }) i = k + 1 continue } if (masked[k] === '{') { + // `#[path = "dir"] mod m { mod a; }` finds `a` under `dir/`, not `m/`; solana-program's + // `#[path = ""] mod non_bpf_modules { mod account_keys; }` right beside the file. + const dir = attrs.flatMap((a) => a.paths).find((p) => p.cfg === null)?.path ?? name depth++ - stack.push({ name, depth, conditional, start: k + 1 }) + if (own !== null) cfgScopes.push({ depth, cfg: own }) + stack.push({ name, dir, depth, conditional, start: k + 1, vis }) i = k + 1 continue } @@ -471,12 +816,26 @@ export function scanRustItems(content, { features = null, test = false } = {}) { if (word === 'use') { const end = masked.indexOf(';', i + 3) const stop = end === -1 ? n : end + const body = code.slice(i + 3, stop) + // A `use` item's tree starts with a name, `::`, `{` or -- in a `macro_rules!` template -- + // `$crate`. Anything else is the word in macro input (syn's `Token![use]`), and a tree + // interpolating a metavariable (`use $m::X;`, quote's `use #path as _serde;`; a raw + // identifier's `r#` is no interpolation) is a template for code the loader can't place: + // neither is an import of this file. + if (!/^\s*[A-Za-z_{:$]/u.test(body) || /(? 1 || p.binding !== p.segments[0])) bindings.add(p.binding) + const im = { segments: p.segments, absolute: p.absolute, binding: p.binding, glob: p.glob, inlinePath: ip, vis, cfg, macro, template } + imports.push(im) + if (hidesCrate(im)) bindings.add(p.binding) } useSpans.push([i, stop]) pending = [] @@ -488,8 +847,12 @@ export function scanRustItems(content, { features = null, test = false } = {}) { const m = EXTERN_CRATE_RE.exec(masked) pending = [] if (m) { - externCrates.push({ name: m[1], inlinePath: inlinePath() }) + const ip = inlinePath() + externCrates.push({ name: m[1], alias: m[2] ?? null, inlinePath: ip }) if (m[2] !== undefined) bindings.add(m[2]) + // `extern crate self as x;` names this crate's root; any other, a crate by its own name. + const selfCrate = m[1] === 'self' + imports.push({ segments: [selfCrate ? 'crate' : m[1]], absolute: !selfCrate, binding: m[2] ?? m[1], glob: false, inlinePath: ip, vis, cfg, macro: i < macroUntil ? macroName : null }) i = EXTERN_CRATE_RE.lastIndex continue } @@ -500,6 +863,9 @@ export function scanRustItems(content, { features = null, test = false } = {}) { i += word.length } closeTo(0, n) + // A `mod` or `macro_rules!` a `macro_rules!` template declares stands where the macro is first + // invoked here (serde's `crate_root! { … macro_rules! tri { … } … mod de; … }`). + for (const m of [...mods, ...macros]) if (m.template !== null) m.offset = calls.get(m.template)?.[0] ?? Infinity // Innermost inline module enclosing an offset (spans nest, so the longest path wins). const inlineAt = (offset) => { @@ -511,12 +877,13 @@ export function scanRustItems(content, { features = null, test = false } = {}) { } const refs = new Map() // `fromUse` marks a path written in a `use` item -- the reliable signal that its lead names a - // crate (an expression path's lead is as likely a module or an imported item). - const addRef = (spec, segments, absolute, ip, fromUse) => { - const key = `${ip.join('::')}\0${spec}` - if (!refs.has(key)) refs.set(key, { spec, segments, absolute, inlinePath: ip, fromUse }) + // crate (an expression path's lead is as likely a module or an imported item). `macroCall` + // marks an invoked path (`$crate::span!(…)`): it names a macro, not the module of that name. + const addRef = (spec, segments, absolute, ip, fromUse, macroCall) => { + const key = `${ip.join('::')}\0${spec}\0${macroCall ? '!' : ''}` + if (!refs.has(key)) refs.set(key, { spec, segments, absolute, inlinePath: ip, fromUse, macroCall }) } - for (const u of uses) addRef(u.spec, u.segments, u.absolute, u.inlinePath, true) + for (const u of uses) addRef(u.spec, u.segments, u.absolute, u.inlinePath, true, false) // Expression-position paths: scan everything outside `use` items (the tree parser owns those; // a regex over `use syn::{parse::Parse}` would take `parse::Parse` for a local module path) and // outside skipped test/doc-only items (a `quickcheck::quickcheck(…)` in a `#[test]` fn body @@ -534,13 +901,28 @@ export function scanRustItems(content, { features = null, test = false } = {}) { const keywordPaths = [] const leadPaths = [] for (const m of exprs.matchAll(LEAD_PATH_RE)) { + // `$trait::fmt(…)` in a `macro_rules!` template: the lead is a metavariable, not a name; + // `$crate::…` is the one interpolation that names something (this crate). + if (exprs[m.index - 1] === '$' && m[1] !== 'crate') continue if (PATH_KEYWORDS.has(m[1])) keywordPaths.push(m) else if (!NON_CRATE_LEADS.has(m[1])) leadPaths.push(m) } - for (const m of [...keywordPaths, ...leadPaths]) addRef(m[0], m[0].split('::'), false, inlineAt(m.index), false) - return { mods, refs: [...refs.values()], externCrates, bindings } + for (const m of [...keywordPaths, ...leadPaths]) { + let after = m.index + m[0].length + while (after < n && WS_RE.test(exprs[after])) after++ + const macroCall = exprs[after] === '!' && exprs[after + 1] !== '=' + addRef(m[0], m[0].split('::'), false, inlineAt(m.index), false, macroCall) + } + const inlineModuleVis = new Map() + for (const s of spans) if (!inlineModuleVis.has(s.path.join('::'))) inlineModuleVis.set(s.path.join('::'), s.vis) + const inlineModules = [...inlineModuleVis.keys()].map((p) => p.split('::')) + return { mods, refs: [...refs.values()], externCrates, bindings, imports, macros, invocations, calls, includes, unfollowed, defined, inlineModules, inlineModuleVis, skipped } } +// Whether a `use` import binds a name that is not the crate of that name: `use std::io;` binds +// `io`, but `use serde_json;` / `use rand::{self, Rng};` bind the crate itself under its own name. +const hidesCrate = (im) => im.binding !== null && (im.segments.length > 1 || im.binding !== im.segments[0]) + // --- Module files ----------------------------------------------------------------------- // Directory holding this file's submodules, per Rust's path rules: siblings for crate roots, @@ -569,13 +951,18 @@ function firstFile(candidates, { knownSources, baseDir }) { // Resolve `mod ;` declared in `fromFile` (inside the inline modules `inlinePath`) to its // file, trying `/.rs` then `//mod.rs`. `roots` are the files walked as crate // roots (the entries) and `pathLoaded` the files a `#[path]` loaded: their submodules are -// siblings. An entry not named main.rs/lib.rs also gets the non-root `/` rule as a -// fallback, so a glob-listed module file still resolves. -export function resolveModPath(modName, fromFile, { knownSources, baseDir, roots, pathLoaded, inlinePath = [] } = {}) { - const dirs = [getModuleDir(fromFile, { root: ownsDir(fromFile, { roots, pathLoaded }) })] - if (roots?.has(fromFile)) dirs.push(getModuleDir(fromFile)) // the same dir for main.rs/lib.rs/mod.rs +// siblings. Both also get the non-root `/` rule as a fallback: a glob-listed module file +// still resolves, and so does a file that a plain `mod` mounts as well as a `#[path]` (each mount +// is its own module to rustc; the loader keeps one resolution, whichever file exists). +export function resolveModPath(modName, fromFile, { knownSources, baseDir, roots, pathLoaded, inlinePath = [], boundary = null } = {}) { + const owns = ownsDir(fromFile, { roots, pathLoaded }) + const dirs = [getModuleDir(fromFile, { root: owns })] + if (owns) dirs.push(getModuleDir(fromFile)) // the same dir for main.rs/lib.rs/mod.rs for (const dir of new Set(dirs)) { - const base = [dir, ...inlinePath, modName].filter(Boolean).join('/') + // An inline module's `#[path = "…"]` may climb (`..`): normalized, and held to `boundary` + // like any explicit path (resolveExplicitModPath). + const base = normalizeRel(dir, [...inlinePath, modName].filter(Boolean).join('/')) + if (base === null || (boundary !== null && !withinDir(base, boundary))) continue const found = firstFile([`${base}.rs`, `${base}/mod.rs`], { knownSources, baseDir }) if (found) return found } @@ -585,21 +972,55 @@ export function resolveModPath(modName, fromFile, { knownSources, baseDir, roots // Resolve a `#[path = "…"]` target the way rustc does: relative to the declaring file's directory, // or -- inside inline modules -- to the file's module dir plus the inline module names. Absolute // or root-escaping paths resolve to nothing (a read would refuse them anyway). -export function resolveExplicitModPath(explicitPath, fromFile, { knownSources, baseDir, roots, pathLoaded, inlinePath = [] } = {}) { +export function resolveExplicitModPath(explicitPath, fromFile, { knownSources, baseDir, roots, pathLoaded, inlinePath = [], boundary = null } = {}) { const dir = inlinePath.length === 0 ? posix.dirname(fromFile) : [getModuleDir(fromFile, { root: ownsDir(fromFile, { roots, pathLoaded }) }), ...inlinePath].filter(Boolean).join('/') const rel = normalizeRel(dir, explicitPath) - return rel === null ? null : firstFile([rel], { knownSources, baseDir }) + if (rel === null || (boundary !== null && !withinDir(rel, boundary))) return null + return firstFile([rel], { knownSources, baseDir }) +} + +// Whether project-relative `rel` lies in directory `dir` (`.`: anywhere in the bundle root). +const withinDir = (rel, dir) => dir === '.' || rel === dir || rel.startsWith(`${dir}/`) + +// The directory a file's `#[path]`, include and build-script paths may reach into: a published +// crate never legitimately names a file outside its own package, so a file of a vendored one is +// held to that package's directory (`vendor/`); the project's own code -- the root package, +// workspace members, path dependencies -- may reach anywhere in the bundle root (`.`). +function boundaryOf(file, ctx) { + return ctx?.isVendored(file) === true ? ctx.packageInfo(file).dir : '.' +} + +// Resolve an include macro's argument (scanRustItems' `{ kind, path, base }`) named in `file` to a +// project-relative path: relative to the file, or to its package's root for a +// `concat!(env!("CARGO_MANIFEST_DIR"), …)` path; null when it escapes the bundle root, or the +// package when the file is a vendored crate's (boundaryOf). +function includeTarget(inc, file, ctx) { + const dir = inc.base === 'manifest' ? ctx?.packageInfo(file)?.dir ?? null : posix.dirname(file) + if (dir === null) return null + const rel = normalizeRel(dir, inc.path) + if (rel === null || !withinDir(rel, boundaryOf(file, ctx))) { + console.warn(`[loader.rust] Refusing include outside its package: ${inc.path} from ${file}`) + return null + } + return rel } // Every file a `mod` declaration can denote, as `{ cfg, file, explicit }`: an unconditional // `#[path]` names one outright; otherwise each `#[cfg_attr(, path = …)]` names one under -// its predicate (`cfg`; the scanner already dropped the variants whose predicate can't hold in -// the build), and the default `.rs`/`/mod.rs` lookup is the fallback (`cfg` null). -// `explicit` marks a file a `#[path]` named (its own submodules then sit beside it). +// its predicate (`cfg`), and the default `.rs`/`/mod.rs` lookup is the fallback (`cfg` +// null). A variant whose predicate can't hold in the build is dropped: the scanner already did so +// for the build it scanned under, and `opts.features` / `opts.test` / `opts.target`, when given, +// describe one here. `explicit` marks a file a `#[path]` named (its own submodules then sit +// beside it). The inline modules the declaration sits in contribute their directories +// (`inlineDirs`: an inline module's own `#[path]` replaces its name, an empty one adds nothing), +// else their names. export function resolveModDecl(decl, fromFile, opts = {}) { - const o = { ...opts, inlinePath: decl.inlinePath } + const o = { ...opts, inlinePath: decl.inlineDirs ?? decl.inlinePath } + const env = opts.features === undefined && opts.test === undefined && opts.target === undefined + ? null + : { features: opts.features ?? null, test: opts.test === true, target: opts.target ?? null } const unconditional = decl.paths.find((p) => p.cfg === null) if (unconditional) { const file = resolveExplicitModPath(unconditional.path, fromFile, o) @@ -607,10 +1028,12 @@ export function resolveModDecl(decl, fromFile, opts = {}) { } const out = [] for (const { path, cfg } of decl.paths) { + if (env !== null && evalCfg(cfg, env) === false) continue const file = resolveExplicitModPath(path, fromFile, o) if (file && !out.some((x) => x.file === file)) out.push({ cfg, file, explicit: true }) } - const fallback = resolveModPath(decl.name, fromFile, o) + // A variant that always applies (`decl.noDefault`) leaves no build for the default lookup. + const fallback = decl.noDefault === true ? null : resolveModPath(decl.name, fromFile, o) if (fallback && !out.some((x) => x.file === fallback)) out.push({ cfg: null, file: fallback, explicit: false }) return out } @@ -646,50 +1069,85 @@ export function crateRoots(sources, explicit = [], cargo = null) { // following `mod` edges breadth-first from `crate`, plus `files`: file → `{ root, modulePath }` // (first root to reach a file claims it). A file is reached under one module path per `mod` spec // (`mod a::b` for a declaration inside inline module `a`); a cfg-variant Map target contributes -// every variant under the same path. -export function buildModuleTrees(sources, resolutions, roots) { +// every variant under the same path, the first naming the module's file in the tree. +// `inlineModules` (file → the paths of its inline `mod x { … }` blocks, from scanRustItems) puts +// those modules in the tree too, owned by the file holding them -- unless a `mod` of the same +// name names a file (`#[cfg(unix)] mod imp;` beside `#[cfg(not(unix))] mod imp { … }`): the file +// is the more telling target, the inline body's is where the path was written anyway. Each file's +// entry also carries `leaves`, the cfg leaves (cfgLeaves) of every `mod` on the way down to it -- +// the declaration's own cfg and the gate macro it sits in (`mountCfgs`: file → spec → target +// file → its declarations' `{ cfg, macro }`; a file declared under several is compiled under any +// one of them: an `any` leaf, see cfgLeaves and gateLeaves) and, for a `mod` with several files, a `variant` leaf naming which (two +// variants of one `mod` never both apply) -- and `parent`, the file that mounted it. +export function buildModuleTrees(sources, resolutions, roots, inlineModules = null, mountCfgs = null) { const trees = new Map() const files = new Map() + const variantsOf = (target) => (target instanceof Map ? [...target] : [['*', target]]) + const mountLeaves = (gates) => (gates === undefined ? [] : anyOf(gates.map(({ cfg, macro }) => [...cfgLeaves(cfg), ...gateLeaves(macro)]))) + const mounted = (path, spec, sub, key, file, several, leaves) => [ + ...leaves, + ...mountLeaves(mountCfgs?.get(path)?.get(spec)?.get(file)), + ...(several ? [{ key: 'variant', site: sub, value: key.replace(/#\d+$/u, ''), neg: false }] : []), + ] for (const root of roots) { if (!sources.has(root)) continue - const tree = new Map() + const tree = new Map([['crate', root]]) trees.set(root, tree) const seen = new Set() - const queue = [[root, 'crate', 0]] + const queue = [[root, 'crate', 0, [], null]] for (let qi = 0; qi < queue.length; qi++) { - const [path, modulePath, depth] = queue[qi] - if (!tree.has(modulePath)) tree.set(modulePath, path) - if (!files.has(path)) files.set(path, { root, modulePath }) + const [path, modulePath, depth, leaves, parent] = queue[qi] + if (!files.has(path)) files.set(path, { root, modulePath, leaves, parent }) if (seen.has(path) || depth >= MAX_MODULE_DEPTH) continue seen.add(path) - for (const [spec, target] of resolutions.get(path) ?? []) { + // Module files first, then the inline modules that hold nested declarations, then the other + // inline modules: a `mod imp;` file beats a `mod imp { … }` of the same name however the + // declarations are ordered. + const specs = [...(resolutions.get(path) ?? [])] + for (const [spec, target] of specs) { + if (!spec.startsWith('mod ')) continue + const parts = spec.slice(4).split('::') + if (parts.length > 1) continue + const sub = `${modulePath}::${parts[0]}` + const variants = variantsOf(target) + for (const [key, file] of variants) { + if (!tree.has(sub)) tree.set(sub, file) + queue.push([file, sub, depth + 1, mounted(path, spec, sub, key, file, variants.length > 1, leaves), path]) + } + } + for (const [spec, target] of specs) { + // An `include!`d file's tokens are this module's: it is walked under the same module path. + if (spec.startsWith('include ')) { + queue.push([target, modulePath, depth + 1, leaves, path]) + continue + } if (!spec.startsWith('mod ')) continue + const parts = spec.slice(4).split('::') + if (parts.length === 1) continue // `mod a::b::name`: `a` and `a::b` are inline modules whose bodies live in this file, so // paths into them (`a::b::x`, `super::` from `name`) resolve to it. - const parts = spec.slice(4).split('::') for (let k = 1; k < parts.length; k++) { const inline = `${modulePath}::${parts.slice(0, k).join('::')}` if (!tree.has(inline)) tree.set(inline, path) } const sub = `${modulePath}::${spec.slice(4)}` - for (const file of target instanceof Map ? target.values() : [target]) queue.push([file, sub, depth + 1]) + const variants = variantsOf(target) + for (const [key, file] of variants) { + if (!tree.has(sub)) tree.set(sub, file) + queue.push([file, sub, depth + 1, mounted(path, spec, sub, key, file, variants.length > 1, leaves), path]) + } + } + for (const inline of inlineModules?.get(path) ?? []) { + for (let k = 1; k <= inline.length; k++) { + const key = `${modulePath}::${inline.slice(0, k).join('::')}` + if (!tree.has(key)) tree.set(key, path) + } } } } return { trees, files } } -// Resolve `crate::a::b::Item` to the deepest module prefix that names a file (trailing item -// names aren't modules); null when none does. -export function resolveUsePath(usePath, moduleTree) { - const parts = usePath.split('::') - for (let i = parts.length; i >= 1; i--) { - const modulePath = parts.slice(0, i).join('::') - if (moduleTree.has(modulePath)) return moduleTree.get(modulePath) - } - return null -} - // The crate a path's lead may name: null for a path keyword or a sysroot crate, and -- unless the // path is absolute (`::name::…`) -- for a name the file imported (`bindings`: `use std::io; // io::stdin()` names no crate `io`). @@ -698,47 +1156,912 @@ const crateLead = ({ segments, absolute }, bindings) => { return head === undefined || NON_CRATE_LEADS.has(head) || (!absolute && bindings.has(head)) ? null : head } -// Resolve one path reference made in `file` to what it names in the bundle: a module file of the -// same crate (`{ kind: 'module', target }`) for `crate::`/`self::`/`super::` paths and for -// 2018-style relative paths whose lead is a child module of the current module; else an in-tree -// crate root (`{ kind: 'crate', name, target }`) via `resolveCrate`, or `{ kind: 'unresolved', -// name }` when the lead could name a crate (crateLead) but nothing in-tree did; null for anything -// else (an item in scope, std, a path above the crate root). -function resolvePathRef(ref, file, { trees, files, resolveCrate, bindings }) { +// --- Path resolution ------------------------------------------------------------------------ +// +// A path is resolved against its crate's module tree, module by module. Where a segment names no +// module of the one reached, the imports written in that module may provide the name: a `use` +// binding it outright, or a glob (`pub use inner::*`) when the module the glob names -- one of +// this crate -- provides it in turn. What a name leads to is a property of (module, name) and of +// how much of the module's imports the asking module may see, so it is computed once per crate +// and shared by every path resolved (`ctx.provided`): resolution stays linear in the number of +// paths, whatever the fan-out of the crate's re-exports. +// +// Results: `{ kind: 'module', modulePath, file }` for a module of the crate; `{ kind: 'item', +// file }` for anything else the crate's files hold (an item of the module reached, a name a +// `use` brought in from the sysroot -- as far as the bundle is concerned it lives where the `use` +// is); `{ kind: 'crate', name, file }` for another in-tree crate; null when nothing here answers. +// A result that went through an import at the top level carries `via: { modulePath, consumed, +// through }`: the module whose import was followed first, how many of the path's own segments +// named modules up to it, and the file holding that import. + +// The module a visibility (scanRustItems' `vis`) written in module `modulePath` reaches: the +// crate root for `pub` / `pub(crate)`, the module itself for a private item or `pub(self)`, its +// parent for `pub(super)`, the ancestor a `pub(in …)` names (`crate::a`, `super::…`, `self`) -- +// when it is one; rustc rejects any other, so that reads as `crate`. +function visibilityScope(vis, modulePath) { + if (vis === null || vis === 'self') return modulePath + if (vis === 'crate') return 'crate' + const parent = modulePath.includes('::') ? modulePath.slice(0, modulePath.lastIndexOf('::')) : 'crate' + if (vis === 'super') return parent + const segments = vis.slice(3).split('::') + let cur + if (segments[0] === 'crate') cur = ['crate'] + else if (segments[0] === 'self') cur = modulePath.split('::') + else if (segments[0] === 'super') cur = parent.split('::') + else return 'crate' + let i = 1 + while (segments[i] === 'super' && cur.length > 1) { + cur.pop() + i++ + } + const scope = [...cur, ...segments.slice(i)].join('::') + return modulePath === scope || modulePath.startsWith(`${scope}::`) ? scope : 'crate' +} + +// --- cfg compatibility ----------------------------------------------------------------------- +// +// Which of a module's several files, imports or glob sources a path written under some cfgs can +// mean: libc's `crate::c_int` from a file under `#[cfg(unix)] mod unix;` is not the one +// `#[cfg(target_os = "fuchsia")] pub use fuchsia::*;` brings in, mio's `sys::Waker` from a file in +// one `cfg_if!` branch is not the `pub use` in another. Each file carries the leaves of the cfgs +// it was mounted under (buildModuleTrees), each import and item those of its own cfg on top; two +// leaf sets that can't hold together are exclusive, and such a candidate is skipped. + +// The conjunctive leaves of a cfg predicate, `{ key, value, neg }`: `all(unix, not(feature = +// "std"))` → `unix` and `!feature="std"`; `not(any(a, b))` → `!a`, `!b`. An `any(…)` of +// alternatives that each decide something is one leaf, `{ key: 'any', alts }`, holding when any +// alternative (a conjunction of leaves) does -- `any(target_os = "linux", target_os = "l4re")` +// can't hold along with `target_os = "aix"` -- as is a negated `all(…)`; one with an alternative +// that decides nothing (`any(unix, libc_core_cvoid)`) decides nothing itself. +function cfgLeaves(pred) { + if (pred === null || pred === undefined) return [] + const p = pred.trim() + const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) + if (!m) { + const kv = /^([\w-]+)\s*(?:=\s*"([^"]*)")?$/u.exec(p) + return kv ? [{ key: kv[1], value: kv[2] ?? null, neg: false }] : [] + } + const parts = splitTopLevel(m[2]).filter((part) => part.trim() !== '').map(cfgLeaves) + if (m[1] === 'all') return parts.flat() + if (m[1] === 'any') return anyOf(parts) + return parts.length === 1 ? negated(parts[0]) : [] +} +// The leaves holding when any of `alts` (conjunctions) does: the one alternative when there is +// one (the same twice counts once), an `any` leaf of them, nothing when one decides nothing. +const anyOf = (alts) => { + const uniq = new Map() + for (const alt of alts) { + if (alt.length === 0) return [] + uniq.set(alt.map(leafKey).toSorted().join('&'), alt) + } + const list = [...uniq.values()] + if (list.length <= 1) return list[0] ?? [] + return [{ key: 'any', value: null, neg: false, alts: list }] +} +// The leaves of a conjunction's negation: one leaf flips, an `any` leaf's negation is every +// alternative's, several leaves negate to any of their negations. +const negated = (leaves) => { + if (leaves.length === 0) return [] + if (leaves.length > 1) return anyOf(leaves.map((l) => negated([l]))) + const [l] = leaves + return l.alts === undefined ? [{ ...l, neg: !l.neg }] : l.alts.flatMap(negated) +} + +// The leaf of a cfg gate the loader can't see into: the body of a `cfg_!` macro (tokio's +// `cfg_io_uring! { … }`, mio's `cfg_os_poll! { … }`), whose `cfg_not_!` is its negation -- +// so what the two declare is never taken together, and neither counts as certainly compiled +// (`cfg_if!` is read branch by branch; any other macro's body is no gate). +const GATE_MACRO_RE = /^cfg_(not_)?(?!if$)(\w+)$/u +function gateLeaves(macro) { + const m = macro === null || macro === undefined ? null : GATE_MACRO_RE.exec(macro) + return m === null ? [] : [{ key: `cfg_${m[2]}!`, value: null, neg: m[1] !== undefined }] +} + +// Keys a target has one value of: `target_os = "linux"` and `target_os = "macos"` never both hold +// (`feature`, `target_family` and `target_has_atomic` may take several). +const SINGLE_VALUED_CFG_KEYS = new Set(['target_os', 'target_arch', 'target_env', 'target_vendor', 'target_abi', 'target_pointer_width', 'target_endian', 'panic']) +// Operating systems whose targets are not `unix`. +const NON_UNIX_OS = new Set(['windows', 'none', 'uefi', 'wasi', 'solana', 'cuda', 'xous', 'zkvm', 'psp', 'unknown']) +const isWindowsLeaf = (l) => (l.key === 'windows' && l.value === null) || ((l.key === 'target_os' || l.key === 'target_family') && l.value === 'windows') +const isUnixLeaf = (l) => (l.key === 'unix' && l.value === null) || (l.key === 'target_family' && l.value === 'unix') + +// Whether two leaf lists (conjunctions) can't hold at once: a leaf and its negation; two values of +// a single-valued key; two variants of one `mod` (`variant` leaves, see buildModuleTrees); Windows +// against unix; an `any` leaf none of whose alternatives can hold with the other. Asked once per +// pair of leaf sets (compatible), over what `prepared` gathers of each list. +function cfgExclusive(a, b) { + const pa = prepared(a) + const pb = prepared(b) + for (const k of pa.negKeys) if (pb.keys.has(k)) return true + for (const [k, v] of pa.values) { + const w = pb.values.get(k) + if (w !== undefined && w !== v) return true + } + if ((pa.windows && (pb.unix || pb.otherOs)) || (pb.windows && (pa.unix || pa.otherOs)) || (pa.unix && pb.nonUnixOs) || (pb.unix && pa.nonUnixOs)) return true + for (const x of pa.anys) if (x.alts.every((alt) => cfgExclusive(alt, b))) return true + for (const y of pb.anys) if (y.alts.every((alt) => cfgExclusive(a, alt))) return true + return false +} +// What cfgExclusive asks of a leaf list, gathered once and kept on it: its plain leaves' keys and +// their negations', the value each positive single-valued key (and `mod` variant site) takes -- +// `MANY` for several, which nothing holds along with -- its Windows / unix standing (a +// `target_os` other than windows, `otherOs`; one of the non-unix ones, `nonUnixOs`) and its `any` +// leaves. +const MANY = Symbol('many') +function prepared(leaves) { + if (leaves.prep !== undefined) return leaves.prep + const prep = { keys: new Set(), negKeys: [], values: new Map(), anys: [], windows: false, unix: false, otherOs: false, nonUnixOs: false } + for (const l of leaves) { + if (l.alts !== undefined) { + prep.anys.push(l) + continue + } + const k = leafKey(l) + prep.keys.add(k) + prep.negKeys.push(l.neg ? k.slice(1) : `!${k}`) + if (l.neg) continue + if (l.value !== null && (SINGLE_VALUED_CFG_KEYS.has(l.key) || l.key === 'variant')) { + const site = l.key === 'variant' ? `variant@${l.site}` : l.key + prep.values.set(site, prep.values.has(site) && prep.values.get(site) !== l.value ? MANY : l.value) + } + if (isWindowsLeaf(l)) prep.windows = true + if (isUnixLeaf(l)) prep.unix = true + if (l.key === 'target_os' && l.value !== null) { + if (l.value !== 'windows') prep.otherOs = true + if (NON_UNIX_OS.has(l.value)) prep.nonUnixOs = true + } + } + leaves.prep = prep + return prep +} + +// A stable text for one leaf; an `any` leaf's names its alternatives. +const leafKey = (l) => (l.alts === undefined ? `${l.neg ? '!' : ''}${l.key}${l.site === undefined ? '' : `@${l.site}`}${l.value === null ? '' : `=${l.value}`}` : `any(${l.alts.map((alt) => alt.map(leafKey).toSorted().join('&')).toSorted().join(',')})`) + +// Leaf sets are interned per tree (`sets`: key → `{ leaves, key }`), one object per distinct set: +// the union of the cfgs along a chain of globs stays small however long the chain, and the +// compatibility verdict for a pair of sets is looked up by key, not computed again. +const NO_LEAVES = { leaves: [], key: '' } +function leafSet(leaves, sets) { + if (leaves.length === 0) return NO_LEAVES + const uniq = new Map() + for (const l of leaves) uniq.set(leafKey(l), l) + const key = [...uniq.keys()].toSorted().join('|') + return sets.get(key) ?? sets.set(key, { leaves: [...uniq.values()], key }).get(key) +} +// A union is asked for again and again along a closure (a glob's leaves with each candidate's): kept +// on the first set, by the second's key. +const unionLeaves = (a, b, sets) => { + if (a.key === '') return b + if (b.key === '' || b.key === a.key) return a + const unions = (a.unions ??= new Map()) + return unions.get(b.key) ?? unions.set(b.key, leafSet([...a.leaves, ...b.leaves], sets)).get(b.key) +} +// The leaves holding when either of two sets does (a module two glob paths reach): an `any` of +// the two -- of their alternatives, when one is an `any` leaf already. +const eitherLeaves = (a, b, sets) => { + if (a.key === b.key) return a + if (a.key === '' || b.key === '') return NO_LEAVES + const altsOf = (s) => (s.leaves.length === 1 && s.leaves[0].alts !== undefined ? s.leaves[0].alts : [s.leaves]) + return leafSet(anyOf([...altsOf(a), ...altsOf(b)]), sets) +} + +// The leaf set of an `asker` -- `{ file, set, keys, compat, sure, doubt }`, the file a path was +// written in, its set the leaves it was mounted under plus, when the build's target is known, the +// target's cfgs (`unix`, `target_os = "linux"`, …: what every file of the build is compiled +// under) -- against a candidate's: exclusive candidates are skipped; no asker (an internal query) +// skips nothing. The verdict is kept per pair of sets in `compat`, shared by the askers of one set. +const compatible = (asker, set) => { + if (asker === undefined || set.key === '') return true + let verdict = asker.compat.get(set.key) + if (verdict === undefined) { + verdict = !cfgExclusive(asker.set.leaves, set.leaves) + asker.compat.set(set.key, verdict) + } + return verdict +} +// Whether every leaf of a candidate's set is among the asker's own (`keys`, by leafKey): the two +// are certainly compiled together, not merely possibly (compatible). libc's aix/mod.rs, mounted +// under `target_os = "aix"`, asks `crate::fsid_t`: the aix glob's, under that same leaf, is the +// answer, rather than the first platform's whose cfg is an `any(…)` nothing rules out. Kept per +// pair in `sure`. An `any` leaf holds when one of its alternatives does, and so does the negation +// of a custom cfg (`not(loom)`: presumably off, see doubtful). An asker under no cfg has nothing +// to prefer by: for it, written order stands (everything not doubtful is as sure). +const holds = (l, keys) => keys.has(leafKey(l)) || (l.alts === undefined ? l.neg && customKey(l.key) : l.alts.some((alt) => alt.every((m) => holds(m, keys)))) +const entailed = (asker, set) => { + if (asker === undefined || asker.set.key === '' || set.key === '') return true + let verdict = asker.sure.get(set.key) + if (verdict === undefined) { + verdict = set.leaves.every((l) => holds(l, asker.keys)) + asker.sure.set(set.key, verdict) + } + return verdict +} +// The cfgs rustc and cargo set: anything else in a positive leaf is a custom `--cfg` (`loom`, +// `docsrs`, `tokio_unstable`, mio's `mio_unsupported_force_poll_poll`), off in a default build +// unless a build script or RUSTFLAGS sets it -- so a candidate under one is `doubtful`: taken +// after a compatible candidate under none (never ruled out: the loader doesn't know the build's +// flags). An `any` leaf is doubtful when each alternative is, or can't hold with the asker. A +// gate macro's leaf (`cfg_x!`) and a `mod` variant's are neither, nor is a leaf the asker holds +// itself (a file under `#[cfg(loom)]` takes the loom candidates). +const KNOWN_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doctest', 'doc', 'debug_assertions', 'overflow_checks', 'panic', 'proc_macro', 'miri', 'sanitize', 'target_feature', 'target_thread_local', 'ub_checks', 'relocation_model', 'fmt_debug', 'clippy', 'rustfmt', 'variant']) +const customKey = (key) => !KNOWN_CFG_KEYS.has(key) && !key.endsWith('!') +const doubtfulLeaf = (l, askerLeaves) => (l.alts === undefined ? !l.neg && customKey(l.key) : l.alts.every((alt) => cfgExclusive(alt, askerLeaves) || alt.some((m) => doubtfulLeaf(m, askerLeaves)))) +// Whether a leaf list has any custom cfg at all -- else no asker finds it doubtful. +const hasCustom = (leaves) => leaves.some((l) => (l.alts === undefined ? !l.neg && customKey(l.key) : l.alts.some(hasCustom))) +const doubtful = (asker, set) => { + if (asker === undefined || set.key === '') return false + if (set.custom === undefined) set.custom = hasCustom(set.leaves) + if (!set.custom) return false + let verdict = asker.doubt.get(set.key) + if (verdict === undefined) { + verdict = set.leaves.some((l) => !holds(l, asker.keys) && doubtfulLeaf(l, asker.set.leaves)) + asker.doubt.set(set.key, verdict) + } + return verdict +} + +// --- memoization ----------------------------------------------------------------------------- + +// Memoization with cycle detection (globClosure; the same bookkeeping serves globSource, pick and +// the candidate lists). A query asked again while it is being computed -- an import cycle: a +// glob into a module whose own globs lead back -- reads as `empty`, and the entry in progress is +// marked hit (`ctx.hits`, by its depth on the stack of computations under way, `ctx.walking`). +// Whatever is computed on top of such a read may be short and is not kept, so a later query from +// elsewhere in the cycle computes it afresh. The entry that was hit is kept once complete, but +// may be short itself (it read nothing for its own name) -- it is marked `cyclic`, with its +// `compute`, for stabilizeClosures to run again once every entry is in place. +function memoized(memo, key, ctx, compute, { empty }) { + const hit = memo.get(key) + if (hit !== undefined) { + if (hit.depth !== undefined) { + ctx.hits.add(hit.depth) + return empty + } + return hit.value + } + const depth = ctx.walking++ + memo.set(key, { depth }) + const value = compute() + const cyclic = ctx.hits.delete(depth) + ctx.walking-- + if (minHit(ctx) < depth) memo.delete(key) + else memo.set(key, cyclic ? { value, cyclic, compute } : { value }) + return value +} + +// Recompute the glob closures that were computed short of a cycle (memoized), each against the +// others as they now stand, until none grows: with every entry complete, a closure is the union +// of its globs' targets and their settled closures, whatever the order the queries came in. The +// lookups made meanwhile (glob sources, imports' answers, candidate lists) are dropped first, as +// they may rest on the short closures. True when anything changed -- the paths resolved so far must be resolved +// again. +function stabilizeClosures(ctx) { + const cyclic = [] + for (const memo of ctx.closures.values()) for (const entry of memo.values()) if (entry.cyclic === true) cyclic.push(entry) + if (cyclic.length === 0) return false + const reset = () => { + ctx.provided.clear() + ctx.opaque.clear() + ctx.opaqueGlobs.clear() + for (const byModule of ctx.imports.values()) { + for (const of of byModule.values()) { + for (const im of [...of.globs, ...[...of.named.values()].flat()]) { + delete im.source + delete im.answers + } + } + } + } + let changed = false + for (let round = 0; round < MAX_STABILIZE_ROUNDS; round++) { + reset() + let grew = false + for (const entry of cyclic) { + const again = entry.compute() + if (again.length !== entry.value.length) { + entry.value = again + grew = true + } + } + if (!grew) break + changed = true + } + reset() + return changed +} +const MAX_STABILIZE_ROUNDS = 8 + +// The shallowest computation under way that was cycled back to, Infinity for none. +function minHit(ctx) { + let min = Infinity + for (const d of ctx.hits) if (d < min) min = d + return min +} + +// Number of leading module-path segments `a` and `b` share (`crate::a::b` / `crate::a::c` → 2). +function commonDepth(a, b) { + const n = Math.min(a.length, b.length) + let i = 0 + let depth = 0 + while (i < n && a[i] === b[i]) { + if (a[i] === ':' && a[i + 1] === ':') { + depth++ + i += 2 + } else { + i++ + } + } + // The segment the match stopped in counts when it is whole in both: each ends there or goes on + // with a `::`. + const wholeA = i === a.length || a.startsWith('::', i) + const wholeB = i === b.length || b.startsWith('::', i) + return wholeA && wholeB ? depth + 1 : depth +} + +// What the imports written in module `at` make of `name`, as seen from module `from` (`from` +// decides which imports apply: a `pub(in scope)` one to a path written inside `scope`, a plain +// `use` to one written in `at` or below). A `use` binding the name outright is followed (bound); +// else the modules of this crate the module's globs reach (globClosure) are searched for it, each +// as seen from the module whose glob named it: as a child module, an item it defines, or through +// a `use` of its own. A glob into the sysroot or into another crate provides nothing: it can't be +// told what it brings in, and claiming everything would hide the crate's own edges. `asker` is +// the file the path was written in, with its cfg leaves (see cfg compatibility): an import, item +// or glob source exclusive with them is passed over. The candidates are listed once per (at, +// name, what `from` may see) -- providedAll -- and the asker picks among them. +function provided(root, at, name, from, ctx, asker, typesOnly = false) { + return pick(providedAll(root, at, name, commonDepth(from, at), ctx), root, ctx, asker, typesOnly) // scopes are ancestors of `at`: `from` is inside those this deep or shallower +} + +// The answer of the first of `candidates` (see providedAll: in written order) whose cfgs the +// asker's entail (entailed) -- else the first compatible with them under no custom cfg, else the +// first compatible at all (doubtful) -- and, for a path's lead or +// prefix (`typesOnly`), in the type namespace: a `fn log` can't lead `log::info`. A child module, +// item or macro answers as it is; an import answers what it leads to (followImport, on the +// asker's behalf, so its cfgs hold along the whole chain), resolved when first asked and then +// kept on the import per asker -- unless the answer came out of an import cycle still being +// resolved (see memoized). An import being followed that is met again leads back to itself (`use +// log;` in every platform file of a module asks the module for `log`, which is these imports): +// nothing here. An import that leads nowhere the bundle can see is passed over, and -- when it is +// a path, not a crate named outright (`use serde;`) -- stands in as the last resort: the name is +// an item here, as far as the bundle knows. A value where a type is wanted (`typesOnly`: a `fn` +// defined here, an import of one) is passed over too, and when nothing else answers, the answer +// is VALUE_ONLY rather than nothing: the name is taken, but not in this namespace. +function pick(candidates, root, ctx, asker, typesOnly = false) { + let unseen = null + let maybe = null // the first compatible answer under no doubt, for when none is entailed + let doubted = null // the first compatible answer under a doubtful cfg, for when there is nothing else + let valueOnly = false // a candidate passed over for being a value where a type is wanted + for (let i = 0; ; i++) { + const c = candidates.get(i) + if (c === undefined) break + if (!compatible(asker, c.leaves)) continue + if (typesOnly && c.ns === 'value') { + valueOnly = true + continue + } + const doubt = doubtful(asker, c.leaves) + const sure = !doubt && entailed(asker, c.leaves) + if (!sure && maybe !== null) continue + if (doubt && doubted !== null) continue + let r + if (c.import === undefined) { + r = c.answer + } else { + const im = c.import + if (im.following !== undefined) { + ctx.hits.add(im.following) + continue + } + const key = `${asker?.set.key ?? ''}\0${typesOnly}` + r = im.answers?.get(key) + if (r === undefined) { + const depth = ctx.walking++ + im.following = depth + r = followImport(im, root, ctx, asker, typesOnly) + im.following = undefined + ctx.walking-- + ctx.hits.delete(depth) + if (minHit(ctx) > depth) (im.answers ??= new Map()).set(key, r) + } + if (r === null) { + if (im.segments.length > 1) unseen ??= c + continue + } + if (r === VALUE_ONLY) { + valueOnly = true + continue + } + } + if (sure) return { ...r, through: c.through } + if (doubt) doubted = { ...r, through: c.through } + else maybe = { ...r, through: c.through } + } + if (maybe !== null) return maybe + if (doubted !== null) return doubted + if (unseen !== null) return { kind: 'item', file: unseen.import.file, through: unseen.through } + return valueOnly ? VALUE_ONLY : null +} + +// Every candidate module `at`'s imports offer for `name` as seen `seeing` levels into it, in +// written order -- `{ answer | import, through, leaves }`: what its `use` items bind (as imports +// to follow, see pick), then what each module its globs reach has (hasAll), under the glob's cfg +// leaves too. The list depends on the crate's structure alone, so it is built once per (at, +// name, seeing) and shared by every asker, who picks the first candidate compatible with its own +// cfgs -- and built lazily (Lazy): a closure holds hundreds of modules, and the first few usually +// answer. A list built while the closure was still being resolved (an import cycle, see +// memoized) is not kept. +function providedAll(root, at, name, seeing, ctx) { + const memo = ctx.provided.get(root) ?? ctx.provided.set(root, new Map()).get(root) + const key = `${at}\0${name}\0${seeing}` + let list = memo.get(key) + if (list === undefined) { + const depth = ctx.walking++ + const closure = globClosure(root, at, seeing, ctx) + ctx.walking-- + list = new Lazy(function* () { + yield* importsOf(root, at, name, seeing, ctx) + // Of the closure's modules (hundreds, in libc), only the ones with something of the name + // (`having`), in closure order. + const having = ctx.having.get(root)?.get(name) + if (having === undefined) return + const positions = [] + for (const module of having) for (const i of closureIndex(closure).get(module) ?? NONE) positions.push(i) + for (const i of positions.toSorted((x, y) => x - y)) { + const reached = closure[i] + for (const c of hasAll(root, reached.module, name, reached.seeing, ctx)) yield { ...c, through: reached.through, leaves: unionLeaves(reached.leaves, c.leaves, ctx.leafSets) } + } + }(), ctx) + if (minHit(ctx) >= depth) memo.set(key, list) + } + return list +} + +// A closure's entries by module (one may be reached at several depths), kept on the closure. +function closureIndex(closure) { + if (closure.index === undefined) { + closure.index = new Map() + closure.forEach((e, i) => (closure.index.get(e.module) ?? closure.index.set(e.module, []).get(e.module)).push(i)) + } + return closure.index +} + +// A list materialized from a generator as far as it is read (`get(i)`, undefined past the end). +// Read again while its generator is producing an element (the element's own lookups led back to +// the same question), it ends there for that reader, and the computation under way is marked hit +// (see memoized) so nothing computed on that short view is kept. +class Lazy { + constructor(gen, ctx) { + this.gen = gen + this.ctx = ctx + this.out = [] + this.running = -1 + } + + get(i) { + if (this.running !== -1) { + this.ctx.hits.add(this.running) + return undefined + } + while (i >= this.out.length) { + this.running = this.ctx.walking + let next + try { + next = this.gen.next() + } finally { + this.running = -1 + } + if (next.done) return undefined + this.out.push(next.value) + } + return this.out[i] + } + + * [Symbol.iterator]() { + for (let i = 0; ; i++) { + const c = this.get(i) + if (c === undefined) return + yield c + } + } +} + +// Everything module `at` has under `name` for one seeing `seeing` levels into it -- what a glob +// into the module brings in -- as candidates in written order: a child module (visible that far), +// the items it defines (the module's own tree file first: an inline `mod imp { }` under one cfg +// beside a `mod imp;` file under another may both define it), at the crate root a `#[macro_export]` +// macro (an item of the root's namespace: zerocopy's `use super::*;` from the root brings +// `into_inner!` in), and its `use` items binding the name (importsOf). +function hasAll(root, at, name, seeing, ctx) { + // Asked for every module a closure reaches, most of which have nothing of the name. + const child = ctx.children.get(root)?.get(at)?.has(name) === true + const defs = ctx.defined.get(root)?.get(at)?.get(name) + const macro = at === 'crate' ? ctx.macros.get(root)?.get(name) : undefined + const imports = importsOf(root, at, name, seeing, ctx) + if (!child && defs === undefined && macro === undefined && imports.length === 0) return NONE + const out = [] + if (child) { + const sub = `${at}::${name}` + if ((ctx.modScope.get(root)?.get(sub) ?? 1) <= seeing) out.push({ answer: { kind: 'module', modulePath: sub, file: ctx.trees.get(root).get(sub) }, leaves: NO_LEAVES }) + } + if (defs !== undefined) { + const own = ctx.trees.get(root).get(at) + for (const d of defs) if (d.scopeDepth <= seeing && d.file === own) out.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns }) + for (const d of defs) if (d.scopeDepth <= seeing && d.file !== own) out.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns }) + } + if (macro !== undefined) out.push({ answer: { kind: 'item', file: macro }, leaves: NO_LEAVES }) + out.push(...imports) + return out +} +const NONE = Object.freeze([]) + +// The file in which module `at` defines item `name` (a `struct`, `fn`, … visible `seeing` levels +// in, under cfgs the asker's allow; in the type namespace when `typesOnly`, see pick), or null. +function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { + const defs = ctx.defined.get(root)?.get(at)?.get(name) + if (defs === undefined) return null + const own = ctx.trees.get(root).get(at) + // Ranked as pick ranks candidates -- sure, then compatible under no custom cfg, then doubtful + // -- the module's own tree file first within a rank, else the first written. + let best = null + let bestRank = Infinity + for (const d of defs) { + if (d.scopeDepth > seeing || !compatible(asker, d.leaves) || (typesOnly && d.ns === 'value')) continue + const tier = doubtful(asker, d.leaves) ? 2 : (entailed(asker, d.leaves) ? 0 : 1) + const rank = tier * 2 + (d.file === own ? 0 : 1) + if (rank < bestRank) { + best = d.file + bestRank = rank + if (rank === 0) break + } + } + return best +} + +// What module `at` (spread over the files of cfg variants, `#[cfg_attr(unix, path = …)]`) makes +// of `name` for a path written in `file`, that file's own imports first: its `use` binding the +// name, then its own globs; only then the module's imports at large (providedAll). For a module +// held in one file the two agree, so the shortcut is taken only when they may not. Memoized like +// providedAll: a `use log;` in such a file asks for `log` in its own module, which is this import. +function providedFrom(root, at, name, ctx, file, asker, typesOnly = false) { + const of = ctx.imports.get(root)?.get(at) + const several = of !== undefined && [...of.globs, ...[...of.named.values()].flat()].some((im) => im.file !== file) + if (!several) return provided(root, at, name, at, ctx, asker, typesOnly) + const memo = ctx.provided.get(root) ?? ctx.provided.set(root, new Map()).get(root) + const seeing = at.split('::').length + const key = `${at}\0${name}\0@${file}` + let list = memo.get(key) + if (list === undefined) { + // Resolving this file's glob sources may ask this very question (`pub use sibling::*;` beside + // `use super::*;`): while the list is in the making it reads as nothing, and is not kept then. + const depth = ctx.walking++ + memo.set(key, { depth }) + const own = [] // this file's globs, with their sources + for (const im of of.globs) { + if (im.file !== file) continue + const source = globSource(im, root, ctx) + if (source?.kind === 'module') own.push([im, source.modulePath]) + } + const cyclic = ctx.hits.delete(depth) + ctx.walking-- + list = new Lazy(function* () { + yield* importsOf(root, at, name, seeing, ctx, file) + for (const [im, module] of own) { + const sees = commonDepth(at, module) + for (const c of hasAll(root, module, name, sees, ctx)) yield { ...c, through: im.file, leaves: unionLeaves(im.leaves, c.leaves, ctx.leafSets) } + for (const c of providedAll(root, module, name, sees, ctx)) yield { ...c, through: im.file, leaves: unionLeaves(im.leaves, c.leaves, ctx.leafSets) } + } + yield* providedAll(root, at, name, seeing, ctx) + }(), ctx) + if (!cyclic && minHit(ctx) >= depth) memo.set(key, list) + else memo.delete(key) + } else if (list.depth !== undefined) { + ctx.hits.add(list.depth) + return null + } + return pick(list, root, ctx, asker, typesOnly) +} + +// The `use` items of module `at` binding `name`, visible to one seeing `seeing` levels into it, +// as candidates to follow (pick), in written order, each with the file holding the import as +// `through`. Several may bind it (`#[cfg(unix)] use a::X;` beside `#[cfg(windows)] use b::X;`). +// `onlyFile` restricts to the imports written in that file: the module may be spread over cfg +// variants (`#[cfg_attr(unix, path = …)]`), each binding the name its own way, and a path written +// in one of them means that one's. +function importsOf(root, at, name, seeing, ctx, onlyFile) { + const named = ctx.imports.get(root)?.get(at)?.named.get(name) + if (named === undefined) return NONE + const out = [] + for (const im of named) { + if (im.scopeDepth > seeing || (onlyFile !== undefined && im.file !== onlyFile)) continue + out.push({ import: im, through: im.file, leaves: im.leaves }) + } + return out +} + +// The modules of this crate that the globs of module `at` visible to one seeing `seeing` levels +// into it reach, transitively (a glob's target module's own globs, as seen from the module whose +// glob named it), depth-first in written order, each `{ module, seeing, through }`: how deep the +// module naming it sees into it, and the file holding the glob of `at` it was reached through. +// Computed once per (at, seeing), a settled closure of a module reached standing in for walking +// on from it; a closure asked for while it is being computed (a glob into a module whose own +// imports name the glob's source) is a cycle, and reads as empty. +function globClosure(root, at, seeing, ctx) { + const memo = ctx.closures.get(root) ?? ctx.closures.set(root, new Map()).get(root) + const key = `${at}\0${seeing}` + return memoized(memo, key, ctx, () => { + const modules = [] + const seen = new Map([[key, null]]) // (module, seeing) → its entry + // `leaves`: the cfgs of the globs walked through to get here, all of which must hold (a leaf + // set). A module reached again by another path (libc's `mod primitives`, glob re-exported + // under a dozen cfgs) is reachable under either -- and so is everything its own globs reach, + // so a path that widens an entry is walked on from it (`widened`); one that adds nothing ends. + const add = (k, entry) => { + const known = seen.get(k) + if (known === undefined) { + seen.set(k, entry) + modules.push(entry) + return 'new' + } + if (known === null) return 'same' + const either = eitherLeaves(known.leaves, entry.leaves, ctx.leafSets) + if (either === known.leaves) return 'same' + known.leaves = either + return 'widened' + } + const reach = (module, sees, through, leaves) => { + const k = `${module}\0${sees}` + if (add(k, { module, seeing: sees, through, leaves }) === 'same') return + const settled = memo.get(k) + if (settled !== undefined && settled.depth === undefined) { + // Already transitive: no walking on from any of them. + for (const m of settled.value) add(`${m.module}\0${m.seeing}`, { module: m.module, seeing: m.seeing, through, leaves: unionLeaves(leaves, m.leaves, ctx.leafSets) }) + return + } + for (const im of ctx.imports.get(root)?.get(module)?.globs ?? []) { + if (im.scopeDepth > sees) continue + const source = globSource(im, root, ctx) + if (source?.kind === 'module') reach(source.modulePath, commonDepth(module, source.modulePath), through ?? im.file, unionLeaves(leaves, im.leaves, ctx.leafSets)) + } + } + for (const im of ctx.imports.get(root)?.get(at)?.globs ?? []) { + if (im.scopeDepth > seeing) continue + const source = globSource(im, root, ctx) + if (source?.kind === 'module') reach(source.modulePath, commonDepth(at, source.modulePath), im.file, im.leaves) + } + return modules + }, { empty: [] }) +} + +// What a glob imports from (followImport), asked once per glob rather than once per closure it +// is part of -- unless the answer came out of an import cycle still being resolved. +function globSource(im, root, ctx) { + if (im.source !== undefined) return im.source + const depth = ctx.walking++ + const source = followImport(im, root, ctx) + ctx.walking-- + if (minHit(ctx) >= depth) im.source = source + return source +} + +// What an import leads to: its path resolved as written in its module, else the crate its lead +// names -- the sysroot's (`use core::fmt`: an item, living here as far as the bundle knows), an +// in-tree one, or null for one that isn't (nor anything the tree can model). +function followImport(im, root, ctx, asker, typesOnly = false) { + if (!im.absolute) { + const r = walkPath(im.segments, root, im.module, ctx, { file: im.file, asker, typesOnly }) + if (r !== null) return r + } + const head = im.segments[0] + if (NON_CRATE_LEADS.has(head)) return { kind: 'item', file: im.file } + const target = ctx.resolveCrate(head, im.file) + return target ? { kind: 'crate', name: head, file: target } : null +} + +// walkPath's answer for a path naming a value-namespace item (a `fn`, `const` or `static`) where +// a type or module is wanted: as a prefix (`log::info!` beside a `fn log`), or as the target of +// an import followed as one (`use crate::util::log; log::info!`). Nothing here -- and not an +// item of the module either: the lead may still name a crate. +const VALUE_ONLY = Object.freeze({ kind: 'value' }) +// Whether module `at` defines `name` only in the value namespace. +const onlyValues = (root, at, name, ctx) => { + const defs = ctx.defined.get(root)?.get(at)?.get(name) + return defs !== undefined && defs.every((d) => d.ns === 'value') +} + +// Resolve `segments`, a path written in module `from` of `root`'s crate, when its lead anchors it +// there: `crate::`, `self::`, `super::`, a child module of `from`, a name `from`'s imports provide +// (`use gen::consts; pub use consts::*;`, `extern crate serde_core as s; s::Value`), or -- for a +// bare name -- a `macro_rules!` of the file (`macro_rules! helper { … } pub(crate) use helper;`). +// Null when it doesn't (an item in scope, a crate: the caller looks the lead up as one). +// Then module by module; a segment naming no module is looked up in the module reached, and the +// walk goes on from a module that provides, or ends at whatever else it provides. A segment +// nothing provides is an item of the module reached, in the file defining it when that is known. +// At the crate root a final segment may name a `#[macro_export]` macro, which lives in the file +// defining it: `$crate::name!` names the macro before anything else (`macroCall`), any other path +// the module or import of that name first. In the module the path is written in, `file`'s own +// imports come first: the module's other files, if any, are cfg variants (providedFrom). `asker` +// (see cfg compatibility) rules out what can't be compiled together with the path. +function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, typesOnly = false } = {}) { + const tree = ctx.trees.get(root) + const head = segments[0] + // A segment with more after it names a module or type-namespace item: a `fn log` doesn't lead + // `log::info!`, the crate does (`typed`, see pick) -- the last one too when the caller wants a + // type (`typesOnly`: an import followed as a path's prefix). + const lookup = (at, name, typed) => (at === from && file !== undefined ? providedFrom(root, at, name, ctx, file, asker, typed) : provided(root, at, name, from, ctx, asker, typed)) + let cur = from.split('::') + let i = 0 + let via + if (head === 'crate') { + cur = ['crate'] + i = 1 + } else if (head === 'self') { + i = 1 + } else if (head !== 'super' && !tree.has(`${from}::${head}`)) { + const typed = segments.length > 1 || typesOnly + const p = lookup(from, head, typed) + if (p === null || p === VALUE_ONLY) { + // An item of the module itself (`enum Kind { … } use Kind::*;`), or a macro of the file. + const own = definedIn(root, from, head, Infinity, ctx, asker, typed) + if (own !== null) return { kind: 'item', file: own } + if (typed && (p === VALUE_ONLY || onlyValues(root, from, head, ctx))) return VALUE_ONLY + return segments.length === 1 && file !== undefined && ctx.fileMacros.get(file)?.has(head) === true ? { kind: 'item', file } : null + } + via = { modulePath: from, consumed: 0, through: p.through } + if (p.kind !== 'module') return { ...p, via } + cur = p.modulePath.split('::') + i = 1 + } + while (segments[i] === 'super') { + cur.pop() + i++ + if (cur.length === 0) return null + } + while (i < segments.length) { + const at = cur.join('::') + const name = segments[i] + const last = i === segments.length - 1 + const macro = at === 'crate' && last ? ctx.macros.get(root)?.get(name) : undefined + if (macro !== undefined && macroCall) return { kind: 'item', file: macro, via } + const child = `${at}::${name}` + if (tree.has(child)) { + cur.push(name) + i++ + continue + } + const typed = !last || typesOnly + const p = lookup(at, name, typed) + if (p === null || p === VALUE_ONLY) { + if (macro !== undefined) return { kind: 'item', file: macro, via } + const def = definedIn(root, at, name, Infinity, ctx, asker, typed) + if (def !== null) return { kind: 'item', file: def, via } + if (typed && (p === VALUE_ONLY || onlyValues(root, at, name, ctx))) return VALUE_ONLY + return { kind: 'item', file: tree.get(at), via } + } + via ??= { modulePath: at, consumed: i, through: p.through } + if (p.kind !== 'module') return { ...p, via } + cur = p.modulePath.split('::') + i++ + } + const modulePath = cur.join('::') + return { kind: 'module', modulePath, file: tree.get(modulePath), via } +} + +// Whether a glob in module `at`, or in a module its globs reach, may bring `lead` in without the +// loader seeing it -- a glob into a crate that isn't in-tree (`use syn::*;` brings syn's +// `punctuated` module in) -- or does bring it in from another in-tree crate: from the module the +// glob's path names there (`use tokio::sync::*;` brings tokio's `sync::mpsc` in), what it has +// (hasAll) or its own globs may bring in, in turn. Then a lead nothing here explains is not one to +// report as a missing crate. A glob the loader sees through -- into a module of this crate, an +// enum, the sysroot, or an in-tree crate's module without the name -- explains nothing, and no +// glob brings in its own crate's name. Memoized per (at, lead); asked again while being answered +// (globs between two crates leading back), it reads as no. Only the globs leading out of the +// crate matter (opaqueGlobsOf), so a closure of hundreds costs a lookup per such glob. +// The globs of `root`'s crate that lead out of it -- into a crate that isn't in-tree (source +// null) or into another in-tree one -- by module, with their sources, found once per crate (and +// again after stabilizeClosures): the few of a closure's hundreds of modules globMayProvide has +// to look at. +function opaqueGlobsOf(root, ctx) { + let index = ctx.opaqueGlobs.get(root) + if (index === undefined) { + index = new Map() + for (const [module, of] of ctx.imports.get(root) ?? []) { + for (const im of of.globs) { + if (NON_CRATE_LEADS.has(im.segments[0])) continue + const source = globSource(im, root, ctx) + if (source === null || source.kind === 'crate') (index.get(module) ?? index.set(module, []).get(module)).push({ im, source }) + } + } + ctx.opaqueGlobs.set(root, index) + } + return index +} + +function globMayProvide(root, at, lead, ctx) { + const memo = ctx.opaque.get(root) ?? ctx.opaque.set(root, new Map()).get(root) + const key = `${at}\0${lead}` + if (memo.has(key)) return memo.get(key) + memo.set(key, false) + // The globs leading out of the crate that `at` sees -- its own and its closure's, at the depth + // each module is seen (a module may be reached at several) -- gathered once per `at`: the ones + // into crates not in-tree by the crate name they bring nothing of (their own), the ones into + // in-tree crates with their sources. + const reachKey = `${at}\0` + let reach = memo.get(reachKey) + if (reach === undefined) { + const seeing = at.split('::').length + const closure = globClosure(root, at, seeing, ctx) + const positions = closureIndex(closure) + reach = { unseen: new Map(), crates: [] } + for (const [module, globs] of opaqueGlobsOf(root, ctx)) { + let sees = module === at ? seeing : -1 + for (const i of positions.get(module) ?? NONE) sees = Math.max(sees, closure[i].seeing) + if (sees === -1) continue + for (const { im, source } of globs) { + if (im.scopeDepth > sees) continue + if (source === null) reach.unseen.set(im.segments[0], (reach.unseen.get(im.segments[0]) ?? 0) + 1) + else reach.crates.push({ im, source }) + } + } + memo.set(reachKey, reach) + } + // What another crate's module `module` (root `file`) makes visible under `lead`, as far as the + // bundle sees: anything, for a crate not in-tree; else what the module has, what its `pub` + // globs bring in from its own crate (providedAll), or may from a third. + const inCrate = (file, module) => !ctx.trees.has(file) || hasAll(file, module, lead, 1, ctx).length > 0 || providedAll(file, module, lead, 1, ctx).get(0) !== undefined || globMayProvide(file, module, lead, ctx) + let answer = false + let unseen = 0 + for (const [name, count] of reach.unseen) if (name !== lead) unseen += count + if (unseen > 0) answer = true + for (const { im, source } of reach.crates) { + if (answer || im.segments[0] === lead) break + // The glob's path on from the crate it names (`sync` of `use tokio::sync::*;`), when the + // lead itself named the crate (no `via`: the path wasn't anchored in this crate first). + const rest = source.via === undefined ? im.segments.slice(1) : [] + const there = rest.length === 0 || !ctx.trees.has(source.file) ? { kind: 'module', modulePath: 'crate' } : walkPath(['crate', ...rest], source.file, 'crate', ctx) + if ((there?.kind === 'module' && inCrate(source.file, there.modulePath)) || (there?.kind === 'crate' && inCrate(there.file, 'crate'))) answer = true + } + memo.set(key, answer) + return answer +} + +// Resolve one path reference made in `file` to what it names in the bundle (see walkPath): a +// module or item of the same crate, an in-tree crate (`{ kind: 'crate', name, file }`, through +// `resolveCrate` when the lead names one, by its own name or as the crate root's `extern crate x +// as y;` aliases it, `::y::…` included), an exported macro named by a one-segment path written at +// the crate root, where such a macro is an item (`pub use anyhow as format_err;` in the anyhow +// crate's lib.rs; a child module reaches it as `crate::anyhow`), or `{ kind: 'unresolved', name }` +// when the lead could name a crate (crateLead) but nothing in-tree did; null for anything else +// (an item in scope, std, a path above the crate root, a name a glob into another crate may well +// provide). `bindings` are the names `file`'s imports bind -- in whichever namespace: one bound +// only as a value (`use crate::util::log;` of a `fn log`) doesn't keep `log::info!` from naming +// the crate. +const NO_BINDINGS = new Set() +function resolvePathRef(ref, file, bindings, ctx) { const { segments, absolute, inlinePath } = ref if (segments.length === 0) return null - const head = segments[0] - const here = files.get(file) + const here = ctx.files.get(file) + const from = here ? [here.modulePath, ...inlinePath].join('::') : null + let valueOnly = false if (!absolute && here) { - const tree = trees.get(here.root) - const current = [...here.modulePath.split('::'), ...inlinePath] - let cur = null - let idx = 0 - if (head === 'crate') { - cur = ['crate'] - idx = 1 - } else if (head === 'self') { - cur = current - idx = 1 - } else if (head === 'super') { - cur = current - } else if (tree.has([...current, head].join('::'))) { - cur = current - } - if (cur) { - while (segments[idx] === 'super') { - cur.pop() - idx++ - if (cur.length === 0) return null - } - const target = resolveUsePath([...cur, ...segments.slice(idx)].join('::'), tree) - return target ? { kind: 'module', target } : null - } - } - const name = crateLead(ref, bindings) - if (name === null) return null - const target = resolveCrate(name, file) - return target ? { kind: 'crate', name, target } : { kind: 'unresolved', name } + const r = walkPath(segments, here.root, from, ctx, { macroCall: ref.macroCall, file, asker: here.asker }) + if (r === VALUE_ONLY) valueOnly = true // the lead is bound here, but as a value: `log::…` names the crate + else if (r !== null) return r + } + const lead = crateLead(ref, valueOnly ? NO_BINDINGS : bindings) + if (lead === null) return null + const name = ctx.externPrelude.get(here?.root)?.get(lead) ?? lead + const target = ctx.resolveCrate(name, file) + if (target) return { kind: 'crate', name, file: target } + const macro = segments.length === 1 && from === 'crate' ? ctx.macros.get(here.root)?.get(lead) : undefined + if (macro !== undefined) return { kind: 'item', file: macro } + return here && globMayProvide(here.root, from, lead, ctx) ? null : { kind: 'unresolved', name } +} + +// `map`: root → name → Set, noting `module` under (root, name). +const note = (map, root, name, module) => { + const byName = map.get(root) ?? map.set(root, new Map()).get(root) + ;(byName.get(name) ?? byName.set(name, new Set()).get(name)).add(module) } // Bundle import keys can't contain '/'; a cfg predicate practically never does, but fail safe. @@ -748,16 +2071,24 @@ const cfgKey = (cfg) => (cfg ?? '*').replaceAll('/', '|') // `benches/*.rs` and their modules), judged against its package dir when a Cargo context knows it. const isTestTarget = (rel, ctx) => (ctx ? ctx.isTestTarget(rel) : isTestTargetPath('.', rel)) +// The build `rel` is scanned under (scanRustItems' options): its crate's resolved features, whether +// it is test code, and the target's cfg set, as far as the Cargo context `ctx` knows them -- none +// for a file compiled for the host (`hostFiles`: a build script and what it reaches), whose +// target the loader doesn't know. +const buildOf = (rel, ctx, hostFiles = null) => ({ features: ctx?.featuresFor(rel) ?? null, test: isTestTarget(rel, ctx), target: hostFiles?.has(rel) === true ? null : ctx?.targetCfgs ?? null }) + // Scan results per `sources` map, so the tree pass reuses the walk's scan of each file when it -// ran under the same build (the same feature set object and test flag). +// ran under the same build (the same feature set object, test flag and target cfg set object) on +// the same content (a caller may edit a file in a map it reuses). const scanCache = new WeakMap() function cachedScan(sources, path, content, build) { if (!scanCache.has(sources)) scanCache.set(sources, new Map()) const cache = scanCache.get(sources) - const hit = cache.get(path) - if (hit && hit.features === build.features && hit.test === build.test) return hit.items + const key = build.templates === undefined ? path : `${path}\0${[...build.templates].join()}` // a rescan with fewer templates, kept beside the plain scan + const hit = cache.get(key) + if (hit && hit.content === content && hit.features === build.features && hit.test === build.test && hit.target === build.target) return hit.items const items = scanRustItems(content, build) - cache.set(path, { ...build, items }) + cache.set(key, { ...build, content, items }) return items } @@ -771,30 +2102,60 @@ function cachedScan(sources, path, content, build) { // same-name `#[cfg]`-gated declarations name different files; `` → module file; // `use ` → crate root. `unresolvedCrates` names the crates `use`/`extern crate` referenced // that nothing in-tree satisfied (registry deps that weren't vendored, typically) -- -// informational, never fatal. A `cargo` context (cargo.js) may be passed in to share one +// informational, never fatal. `wantedRoots` are in-tree crate roots the paths named that +// `sources` lacks (the walk took their names for bound items): loaded and built again, they +// complete the bundle. A `cargo` context (cargo.js) may be passed in to share one // between the walk and this pass; it also carries each crate's resolved features, which decide -// `#[cfg(feature = …)]`. -export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null } = {}) { +// `#[cfg(feature = …)]`, and the build target's cfg set when one was given, which decides +// `#[cfg(unix)]` and the like. +export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null, formats = null, hostFiles = null } = {}) { const ctx = cargo ?? (baseDir ? createCargoContext(baseDir, { entries: roots }) : null) const rootSet = crateRoots(sources, roots, ctx) const resolutions = new Map() const missing = [] - const unresolvedCrates = new Set() const scanned = new Map() for (const [path, content] of sources) { - scanned.set(path, cachedScan(sources, path, content, { features: ctx?.featuresFor(path) ?? null, test: isTestTarget(path, ctx) })) + if (formats?.has(path)) continue // an `include_str!` / `include_bytes!` asset: carried, not Rust + scanned.set(path, cachedScan(sources, path, content, buildOf(path, ctx, hostFiles))) + } + // A package with a `macro_rules!` of a template macro's name (its own `quote!`, in whichever of + // its files) invokes that one everywhere in the package: no template there, so the files of the + // package that skipped a body of that name are scanned again with the name dropped from the set + // (the package: the Cargo context's, or the whole bundle without one). + const packageOf = (path) => ctx?.packageInfo(path)?.dir ?? '.' + const ownTemplates = new Map() // package → the template names it defines + for (const [path, items] of scanned) { + for (const m of items.macros) if (TEMPLATE_MACROS.has(m.name)) (ownTemplates.get(packageOf(path)) ?? ownTemplates.set(packageOf(path), new Set()).get(packageOf(path))).add(m.name) + } + const reducedTemplates = new Map() // per package's own set: TEMPLATE_MACROS less those, one object + for (const [path, items] of scanned) { + const own = items.skipped.size > 0 ? ownTemplates.get(packageOf(path)) : undefined + if (own === undefined || own.isDisjointFrom(items.skipped)) continue + const templates = reducedTemplates.get(own) ?? reducedTemplates.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) + scanned.set(path, cachedScan(sources, path, sources.get(path), { ...buildOf(path, ctx, hostFiles), templates })) } - // Files a `#[path]` names own their directory (see getModuleDir), so find those before resolving - // any default `mod` lookup; a path-loaded file may itself hold inline modules with `#[path]`s, - // hence the loop to a fixed point. + // An `include!`d file's items join the includer's module (buildModuleTrees), but a `mod` it + // declares resolves beside the included file itself, as rustc does: like a `#[path]`-loaded + // file, it owns its directory (pathLoaded). const pathLoaded = new Set() + for (const [path, items] of scanned) { + for (const inc of items.includes) { + if (inc.kind !== 'include') continue + const file = includeTarget(inc, path, ctx) + if (file !== null && file !== path && sources.has(file)) pathLoaded.add(file) + } + } + // Files a `#[path]` names own their directory too (see getModuleDir), so find those before + // resolving any default `mod` lookup; a path-loaded file may itself hold inline modules with + // `#[path]`s, hence the loop to a fixed point. const modOpts = { knownSources: sources, roots: rootSet, pathLoaded } + const declsOf = (path, decl) => resolveModDecl(decl, path, { ...modOpts, boundary: boundaryOf(path, ctx) }) for (let grew = true; grew;) { grew = false for (const [path, items] of scanned) { for (const decl of items.mods) { if (decl.paths.length === 0) continue - for (const t of resolveModDecl(decl, path, modOpts)) { + for (const t of declsOf(path, decl)) { if (t.explicit && !pathLoaded.has(t.file)) { pathLoaded.add(t.file) grew = true @@ -803,14 +2164,36 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul } } } + // Bare macro invocations per file, for the include macros a `macro_rules!` body holds: rustc + // resolves those relative to each invoking file (recorded there, as `include_str `). + const invokers = new Map() + for (const [path, items] of scanned) { + for (const name of items.invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(path) + } + const includeEdges = new Map() // file → [{ kind, path, base, conditional }] from macro templates it invokes + for (const [, items] of scanned) { + for (const m of items.macros) { + if (m.includes.length === 0) continue + for (const f of invokers.get(m.name) ?? []) { + const list = includeEdges.get(f) ?? includeEdges.set(f, []).get(f) + for (const inc of m.includes) list.push({ ...inc, conditional: true }) + } + } + } + // file → spec → target file → the declarations mounting it, each its cfg (null for an ungated + // one) and the macro body it sits in, for the cfg leaves each file carries (buildModuleTrees). + const mountCfgs = new Map() for (const [path, items] of scanned) { // spec -> Map: each declaration's files under its cfg predicate ('*' when - // ungated), so `#[cfg(unix)] #[path = "u.rs"] mod imp;` beside `#[cfg(windows)] #[path = - // "w.rs"] mod imp;` keeps both files. The first declaration of a key wins. + // ungated; `!` for one a macro emits under a cfg the loader can't see, tokio's + // `cfg_has_atomic_u64! { mod imp; }`), so `#[cfg(unix)] #[path = "u.rs"] mod imp;` beside + // `#[cfg(windows)] #[path = "w.rs"] mod imp;` keeps both files. The first declaration of a + // key wins; another file under the same key gets the key numbered (`*#2`) rather than lost. const byCfg = new Map() + const cfgsHere = mountCfgs.set(path, new Map()).get(path) for (const decl of items.mods) { const spec = `mod ${[...decl.inlinePath, decl.name].join('::')}` - const targets = resolveModDecl(decl, path, modOpts) + const targets = declsOf(path, decl) if (targets.length === 0) { // conditional && unresolved: cfg-gated module, may be compiled out -- tolerated. if (!decl.conditional) { @@ -820,46 +2203,336 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul continue } const keyed = byCfg.get(spec) ?? byCfg.set(spec, new Map()).get(spec) + const byFile = cfgsHere.get(spec) ?? cfgsHere.set(spec, new Map()).get(spec) for (const t of targets) { - const key = cfgKey(t.cfg ?? decl.cfg) + const cfg = t.cfg ?? decl.cfg ?? null + const base = cfgKey(cfg ?? (decl.macro === null ? null : `${decl.macro}!`)) + let key = base + for (let k = 2; keyed.has(key) && keyed.get(key) !== t.file; k++) key = `${base}#${k}` if (!keyed.has(key)) keyed.set(key, t.file) + ;(byFile.get(t.file) ?? byFile.set(t.file, []).get(t.file)).push({ cfg, macro: decl.macro }) } } - // A single file is a flat string; cfg variants stay a Map. + // A single file is a flat string -- also when every cfg names the same file (libc's `mod + // primitives` under a dozen); cfg variants stay a Map. const specMap = new Map() - for (const [spec, keyed] of byCfg) specMap.set(spec, keyed.size === 1 ? keyed.values().next().value : keyed) + for (const [spec, keyed] of byCfg) { + const distinct = new Set(keyed.values()) + specMap.set(spec, distinct.size === 1 ? distinct.values().next().value : keyed) + } + // `include!("generated/consts.rs")` and the asset macros name a file relative to this one (or + // to the package root), the ones in a macro body it invokes relative to this one too. + for (const inc of [...items.includes, ...(includeEdges.get(path) ?? [])]) { + const file = includeTarget(inc, path, ctx) + if (file !== null && file !== path && sources.has(file)) specMap.set(`${inc.kind} ${inc.path}`, file) + else if (!inc.conditional) console.warn(`[loader.rust] Missing include: ${inc.path} from ${path}`) + } + if (items.unfollowed > 0) console.warn(`[loader.rust] ${items.unfollowed} include${items.unfollowed === 1 ? '' : 's'} in ${path} name${items.unfollowed === 1 ? 's' : ''} a path the loader can't follow (not a literal)`) resolutions.set(path, specMap) } - const { trees, files } = buildModuleTrees(sources, resolutions, rootSet) - for (const [path, items] of scanned) { + const { trees, files } = buildModuleTrees(sources, resolutions, rootSet, new Map([...scanned].map(([p, items]) => [p, items.inlineModules])), mountCfgs) + // What each file asks as: itself, with the cfg leaves it was mounted under (see cfg + // compatibility), as an interned leaf set (leafSets: every set of this tree, by key). + const leafSets = new Map() + const askerSets = new Map() // per leaf set: what every asker of that set shares + // With the build's target known, its cfgs hold for every file compiled for it (not a build + // script's, compiled for the host): a `unix` asker never takes a `windows` candidate, a + // `target_os = "linux"` one prefers the `any(android, linux, …)` branch (see compatible). + const targetLeaves = leafSet([...(ctx?.targetCfgs ?? [])].flatMap((cfg) => { + const m = /^([\w-]+)(?:="([^"]*)")?$/u.exec(cfg) + return m && TARGET_CFG_KEYS.has(m[1]) ? [{ key: m[1], value: m[2] ?? null, neg: false }] : [] + }), leafSets) + for (const [path, f] of files) { + f.leaves = leafSet(f.leaves, leafSets) + const set = hostFiles?.has(path) === true ? f.leaves : unionLeaves(f.leaves, targetLeaves, leafSets) + const shared = askerSets.get(set.key) ?? askerSets.set(set.key, { keys: new Set(set.leaves.map(leafKey)), compat: new Map(), sure: new Map(), doubt: new Map() }).get(set.key) + f.asker = { file: path, set, ...shared } + } + // Per crate root, each module's child modules by name (every module's parent is in its tree). + const children = new Map() + for (const [root, tree] of trees) { + const byParent = new Map() + for (const modulePath of tree.keys()) { + const cut = modulePath.lastIndexOf('::') + if (cut === -1) continue + const parent = modulePath.slice(0, cut) + ;(byParent.get(parent) ?? byParent.set(parent, new Set()).get(parent)).add(modulePath.slice(cut + 2)) + } + children.set(root, byParent) + } + // Per crate root, what a path may continue along where a segment names no module: the imports + // of each module (by module path: `named` by the name each binds, and `globs`; each with the + // `file` holding it, its `module`, how deep in the tree the module its visibility reaches sits, + // `scopeDepth`: 1 for a `pub`, the module's own depth for a plain `use`, and its cfg `leaves`: + // its file's plus its own cfg's), the items each module defines (`defined`: module → name → + // [{ file, scopeDepth, ns, leaves }], `ns` the namespace: `value` for a fn/const/static, which no + // path continues past, `type` for the rest), how far each module's own visibility reaches (`modScope`: + // module → scopeDepth, for what a glob may bring in) and the `#[macro_export]` macros (by name → + // file). `fileMacros` is file → the `macro_rules!` names it defines. `externPrelude` (root → + // alias → crate) holds the root's `extern crate x as y;`, in scope crate-wide. + const imports = new Map() + const defined = new Map() + const modScope = new Map() + const macros = new Map() + const fileMacros = new Map() + const externPrelude = new Map() + const perRoot = (map, root) => map.get(root) ?? map.set(root, new Map()).get(root) + const scopeDepthOf = (vis, modulePath) => visibilityScope(vis, modulePath).split('::').length + // Bare `name!` invocations resolve in textual scope, as rustc has it: a `macro_rules!` is in + // scope from its definition to the end of its file, in the files the `mod`s after it mount (and + // theirs, on down), and -- when its module is mounted with `#[macro_use]` -- in the mounting + // file after that `mod`, and on from there. `mounts` is file → `{ parent, offset }`, the `mod` + // (or `include!`) that first mounted it; `macroUseMods` file → its `#[macro_use] mod`s, each + // `{ offset, files }`. + const mounts = new Map() + const macroUseMods = new Map() + const filesOf = (target) => (target instanceof Map ? [...target.values()] : [target]) + // Where a `mod` stands for textual scope: in its file at its offset -- or, for one declared in + // a `macro_rules!` template this file never invokes (serde's core/crate_root.rs declares `pub + // mod de;` inside `crate_root!`, which lib.rs invokes), in the first file (tree order) invoking + // the macro, at that invocation: rustc emits the item there. + const callerMemo = new Map() + const firstCaller = (name) => { + if (!callerMemo.has(name)) callerMemo.set(name, files.keys().find((f) => scanned.get(f)?.calls.has(name) === true) ?? null) + return callerMemo.get(name) + } + const mountOf = (path, decl) => { + const caller = decl.template !== null && decl.offset === Infinity ? firstCaller(decl.template) : null + return caller === null ? { parent: path, offset: decl.offset, at: decl.at } : { parent: caller, offset: scanned.get(caller).calls.get(decl.template)[0], at: decl.at } + } + // In tree order (`files`: each crate's, from its root down, the variants of a `mod` in the order + // declared), whatever order the sources came in: among several files defining or importing a + // name for one module, the first is the tree's own, and the answers are the same for any order + // the caller lists the files in. + for (const [path, here] of files) { + const items = scanned.get(path) + if (items === undefined) continue + for (const im of items.imports) { + // A `use $crate::…` in a `macro_rules!` template is the invoking module's, wherever that is. + // (A `use` without `$crate` in one is kept: serde wraps its whole root in a macro, whose + // `mod lib { pub use core::fmt; … }` is what every file's `use crate::lib::*` names.) + if (im.template === true) continue + const modulePath = [here.modulePath, ...im.inlinePath].join('::') + const byModule = perRoot(imports, here.root) + const of = byModule.get(modulePath) ?? byModule.set(modulePath, { named: new Map(), globs: [] }).get(modulePath) + const entry = { ...im, file: path, module: modulePath, scopeDepth: scopeDepthOf(im.vis, modulePath), leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(im.cfg), ...gateLeaves(im.macro)], leafSets), leafSets) } + if (im.glob) of.globs.push(entry) + else if (im.binding !== null) (of.named.get(im.binding) ?? of.named.set(im.binding, []).get(im.binding)).push(entry) + } + for (const d of items.defined) { + const modulePath = [here.modulePath, ...d.inlinePath].join('::') + const byModule = perRoot(defined, here.root) + const names = byModule.get(modulePath) ?? byModule.set(modulePath, new Map()).get(modulePath) + ;(names.get(d.name) ?? names.set(d.name, []).get(d.name)).push({ file: path, scopeDepth: scopeDepthOf(d.vis, modulePath), ns: d.ns, leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(d.cfg), ...gateLeaves(d.macro)], leafSets), leafSets) }) + } + const scopes = perRoot(modScope, here.root) const specMap = resolutions.get(path) + for (const decl of items.mods) { + const parent = [here.modulePath, ...decl.inlinePath].join('::') + const modulePath = `${parent}::${decl.name}` + if (!scopes.has(modulePath)) scopes.set(modulePath, scopeDepthOf(decl.vis, parent)) + const target = specMap.get(`mod ${[...decl.inlinePath, decl.name].join('::')}`) + if (target === undefined) continue + const mountedFiles = filesOf(target).filter((f) => f !== path) + const mount = mountOf(path, decl) + for (const f of mountedFiles) if (!mounts.has(f)) mounts.set(f, mount) + if (decl.macroUse && mountedFiles.length > 0) (macroUseMods.get(mount.parent) ?? macroUseMods.set(mount.parent, []).get(mount.parent)).push({ offset: mount.offset, at: mount.at, files: mountedFiles }) + } + for (const [spec, target] of specMap) { + if (spec.startsWith('include ') && !mounts.has(target) && target !== path) mounts.set(target, { parent: path, offset: Infinity, at: Infinity }) + } + for (const [inline, vis] of items.inlineModuleVis) { + const modulePath = `${here.modulePath}::${inline}` + if (!scopes.has(modulePath)) scopes.set(modulePath, scopeDepthOf(vis, modulePath.slice(0, modulePath.lastIndexOf('::')))) + } + fileMacros.set(path, new Set(items.macros.map((m) => m.name))) + for (const m of items.macros) { + if (!m.exported) continue + const byName = perRoot(macros, here.root) + if (!byName.has(m.name)) byName.set(m.name, path) + } + if (here.modulePath === 'crate') { + const prelude = perRoot(externPrelude, here.root) + for (const e of items.externCrates) { + if (e.inlinePath.length === 0 && !NON_CRATE_LEADS.has(e.name) && !prelude.has(e.alias ?? e.name)) prelude.set(e.alias ?? e.name, e.name) + } + } + } + // The macros a file mounted with `#[macro_use]` hands the file mounting it: its own and, in + // turn, its own `#[macro_use] mod`s' (name → defining file). + const exportsMemo = new Map() + const macroExports = (file, guard = new Set()) => { + if (exportsMemo.has(file)) return exportsMemo.get(file) + if (guard.has(file)) return new Map() + guard.add(file) + const out = new Map() + for (const { files: mountedFiles } of macroUseMods.get(file) ?? []) for (const f of mountedFiles) for (const [name, where] of macroExports(f, guard)) out.set(name, where) + for (const name of fileMacros.get(file) ?? []) out.set(name, file) + exportsMemo.set(file, out) + return out + } + // A file's macro events in source order, each `{ offset, at, macros }` (name → defining file): + // its own `macro_rules!` and what its `#[macro_use] mod`s hand it. `offset` is where an item + // stands, `at` where it is written: the same for most, the invocation and the place in the + // template for what a `macro_rules!` emits (serde's `crate_root! { … macro_rules! tri { … } … + // pub mod de; … }`: `tri` is in scope in `de`, both standing at the invocation). + const eventsMemo = new Map() + const fileEvents = (file) => { + let events = eventsMemo.get(file) + if (events === undefined) { + events = [ + ...(scanned.get(file)?.macros ?? []).map((m) => ({ offset: m.offset, at: m.at, macros: new Map([[m.name, file]]) })), + ...(macroUseMods.get(file) ?? []).map((u) => ({ offset: u.offset, at: u.at, macros: new Map(u.files.flatMap((f) => [...macroExports(f)])) })), + ].toSorted((a, b) => a.offset - b.offset || a.at - b.at) + eventsMemo.set(file, events) + } + return events + } + const before = (e, offset, at) => e.offset < offset || (e.offset === offset && e.at < at) + // The macros in textual scope in `file` before position (`offset`, `at`) -- (Infinity, + // Infinity): its end -- on top of `scope`, what it sees from above: the later definition of a + // name shadows the earlier, whichever kind each is. + const textualScope = (file, offset, at, scope) => { + const out = new Map(scope) + for (const e of fileEvents(file)) { + if (!before(e, offset, at)) break + for (const [name, where] of e.macros) out.set(name, where) + } + return out + } + // The file defining the macro `name` in scope at (`offset`, `at`) in `file`: the last event + // before that point defining it, else what the file sees from above (`scope`); a definition + // later in the file doesn't reach an earlier call. + const macroAt = (file, name, offset, at, scope) => { + const events = fileEvents(file) + for (let i = events.length - 1; i >= 0; i--) { + if (!before(events[i], offset, at)) continue + const where = events[i].macros.get(name) + if (where !== undefined) return where + } + return scope.get(name) + } + // What a file sees from above: what its parent sees from above, and the parent's textual scope + // at the `mod` mounting it. + const inheritedMemo = new Map() + const inherited = (file, guard = new Set()) => { + if (inheritedMemo.has(file)) return inheritedMemo.get(file) + const mount = mounts.get(file) + if (mount === undefined || guard.has(file)) return new Map() + guard.add(file) + const out = textualScope(mount.parent, mount.offset, mount.at, inherited(mount.parent, guard)) + inheritedMemo.set(file, out) + return out + } + // Crate names resolve per package (a path dep's, a rename), so per file; the same (name, file) + // comes up once per reference to the crate, and every glob import's source resolution asks. + const crates = new Map() + const resolveCrate = (name, from) => { + const key = `${name}\0${from}` + if (!crates.has(key)) crates.set(key, ctx?.resolveCrate(name, from) ?? resolveVendoredCrate(name, { knownSources: sources })) + return crates.get(key) + } + // One context for every path of every file: `provided` memoizes across them (see walkPath). + // Per crate root, name → the modules with anything of the name (a child module, an item, an + // import binding it; the root for an exported macro): what hasAll may answer for. + const having = new Map() + for (const [root, byParent] of children) for (const [parent, names] of byParent) for (const name of names) note(having, root, name, parent) + for (const [root, byModule] of defined) for (const [module, names] of byModule) for (const name of names.keys()) note(having, root, name, module) + for (const [root, byModule] of imports) for (const [module, of] of byModule) for (const name of of.named.keys()) note(having, root, name, module) + for (const [root, byName] of macros) for (const name of byName.keys()) note(having, root, name, 'crate') + const pathCtx = { trees, children, files, resolveCrate, imports, defined, modScope, macros, fileMacros, externPrelude, leafSets, having, provided: new Map(), closures: new Map(), opaque: new Map(), opaqueGlobs: new Map(), walking: 0, hits: new Set() } + // Every path of every file, on top of the `mod` / include edges (`resolutions`): once, and once + // more when the first pass left glob closures short of an import cycle (stabilizeClosures). + const resolveAll = () => { + const out = new Map() + const unresolved = new Set() + const wanted = new Set() + for (const [path, items] of scanned) { + const specMap = new Map(resolutions.get(path)) + out.set(path, specMap) + resolveFile(path, items, specMap, unresolved, wanted) + } + return { out, unresolved, wanted } + } + const resolveFile = (path, items, specMap, unresolved, wanted) => { // Only edges to bundled files, never to self, first spelling wins. const add = (spec, target) => { if (target && target !== path && sources.has(target) && !specMap.has(spec)) specMap.set(spec, target) } - const resolveCrate = (name, from) => ctx?.resolveCrate(name, from) ?? resolveVendoredCrate(name, { knownSources: sources }) + // A crate resolved to a root the walk didn't load (its name was bound in the walking file as + // a value, see resolvePathRef): the caller may load it and build again. + const addCrate = (name, target) => { + if (target && !sources.has(target)) wanted.add(target) + add(`use ${name}`, target) + } // A crate name nothing in-tree satisfied -- sysroot crates aside, and an uppercase lead is an // `Enum::Variant` import, not a crate. const noteUnresolved = (name) => { - if (!OTHER_SYSROOT_CRATES.has(name) && /^[a-z_]/u.test(name)) unresolvedCrates.add(name) + if (!OTHER_SYSROOT_CRATES.has(name) && /^[a-z_]/u.test(name)) unresolved.add(name) } + const here = files.get(path) + const via = new Map() // file → the prefix of a path that went through an import of a module there for (const ref of items.refs) { - const r = resolvePathRef(ref, path, { trees, files, resolveCrate, bindings: items.bindings }) + const r = resolvePathRef(ref, path, items.bindings, pathCtx) if (r === null) continue - if (r.kind === 'module') add(ref.spec, r.target) - else if (r.kind === 'crate') add(`use ${r.name}`, r.target) - // Only a `use` reliably says its lead names a crate (see scanRustItems' `fromUse`). - else if (ref.fromUse) noteUnresolved(r.name) + if (r.kind === 'crate') addCrate(r.name, r.file) + else if (r.kind === 'unresolved') { + // Only a `use` reliably says its lead names a crate (see scanRustItems' `fromUse`). + if (ref.fromUse) noteUnresolved(r.name) + continue + } else add(ref.macroCall ? `${ref.spec}!` : ref.spec, r.file) // `crate::helper!()` beside `crate::helper()`: two edges + // The module an import was followed from, when the path's own segments name it (`super`, + // `crate::util`) and its file is neither this one nor the crate root, which every file of + // the crate hangs off anyway. + const v = r.via + if (v && v.consumed > 0 && v.through !== path && v.through !== here.root && v.through !== r.file && !via.has(v.through)) { + via.set(v.through, ref.segments.slice(0, v.consumed).join('::')) + } } for (const { name } of items.externCrates) { if (NON_CRATE_LEADS.has(name)) continue // `extern crate self as x;` / `extern crate alloc;` const target = resolveCrate(name, path) - if (target) add(`use ${name}`, target) + if (target) addCrate(name, target) else noteUnresolved(name) } + // A macro invoked by bare name and defined by `macro_rules!` in textual scope (see mounts): + // this file's own, what the files above hand down (bytes' `fmt_impl!`, from fmt/mod.rs in + // fmt/debug.rs), what a `#[macro_use] mod` of this file brings in (tokio's `ready!`, from + // macros/ready.rs), and -- in the crate root's file, whose items they are -- the exported ones; + // else one the module's imports bring in by path (`use crate::combinator::dispatch;`, a `use + // super::*` from the root). (A `macro_rules! vec` in some sibling file of the crate is not + // what `vec!` names.) + if (here !== undefined) { + const above = inherited(path) + const exported = here.modulePath === 'crate' ? macros.get(here.root) : undefined + for (const [name, offsets] of items.calls) { + // Scope is decided at each call: a `macro_rules!` later in the file doesn't shadow what an + // earlier `#[macro_use] mod` handed an earlier call. + const found = new Set() + for (const offset of offsets) { + let macroFile = macroAt(path, name, offset, offset, above) ?? exported?.get(name) + if (macroFile === undefined) { + const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { macroCall: true, file: path, asker: here.asker })?.file + if (byPath !== undefined && fileMacros.get(byPath)?.has(name) === true) macroFile = byPath // a file that defines a macro of that name, not a `fn write` beside `write!` + } + if (macroFile !== undefined) found.add(macroFile) + } + for (const macroFile of found) add(`${name}!`, macroFile) + } + } + // A module whose import a path went through is a dependency of the file too, whatever the + // import led to (bitflags' `$crate::__private::serde::Serialize` needs the root's `__private` + // as much as serde). When nothing else in the file points at that module's file, the path's + // prefix naming the module records it (`crate::__private` → lib.rs); the bare crate root is + // implied by the module tree. + const targets = new Set() + for (const t of specMap.values()) for (const f of t instanceof Map ? t.values() : [t]) targets.add(f) + for (const [owner, spec] of via) if (!targets.has(owner)) add(spec, owner) } - return { sources, resolutions, missing, unresolvedCrates } + let { out, unresolved, wanted } = resolveAll() + if (stabilizeClosures(pathCtx)) ({ out, unresolved, wanted } = resolveAll()) + return { sources, resolutions: out, missing, unresolvedCrates: unresolved, wantedRoots: [...wanted] } } // --- Disk walk -------------------------------------------------------------------------- @@ -867,14 +2540,37 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // Walk from `entries` (the crate roots), following `mod` declarations and references to in-tree // crates, reading each reachable file once; a missing file warns and is skipped without aborting // the walk (buildRustTree decides what is fatal). `cargo` (cargo.js) resolves crate names and -// carries each crate's features; one is created for `entries` when not passed in. -export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null } = {}) { - const sources = new Map() +// carries each crate's features; one is created for `entries` when not passed in. Pass `sources` +// to walk further roots into an earlier walk's map (files already there are not read again). +export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, formats = null, sources = new Map(), hostFiles = new Set() } = {}) { const realBase = realpathSync(baseDir) const roots = new Set(entries) const pathLoaded = new Set() // files a `#[path]` named: their submodules sit beside them + const assets = new Map() // files an `include_str!` / `include_bytes!` names: carried as they are, not scanned + const rustFiles = new Set(entries) // files named as Rust source (a `mod`, an `include!`): scanned, never an asset const ctx = cargo ?? createCargoContext(baseDir, { entries }) const modOpts = { baseDir, roots, pathLoaded } + // file → the directory it must really lie in (boundaryOf, from the file that named it): a + // vendored crate's files stay in its package, symlinks included. + const boundaries = new Map(entries.map((e) => [e, boundaryOf(e, ctx)])) + // An include macro in a `macro_rules!` body names a file relative to each file invoking the + // macro: definitions and invocations arrive in any order, so each side is kept and paired. + const templates = new Map() // macro name → its body's includes + const invokers = new Map() // macro name → the files invoking it by bare name + // A package defining a `macro_rules!` of a template macro's name (its own `quote!`) invokes that + // one everywhere in the package (see buildRustTree): its files are scanned without the name in + // the template set -- again, for the ones scanned before the definition was met. + const packageOf = (rel) => ctx.packageInfo(rel)?.dir ?? '.' + const ownTemplates = new Map() // package → the template names it defines + const skippedIn = new Map() // package → its files that skipped a template body + const reduced = new Map() // per package's own set: TEMPLATE_MACROS less those, one object + const scanFor = (relPath, content) => { + const own = ownTemplates.get(packageOf(relPath)) + const build = buildOf(relPath, ctx, hostFiles) + if (own === undefined) return cachedScan(sources, relPath, content, build) + const set = reduced.get(own) ?? reduced.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) + return cachedScan(sources, relPath, content, { ...build, templates: set }) + } const processWave = async (wave) => { const toLoad = [...new Set(wave)].filter((p) => !sources.has(p)) @@ -883,7 +2579,23 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null toLoad.map(async (relPath) => { try { assertRealPathWithinBase(realBase, baseDir, relPath) - return [relPath, await readFile(join(baseDir, relPath), 'utf8')] + // A vendored crate's file must really lie in its package: a symlink out of it (to the + // project's `.env`, say) is refused like an include naming the target outright. + const within = boundaries.get(relPath) + if (within !== undefined && within !== '.') { + const rel = relative(realpathSync(join(baseDir, within)), realpathSync(join(baseDir, relPath))).split(/[\\/]/u).join('/') + if (rel.startsWith('..') || isAbsolute(rel)) { + console.warn(`[loader.rust] Refusing file outside its package: ${relPath} (a link out of ${within})`) + return null + } + } + const kind = assets.get(relPath) + if (kind === undefined) return [relPath, await readFile(join(baseDir, relPath), 'utf8'), false] + // `include_str!` embeds UTF-8 text (rustc rejects anything else); `include_bytes!` any bytes. + const buf = await readFile(join(baseDir, relPath)) + const utf8 = kind === 'include_str' || isUtf8(buf) + formats?.set(relPath, utf8 ? 'resource' : 'resource:base64') + return [relPath, utf8 ? buf.toString('utf8') : buf.toString('base64'), true] } catch (err) { if (err.code === 'ENOENT') { console.warn(`[loader.rust] Missing file: ${relPath}`) @@ -899,18 +2611,72 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null }) ) const next = [] - for (const entry of reads) { - if (!entry) continue - const [relPath, content] = entry - sources.set(relPath, content) - const build = { features: ctx.featuresFor(relPath), test: isTestTarget(relPath, ctx) } - const { mods, refs, externCrates, bindings } = cachedScan(sources, relPath, content, build) + // What a Rust file names, queued for the next wave. Run again for a file whose package turned + // out to define a template macro (below): the file's `quote!` bodies are its own then. + const handle = (relPath, content) => { + // What this file names is compiled as it is: for the host when it is a build script's, and + // -- for a vendored crate's -- within its package (`boundary`). + const host = hostFiles.has(relPath) + const boundary = boundaryOf(relPath, ctx) + const queue = (file, { rust, within = boundary }) => { + if (rust) { + rustFiles.add(file) + // Named as Rust source after being carried as an asset: read again, as source. + if (assets.delete(file) && sources.has(file)) { + sources.delete(file) + formats?.delete(file) + } + } + if (!boundaries.has(file)) boundaries.set(file, within) + if (sources.has(file)) return + if (host) hostFiles.add(file) + next.push(file) + } + const includeOf = (inc, from) => { + const file = includeTarget(inc, from, ctx) + if (file === null || file === from) return + // A FIFO or device named would hang the read; a missing file is the tree pass's to report. + if (!sources.has(file) && !isFile(join(baseDir, file))) return + if (inc.kind === 'include') { + pathLoaded.add(file) // its own `mod`s resolve beside it, as rustc has it + queue(file, { rust: true }) + } else if (!rustFiles.has(file) && !sources.has(file)) { + assets.set(file, inc.kind) + queue(file, { rust: false }) + } + } + const items = scanFor(relPath, content) + const { mods, refs, externCrates, bindings, includes, macros, invocations } = items + const pkg = packageOf(relPath) + if (items.skipped.size > 0) (skippedIn.get(pkg) ?? skippedIn.set(pkg, new Set()).get(pkg)).add(relPath) + for (const m of macros) { + if (!TEMPLATE_MACROS.has(m.name)) continue + const own = ownTemplates.get(pkg) ?? ownTemplates.set(pkg, new Set()).get(pkg) + if (own.has(m.name)) continue + own.add(m.name) + for (const f of skippedIn.get(pkg) ?? []) if (f !== relPath) handle(f, sources.get(f)) + } + // An `include!`d file's `mod` declarations resolve beside that file (it is in `pathLoaded` + // from when it was queued), within its package. for (const decl of mods) { - for (const { file, explicit } of resolveModDecl(decl, relPath, modOpts)) { + for (const { file, explicit } of resolveModDecl(decl, relPath, { ...modOpts, boundary })) { if (explicit) pathLoaded.add(file) - if (!sources.has(file)) next.push(file) + queue(file, { rust: true }) } } + // `include!("x.rs")` splices Rust source (scanned next wave, under this module); the asset + // macros name a file to carry as it is. A build-output path (`concat!(env!("OUT_DIR"), …)`) + // never gets here. + for (const inc of includes) includeOf(inc, relPath) + for (const m of macros) { + if (m.includes.length === 0) continue + templates.set(m.name, m.includes) + for (const f of invokers.get(m.name) ?? []) for (const inc of m.includes) includeOf(inc, f) + } + for (const name of invocations) { + ;(invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(relPath) + for (const inc of templates.get(name) ?? []) includeOf(inc, relPath) + } // A reference to an in-tree crate pulls its root in (a crate root by role, whatever its // name -- `[lib] path` may point anywhere); the root's own `mod` edges follow next wave. const leads = new Set() @@ -923,9 +2689,15 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null const lib = ctx.resolveCrate(name, relPath) if (!lib) continue roots.add(lib) - if (!sources.has(lib)) next.push(lib) + queue(lib, { rust: true, within: boundaryOf(lib, ctx) }) } } + for (const entry of reads) { + if (!entry) continue + const [relPath, content, asset] = entry + sources.set(relPath, content) + if (!asset) handle(relPath, content) + } await processWave(next) } @@ -958,6 +2730,7 @@ export async function loadRust(rsTxtFile) { for (const line of lines) assertWithinBase(baseDir, line, 'Entry path') const cargo = createCargoContext(baseDir, { entries: lines }) - const sources = await collectRustFilesFromDisk(baseDir, lines, { cargo }) - return buildRustTree(sources, { roots: lines, baseDir, cargo }) + const formats = new Map() + const sources = await collectRustFilesFromDisk(baseDir, lines, { cargo, formats }) + return { ...buildRustTree(sources, { roots: lines, baseDir, cargo, formats }), formats } } diff --git a/tests/bundle-cmd.test.js b/tests/bundle-cmd.test.js index a467e74c..29ed11a0 100644 --- a/tests/bundle-cmd.test.js +++ b/tests/bundle-cmd.test.js @@ -3947,11 +3947,61 @@ test('CLI: bundle rejects the --cargo-* feature flags for a non-Rust bundle, and t.assert.match(runCli(['bundle', '--cargo-features=x', 'main.sh'], { cwd }).stderr, /--cargo-features is only valid for Rust bundles/u) t.assert.match(runCli(['bundle', '--cargo-no-default-features', 'main.sh'], { cwd }).stderr, /--cargo-no-default-features is only valid for Rust bundles/u) t.assert.match(runCli(['bundle', '--cargo-all-features', 'main.sh'], { cwd }).stderr, /--cargo-all-features is only valid for Rust bundles/u) + t.assert.match(runCli(['bundle', '--cargo-target=host', 'main.sh'], { cwd }).stderr, /--cargo-target is only valid for Rust bundles/u) + t.assert.match(runCli(['bundle', '--cargo-manifests', 'main.sh'], { cwd }).stderr, /--cargo-manifests is only valid for Rust bundles/u) const empty = runCli(['bundle', '--cargo-features=,', 'src/main.rs'], { cwd: join(rustFixtures, 'features') }) t.assert.equal(empty.status, 1) t.assert.match(empty.stderr, /--cargo-features must list at least one feature/u) + const target = runCli(['bundle', '--cargo-target=x86_64 linux', 'src/main.rs'], { cwd: join(rustFixtures, 'features') }) + t.assert.equal(target.status, 1) + t.assert.match(target.stderr, /--cargo-target must be a target triple or "host"/u) }) +test('buildBundle rejects --cargo-target and --cargo-manifests for a non-Rust bundle', async (t) => { + await t.assert.rejects( + () => buildBundle({ cwd: join(bashFixtures, 'basic'), entries: ['main.sh'], cargoTarget: 'host' }), + /--cargo-target is only valid for Rust bundles/u, + ) + await t.assert.rejects( + () => buildBundle({ cwd: join(bashFixtures, 'basic'), entries: ['main.sh'], cargoManifests: true }), + /--cargo-manifests is only valid for Rust bundles/u, + ) +}) + +test('CLI: bundle --cargo-manifests adds the package manifest, lockfile and build script to a Rust bundle', withTmp((t, tmp) => { + const outPath = join(tmp, 'out.stasis.code.br') + const cwd = join(rustFixtures, 'includes') + const plain = runCli(['bundle', '-o', outPath, 'src/lib.rs'], { cwd }) + t.assert.equal(plain.status, 0, plain.stderr) + t.assert.match(plain.stderr, /Bundled 7 files in 1 package/u) + const withManifests = runCli(['bundle', '--cargo-manifests', '-o', outPath, 'src/lib.rs'], { cwd }) + t.assert.equal(withManifests.status, 0, withManifests.stderr) + t.assert.match(withManifests.stderr, /Bundled 11 files in 1 package/u) + const parsed = Bundle.parse(brotliDecompressSync(readFileSync(outPath)).toString('utf8')) + t.assert.deepEqual([...parsed.sources.keys()].toSorted(), [ + 'Cargo.lock', 'Cargo.toml', 'README.md', 'build.rs', 'build/helper.rs', 'data/blob.bin', 'data/table.txt', + 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs', + ]) + t.assert.equal(parsed.formats.get('Cargo.toml'), 'resource') + t.assert.equal(parsed.formats.get('build.rs'), 'rust') + t.assert.equal(parsed.imports.get('rust').get('build.rs').get('mod helper'), 'build/helper.rs') + t.assert.deepEqual([...parsed.entries], ['src/lib.rs']) +})) + +const hasRustc = spawnSync('rustc', ['--version'], { stdio: 'ignore' }).status === 0 + +test('CLI: bundle --cargo-target keeps only the named target\'s #[cfg_attr(…, path)] variant', { skip: hasRustc ? false : 'rustc not on PATH' }, withTmp((t, tmp) => { + const outPath = join(tmp, 'out.stasis.code.br') + const r = runCli(['bundle', '--cargo-target=x86_64-unknown-linux-gnu', '-o', outPath, 'src/lib.rs'], { cwd: join(rustFixtures, 'path-attr') }) + t.assert.equal(r.status, 0, r.stderr) + const parsed = Bundle.parse(brotliDecompressSync(readFileSync(outPath)).toString('utf8')) + t.assert.ok(parsed.sources.has('src/sys/unix.rs')) + // Neither the windows variant nor the default `src/sys.rs`: the unix `#[path]` applies outright. + t.assert.ok(!parsed.sources.has('src/sys/windows.rs')) + t.assert.ok(!parsed.sources.has('src/sys.rs')) + t.assert.equal(parsed.imports.get('rust').get('src/lib.rs').get('mod sys'), 'src/sys/unix.rs') +})) + test('CLI: EXODUS_STASIS_DEBUG=1 prints the resolved Rust features per package', (t) => { const r = runCli(['bundle', '-o', '/dev/null', 'src/main.rs'], { cwd: join(rustFixtures, 'features'), env: { ...cleanEnv, EXODUS_STASIS_DEBUG: '1' } }) t.assert.equal(r.status, 0, r.stderr) diff --git a/tests/fixtures/rust-bundle/includes/Cargo.lock b/tests/fixtures/rust-bundle/includes/Cargo.lock new file mode 100644 index 00000000..fc3da2fe --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "inc" +version = "0.1.0" diff --git a/tests/fixtures/rust-bundle/includes/Cargo.toml b/tests/fixtures/rust-bundle/includes/Cargo.toml new file mode 100644 index 00000000..28a41d09 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "inc" +version = "0.1.0" +edition = "2021" diff --git a/tests/fixtures/rust-bundle/includes/README.md b/tests/fixtures/rust-bundle/includes/README.md new file mode 100644 index 00000000..898d009c --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/README.md @@ -0,0 +1,3 @@ +# inc + +A fixture crate. diff --git a/tests/fixtures/rust-bundle/includes/build.rs b/tests/fixtures/rust-bundle/includes/build.rs new file mode 100644 index 00000000..9561dc40 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/build.rs @@ -0,0 +1,8 @@ +// The package's build script: a crate root of its own, with a module beside it. +#[path = "build/helper.rs"] +mod helper; + +fn main() { + helper::emit(); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/tests/fixtures/rust-bundle/includes/build/helper.rs b/tests/fixtures/rust-bundle/includes/build/helper.rs new file mode 100644 index 00000000..6b1e61a9 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/build/helper.rs @@ -0,0 +1 @@ +pub fn emit() {} diff --git a/tests/fixtures/rust-bundle/includes/data/blob.bin b/tests/fixtures/rust-bundle/includes/data/blob.bin new file mode 100644 index 0000000000000000000000000000000000000000..717c6b3086b9625de0ba7338b4f3d5fdc3ca8a0b GIT binary patch literal 4 LcmZSh|Bn#>1pWd4 literal 0 HcmV?d00001 diff --git a/tests/fixtures/rust-bundle/includes/data/table.txt b/tests/fixtures/rust-bundle/includes/data/table.txt new file mode 100644 index 00000000..cfa20f81 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/data/table.txt @@ -0,0 +1,2 @@ +a,b +1,2 diff --git a/tests/fixtures/rust-bundle/includes/src/gated.rs b/tests/fixtures/rust-bundle/includes/src/gated.rs new file mode 100644 index 00000000..e3b036e2 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/src/gated.rs @@ -0,0 +1,4 @@ +#![cfg(feature = "never")] + +mod ghost; +use phantom_crate::Thing; diff --git a/tests/fixtures/rust-bundle/includes/src/generated/consts.rs b/tests/fixtures/rust-bundle/includes/src/generated/consts.rs new file mode 100644 index 00000000..8dc64d84 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/src/generated/consts.rs @@ -0,0 +1 @@ +pub const LIMIT: u32 = 3; diff --git a/tests/fixtures/rust-bundle/includes/src/lib.rs b/tests/fixtures/rust-bundle/includes/src/lib.rs new file mode 100644 index 00000000..2d51fdbc --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/src/lib.rs @@ -0,0 +1,18 @@ +#![doc = include_str!("../README.md")] + +#[macro_use] +mod macros; + +include!("generated/consts.rs"); + +pub const TABLE: &str = include_str!("../data/table.txt"); +pub const BLOB: &[u8] = include_bytes!("../data/blob.bin"); + +pub fn go() -> u32 { + ready!(1) + LIMIT +} + +#[cfg(test)] +const TEST_ONLY: &str = include_str!("../data/test-only.txt"); + +mod gated; diff --git a/tests/fixtures/rust-bundle/includes/src/macros.rs b/tests/fixtures/rust-bundle/includes/src/macros.rs new file mode 100644 index 00000000..f7692b32 --- /dev/null +++ b/tests/fixtures/rust-bundle/includes/src/macros.rs @@ -0,0 +1,5 @@ +macro_rules! ready { + ($e:expr) => { + $e + }; +} diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index 5b85578e..f4332814 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -1,20 +1,24 @@ import { test } from 'node:test' import { spawnSync } from 'node:child_process' -import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { homedir, tmpdir } from 'node:os' +import { dirname, join, parse } from 'node:path' import { fileURLToPath } from 'node:url' import { createCargoContext, + evalCfg, findCargoLock, parseCargoLock, parseCargoManifest, parseFeatureList, + parseRustcCfg, resolutionFromMetadata, + rustcTargetCfgs, satisfiesCargoReq, } from '../stasis/src/loaders/cargo.js' import { buildRustBundle } from '../stasis/src/cmd/bundle.js' +import { buildRustTree } from '../stasis/src/loaders/rust.js' const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'rust-bundle') const featuresFixture = join(fixtures, 'features') @@ -33,9 +37,10 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin '[build-dependencies]', 'cc = "1"', "[target.'cfg(unix)'.dependencies]", 'nix = "0.29"', '[features]', 'default = ["std"]', 'std = []', 'full = [', ' "std",', ' "dep:opt",', ' "sub/two",', ' "opt?/extra",', ']', - '[patch.crates-io]', 'plain = { path = "patches/plain" }', + '[patch.crates-io]', 'plain = { path = "patches/plain" }', 'dotted.path = "patches/dotted"', + '[patch.crates-io.subbed]', 'path = "patches/subbed"', ].join('\n')) - t.assert.deepEqual(m.package, { name: 'app', version: '0.1.0', versionFromWorkspace: false, edition: '2021' }) + t.assert.deepEqual(m.package, { name: 'app', version: '0.1.0', versionFromWorkspace: false, edition: '2021', editionFromWorkspace: false, build: null }) t.assert.equal(m.resolver, '2') t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']]]) const dep = (k) => { @@ -51,10 +56,20 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin t.assert.deepEqual(dep('opt'), { version: '1', kinds: { normal: { optional: true, defaultFeatures: false, features: ['a'] } } }) t.assert.deepEqual(dep('sub'), { version: '2', kinds: { normal: { optional: false, defaultFeatures: null, features: ['one', 'two'] } } }) t.assert.deepEqual(Object.keys(dep('cc').kinds), ['build']) - t.assert.deepEqual(Object.keys(dep('nix').kinds), ['normal']) + // A target-specific table is a request of its own, beside the plain one. + t.assert.deepEqual(Object.keys(dep('nix').kinds), ['normal@cfg(unix)']) // the key is the `use` spelling, the name the manifest's (an optional dep's implicit feature name) t.assert.deepEqual([m.deps.get('pm_crate').key, m.deps.get('pm_crate').name, m.deps.get('pm_crate').kinds.get('normal').optional], ['pm_crate', 'pm-crate', true]) - t.assert.deepEqual([...m.patches], [['plain', 'patches/plain']]) + // `[patch.]` entries in every spelling: inline table, dotted key, sub-table. + t.assert.deepEqual([...m.patches], [['plain', 'patches/plain'], ['dotted', 'patches/dotted'], ['subbed', 'patches/subbed']]) +}) + +test('parseCargoManifest reads a workspace-inherited edition', (t) => { + const m = parseCargoManifest(['[workspace]', '[workspace.package]', 'edition = "2021"', '[package]', 'name = "app"', 'version = "0.1.0"', 'edition.workspace = true'].join('\n')) + t.assert.equal(m.package.edition, null) + t.assert.equal(m.package.editionFromWorkspace, true) + t.assert.equal(m.workspacePackage.edition, '2021') + t.assert.equal(parseCargoManifest('[package]\nname = "app"\nedition = { workspace = true }\n').package.editionFromWorkspace, true) }) test('parseCargoManifest splits dotted dependency keys and survives multi-line strings', (t) => { @@ -88,8 +103,8 @@ test('parseCargoManifest reads a pair by the table it lands in, whichever way th '[patch.crates-io.plain]', 'path = "patches/plain"', '[workspace.package]', 'version = "0.9.0"', // a [workspace.*] table alone makes this a workspace root ].join('\n')) - t.assert.deepEqual(m.package, { name: 'app', version: null, versionFromWorkspace: true, edition: null }) - t.assert.deepEqual([m.deps.get('winapi').version, [...m.deps.get('winapi').kinds.keys()]], ['0.3', ['dev']]) + t.assert.deepEqual(m.package, { name: 'app', version: null, versionFromWorkspace: true, edition: null, editionFromWorkspace: false, build: null }) + t.assert.deepEqual([m.deps.get('winapi').version, [...m.deps.get('winapi').kinds.keys()]], ['0.3', ['dev@cfg(windows)']]) t.assert.deepEqual([...m.patches], [['plain', 'patches/plain']]) t.assert.deepEqual([m.isWorkspace, m.workspacePackage.version], [true, '0.9.0']) }) @@ -120,14 +135,24 @@ test('parseFeatureList splits cargo\'s repeatable, comma- or space-separated fea t.assert.deepEqual(parseFeatureList([]), []) }) -test('findCargoLock finds the lock in the bundle root or an ancestor (a member dir\'s is its workspace\'s)', (t) => { +test('findCargoLock finds the lock beside the workspace root (a member dir\'s is its workspace\'s), never one further up', (t) => { const tmp = mkdtempSync(join(tmpdir(), 'stasis-lock-')) try { - mkdirSync(join(tmp, 'crates', 'app'), { recursive: true }) - t.assert.equal(findCargoLock(join(tmp, 'crates', 'app')), null) - writeFileSync(join(tmp, 'Cargo.lock'), 'version = 3\n') - t.assert.equal(findCargoLock(join(tmp, 'crates', 'app')), join(tmp, 'Cargo.lock')) - t.assert.equal(findCargoLock(tmp), join(tmp, 'Cargo.lock')) + mkdirSync(join(tmp, 'ws', 'crates', 'app'), { recursive: true }) + writeFileSync(join(tmp, 'Cargo.lock'), 'version = 3\n') // some other project's, above the workspace + writeFileSync(join(tmp, 'ws', 'Cargo.toml'), '[workspace]\nmembers = ["crates/app"]\n') + writeFileSync(join(tmp, 'ws', 'crates', 'app', 'Cargo.toml'), '[package]\nname = "app"\nversion = "0.1.0"\n') + t.assert.equal(findCargoLock(join(tmp, 'ws', 'crates', 'app')), null) + t.assert.equal(findCargoLock(join(tmp, 'ws')), null) + writeFileSync(join(tmp, 'ws', 'Cargo.lock'), 'version = 3\n') + t.assert.equal(findCargoLock(join(tmp, 'ws', 'crates', 'app')), join(tmp, 'ws', 'Cargo.lock')) + t.assert.equal(findCargoLock(join(tmp, 'ws')), join(tmp, 'ws', 'Cargo.lock')) + // A package outside any workspace: its own lock, and nothing above it. + mkdirSync(join(tmp, 'solo')) + writeFileSync(join(tmp, 'solo', 'Cargo.toml'), '[package]\nname = "solo"\nversion = "0.1.0"\n') + t.assert.equal(findCargoLock(join(tmp, 'solo')), null) + writeFileSync(join(tmp, 'solo', 'Cargo.lock'), 'version = 3\n') + t.assert.equal(findCargoLock(join(tmp, 'solo')), join(tmp, 'solo', 'Cargo.lock')) } finally { rmSync(tmp, { recursive: true, force: true }) } @@ -159,6 +184,13 @@ test('satisfiesCargoReq implements Cargo requirement semantics', (t) => { t.assert.equal(satisfiesCargoReq('0.0.4', '0.0.3'), false) t.assert.equal(satisfiesCargoReq('0.5.0', '^0'), true) t.assert.equal(satisfiesCargoReq('1.0.0', '^0'), false) + // a wildcard with an operator keeps the operator: the `*` only omits the component + t.assert.equal(satisfiesCargoReq('3.0.0', '>=1.*'), true) + t.assert.equal(satisfiesCargoReq('1.5.0', '>1.*'), false) + t.assert.equal(satisfiesCargoReq('2.0.0', '>1.*'), true) + t.assert.equal(satisfiesCargoReq('1.2.9', '=1.2.*'), true) + t.assert.equal(satisfiesCargoReq('1.3.0', '=1.2.*'), false) + t.assert.equal(satisfiesCargoReq('2.0.0', '1.*'), false) // tilde, wildcard, exact, comparisons, several comparators t.assert.equal(satisfiesCargoReq('1.2.9', '~1.2.3'), true) t.assert.equal(satisfiesCargoReq('1.3.0', '~1.2.3'), false) @@ -234,6 +266,127 @@ test('createCargoContext falls back to the requirement when there is no Cargo.lo } }) +// A throwaway project: `files` maps project-relative paths to their text. +const writeProject = (files) => { + const tmp = mkdtempSync(join(tmpdir(), 'stasis-cargo-')) + for (const [rel, text] of Object.entries(files)) { + mkdirSync(dirname(join(tmp, rel)), { recursive: true }) + writeFileSync(join(tmp, rel), text) + } + return tmp +} +const withProject = (files, fn) => { + const tmp = writeProject(files) + try { + return fn(tmp) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} +const captureWarningsAsync = async (fn) => { + const original = console.warn + const warnings = [] + console.warn = (...args) => warnings.push(args.join(' ')) + try { + return { result: await fn(), warnings } + } finally { + console.warn = original + } +} +const withProjectAsync = async (files, fn) => { + const tmp = writeProject(files) + try { + return await fn(tmp) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} + +test('createCargoContext applies a bare --cargo-features name to every root package that has it', (t) => { + withProject({ + 'Cargo.toml': '[workspace]\nmembers = ["a", "b"]\n', + 'a/Cargo.toml': '[package]\nname = "a"\nversion = "0.1.0"\nedition = "2021"\n[features]\nserde = []\n', + 'a/src/lib.rs': '', + 'b/Cargo.toml': '[package]\nname = "b"\nversion = "0.1.0"\nedition = "2021"\n[features]\nserde = []\n', + 'b/src/lib.rs': '', + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['a/src/lib.rs', 'b/src/lib.rs'], features: ['serde'] }) + t.assert.deepEqual(enabledOf(cargo), { a: ['serde'], b: ['serde'] }) + }) +}) + +test('createCargoContext keeps a target-specific dependency table apart from the plain one', (t) => { + withProject({ + 'Cargo.toml': ['[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', + '[dependencies]', 'dep = "1"', + "[target.'cfg(unix)'.dependencies]", 'dep = { version = "1", default-features = false, optional = true }', + "[target.'cfg(windows)'.dependencies]", 'win = "1"'].join('\n'), + 'src/main.rs': '', + 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\n[features]\ndefault = ["std"]\nstd = []\n', + 'vendor/dep/src/lib.rs': '', + 'vendor/win/Cargo.toml': '[package]\nname = "win"\nversion = "1.0.0"\n[features]\ndefault = ["api"]\napi = []\n', + 'vendor/win/src/lib.rs': '', + }, (tmp) => { + // The plain `dep = "1"` keeps its defaults and stays in the graph; the unix table's optional, + // default-less request is a second one. A target-only dependency counts (target cfgs are unknown). + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.deepEqual(enabledOf(cargo), { '.': [], 'vendor/dep': ['default', 'std'], 'vendor/win': ['api', 'default'] }) + }) +}) + +test('createCargoContext picks the feature resolver from a workspace-inherited edition', (t) => { + const files = (edition) => ({ + 'Cargo.toml': ['[workspace]', '[workspace.package]', `edition = "${edition}"`, '[package]', 'name = "app"', 'version = "0.1.0"', 'edition.workspace = true', + '[dev-dependencies]', 'devdep = "1"'].join('\n'), + 'src/main.rs': '', + 'vendor/devdep/Cargo.toml': '[package]\nname = "devdep"\nversion = "1.0.0"\n', + 'vendor/devdep/src/lib.rs': '', + }) + // edition 2021 → resolver 2: the root's dev-dependencies stay out of a normal build + withProject(files('2021'), (tmp) => t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': [] })) + // edition 2018 → resolver 1: they join + withProject(files('2018'), (tmp) => t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': [], 'vendor/devdep': [] })) +}) + +test('createCargoContext applies a weak `dep?/feat` once the dependency is active, whichever table activates it', (t) => { + const files = (deps) => ({ + 'Cargo.toml': ['[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', + '[features]', 'default = ["std"]', 'std = ["dep?/std"]', 'with-dep = ["dep:dep"]', ...deps].join('\n'), + 'src/main.rs': '', + 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\n[features]\nstd = []\n', + 'vendor/dep/src/lib.rs': '', + }) + // optional and inactive: the weak feature asks nothing + withProject(files(['[dependencies]', 'dep = { version = "1", optional = true }']), (tmp) => { + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'] }) + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], features: ['with-dep'] })), { '.': ['default', 'std', 'with-dep'], 'vendor/dep': ['std'] }) + }) + // required without default features: active from the start + withProject(files(['[dependencies]', 'dep = { version = "1", default-features = false }']), (tmp) => { + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) + }) + // required only on a target: the target table counts + withProject(files(["[target.'cfg(unix)'.dependencies]", 'dep = { version = "1", default-features = false }']), (tmp) => { + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) + }) +}) + +test('createCargoContext applies a weak `dep?/feat` from --cargo-features once `default` has activated the dependency', (t) => { + withProject({ + 'Cargo.toml': ['[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', + '[features]', 'default = ["with-dep"]', 'with-dep = ["dep:dep"]', + '[dependencies]', 'dep = { version = "1", optional = true }'].join('\n'), + 'src/main.rs': '', + 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\n[features]\nstd = []\n', + 'vendor/dep/src/lib.rs': '', + }, (tmp) => { + // `dep` is activated by `default` inside the fixed-point loop; the flag has to wait for it. + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], features: ['dep?/std'] })), { '.': ['default', 'with-dep'], 'vendor/dep': ['std'] }) + // and asks nothing while the dependency stays inactive + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], features: ['dep?/std'], noDefaultFeatures: true })), { '.': [] }) + }) +}) + test('createCargoContext honours the root feature flags: --features (incl. pkg/feat), --no-default-features, --all-features', (t) => { const withSerde = createCargoContext(featuresFixture, { entries: ['src/main.rs'], features: ['with-serde'] }) t.assert.deepEqual(enabledOf(withSerde), { @@ -355,8 +508,413 @@ test('createCargoContext({ cargo: true }) fails loudly when cargo cannot run', { t.assert.throws(() => createCargoContext(featuresFixture, { entries: ['src/main.rs'], cargo: true }), /cargo metadata could not run/u) }) +test('parseCargoManifest reads [package] build', (t) => { + t.assert.equal(parseCargoManifest('[package]\nname = "a"\nversion = "0.1.0"\nbuild = "build/main.rs"\n').package.build, 'build/main.rs') + t.assert.equal(parseCargoManifest('[package]\nname = "a"\nversion = "0.1.0"\nbuild = false\n').package.build, false) + t.assert.equal(parseCargoManifest('[package]\nname = "a"\nversion = "0.1.0"\n').package.build, null) +}) + +test('createCargoContext names each package\'s manifests, build script, and the root\'s lockfile and cargo config', (t) => { + withProject({ + 'Cargo.toml': '[workspace]\nmembers = ["crates/app", "crates/nobuild"]\n', + 'Cargo.lock': 'version = 3\n', + '.cargo/config.toml': '[build]\ntarget-dir = "out"\n', + 'crates/app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'crates/app/build.rs': 'fn main() {}\n', + 'crates/app/src/main.rs': '', + 'crates/nobuild/Cargo.toml': '[package]\nname = "nobuild"\nversion = "0.1.0"\nedition = "2021"\nbuild = false\n', + 'crates/nobuild/build.rs': 'fn main() {}\n', + 'crates/nobuild/src/lib.rs': '', + 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\nbuild = "scripts/build.rs"\n', + 'vendor/dep/scripts/build.rs': 'fn main() {}\n', + 'vendor/dep/src/lib.rs': '', + 'vendor/plain/Cargo.toml': '[package]\nname = "plain"\nversion = "1.0.0"\n', + 'vendor/plain/src/lib.rs': '', + 'loose.rs': '', + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['crates/app/src/main.rs'] }) + const paths = (file) => cargo.buildFilesFor(file).map((f) => `${f.kind}:${f.path}`) + t.assert.deepEqual(paths('crates/app/src/main.rs'), ['manifest:crates/app/Cargo.toml', 'manifest:Cargo.toml', 'lock:Cargo.lock', 'config:.cargo/config.toml']) + t.assert.deepEqual(paths('vendor/dep/src/lib.rs'), ['manifest:vendor/dep/Cargo.toml', 'manifest:Cargo.toml', 'lock:Cargo.lock', 'config:.cargo/config.toml']) // the root workspace is above it too + t.assert.deepEqual(paths('loose.rs'), []) // no [package] claims it: the root manifest is a bare [workspace] + t.assert.equal(cargo.buildScriptOf('crates/app/src/main.rs'), 'crates/app/build.rs') + t.assert.equal(cargo.buildScriptOf('crates/nobuild/src/lib.rs'), null) // `build = false` + t.assert.equal(cargo.buildScriptOf('vendor/dep/src/lib.rs'), 'vendor/dep/scripts/build.rs') + t.assert.equal(cargo.buildScriptOf('vendor/plain/src/lib.rs'), null) // no build.rs on disk + t.assert.equal(cargo.buildScriptOf('loose.rs'), null) + t.assert.equal(cargo.isVendored('vendor/dep/src/lib.rs'), true) + t.assert.equal(cargo.isVendored('crates/app/src/main.rs'), false) + // A member bundled on its own: the workspace's lockfile lies above the bundle root. + const member = createCargoContext(join(tmp, 'crates/app'), { entries: ['src/main.rs'] }) + t.assert.deepEqual(member.buildFilesFor('src/main.rs'), [{ path: 'Cargo.toml', kind: 'manifest' }]) + }) + // A workspace nested below the bundle root: its lockfile and config, not the root's (absent). + withProject({ + 'rust/Cargo.toml': '[workspace]\nmembers = ["app"]\n', + 'rust/Cargo.lock': 'version = 3\n', + 'rust/.cargo/config': '[build]\n', // the extensionless file, which cargo prefers when both exist + 'rust/.cargo/config.toml': '[build]\n', + 'rust/app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n', + 'rust/app/src/main.rs': '', + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['rust/app/src/main.rs'] }) + t.assert.deepEqual(cargo.buildFilesFor('rust/app/src/main.rs').map((f) => f.path), ['rust/app/Cargo.toml', 'rust/Cargo.toml', 'rust/Cargo.lock', 'rust/.cargo/config']) + }) +}) + +test('buildRustBundle --cargo-manifests carries the manifests, lockfile and cargo config as written, or refuses them', async (t) => { + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = { git = "https://x-access-token:TOKEN@github.com/o/foo.git", branch = "main" }\n', + 'Cargo.lock': 'version = 3\n\n[[package]]\nname = "foo"\nversion = "1.0.0"\nsource = "git+https://x-access-token:TOKEN@github.com/o/foo.git?branch=main&access_token=TOKEN#0123abcd"\n', + '.cargo/config.toml': '# vendoring\n[source.crates-io]\nreplace-with = "vendored-sources"\n[source.vendored-sources]\ndirectory = "vendor"\n[registries.private]\ntoken = "t0k3n"\n[target.x86_64-unknown-linux-gnu]\nrunner = "qemu-x86_64"\n', + 'src/main.rs': 'fn main() {}\n', + } + await withProjectAsync(files, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true }) + // Byte for byte, whatever they hold: stasis doesn't edit what it carries. + for (const p of ['Cargo.toml', 'Cargo.lock', '.cargo/config.toml']) t.assert.equal(bundle.sources.get(p), files[p], p) + // A file that isn't UTF-8 text can't be carried as written: refused, not altered. + writeFileSync(join(tmp, 'Cargo.lock'), Buffer.concat([Buffer.from('version = 3\n# '), Buffer.from([0xff, 0xfe]), Buffer.from('\n')])) + await t.assert.rejects(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true }), { message: 'Rust manifest is not valid UTF-8: Cargo.lock' }) + }) +}) + +test('buildRustBundle holds a vendored crate\'s includes, #[path]s and build script to its own package', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nevil = "1"\n', + '.git/config': '[remote "origin"]\n\turl = https://user:token@example.com/repo.git\n', + '.env': 'SECRET=1\n', + 'secret.rs': 'pub const KEY: &str = "k";\n', + 'LICENSE': 'MIT\n', + 'src/main.rs': 'use evil::f;\nconst L: &str = include_str!("../LICENSE");\n', + 'vendor/evil/Cargo.toml': '[package]\nname = "evil"\nversion = "1.0.0"\nbuild = "../../.env"\n', + 'vendor/evil/src/lib.rs': [ + 'pub fn f() {}', + 'const A: &str = include_str!("../../../.git/config");', + 'const B: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/../../.env"));', + '#[cfg(steal)]', // gated on a cfg the loader can't decide: tolerated when nothing is found + '#[path = "../../../secret.rs"]', + 'mod stolen;', + 'const OK: &str = include_str!("../README.md");', + 'mod fine;', + ].join('\n'), + 'vendor/evil/README.md': 'evil\n', + 'vendor/evil/src/fine.rs': '', + }, async (tmp) => { + const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true })) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), [ + 'Cargo.toml', 'LICENSE', 'src/main.rs', // the project's own include may reach anywhere in the root + 'vendor/evil/Cargo.toml', 'vendor/evil/README.md', 'vendor/evil/src/fine.rs', 'vendor/evil/src/lib.rs', + ]) + t.assert.ok(warnings.some((w) => w.includes('Refusing include outside its package: ../../../.git/config from vendor/evil/src/lib.rs')), warnings.join('\n')) + t.assert.ok(warnings.some((w) => w.includes('Refusing include outside its package: ../../.env from vendor/evil/src/lib.rs')), warnings.join('\n')) + t.assert.ok(warnings.some((w) => w.includes('Refusing build script outside its package: ../../.env in vendor/evil')), warnings.join('\n')) + // Without the gate, a `#[path]` outside the package is a missing module: fatal for the bundle. + writeFileSync(join(tmp, 'vendor/evil/src/lib.rs'), '#[path = "../../../secret.rs"]\nmod stolen;\n') + await t.assert.rejects( + () => captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true })), + /Unresolved module: mod stolen from vendor\/evil\/src\/lib\.rs/u, + ) + }) +}) + +test('buildRustBundle holds a vendored crate to its package through symlinks, an inline module\'s #[path] and its [lib] path', { skip: process.platform === 'win32' ? 'symlinks' : false }, async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nevil = "1"\nevil2 = "1"\n', + '.env': 'SECRET=1\n', + 'private_mod.rs': 'pub const KEY: u8 = 1;\n', + 'outside_dir/mod.rs': 'pub const OUT: u8 = 2;\n', + 'src/main.rs': 'use evil::f;\nuse evil2::g;\n', + 'vendor/evil/Cargo.toml': '[package]\nname = "evil"\nversion = "1.0.0"\n', + 'vendor/evil/src/lib.rs': [ + 'pub fn f() {}', + '#[cfg(steal)] #[path = "../../.."] mod m { mod private_mod; }', // an inline module's #[path] climbing out: refused (gated: tolerated) + '#[cfg(steal)] mod linked_file;', // a symlink to a project file + '#[cfg(steal)] mod linked_dir;', // a symlink to a project directory + '#[cfg(steal)] const E: &str = include_str!("env_link");', // a symlink to the project's .env + ].join('\n'), + 'vendor/evil2/Cargo.toml': '[package]\nname = "evil2"\nversion = "1.0.0"\n[lib]\npath = "../../private_mod.rs"\n', + }, async (tmp) => { + symlinkSync(join(tmp, 'private_mod.rs'), join(tmp, 'vendor/evil/src/linked_file.rs')) + symlinkSync(join(tmp, 'outside_dir'), join(tmp, 'vendor/evil/src/linked_dir')) + symlinkSync(join(tmp, '.env'), join(tmp, 'vendor/evil/src/env_link')) + const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true })) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['Cargo.toml', 'src/main.rs', 'vendor/evil/Cargo.toml', 'vendor/evil/src/lib.rs']) + for (const expected of [ + 'Refusing file outside its package: vendor/evil/src/linked_file.rs (a link out of vendor/evil)', + 'Refusing file outside its package: vendor/evil/src/linked_dir/mod.rs (a link out of vendor/evil)', + 'Refusing file outside its package: vendor/evil/src/env_link (a link out of vendor/evil)', + 'Refusing lib path outside its package: ../../private_mod.rs in vendor/evil2', + ]) t.assert.ok(warnings.some((w) => w.includes(expected)), `${expected}\n${warnings.join('\n')}`) + t.assert.ok(!warnings.some((w) => w.includes('private_mod.rs') && w.includes('Refusing file')), warnings.join('\n')) // the inline #[path] never names it: no file to refuse + // Ungated, every refused module is missing: fatal. + writeFileSync(join(tmp, 'vendor/evil/src/lib.rs'), 'pub fn f() {}\n#[path = "../../.."] mod m { mod private_mod; }\nmod linked_file;\nmod linked_dir;\n') + await t.assert.rejects( + () => captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'], cargoManifests: true })), + /Unresolved module: mod m::private_mod from vendor\/evil\/src\/lib\.rs[\s\S]*mod linked_file[\s\S]*mod linked_dir/u, + ) + }) +}) + +test('buildRustBundle scans a file named by both `mod` and `include_str!` as Rust', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': 'const SRC: &str = include_str!("shown.rs");\nmod shown;\nmod util;\n', + 'src/shown.rs': 'use crate::util::helper;\npub fn f() { helper() }\n', + 'src/util.rs': 'pub fn helper() {}\n', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.equal(bundle.formats.get('src/shown.rs'), 'rust') // a module first, its text second + t.assert.deepEqual(Object.fromEntries(bundle.imports.get('rust').get('src/shown.rs')), { 'crate::util::helper': 'src/util.rs' }) + t.assert.deepEqual(Object.fromEntries(bundle.imports.get('rust').get('src/lib.rs')), { 'include_str shown.rs': 'src/shown.rs', 'mod shown': 'src/shown.rs', 'mod util': 'src/util.rs' }) + // Named the other way round, too. + writeFileSync(join(tmp, 'src/lib.rs'), 'mod util;\nmod shown;\nconst SRC: &str = include_str!("shown.rs");\n') + const swapped = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.equal(swapped.formats.get('src/shown.rs'), 'rust') + }) +}) + +test('buildRustBundle --cargo-manifests compiles build scripts for the host, not --cargo-target', async (t) => { + await withProjectAsync({ + 'Cargo.toml': [ + '[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', 'build = "build.rs"', + "[target.'cfg(windows)'.dependencies]", 'win = "1"', + "[target.'cfg(windows)'.build-dependencies]", 'winbuild = "1"', + ].join('\n'), + 'build.rs': '#[cfg(windows)]\nmod registry;\nfn main() { winbuild::probe(); }\n', + 'registry.rs': '', + 'src/lib.rs': '#[cfg(windows)]\nmod win;\n#[cfg(windows)]\nfn f() { win::g(); }\n', + 'src/win.rs': 'pub fn g() {}\n', + 'vendor/win/Cargo.toml': '[package]\nname = "win"\nversion = "1.0.0"\n', + 'vendor/win/src/lib.rs': '', + 'vendor/winbuild/Cargo.toml': '[package]\nname = "winbuild"\nversion = "1.0.0"\n', + 'vendor/winbuild/src/lib.rs': 'pub fn probe() {}\n', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoManifests: true, cargoTarget: LINUX }) + // The library drops its Windows module; the build script keeps its own (the host may be Windows), + // and its Windows-only build-dependency counts. + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['Cargo.toml', 'build.rs', 'registry.rs', 'src/lib.rs', 'vendor/winbuild/Cargo.toml', 'vendor/winbuild/src/lib.rs']) + const edges = bundle.imports.get('rust') + t.assert.deepEqual(Object.fromEntries(edges.get('build.rs')), { 'mod registry': 'registry.rs', 'use winbuild': 'vendor/winbuild/src/lib.rs' }) + t.assert.deepEqual(Object.fromEntries(edges.get('src/lib.rs')), {}) + }) +}) + +test('buildRustBundle --cargo-manifests carries manifests, the lockfile and cargo config, and walks build scripts as crate roots', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[workspace]\nmembers = ["app"]\n', + 'Cargo.lock': 'version = 3\n', + '.cargo/config.toml': '[source.crates-io]\nreplace-with = "vendored-sources"\n[source.vendored-sources]\ndirectory = "vendor"\n', + 'app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\ndep = "1"\n[build-dependencies]\ncc = "1"\n', + 'app/build.rs': 'mod gen;\nfn main() { cc::Build::new(); gen::run(); }\n', + 'app/gen.rs': 'pub fn run() {}\n', + 'app/src/main.rs': 'fn main() { dep::f(); }\n', + 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\n[build-dependencies]\nautocfg = "1"\n', + 'vendor/dep/build.rs': 'fn main() { autocfg::new(); }\n', + 'vendor/dep/src/lib.rs': 'pub fn f() {}\n', + 'vendor/cc/Cargo.toml': '[package]\nname = "cc"\nversion = "1.0.0"\n', + 'vendor/cc/src/lib.rs': 'pub struct Build;\n', + 'vendor/autocfg/Cargo.toml': '[package]\nname = "autocfg"\nversion = "1.0.0"\n', + 'vendor/autocfg/src/lib.rs': 'pub fn new() {}\n', + 'vendor/unused/Cargo.toml': '[package]\nname = "unused"\nversion = "1.0.0"\n', + 'vendor/unused/src/lib.rs': '', + }, async (tmp) => { + const plain = await buildRustBundle({ cwd: tmp, entries: ['app/src/main.rs'] }) + t.assert.deepEqual([...plain.sources.keys()].toSorted(), ['app/src/main.rs', 'vendor/dep/src/lib.rs']) + const bundle = await buildRustBundle({ cwd: tmp, entries: ['app/src/main.rs'], cargoManifests: true }) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), [ + '.cargo/config.toml', 'Cargo.lock', 'Cargo.toml', + 'app/Cargo.toml', 'app/build.rs', 'app/gen.rs', 'app/src/main.rs', + 'vendor/autocfg/Cargo.toml', 'vendor/autocfg/src/lib.rs', // dep's build-dependency, through dep's build script + 'vendor/cc/Cargo.toml', 'vendor/cc/src/lib.rs', // app's build-dependency, through app's build script + 'vendor/dep/Cargo.toml', 'vendor/dep/build.rs', 'vendor/dep/src/lib.rs', + ]) + t.assert.equal(bundle.formats.get('app/Cargo.toml'), 'resource') + t.assert.equal(bundle.formats.get('Cargo.lock'), 'resource') + t.assert.equal(bundle.formats.get('.cargo/config.toml'), 'resource') + t.assert.equal(bundle.formats.get('app/build.rs'), 'rust') + t.assert.equal(bundle.entries.has('app/build.rs'), false) // a crate root of the walk, not an entry of the bundle + const edges = bundle.imports.get('rust') + t.assert.deepEqual(Object.fromEntries(edges.get('app/build.rs')), { 'mod gen': 'app/gen.rs', 'gen::run': 'app/gen.rs', 'use cc': 'vendor/cc/src/lib.rs' }) + t.assert.deepEqual(Object.fromEntries(edges.get('vendor/dep/build.rs')), { 'use autocfg': 'vendor/autocfg/src/lib.rs' }) + // Buckets: each manifest sits with its package; the root-level files in the workspace bucket. + t.assert.deepEqual(Object.keys(bundle.modules.get('vendor/dep').files).toSorted(), ['Cargo.toml', 'build.rs', 'src/lib.rs']) + t.assert.equal(bundle.modules.get('vendor/dep').ecosystem, 'cargo') + t.assert.deepEqual(Object.keys(bundle.modules.get('app').files).toSorted(), ['Cargo.toml', 'build.rs', 'gen.rs', 'src/main.rs']) + t.assert.deepEqual(Object.keys(bundle.modules.get('.').files).toSorted(), ['.cargo/config.toml', 'Cargo.lock', 'Cargo.toml']) + }) +}) + +test('parseRustcCfg reads rustc --print cfg output as a set of leaves', (t) => { + t.assert.deepEqual([...parseRustcCfg('debug_assertions\npanic="unwind"\ntarget_os="linux"\n\nunix\n')], ['debug_assertions', 'panic="unwind"', 'target_os="linux"', 'unix']) +}) + +// A Linux target as rustc would describe it, for tests that must not depend on rustc. +const LINUX = { triple: 'x86_64-unknown-linux-gnu', cfgs: new Set(['unix', 'target_os="linux"', 'target_family="unix"', 'target_arch="x86_64"', 'target_env="gnu"', 'target_pointer_width="64"']) } + +test('createCargoContext decides target-specific dependency tables when the target is known', (t) => { + const files = { + 'Cargo.toml': ['[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', + "[target.'cfg(windows)'.dependencies]", 'win = "1"', + "[target.'cfg(unix)'.dependencies]", 'nix = "1"', + '[target.x86_64-unknown-linux-gnu.dependencies]', 'gnu = "1"', + '[target.aarch64-apple-darwin.dependencies]', 'mac = "1"', + "[target.'cfg(loom)'.dependencies]", 'loom = "1"'].join('\n'), + 'src/main.rs': '', + } + for (const name of ['win', 'nix', 'gnu', 'mac', 'loom']) { + files[`vendor/${name}/Cargo.toml`] = `[package]\nname = "${name}"\nversion = "1.0.0"\n[features]\ndefault = ["std"]\nstd = []\n` + files[`vendor/${name}/src/lib.rs`] = '' + } + withProject(files, (tmp) => { + // Without a target every table counts. + t.assert.deepEqual(Object.keys(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] }))).toSorted(), ['.', 'vendor/gnu', 'vendor/loom', 'vendor/mac', 'vendor/nix', 'vendor/win']) + const linux = createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }) + t.assert.equal(linux.targetTriple, 'x86_64-unknown-linux-gnu') + t.assert.ok(linux.targetCfgs.has('unix')) + // `cfg(loom)` is a custom cfg the loader can't decide: kept. + t.assert.deepEqual(Object.keys(enabledOf(linux)).toSorted(), ['.', 'vendor/gnu', 'vendor/loom', 'vendor/nix']) + const plain = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.equal(plain.targetTriple, null) + t.assert.equal(plain.targetCfgs, null) + }) +}) + +test('buildRustBundle with a target leaves out the other targets\' modules and path variants', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': '#[cfg_attr(unix, path = "sys/unix.rs")]\n#[cfg_attr(windows, path = "sys/windows.rs")]\nmod sys;\n#[cfg(windows)]\nmod win;\n#[cfg(not(windows))]\nmod other;\n', + 'src/sys/unix.rs': '', + 'src/sys/windows.rs': '', + 'src/win.rs': '', + 'src/other.rs': '', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX }) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/lib.rs', 'src/other.rs', 'src/sys/unix.rs']) + const lib = bundle.imports.get('rust').get('src/lib.rs') + t.assert.equal(lib.get('mod sys'), 'src/sys/unix.rs') // one file: no cfg-keyed map + t.assert.equal(lib.get('mod other'), 'src/other.rs') + const all = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.deepEqual([...all.sources.keys()].toSorted(), ['src/lib.rs', 'src/other.rs', 'src/sys/unix.rs', 'src/sys/windows.rs', 'src/win.rs']) + }) +}) + +test('rustcTargetCfgs runs $RUSTC when set: a target only another toolchain knows', { skip: process.platform === 'win32' ? 'POSIX shell script' : false }, (t) => { + // Stands in for Solana's platform-tools rustc, which knows `sbf-solana-solana` where a rustup one does not. + const tmp = mkdtempSync(join(tmpdir(), 'stasis-rustc-')) + const fake = join(tmp, 'rustc') + writeFileSync(fake, [ + '#!/bin/sh', + `printf '%s\\n%s\\n' "$PWD" "$RUSTUP_AUTO_INSTALL" > "${tmp}/ran"`, + 'case "$*" in', + ' "-vV") printf "rustc 1.79.0-dev\\nhost: sbf-solana-solana\\n" ;;', + ' "--print cfg --target sbf-solana-solana") printf \'target_arch="sbf"\\ntarget_os="solana"\\ntarget_family="solana"\\ntarget_pointer_width="64"\\ntarget_endian="little"\\n\' ;;', + ' *) echo "unknown target" >&2; exit 1 ;;', + 'esac', + ].join('\n'), { mode: 0o755 }) + const previous = process.env.RUSTC + process.env.RUSTC = fake + try { + const sbf = rustcTargetCfgs('sbf-solana-solana') + t.assert.equal(sbf.triple, 'sbf-solana-solana') + t.assert.deepEqual([...sbf.cfgs], ['target_arch="sbf"', 'target_os="solana"', 'target_family="solana"', 'target_pointer_width="64"', 'target_endian="little"']) + t.assert.equal(rustcTargetCfgs('host').triple, 'sbf-solana-solana') + t.assert.throws(() => rustcTargetCfgs('x86_64-unknown-linux-gnu'), /rustc --print cfg --target x86_64-unknown-linux-gnu failed \(exit 1\):\nunknown target/u) + // `#[cfg(not(target_os = "solana"))]` is dead under it. + t.assert.equal(evalCfg('not(target_os = "solana")', { target: sbf.cfgs }), false) + // rustc ran from the user's home directory (never the project being bundled, whose + // `rust-toolchain` file could name any binary, nor a world-writable temp dir), with rustup's + // auto-install off. + t.assert.deepEqual(readFileSync(join(tmp, 'ran'), 'utf8').trim().split('\n'), [homedir(), '0']) + mkdirSync(join(tmp, 'project', 'home'), { recursive: true }) + mkdirSync(join(tmp, 'elsewhere')) + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'project')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], homedir()) + // A home directory that is the bundle root, or inside it, would let the project's toolchain + // file choose: the filesystem root then. + const home = process.env.HOME + process.env.HOME = join(tmp, 'project', 'home') + try { + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'project')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], parse(process.cwd()).root) + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'project', 'home')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], parse(process.cwd()).root) + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'elsewhere')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], join(tmp, 'project', 'home')) + // Through a link, either way round: real paths decide. + symlinkSync(join(tmp, 'project', 'home'), join(tmp, 'home-link')) + symlinkSync(join(tmp, 'project'), join(tmp, 'project-link')) + process.env.HOME = join(tmp, 'home-link') + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'project')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], parse(process.cwd()).root) + process.env.HOME = join(tmp, 'project', 'home') + rustcTargetCfgs('sbf-solana-solana', join(tmp, 'project-link')) + t.assert.equal(readFileSync(join(tmp, 'ran'), 'utf8').split('\n')[0], parse(process.cwd()).root) + } finally { + if (home === undefined) delete process.env.HOME + else process.env.HOME = home + } + } finally { + if (previous === undefined) delete process.env.RUSTC + else process.env.RUSTC = previous + rmSync(tmp, { recursive: true, force: true }) + } +}) + +const hasRustc = spawnSync('rustc', ['--version'], { stdio: 'ignore' }).status === 0 + +test('rustcTargetCfgs asks rustc for a target\'s cfg set, and for the host triple', { skip: hasRustc ? false : 'rustc not on PATH' }, (t) => { + const host = rustcTargetCfgs('host') + t.assert.match(host.triple, /^\w+-/u) + t.assert.ok([...host.cfgs].some((c) => c.startsWith('target_os='))) + t.assert.ok(host.cfgs.has('unix') || host.cfgs.has('windows')) + // Any target rustc knows, without its standard library installed. + const wasm = rustcTargetCfgs('wasm32-unknown-unknown') + t.assert.equal(wasm.triple, 'wasm32-unknown-unknown') + t.assert.ok(wasm.cfgs.has('target_arch="wasm32"')) + t.assert.ok(!wasm.cfgs.has('unix') && !wasm.cfgs.has('windows')) + t.assert.deepEqual(createCargoContext(featuresFixture, { entries: ['src/main.rs'], target: 'wasm32-unknown-unknown' }).targetCfgs, wasm.cfgs) + t.assert.throws(() => rustcTargetCfgs('no-such-target'), /rustc --print cfg --target no-such-target failed/u) +}) + +test('createCargoContext finds vendored crates in the directory .cargo/config.toml names', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\ndep = "1"\n', + '.cargo/config.toml': '[source.crates-io]\nreplace-with = "vendored-sources"\n\n[source.vendored-sources]\ndirectory = "third_party/crates"\n', + 'src/main.rs': '', + 'third_party/crates/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\n', + 'third_party/crates/dep/src/lib.rs': '', + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.equal(cargo.vendorDir, 'third_party/crates') + t.assert.equal(cargo.resolveCrate('dep', 'src/main.rs'), 'third_party/crates/dep/src/lib.rs') + t.assert.deepEqual(enabledOf(cargo), { '.': [], 'third_party/crates/dep': [] }) + }) + t.assert.equal(createCargoContext(featuresFixture, { entries: ['src/main.rs'] }).vendorDir, 'vendor') + // however the table is spelled out + const manifest = '[package]\nname = "app"\nversion = "0.1.0"\n' + withProject({ 'Cargo.toml': manifest, '.cargo/config': '[source]\nvendored-sources = { directory = "deps/" }\n' }, (tmp) => { + t.assert.equal(createCargoContext(tmp).vendorDir, 'deps') + }) + // A cargo config, Cargo.toml or Cargo.lock that exists but isn't TOML stops the build, naming it. + withProject({ 'Cargo.toml': manifest, '.cargo/config.toml': '[source.vendored-sources]\ndirectory = third_party\n' }, (tmp) => { + t.assert.throws(() => createCargoContext(tmp), { name: 'TomlError', message: '.cargo/config.toml: expected a value, found "third_party" at line 2' }) + }) + withProject({ 'Cargo.toml': '[package]\nname = "app"\nname = "again"\n', 'src/main.rs': '' }, (tmp) => { + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/main.rs'] }).packageInfo('src/main.rs'), { name: 'TomlError', message: 'Cargo.toml: duplicate key "name" at line 3' }) + }) +}) + // --- bundling --- +test('buildRustBundle carries include!d source and include_str!/include_bytes! assets as resources', async (t) => { + const bundle = await buildRustBundle({ cwd: join(fixtures, 'includes'), entries: ['src/lib.rs'] }) + t.assert.deepEqual(sorted(bundle.sources.keys()), ['README.md', 'data/blob.bin', 'data/table.txt', 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs']) + t.assert.deepEqual([bundle.formats.get('src/generated/consts.rs'), bundle.formats.get('data/table.txt'), bundle.formats.get('data/blob.bin')], ['rust', 'resource', 'resource:base64']) + t.assert.equal(bundle.sources.get('data/blob.bin'), Buffer.from([0, 0xff, 0xfe, 1]).toString('base64')) + t.assert.deepEqual(Object.keys(bundle.modules.get('.').files).toSorted(), ['README.md', 'data/blob.bin', 'data/table.txt', 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs']) +}) + test('buildRustBundle leaves feature-gated code that is off out of the bundle, per crate and per version', async (t) => { const bundle = await buildRustBundle({ cwd: featuresFixture, entries: ['src/main.rs'] }) t.assert.deepEqual(sorted(bundle.sources.keys()), [ @@ -422,3 +980,65 @@ test('buildRustBundle treats a feature that is on as firm: a missing gated modul rmSync(tmp, { recursive: true, force: true }) } }) + +test('buildRustBundle loads a crate root the tree pass asks for: a crate whose name the file also binds as a value', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nlog = "0.4"\n', + 'src/main.rs': 'mod util;\nuse crate::util::log;\nfn main() { log(); log::info!("x"); }\n', + 'src/util.rs': 'pub fn log() {}\n', + 'vendor/log/Cargo.toml': '[package]\nname = "log"\nversion = "0.4.0"\n', + 'vendor/log/src/lib.rs': 'pub mod macros;\n', + 'vendor/log/src/macros.rs': '', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/main.rs'] }) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/main.rs', 'src/util.rs', 'vendor/log/src/lib.rs', 'vendor/log/src/macros.rs']) + t.assert.deepEqual(Object.fromEntries(bundle.imports.get('rust').get('src/main.rs')), { 'mod util': 'src/util.rs', 'crate::util::log': 'src/util.rs', 'use log': 'vendor/log/src/lib.rs' }) + }) +}) + +test('buildRustBundle reads a package-defined quote! body as macro input while walking, whichever file defines it first', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': '#[macro_use]\nmod m;\nmod user;\n', + 'src/m.rs': '#[macro_use]\nmod inner;\n', // the definition is a wave deeper than user.rs: met after user.rs was scanned + 'src/m/inner.rs': 'macro_rules! quote { ($($t:tt)*) => {} }\n', + 'src/user.rs': 'quote! { mod hidden; }\n', + 'src/user/hidden.rs': 'pub fn h() {}\n', + 'vendor/other/Cargo.toml': '[package]\nname = "other"\nversion = "1.0.0"\n', + 'vendor/other/src/lib.rs': 'fn g() { quote! { mod nothere; } }\n', // another package: a template, skipped + }, async (tmp) => { + const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/lib.rs', 'vendor/other/src/lib.rs'] })) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/lib.rs', 'src/m.rs', 'src/m/inner.rs', 'src/user.rs', 'src/user/hidden.rs', 'vendor/other/src/lib.rs']) + t.assert.deepEqual(Object.fromEntries(bundle.imports.get('rust').get('src/user.rs')), { 'mod hidden': 'src/user/hidden.rs', 'quote!': 'src/m/inner.rs' }) + t.assert.deepEqual(warnings, []) + }) +}) + +test('buildRustTree with a target holds every file to the target\'s cfgs: mio\'s Waker is the linux one, not the first written', (t) => { + // mio's sys/unix/mod.rs declares the selector (with its own `Waker` for the poll backend) + // before the waker module; a Linux build has the eventfd waker. + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': 'mod sys;\nmod waker;\n', + 'src/sys/mod.rs': '#[cfg(unix)]\ncfg_os_poll! {\n mod unix;\n pub use self::unix::*;\n}\n', + 'src/sys/unix/mod.rs': 'cfg_os_poll! {\n #[cfg_attr(any(mio_unsupported_force_poll_poll, target_os = "aix", target_os = "solaris"), path = "selector/poll.rs")]\n #[cfg_attr(all(not(mio_unsupported_force_poll_poll), any(target_os = "android", target_os = "linux")), path = "selector/epoll.rs")]\n mod selector;\n pub(crate) use self::selector::*;\n #[cfg_attr(any(mio_unsupported_force_waker_pipe, target_os = "aix", target_os = "solaris"), path = "waker/pipe.rs")]\n #[cfg_attr(all(not(mio_unsupported_force_waker_pipe), any(target_os = "android", target_os = "linux")), path = "waker/eventfd.rs")]\n mod waker;\n pub(crate) use self::waker::Waker;\n}\n', + 'src/sys/unix/selector/poll.rs': 'pub struct Selector;\npub struct Waker;\n', + 'src/sys/unix/selector/epoll.rs': 'pub struct Selector;\n', + 'src/sys/unix/waker/pipe.rs': 'pub struct Waker;\n', + 'src/sys/unix/waker/eventfd.rs': 'pub struct Waker;\n', + 'src/waker.rs': 'use crate::sys;\nfn f() { sys::Waker::new(); sys::Selector::new(); }\n', + } + withProject(files, (tmp) => { + const entries = ['src/lib.rs'] + const sources = new Map(Object.entries(files).filter(([f]) => f.endsWith('.rs'))) + const plain = buildRustTree(sources, { roots: entries, baseDir: tmp, cargo: createCargoContext(tmp, { entries }) }) + // Without a target nothing rules either waker file out (`any(mio_unsupported_force_waker_pipe, + // aix, solaris)` may hold on aix): the first variant declared. With a Linux target the aix and + // solaris alternatives are out, the custom cfg is presumably off, and the eventfd branch's + // `any(android, linux)` holds: that one. + t.assert.equal(Object.fromEntries(plain.resolutions.get('src/waker.rs'))['sys::Waker::new'], 'src/sys/unix/waker/pipe.rs') + const linux = buildRustTree(sources, { roots: entries, baseDir: tmp, cargo: createCargoContext(tmp, { entries, target: LINUX }) }) + t.assert.equal(Object.fromEntries(linux.resolutions.get('src/waker.rs'))['sys::Waker::new'], 'src/sys/unix/waker/eventfd.rs') + t.assert.equal(Object.fromEntries(linux.resolutions.get('src/waker.rs'))['sys::Selector::new'], 'src/sys/unix/selector/epoll.rs') + }) +}) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 123e7763..01f5a011 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -2,13 +2,12 @@ import { test } from 'node:test' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' -import { createCargoContext, parseCargoManifest } from '../stasis/src/loaders/cargo.js' +import { createCargoContext, evalCfg, parseCargoManifest } from '../stasis/src/loaders/cargo.js' import { buildModuleTrees, buildRustTree, collectRustFilesFromDisk, crateRoots, - evalCfg, getModuleDir, lexRust, loadRust, @@ -16,7 +15,6 @@ import { resolveExplicitModPath, resolveModDecl, resolveModPath, - resolveUsePath, resolveVendoredCrate, scanRustItems, } from '../stasis/src/loaders/rust.js' @@ -68,6 +66,18 @@ test('lexRust handles raw strings, byte strings, char literals and lifetimes', ( t.assert.match(masked, /mod z;$/u) }) +test('lexRust keeps UTF-16 offsets aligned across astral chars in strings and comments', (t) => { + // An emoji is two UTF-16 units: blanking it as one space would shift every later offset. + const src = 'const S: &str = "😀 smile"; // 🎉\nuse serde::Serialize;\n#[path = "x.rs"]\nmod m;\n' + const { code, masked } = lexRust(src) + t.assert.equal(code.length, src.length) + t.assert.equal(masked.length, src.length) + t.assert.match(masked, /" "; {6}\nuse serde::Serialize;/u) + const items = scanRustItems(src) + t.assert.deepEqual(items.refs.map((r) => r.spec), ['serde::Serialize']) + t.assert.deepEqual(items.mods[0].paths, [{ path: 'x.rs', cfg: null }]) +}) + // --- use trees --- test('parseUseTree flattens brace groups (incl. nested and multi-line) into one path each', (t) => { @@ -91,9 +101,188 @@ test('scanRustItems finds external mod declarations (incl. pub / pub(crate)) and ['foo', 'bar', 'baz', 'real']) }) -test('scanRustItems records an external mod declared inside inline modules with its inline path', (t) => { +test('scanRustItems records an external mod declared inside inline modules with its inline path, and the directories those stand for', (t) => { const { mods } = scanRustItems('mod outer {\n pub mod inner;\n mod deep {\n mod leaf;\n }\n}\nmod top;\n') - t.assert.deepEqual(mods.map((m) => [m.name, m.inlinePath]), [['inner', ['outer']], ['leaf', ['outer', 'deep']], ['top', []]]) + t.assert.deepEqual(mods.map((m) => [m.name, m.inlinePath, m.inlineDirs]), [['inner', ['outer'], ['outer']], ['leaf', ['outer', 'deep'], ['outer', 'deep']], ['top', [], []]]) + // solana-program: `#[path = ""] mod non_bpf_modules { mod account_keys; }` puts account_keys.rs + // beside the file; a non-empty path names the directory in the module name's stead. + const solana = scanRustItems('#[cfg(not(target_os = "solana"))]\n#[path = ""]\nmod non_bpf_modules {\n mod account_keys;\n #[path = "thread_files"]\n mod thread { mod local_data; }\n}\n') + t.assert.deepEqual(solana.mods.map((m) => [m.name, m.inlinePath, m.inlineDirs]), [ + ['account_keys', ['non_bpf_modules'], ['']], + ['local_data', ['non_bpf_modules', 'thread'], ['', 'thread_files']], + ]) +}) + +test('scanRustItems takes `use` for an import only when a use tree follows it', (t) => { + // syn: `Token![use]` in macro input; serde_derive: `quote! { use #path as _serde; }`, a template + // for generated code; a macro_rules! template interpolating a metavariable. None imports anything. + const src = [ + 'fn parse(input: ParseStream) -> Result {', + ' if input.peek(Token![use]) { let precise_capture_begin = input.parse::()?; }', + ' Ok(true)', + '}', + 'fn wrap(path: &Path) -> TokenStream { quote! { use #path as _serde; #[allow(unused)] use _serde::__private::Ok; } }', + 'macro_rules! m { ($m:ident, $trait:ident) => { use $m::Thing; use $crate::real::Item; use ::libc::c_int; $trait::fmt(self); $crate::util::go(); } }', + 'pub use crate::{r#ref::*, split_at::SplitAt};', // zerocopy: a raw identifier is no interpolation + 'use std::io::{self, Read};', + ].join('\n') + const { refs, imports, bindings } = scanRustItems(src) + t.assert.deepEqual(refs.filter((r) => r.fromUse).map((r) => r.spec), ['crate::real::Item', '::libc::c_int', 'crate::ref', 'crate::split_at::SplitAt', 'std::io', 'std::io::Read']) + t.assert.deepEqual(refs.filter((r) => !r.fromUse).map((r) => r.spec), ['crate::util::go']) // `$trait::fmt` is a metavariable's + t.assert.deepEqual(imports.map((im) => im.binding), ['Item', 'c_int', null, 'SplitAt', 'io', 'Read']) + t.assert.deepEqual([...bindings], ['Item', 'c_int', 'SplitAt', 'io', 'Read']) +}) + +test('scanRustItems gates the declarations inside a cfg-gated block or inline module on its cfg', (t) => { + const src = [ + 'cfg_if::cfg_if! {', + ' if #[cfg(unix)] { #[path = "u.rs"] mod imp; } else if #[cfg(windows)] { mod imp; } else { mod imp; }', + '}', + '#[cfg(feature = "std")] mod outer { mod a; #[cfg(unix)] mod b; }', + 'cfg_has_atomic! { mod atomic; }', + 'mod plain;', + ].join('\n') + // A later `cfg_if!` branch applies only when the earlier ones' cfgs don't hold. + t.assert.deepEqual(scanRustItems(src).mods.map((m) => [m.name, m.cfg, m.conditional, m.macro]), [ + ['imp', 'unix', true, 'cfg_if'], + ['imp', 'all(not(unix), windows)', true, 'cfg_if'], + ['imp', 'all(not(unix), not(windows))', true, 'cfg_if'], + ['a', 'feature = "std"', true, null], + ['b', 'all(feature = "std", unix)', true, null], + ['atomic', null, true, 'cfg_has_atomic'], + ['plain', null, false, null], + ]) + // With the feature on, `outer`'s scope is as firm as none; a dead one skips the whole block. + t.assert.deepEqual(scanRustItems(src, { features: new Set(['std']) }).mods.filter((m) => m.inlinePath.length > 0).map((m) => [m.name, m.cfg, m.conditional]), [['a', 'feature = "std"', false], ['b', 'all(feature = "std", unix)', true]]) + t.assert.deepEqual(scanRustItems(src, { features: new Set() }).mods.map((m) => m.name), ['imp', 'imp', 'imp', 'atomic', 'plain']) +}) + +test('buildRustTree keeps every file of same-name mods a cfg_if! or cfg macros declare, keyed by branch cfg or macro', (t) => { + // tokio's atomic_u64: `cfg_has_atomic_u64! { #[path = "native.rs"] mod imp; }` beside + // `cfg_not_has_atomic_u64! { #[path = "as_mutex.rs"] mod imp; }`, the second file with modules + // and paths of its own. + const sources = new Map([ + ['src/lib.rs', 'mod atomic;\nmod sys;\n'], + ['src/atomic.rs', 'cfg_has_atomic_u64! {\n #[path = "atomic_native.rs"]\n mod imp;\n}\ncfg_not_has_atomic_u64! {\n #[path = "atomic_as_mutex.rs"]\n mod imp;\n}\npub(crate) use imp::AtomicU64;\n'], + ['src/atomic_native.rs', 'pub struct AtomicU64;\n'], + ['src/atomic_as_mutex.rs', 'cfg_has_const_mutex_new! {\n #[path = "static_const_new.rs"]\n mod static_macro;\n}\ncfg_not_has_const_mutex_new! {\n #[path = "static_once_cell.rs"]\n mod static_macro;\n}\npub(crate) use static_macro::StaticAtomicU64;\npub struct AtomicU64;\n'], + ['src/static_const_new.rs', 'pub struct StaticAtomicU64;\n'], + ['src/static_once_cell.rs', 'pub struct StaticAtomicU64;\n'], + ['src/sys.rs', 'cfg_if::cfg_if! {\n if #[cfg(unix)] { #[path = "sys/unix.rs"] mod imp; } else { #[path = "sys/other.rs"] mod imp; }\n}\npub use imp::*;\n'], + ['src/sys/unix.rs', 'pub fn name() {}\n'], + ['src/sys/other.rs', 'pub fn name() {}\n'], + ]) + const { resolutions, missing, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([missing, [...unresolvedCrates]], [[], []]) // `static_macro` is a module, not a crate + t.assert.deepEqual(edges(resolutions.get('src/atomic.rs')), { + 'mod imp': { 'cfg_has_atomic_u64!': 'src/atomic_native.rs', 'cfg_not_has_atomic_u64!': 'src/atomic_as_mutex.rs' }, + 'imp::AtomicU64': 'src/atomic_native.rs', + }) + // The second variant's own declarations and paths resolve: it is in the module tree. + t.assert.deepEqual(edges(resolutions.get('src/atomic_as_mutex.rs')), { + 'mod static_macro': { 'cfg_has_const_mutex_new!': 'src/static_const_new.rs', 'cfg_not_has_const_mutex_new!': 'src/static_once_cell.rs' }, + 'static_macro::StaticAtomicU64': 'src/static_const_new.rs', + }) + // The `else` branch is keyed by what it means: the earlier cfg not holding. + t.assert.deepEqual(edges(resolutions.get('src/sys.rs')), { 'mod imp': { unix: 'src/sys/unix.rs', 'not(unix)': 'src/sys/other.rs' }, imp: 'src/sys/unix.rs' }) +}) + +test('buildRustTree resolves an include!d file\'s mods beside that file, and anchors its paths in the including module', (t) => { + // rustc: `mod bar;` in an `include!`d file looks for `bar.rs` beside the included file (its + // directory, as for a `#[path]`-loaded file: no `/` step), and `bar` is the includer's + // child module -- checked against rustc 1.94. + const sources = new Map([ + ['src/lib.rs', 'mod net;\npub struct Top;\n'], + ['src/net/mod.rs', 'include!("gen/list.rs");\npub struct Conn;\n'], + ['src/net/gen/list.rs', 'mod codes;\nuse self::codes::Code;\nuse super::Top;\nfn f() { crate::net::Conn::new(); codes::lookup(); }\n'], + ['src/net/gen/codes.rs', 'pub struct Code;\npub fn lookup() {}\npub mod sub;\nuse crate::net::codes::Code as C;\n'], + ['src/net/gen/codes/sub.rs', ''], + ['src/net/codes.rs', 'pub struct Decoy;\n'], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual(edges(resolutions.get('src/net/gen/list.rs')), { + 'mod codes': 'src/net/gen/codes.rs', // beside the included file, not src/net/codes.rs nor src/net/gen/list/codes.rs + 'self::codes::Code': 'src/net/gen/codes.rs', + 'super::Top': 'src/lib.rs', // `super` of `crate::net`, the module these tokens are in, is the root + 'crate::net::Conn::new': 'src/net/mod.rs', + 'codes::lookup': 'src/net/gen/codes.rs', + }) + t.assert.deepEqual(edges(resolutions.get('src/net/gen/codes.rs')), { 'mod sub': 'src/net/gen/codes/sub.rs' }) // `crate::net::codes::Code` is this file: no self edge +}) + +test('scanRustItems keeps a file whose macro body holds a false inner #![cfg]', (t) => { + const items = scanRustItems('m!(\n #![cfg(test)]\n mod x;\n);\nmod real;\n') + t.assert.deepEqual(items.mods.map((m) => m.name), ['x', 'real']) +}) + +test('scanRustItems reads every include argument form, and counts the ones it cannot follow', (t) => { + const items = scanRustItems([ + 'const A: &str = include_str!("with \\"quote\\" and \\\\ back.txt");', + 'const B: &str = include_str!(r#"raw "quoted".txt"#);', + 'const C: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/blob.bin"));', + 'const D: &str = include_str!(concat!("../", "x.txt"));', + 'const E: &str = include_str!(concat!(env!("OUT_DIR"), "/gen.rs"));', + 'macro_rules! doc { () => { include_str!("../README.md") } }', + ].join('\n')) + t.assert.deepEqual(items.includes, [ + { kind: 'include_str', path: 'with "quote" and \\ back.txt', base: 'file', conditional: false }, + { kind: 'include_str', path: 'raw "quoted".txt', base: 'file', conditional: false }, + { kind: 'include_bytes', path: 'data/blob.bin', base: 'manifest', conditional: false }, + ]) + t.assert.equal(items.unfollowed, 1) // `concat!("../", "x.txt")`; the OUT_DIR one is build output + // The include in a macro body is the macro's: resolved relative to whichever file invokes it. + t.assert.deepEqual(items.macros.map(({ name, exported, includes }) => ({ name, exported, includes })), [{ name: 'doc', exported: false, includes: [{ kind: 'include_str', path: '../README.md', base: 'file' }] }]) +}) + +test('buildRustTree resolves an include in a macro_rules! body relative to each invoking file, and a manifest-relative one from the package root', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\nmod deep;\nconst B: &[u8] = include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/blob.bin"));\n'], + ['src/macros.rs', 'macro_rules! doc_of { () => { include_str!("data.txt") } }\n'], + ['src/deep/mod.rs', 'pub fn show() -> &\'static str { doc_of!() }\n'], + ['src/deep/data.txt', 'from deep\n'], + ['src/data.txt', 'from src\n'], + ['data/blob.bin', 'x'], + ]) + const formats = new Map([['src/deep/data.txt', 'resource'], ['src/data.txt', 'resource'], ['data/blob.bin', 'resource']]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'], formats }) + t.assert.deepEqual(edges(resolutions.get('src/deep/mod.rs')), { 'doc_of!': 'src/macros.rs', 'include_str data.txt': 'src/deep/data.txt' }) + t.assert.equal(resolutions.get('src/macros.rs').size, 0) // not relative to the defining file + // Without a Cargo context the package root is unknown: the manifest-relative include is not placed. + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod macros': 'src/macros.rs', 'mod deep': 'src/deep/mod.rs' }) +}) + +test('buildRustTree gives a bare macro call an edge only to a macro in its textual scope', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\nmod other;\nmod user;\n'], + ['src/macros.rs', 'macro_rules! ready { () => {} }\n'], + ['src/other.rs', 'macro_rules! vec { () => {} }\n#[macro_export]\nmacro_rules! exported { () => {} }\n'], + ['src/user.rs', 'fn f() { ready!(); vec![1]; exported!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + // `vec!` is std's: other.rs's private `macro_rules! vec` is not in scope here. Nor is its + // exported one by bare name -- rustc: "cannot find macro `exported` in this scope … have you + // added the `#[macro_use]` on the module?" -- though `crate::exported!` and the root file see it. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'ready!': 'src/macros.rs' }) +}) + +test('buildRustTree finds the modules of an inline module with its own #[path]', (t) => { + const sources = new Map([ + ['src/lib.rs', 'pub mod message;\n'], + ['src/message/mod.rs', '#[cfg(not(target_os = "solana"))]\n#[path = ""]\nmod non_bpf_modules {\n mod account_keys;\n mod versions;\n pub use {account_keys::*, versions::*};\n}\n#[cfg(not(target_os = "solana"))]\npub use non_bpf_modules::*;\n'], + ['src/message/account_keys.rs', 'pub struct AccountKeys;\n'], + ['src/message/versions/mod.rs', 'pub enum VersionedMessage {}\n'], + ['src/user.rs', ''], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual(edges(resolutions.get('src/message/mod.rs')), { + 'mod non_bpf_modules::account_keys': 'src/message/account_keys.rs', + 'mod non_bpf_modules::versions': 'src/message/versions/mod.rs', + account_keys: 'src/message/account_keys.rs', // `pub use {account_keys::*, …}` inside the inline module + versions: 'src/message/versions/mod.rs', + // `pub use non_bpf_modules::*` names the inline module, which lives in this very file: no edge. + }) }) test('scanRustItems marks #[cfg]-gated modules (and those inside a gated inline module) conditional, with their predicate', (t) => { @@ -114,7 +303,7 @@ test('scanRustItems marks #[cfg]-gated modules (and those inside a gated inline ['always', 'not(test)', false], ['both', 'all(unix, feature = "z")', true], ['real', null, false], - ['imp', null, true], + ['imp', 'unix', true], // gated by the inline module it sits in ]) // With x off the cfg_attr never applies its cfg: the module is unconditional, not skipped. const off = scanRustItems('#[cfg_attr(feature = "x", cfg(feature = "y"))]\nmod m;\n', { features: new Set() }).mods @@ -159,6 +348,208 @@ test('evalCfg decides test/doc-only predicates and leaves target/feature ones un t.assert.equal(evalCfg('any()'), false) }) +test('evalCfg decides target predicates against a target cfg set, profile and custom ones never', (t) => { + const target = new Set(['unix', 'target_os="linux"', 'target_family="unix"', 'target_arch="x86_64"', 'target_pointer_width="64"', 'target_feature="sse2"', 'target_has_atomic="64"', 'debug_assertions', 'panic="unwind"']) + t.assert.equal(evalCfg('unix', { target }), true) + t.assert.equal(evalCfg('windows', { target }), false) + t.assert.equal(evalCfg('target_os = "linux"', { target }), true) + t.assert.equal(evalCfg('target_os="macos"', { target }), false) + t.assert.equal(evalCfg('all(unix, target_pointer_width = "64")', { target }), true) + t.assert.equal(evalCfg('any(windows, target_arch = "aarch64")', { target }), false) + t.assert.equal(evalCfg('not(target_os = "windows")', { target }), true) + t.assert.equal(evalCfg('all(unix, feature = "std")', { target }), null) // the feature is undecided without a feature set + t.assert.equal(evalCfg('all(unix, feature = "std")', { target, features: new Set() }), false) + // Target features are the build's to add (`-C target-cpu=native`) or take away (`-C + // target-feature=-crt-static`): undecided either way, as is `target_thread_local`, which a + // stable rustc never prints. + t.assert.equal(evalCfg('target_feature = "sse2"', { target }), null) + t.assert.equal(evalCfg('target_feature = "avx2"', { target }), null) + t.assert.equal(evalCfg('not(target_feature = "avx2")', { target }), null) + t.assert.equal(evalCfg('target_thread_local', { target }), null) + // Profile cfgs are printed by rustc but set by the build; custom cfgs are anyone's. + t.assert.equal(evalCfg('debug_assertions', { target }), null) + t.assert.equal(evalCfg('panic = "unwind"', { target }), null) + t.assert.equal(evalCfg('loom', { target }), null) + t.assert.equal(evalCfg('target_os = "linux"'), null) // no target: undecided, as before +}) + +test('scanRustItems drops target-gated items and cfg_attr path variants for another target', (t) => { + const src = [ + '#[cfg(windows)]', 'mod win;', + '#[cfg(unix)]', 'mod nix;', + '#[cfg_attr(unix, path = "sys/unix.rs")]', '#[cfg_attr(windows, path = "sys/windows.rs")]', 'mod sys;', + '#[cfg(target_os = "linux")] use libc::epoll_create1;', + '#[cfg(target_os = "macos")] use libc::kqueue;', + 'mod plain;', + ].join('\n') + const linux = new Set(['unix', 'target_os="linux"', 'target_family="unix"']) + const items = scanRustItems(src, { target: linux }) + t.assert.deepEqual(items.mods.map((m) => [m.name, m.conditional, m.paths]), [ + ['nix', false, []], // decidably on: as firm as no cfg + ['sys', false, [{ path: 'sys/unix.rs', cfg: null }]], // the variant that holds is the `#[path]` + ['plain', false, []], + ]) + t.assert.deepEqual(items.refs.map((r) => r.spec), ['libc::epoll_create1']) + // Without a target both variants and both modules stay, undecided. + t.assert.deepEqual(scanRustItems(src).mods.map((m) => [m.name, m.conditional]), [['win', true], ['nix', true], ['sys', false], ['plain', false]]) +}) + +test('scanRustItems skips a dead item that starts with no word: a block, a pattern arm, a tuple type, a `*` statement', (t) => { + const src = [ + 'fn f(x: &str) -> String {', + ' #[cfg(feature = "std")]', + ' { std_only::init(); }', + ' #[cfg(feature = "std")]', + ' *counter::GLOBAL.lock() += 1;', + ' match x {', + ' #[cfg(feature = "json")]', + ' "json" => serde_json::to_string(x).unwrap(),', + ' #[cfg(feature = "yaml")]', + ' &"yaml" => serde_yaml::to_string(x).unwrap(),', + ' #[cfg(feature = "pair")]', + ' (a, b) => pair::join(a, b),', + ' #[cfg(feature = "slice")]', + ' [first, ..] => slice::first(first),', + ' #[cfg(feature = "num")]', + ' 1 => num::one(),', + ' _ => live::fallback(x),', + ' }', + '}', + 'struct Tuple(#[cfg(feature = "pair")] (A, pair::B), live::C);', + 'mod after;', + ].join('\n') + const { mods, refs } = scanRustItems(src, { features: new Set() }) + t.assert.deepEqual(mods.map((m) => m.name), ['after']) + t.assert.deepEqual(refs.map((r) => r.spec).toSorted(), ['live::C', 'live::fallback']) +}) + +test('scanRustItems skips a dead item whole whatever commas its generics or where clause hold, and a dead field through its generic type', (t) => { + const src = [ + '#[cfg(feature = "serde")]', + "fn deser<'de, D>(d: D) -> Result, D::Error>", + 'where', + " D: serde::Deserializer<'de>,", + '{', + ' let s = as serde::Deserialize>::deserialize(d)?;', + '}', + '#[cfg(feature = "serde")]', + "impl<'de, A: Array> Deserialize<'de> for SmallVec { fn f() { use serde::de::Error; } }", + '#[cfg(all(tokio_unstable, feature = "tracing"))]', + "pub struct Builder<'a, T> { builder: super::Builder<'a>, x: T }", + 'pub struct Live {', + ' map: HashMap,', + ' #[cfg(feature = "serde")] extra: Option>,', + ' next: u8,', + '}', + 'mod after;', + 'fn live() -> Result<(), E> { crate::real::go(); }', + ].join('\n') + const { mods, refs } = scanRustItems(src, { features: new Set() }) + t.assert.deepEqual(mods.map((m) => m.name), ['after']) + t.assert.deepEqual(refs.map((r) => r.spec), ['crate::real::go']) +}) + +test('scanRustItems blanks the attributes of a dead item and a cfg_attr whose predicate never holds', (t) => { + const src = [ + '#[cfg(feature = "serde")]', + '#[derive(serde::Serialize, serde::Deserialize)]', + 'pub struct Gated;', + '#[cfg_attr(feature = "serde", derive(serde::Serialize))]', + '#[derive(Clone)]', + 'pub struct Live;', + '#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]', + 'pub struct Fuzzed;', + ].join('\n') + const off = scanRustItems(src, { features: new Set(['arbitrary']) }) + // Only the `cfg_attr` whose feature is on names anything live. + t.assert.deepEqual(off.refs.map((r) => r.spec), ['arbitrary::Arbitrary']) + // Undecided features keep the attribute's paths (the crate may well be compiled with them). + t.assert.deepEqual(scanRustItems(src).refs.map((r) => r.spec).toSorted(), ['arbitrary::Arbitrary', 'serde::Deserialize', 'serde::Serialize']) +}) + +test('scanRustItems records use imports with their visibility, macro_rules! definitions and inline modules', (t) => { + const items = scanRustItems([ + 'pub mod __private {', + ' pub use serde_core as serde;', + ' use secret::Thing;', + ' pub(super) use crate::x::Y;', + ' pub(in crate::__private) use crate::x::Z;', + '}', + 'pub use external::*;', + 'pub(crate) use crate::a::{B, c as d};', + 'pub(self) extern crate serde_core as s;', + 'extern crate self as me;', + '#[macro_export]', + '#[doc(hidden)]', + 'macro_rules! exported { () => {} }', + '#[macro_export(local_inner_macros)]', + 'macro_rules! exported_too { () => {} }', + '#[cfg_attr(feature = "std", macro_export)]', + 'macro_rules! exported_when { () => {} }', + '#[cfg_attr(feature = "std", macro_export)]', + '#[cfg(not(feature = "std"))]', + 'macro_rules! never { () => {} }', + 'macro_rules! local { () => {} }', + ].join('\n'), { features: new Set(['std']) }) + t.assert.deepEqual(items.imports.map((r) => [r.inlinePath.join('::'), r.segments.join('::'), r.absolute, r.binding, r.glob, r.vis]), [ + ['__private', 'serde_core', false, 'serde', false, 'crate'], + ['__private', 'secret::Thing', false, 'Thing', false, null], + ['__private', 'crate::x::Y', false, 'Y', false, 'super'], + ['__private', 'crate::x::Z', false, 'Z', false, 'in crate::__private'], + ['', 'external', false, null, true, 'crate'], + ['', 'crate::a::B', false, 'B', false, 'crate'], + ['', 'crate::a::c', false, 'd', false, 'crate'], + ['', 'serde_core', true, 's', false, 'self'], // `extern crate x as y` is `use ::x as y` + ['', 'crate', false, 'me', false, null], // `extern crate self as me` is `use crate as me` + ]) + t.assert.deepEqual(items.macros.map(({ name, exported, includes }) => ({ name, exported, includes })), [ + { name: 'exported', exported: true, includes: [] }, + { name: 'exported_too', exported: true, includes: [] }, + { name: 'exported_when', exported: true, includes: [] }, + { name: 'local', exported: false, includes: [] }, + ]) + t.assert.ok(items.macros.every((m) => Number.isInteger(m.offset)) && items.macros[0].offset < items.macros[1].offset) + t.assert.deepEqual(items.inlineModules, [['__private']]) +}) + +test('scanRustItems marks an invoked path as a macro call', (t) => { + const { refs } = scanRustItems('fn f() { $crate::span!(1); crate::span::Span::new(); if crate::flag::x != 1 { crate::m! { } } }\n') + t.assert.deepEqual(refs.map((r) => [r.spec, r.macroCall]), [['crate::span', true], ['crate::span::Span::new', false], ['crate::flag::x', false], ['crate::m', true]]) +}) + +test('scanRustItems records include macros with a literal path and bare macro invocations, and honours an inner #![cfg]', (t) => { + const items = scanRustItems([ + 'include!("generated/consts.rs");', + 'const T: &str = include_str!("../data/table.txt");', + 'const B: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/blob.bin"));', + 'fn f() { ready!(x); println!("{}", a != b); }', + 'mod dead { #![cfg(feature = "never")] mod ghost; use phantom::X; }', + 'mod live;', + ].join('\n'), { features: new Set() }) + t.assert.deepEqual(items.includes, [ + { kind: 'include', path: 'generated/consts.rs', base: 'file', conditional: false }, + { kind: 'include_str', path: '../data/table.txt', base: 'file', conditional: false }, // the build-output one is no literal + ]) + t.assert.equal(items.unfollowed, 0) // `OUT_DIR` is build output: nothing to warn about + // Inside attributes: the module's own `#![doc = …]`, a live item's `#[doc = …]`, not a dead item's. + const attrs = scanRustItems([ + '#![doc = include_str!("../README.md")]', + '#[doc = include_str!("docs/live.md")]', + 'pub fn live() {}', + '#[cfg(feature = "never")]', + '#[doc = include_str!("docs/dead.md")]', + 'pub fn dead() {}', + ].join('\n'), { features: new Set() }) + t.assert.deepEqual(attrs.includes.map((inc) => inc.path), ['../README.md', 'docs/live.md']) + t.assert.deepEqual([...items.invocations].toSorted(), ['concat', 'env', 'include', 'include_bytes', 'include_str', 'println', 'ready']) // `a != b` is no invocation + t.assert.deepEqual(items.mods.map((m) => m.name), ['live']) // the inline module's inner cfg can't hold: emptied + t.assert.deepEqual(items.refs.map((r) => r.spec), []) + // At the top of a file such an inner cfg empties the whole file. + const gated = scanRustItems('#![cfg(feature = "never")]\nmod ghost;\nuse phantom::X;\n', { features: new Set() }) + t.assert.deepEqual([gated.mods, gated.refs], [[], []]) + t.assert.deepEqual(scanRustItems('#![cfg(feature = "never")]\nmod ghost;\n').mods.map((m) => m.name), ['ghost']) // undecided: kept +}) + test('scanRustItems skips test/doc-only items whole: their mods, uses and paths are never recorded', (t) => { const { mods, refs, externCrates } = scanRustItems([ '#[cfg(test)]', 'mod tests;', @@ -318,10 +709,22 @@ test('resolveModDecl lists cfg_attr variants under their predicate plus the defa { cfg: 'windows', file: 'src/sys/windows.rs', explicit: true }, { cfg: null, file: 'src/sys.rs', explicit: false }, ]) - // A variant whose predicate can't hold in the build never reaches the resolver: the scanner drops it. + // Given the build (`test`, `features`), a variant whose predicate can't hold is dropped here too. + const withMock = { ...decl, paths: [...decl.paths, { path: 'sys/mock.rs', cfg: 'test' }] } + t.assert.deepEqual(resolveModDecl(withMock, 'src/lib.rs', { knownSources: known, test: false }).map((x) => x.file), ['src/sys/unix.rs', 'src/sys/windows.rs', 'src/sys.rs']) + t.assert.deepEqual(resolveModDecl(withMock, 'src/lib.rs', { knownSources: known, test: true }).map((x) => x.file), ['src/sys/unix.rs', 'src/sys/windows.rs', 'src/sys/mock.rs', 'src/sys.rs']) + // Without one, the scanner is trusted to have dropped them for the build it scanned under. const src = '#[cfg_attr(unix, path = "sys/unix.rs")]\n#[cfg_attr(test, path = "sys/mock.rs")]\nmod sys;\n' t.assert.deepEqual(scanRustItems(src).mods[0].paths, [{ path: 'sys/unix.rs', cfg: 'unix' }]) - t.assert.deepEqual(scanRustItems(src, { test: true }).mods[0].paths, [{ path: 'sys/unix.rs', cfg: 'unix' }, { path: 'sys/mock.rs', cfg: 'test' }]) + // rustc applies the first variant whose predicate holds. One that holds in the scanned build ends + // the list and rules out the default lookup; it is the `#[path]` outright only when nothing + // undecided precedes it. + const inTest = scanRustItems(src, { test: true }).mods[0] + t.assert.deepEqual([inTest.paths, inTest.noDefault], [[{ path: 'sys/unix.rs', cfg: 'unix' }, { path: 'sys/mock.rs', cfg: 'test' }], true]) + t.assert.deepEqual(resolveModDecl(inTest, 'src/lib.rs', { knownSources: known }).map((x) => x.file), ['src/sys/unix.rs', 'src/sys/mock.rs']) + const mockFirst = scanRustItems('#[cfg_attr(test, path = "sys/mock.rs")]\n#[cfg_attr(unix, path = "sys/unix.rs")]\nmod sys;\n', { test: true }).mods[0] + t.assert.deepEqual([mockFirst.paths, mockFirst.noDefault], [[{ path: 'sys/mock.rs', cfg: null }], true]) + t.assert.equal(scanRustItems(src).mods[0].noDefault, false) // an unconditional #[path] is authoritative: no default lookup const explicit = { name: 'seed', inlinePath: [], conditional: false, paths: [{ path: 'sys/unix.rs', cfg: null }] } t.assert.deepEqual(resolveModDecl(explicit, 'src/lib.rs', { knownSources: known }), [{ cfg: null, file: 'src/sys/unix.rs', explicit: true }]) @@ -348,7 +751,7 @@ test('parseCargoManifest reads package, lib, dependencies in every shape, and wo '[workspace.package]', 'version = "0.9.0"', '[workspace.dependencies]', 'shared = { path = "crates/shared" }', ].join('\n')) - t.assert.deepEqual(m.package, { name: 'my-app', version: null, versionFromWorkspace: true, edition: '2021' }) + t.assert.deepEqual(m.package, { name: 'my-app', version: null, versionFromWorkspace: true, edition: '2021', editionFromWorkspace: false, build: null }) t.assert.deepEqual(m.lib, { name: 'myapp_lib', path: 'src/the_lib.rs' }) const dep = (k) => { const d = m.deps.get(k) @@ -356,7 +759,7 @@ test('parseCargoManifest reads package, lib, dependencies in every shape, and wo } t.assert.deepEqual([...m.deps.keys()].toSorted(), ['inline_sub', 'nix', 'serde', 'shared', 'tempfile', 'tools', 'util']) t.assert.deepEqual(dep('inline_sub'), { path: '../sub', package: null, workspace: false, kinds: ['normal'] }) - t.assert.deepEqual(dep('nix'), { path: '../nix', package: null, workspace: false, kinds: ['normal'] }) + t.assert.deepEqual(dep('nix'), { path: '../nix', package: null, workspace: false, kinds: ['normal@cfg(unix)'] }) // a target table: its own request t.assert.deepEqual(dep('serde'), { path: null, package: null, workspace: false, kinds: ['normal'] }) t.assert.deepEqual(dep('shared'), { path: null, package: null, workspace: true, kinds: ['normal'] }) t.assert.deepEqual(dep('tempfile'), { path: null, package: null, workspace: false, kinds: ['dev'] }) @@ -502,7 +905,7 @@ test('buildModuleTrees maps module paths to files per crate root, and files back t.assert.equal(tree.get('crate'), 'src/main.rs') t.assert.equal(tree.get('crate::foo'), 'src/foo.rs') t.assert.equal(tree.get('crate::foo::bar'), 'src/foo/bar.rs') - t.assert.deepEqual(files.get('src/foo/bar.rs'), { root: 'src/main.rs', modulePath: 'crate::foo::bar' }) + t.assert.deepEqual(files.get('src/foo/bar.rs'), { root: 'src/main.rs', modulePath: 'crate::foo::bar', leaves: [], parent: 'src/foo.rs' }) }) test('buildModuleTrees keeps a lib and a bin apart (no shared `crate` key) whatever the entry order', (t) => { @@ -536,22 +939,13 @@ test('buildModuleTrees handles a pathologically deep mod chain without overflowi t.assert.ok(tree.size <= 1002, `module-tree depth should be capped, got ${tree.size}`) }) -test('resolveUsePath resolves to the deepest matching module', (t) => { - const tree = new Map([ - ['crate', 'src/main.rs'], - ['crate::foo', 'src/foo.rs'], - ]) - // crate::foo::Greeter -> the foo module (Greeter is an item, not a module) - t.assert.equal(resolveUsePath('crate::foo::Greeter', tree), 'src/foo.rs') - t.assert.equal(resolveUsePath('crate::missing::Thing', tree), 'src/main.rs') -}) - // --- tree --- test('buildRustTree records mod edges and crate:: use edges', async (t) => { const sources = await collectRustFilesFromDisk(join(fixtures, 'use-crate'), ['src/main.rs']) const tree = buildRustTree(sources) - t.assert.deepEqual(Object.keys(tree).toSorted(), ['missing', 'resolutions', 'sources', 'unresolvedCrates']) + t.assert.deepEqual(Object.keys(tree).toSorted(), ['missing', 'resolutions', 'sources', 'unresolvedCrates', 'wantedRoots']) + t.assert.deepEqual(tree.wantedRoots, []) t.assert.deepEqual(tree.missing, []) t.assert.deepEqual([...tree.unresolvedCrates], []) @@ -579,6 +973,642 @@ test('buildRustTree records an edge per path of a brace-grouped / multi-line use t.assert.equal(main['crate::a'], 'src/a.rs') // `pub use crate::a::*` }) +test('buildRustTree follows pub use re-exports and #[macro_export] macros, so `$crate::` paths land where the item lives', (t) => { + // bitflags' layout: a root `__private` module re-exporting each module's `__private`, `pub use + // external::*` at the root, and `#[macro_export]` macros in external.rs invoking each other as + // `$crate::name!` and naming `$crate::__private::serde::…` / `$crate::serde::…`. + const sources = new Map([ + ['src/lib.rs', [ + 'pub mod __private {', + ' pub use crate::{external::__private::*, traits::__private::*};', + '}', + 'pub use external::*;', + 'mod external;', + 'mod traits;', + ].join('\n')], + ['src/external.rs', [ + 'pub(crate) mod __private {', + ' pub use serde_core as serde;', + '}', + 'pub mod serde;', + '#[macro_export]', + 'macro_rules! __impl_external {', + ' () => {', + ' $crate::__impl_external_serde! {}', + ' impl $crate::__private::serde::Serialize for X { fn f() { $crate::serde::serialize() } }', + ' $crate::__private::core::result::Result::Ok(())', + ' };', + '}', + '#[macro_export]', + 'macro_rules! __impl_external_serde { () => {} }', + ].join('\n')], + ['src/external/serde.rs', 'pub fn serialize() {}\n'], + ['src/traits.rs', 'pub(crate) mod __private {\n pub use core;\n}\n'], + ['vendor/serde_core/src/lib.rs', 'pub trait Serialize {}\n'], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual(edges(resolutions.get('src/external.rs')), { + 'mod serde': 'src/external/serde.rs', + 'crate::serde::serialize': 'src/external/serde.rs', // through the root's `pub use external::*` + 'use serde_core': 'vendor/serde_core/src/lib.rs', // `crate::__private::serde::Serialize`, through two re-exports + // `$crate::__private::core::…`: `core` is bound by traits.rs's `pub use core;`, reached through the + // root's glob; the sysroot item lives there as far as the bundle knows. The root's `__private` + // module the path went through is lib.rs, the crate root: never recorded as a via edge. + 'crate::__private::core::result::Result::Ok': 'src/traits.rs', + // `$crate::__impl_external_serde!` is defined in this very file: a self edge, dropped. + }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { + 'crate::external::__private': 'src/external.rs', + 'crate::traits::__private': 'src/traits.rs', + external: 'src/external.rs', + 'mod external': 'src/external.rs', + 'mod traits': 'src/traits.rs', + }) +}) + +test('buildRustTree keeps a path whose re-export chain leaves the bundle on the last module that bound it', (t) => { + // serde: `$crate::__private::Result` in macros.rs; private/mod.rs re-exports it from the `lib` + // module, which re-exports `core`. The item is core's; the bundle's files hold the chain of + // imports that name it, and the path depends on each. + const sources = new Map([ + ['src/lib.rs', ['mod lib {', ' mod core { pub use core::*; }', ' pub use self::core::result;', '}', '#[path = "private/mod.rs"]', 'pub mod __private;', 'mod macros;', 'mod other;'].join('\n')], + ['src/private/mod.rs', 'pub use crate::lib::result::Result::{self, Err, Ok};\npub use thiserror_impl::Error;\n'], + ['src/macros.rs', 'macro_rules! forward { () => { $crate::__private::Result::Ok(()); $crate::__private::Error } }\n'], + ['src/other.rs', 'fn f() -> crate::__private::Result<()> { todo!() }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/macros.rs')), { + // `Result` is bound by the `lib` module's `pub use self::core::result` (its `core::*` glob into + // the sysroot can't place `result`): an item of lib.rs, as far as the bundle knows … + 'crate::__private::Result::Ok': 'src/lib.rs', + // … and `Error` by private/mod.rs's import from a crate that isn't vendored: an item there. + 'crate::__private::Error': 'src/private/mod.rs', + }) + // With nothing else pointing at private/mod.rs, the `__private` module the path went through is recorded. + t.assert.deepEqual(edges(resolutions.get('src/other.rs')), { 'crate::__private::Result': 'src/lib.rs', 'crate::__private': 'src/private/mod.rs' }) +}) + +test('buildRustTree records the module a path went through when its import leads on elsewhere and nothing else points there', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod util;\nmod types;\nmod user;\nmod other;\n'], + ['src/util.rs', 'pub use serde_core as serde;\npub use crate::types::Thing;\npub fn helper() {}\n'], + ['src/types.rs', 'pub struct Thing;\n'], + ['src/user.rs', 'use crate::util::serde::Serialize;\nuse crate::util::Thing;\n'], + ['src/other.rs', 'use crate::util::Thing;\nuse crate::util::helper;\n'], + ['vendor/serde_core/src/lib.rs', 'pub trait Serialize {}\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'use serde_core': 'vendor/serde_core/src/lib.rs', // where `util::serde` leads + 'crate::util::Thing': 'src/types.rs', // where `util::Thing` leads + 'crate::util': 'src/util.rs', // and the module both went through, which nothing else here names + }) + // `crate::util::helper` already points at util.rs: no prefix edge needed. + t.assert.deepEqual(edges(resolutions.get('src/other.rs')), { 'crate::util::Thing': 'src/types.rs', 'crate::util::helper': 'src/util.rs' }) +}) + +test('buildRustTree records include edges and edges to the file defining a macro invoked by bare name', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\ninclude!("generated/consts.rs");\nconst T: &str = include_str!("../data/table.txt");\nfn f() { ready!(1); local!(); vec![1]; }\nmacro_rules! local { () => {} }\n'], + ['src/macros.rs', 'macro_rules! ready { ($e:expr) => { $e } }\n'], + ['src/generated/consts.rs', 'pub const LIMIT: u32 = 3;\n'], + ['data/table.txt', 'a,b\n'], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'], formats: new Map([['data/table.txt', 'resource']]) }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { + 'mod macros': 'src/macros.rs', + 'include generated/consts.rs': 'src/generated/consts.rs', + 'include_str ../data/table.txt': 'data/table.txt', + 'ready!': 'src/macros.rs', // `local!` is defined in this very file, `vec!` nowhere in the crate: no edges + }) + t.assert.ok(!resolutions.has('data/table.txt')) // an asset is carried, not scanned +}) + +test('collectRustFilesFromDisk follows include!, carries include_str!/include_bytes! assets with their format, and leaves a dead include alone', async (t) => { + const baseDir = join(fixtures, 'includes') + const formats = new Map() + const sources = await collectRustFilesFromDisk(baseDir, ['src/lib.rs'], { formats }) + t.assert.deepEqual([...sources.keys()].toSorted(), ['README.md', 'data/blob.bin', 'data/table.txt', 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs']) + t.assert.deepEqual([...formats].toSorted(), [['README.md', 'resource'], ['data/blob.bin', 'resource:base64'], ['data/table.txt', 'resource']]) + t.assert.equal(sources.get('data/blob.bin'), Buffer.from([0, 0xff, 0xfe, 1]).toString('base64')) + t.assert.equal(sources.get('data/table.txt'), 'a,b\n1,2\n') + // src/gated.rs opens with `#![cfg(feature = "never")]`: carried, but its `mod ghost;` and its + // `use phantom_crate::…` are compiled out, so nothing is missing or unresolved. + const { missing, unresolvedCrates, resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'], baseDir, formats }) + t.assert.deepEqual([missing, [...unresolvedCrates]], [[], []]) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { + 'include_str ../README.md': 'README.md', // the crate docs' `#![doc = include_str!(…)]` + 'mod macros': 'src/macros.rs', + 'mod gated': 'src/gated.rs', + 'include generated/consts.rs': 'src/generated/consts.rs', + 'include_str ../data/table.txt': 'data/table.txt', + 'include_bytes ../data/blob.bin': 'data/blob.bin', + 'ready!': 'src/macros.rs', + }) +}) + +test('buildRustTree lets a glob provide a name only from a module of this crate that has it', (t) => { + // A glob into the sysroot or into a crate that isn't in-tree can't be told what it brings in: + // it claims nothing, so the file's crate edges and unresolved reports stay, and a `crate::Name` + // nothing places stays on the module reached. + const sources = new Map([ + ['src/lib.rs', ['mod a;', 'mod b;', 'mod c;', 'mod user;', 'pub use a::*;', 'pub use b::*;', 'pub use std::io::prelude::*;', 'pub use rayon::prelude::*;'].join('\n')], + ['src/a.rs', 'pub mod inner;\npub use crate::c::Item;\npub struct Own;\n'], + ['src/a/inner.rs', 'pub fn f() {}\n'], + ['src/b.rs', 'use core::arch::x86_64::*;\nfn f() { serde::to_string(); _mm_pause(); }\nmod imp { use super::*; use serde_json::json; }\n'], + ['src/c.rs', 'pub struct Item;\n'], + ['src/user.rs', 'use crate::Item;\nuse crate::inner::f;\nuse crate::Own;\nuse crate::Read;\nuse serde::Serialize;\n'], + ['vendor/serde/src/lib.rs', 'pub trait Serialize {}\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/b.rs')), { 'use serde': 'vendor/serde/src/lib.rs' }) // `use core::arch::x86_64::*` hides nothing + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::Item': 'src/c.rs', // through `pub use a::*`, then a's `pub use crate::c::Item` + 'crate::inner::f': 'src/a/inner.rs', // through `pub use a::*`: a child module of a + 'crate::Own': 'src/a.rs', // through `pub use a::*`: a struct a.rs defines + 'crate::Read': 'src/lib.rs', // from the sysroot, through `pub use std::io::prelude::*` + 'use serde': 'vendor/serde/src/lib.rs', + }) + t.assert.deepEqual([...unresolvedCrates].toSorted(), ['rayon', 'serde_json']) +}) + +test('buildRustTree places an item a module defines behind a glob, before a later glob\'s re-export of the name', (t) => { + // libc: every platform module is glob re-exported from the root; `crate::sigset_t` is the struct + // the first such module defines (inside its `s! { … }`), not a later platform's `pub use` of it. + const sources = new Map([ + ['src/lib.rs', 'mod fuchsia;\nmod unix;\nmod user;\npub use crate::fuchsia::*;\npub use crate::unix::*;\n'], + ['src/fuchsia.rs', 's! { pub struct sigset_t { bits: u64 } }\npub type c_int = i32;\nfn f() { crate::sigset_t::default(); }\n'], + ['src/unix.rs', 'pub use self::generic::sigset_t;\nmod generic;\npub const X: crate::c_int = 1;\n'], + ['src/unix/generic.rs', 'pub struct sigset_t;\n'], + ['src/user.rs', 'use crate::sigset_t;\nuse crate::c_int;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::sigset_t': 'src/fuchsia.rs', 'crate::c_int': 'src/fuchsia.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/unix.rs')), { 'mod generic': 'src/unix/generic.rs', 'self::generic::sigset_t': 'src/unix/generic.rs', 'crate::c_int': 'src/fuchsia.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/fuchsia.rs')), {}) // its own struct: no self edge +}) + +test('buildRustTree keeps a private mod and an impl\'s associated items out of what a glob brings in', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod user;\npub use a::*;\n'], + ['src/a.rs', 'mod inner;\npub mod shown;\npub struct S;\nimpl S { pub fn len(&self) -> usize { 0 } pub const N: u8 = 1; type Out = u8; }\nextern "C" { pub fn getpid() -> i32; }\nfn helper() { fn local() {} }\n'], + ['src/a/inner.rs', 'pub struct Hidden;\n'], + ['src/a/shown.rs', ''], + ['src/user.rs', 'use crate::inner::Hidden;\nuse crate::shown;\nuse crate::len;\nuse crate::N;\nuse crate::getpid;\nuse crate::local;\nuse crate::helper;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::inner::Hidden': 'src/lib.rs', // `mod inner` is private to a: not re-exported by `pub use a::*` + 'crate::shown': 'src/a/shown.rs', + 'crate::len': 'src/lib.rs', // a method, not an item of a + 'crate::N': 'src/lib.rs', + 'crate::getpid': 'src/a.rs', // an `extern "C"` block's fn is the module's + 'crate::local': 'src/lib.rs', // a fn's local item is not + 'crate::helper': 'src/lib.rs', // private + }) +}) + +test('buildRustTree resolves a fn call at the root through a glob before an exported macro of that name, and `use log` as the crate', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod facade;\nmod macros;\nmod user;\npub use facade::*;\nuse log::info;\n'], + ['src/facade.rs', 'pub fn helper() {}\n'], + ['src/macros.rs', '#[macro_export]\nmacro_rules! helper { () => {} }\nmacro_rules! log { () => {} }\n'], + ['src/user.rs', 'fn f() { crate::helper(); $crate::helper!(); }\nuse log;\nuse log::debug;\n'], + ['vendor/log/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::helper': 'src/facade.rs', // the fn call: the glob's re-export + 'crate::helper!': 'src/macros.rs', // the invocation: the macro, under a spec of its own + 'use log': 'vendor/log/src/lib.rs', // a private `macro_rules! log` in macros.rs is not what `use log` names + }) + t.assert.equal(edges(resolutions.get('src/lib.rs'))['use log'], 'vendor/log/src/lib.rs') +}) + +test('buildRustTree does not report a lead a glob into another crate may provide', (t) => { + // syn: `use syn::*;` brings syn's `punctuated` module in; `use punctuated::Punctuated` names no crate. + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\n'], + ['src/a.rs', 'use syn::*;\nuse punctuated::Punctuated;\nuse missing_crate::X;\nmod inner { use super::*; use token::Comma; }\n'], + ['src/b.rs', 'use std::io::prelude::*;\nuse Kind::*;\nuse other_missing::Y;\nenum Kind { A }\n'], + ['src/c.rs', 'use proc_macro2::*;\nuse unseen::Z;\n'], + ['vendor/syn/src/lib.rs', 'pub mod punctuated;\npub mod token;\n'], + ['vendor/syn/src/punctuated.rs', ''], + ['vendor/syn/src/token.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + // syn is in-tree, so what its glob brings in is known: `punctuated` and `token`, not + // `missing_crate` (tracing's `use tracing_core::*;` must not hide its `tracing_attributes`). A + // glob into a crate that isn't (proc_macro2 here) may bring anything in, `unseen` included; a + // sysroot glob or an enum's brings in no crate-like lead. + t.assert.deepEqual([...unresolvedCrates].toSorted(), ['missing_crate', 'other_missing', 'proc_macro2']) + t.assert.deepEqual(edges(resolutions.get('src/a.rs')), { 'use syn': 'vendor/syn/src/lib.rs' }) +}) + +test('buildRustTree follows the crate root\'s `extern crate … as` alias from any module', (t) => { + const sources = new Map([ + ['src/lib.rs', 'extern crate serde_core as s;\nextern crate missing_dep as md;\nmod a;\n'], + ['src/a.rs', 'use s::Value;\nfn f() { s::to_value(); md::x(); }\nuse md::Y;\n'], + ['vendor/serde_core/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/a.rs')), { 'use serde_core': 'vendor/serde_core/src/lib.rs' }) + t.assert.deepEqual([...unresolvedCrates], ['missing_dep']) // by its crate name, not the alias +}) + +test('buildRustTree resolves `self::` paths and glob-provided names in a cfg-variant file through that file\'s own imports', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[cfg_attr(unix, path = "sys/unix.rs")]\n#[cfg_attr(windows, path = "sys/windows.rs")]\nmod sys;\nmod u;\nmod w;\n'], + ['src/sys/unix.rs', 'use crate::u::Handle;\nuse crate::u::*;\nfn f() { self::Handle::new(); self::helper::go(); }\n'], + ['src/sys/windows.rs', 'use crate::w::Handle;\nuse crate::w::*;\nfn f() { self::Handle::new(); self::helper::go(); }\n'], + ['src/u.rs', 'pub struct Handle;\npub mod helper { pub fn go() {} }\n'], + ['src/w.rs', 'pub struct Handle;\npub mod helper { pub fn go() {} }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/sys/unix.rs')), { 'crate::u::Handle': 'src/u.rs', 'crate::u': 'src/u.rs', 'self::Handle::new': 'src/u.rs', 'self::helper::go': 'src/u.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/sys/windows.rs')), { 'crate::w::Handle': 'src/w.rs', 'crate::w': 'src/w.rs', 'self::Handle::new': 'src/w.rs', 'self::helper::go': 'src/w.rs' }) +}) + +test('buildRustTree prefers a mod file to an inline module of the same name whatever the declaration order, nested mods included', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[cfg(not(unix))]\nmod imp { mod inner; pub fn f() {} }\n#[cfg(unix)]\nmod imp;\nmod user;\n'], + ['src/imp.rs', 'pub fn f() {}\n'], + ['src/imp/inner.rs', ''], + ['src/user.rs', 'use crate::imp::f;\n'], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::imp::f': 'src/imp.rs' }) +}) + +test('scanRustItems skips the else branches of a dead `if`, and takes a bare macro call after a single `:` for one', (t) => { + const items = scanRustItems([ + 'fn f() {', + ' #[cfg(feature = "never")]', + ' if a { serde::to_string(); } else if b { serde_json::json!(1); } else { toml::from_str(); }', + ' let s = S { a:helper!(), b: other!() };', + ' let p = std::path::Path::new("x");', + '}', + ].join('\n'), { features: new Set() }) + t.assert.deepEqual(items.refs.map((r) => r.spec), []) + t.assert.deepEqual([...items.invocations].toSorted(), ['helper', 'other']) +}) + +test('buildRustTree resolves through several globs of one module, and the first import that leads somewhere', (t) => { + const sources = new Map([ + ['src/lib.rs', ['mod foo;', 'mod b;', 'mod user;', 'pub use foo::a::*;', 'pub use foo::b::*;', '#[cfg(unix)]', 'pub use notvendored::X;', '#[cfg(windows)]', 'pub use crate::b::X;'].join('\n')], + ['src/foo.rs', 'pub mod a;\npub mod b;\n'], + ['src/foo/a.rs', 'pub struct A;\n'], + ['src/foo/b.rs', 'pub mod deep;\n'], + ['src/foo/b/deep.rs', ''], + ['src/b.rs', 'pub struct X;\n'], + ['src/user.rs', 'use crate::deep::D;\nuse crate::X;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::deep::D': 'src/foo/b/deep.rs', // `deep` through the second glob + 'crate::X': 'src/b.rs', // the `#[cfg(windows)]` import: the `#[cfg(unix)]` one leads to a crate that isn't in-tree + }) +}) + +test('buildRustTree prefers a mod file to an inline module of the same name, and the variant an import was followed in', (t) => { + const sources = new Map([ + ['src/lib.rs', ['#[cfg(unix)]', 'mod imp;', '#[cfg(not(unix))]', 'mod imp { pub fn f() {} }', '#[cfg_attr(unix, path = "sys/unix.rs")]', '#[cfg_attr(windows, path = "sys/windows.rs")]', 'mod sys;', 'mod user;', 'mod u;', 'mod w;'].join('\n')], + ['src/imp.rs', 'pub fn f() {}\n'], + ['src/sys/unix.rs', 'pub use crate::u::Handle;\nfn f() { helper::go(); }\nuse crate::u::helper;\n'], + ['src/sys/windows.rs', 'pub use crate::w::Handle;\npub use crate::w::Other;\nfn f() { helper::go(); }\nuse crate::w::helper;\n'], + ['src/u.rs', 'pub struct Handle;\npub mod helper { pub fn go() {} }\n'], + ['src/w.rs', 'pub struct Handle;\npub struct Other;\npub mod helper { pub fn go() {} }\n'], + ['src/user.rs', 'use crate::imp::f;\nuse crate::sys::Handle;\nuse crate::sys::Other;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs'))['mod sys'], { unix: 'src/sys/unix.rs', windows: 'src/sys/windows.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::imp::f': 'src/imp.rs', + 'crate::sys::Handle': 'src/u.rs', // the first variant's import + 'crate::sys': 'src/sys/unix.rs', // the module the paths went through, in the file whose import was followed first + 'crate::sys::Other': 'src/w.rs', // only windows.rs binds `Other`: its import is the one followed + }) + // A path anchored on the module's own import names no module of its own: no edge for the module, + // whichever variant holds the import. + t.assert.deepEqual(edges(resolutions.get('src/sys/windows.rs')), { 'crate::w::Handle': 'src/w.rs', 'crate::w::Other': 'src/w.rs', 'helper::go': 'src/w.rs', 'crate::w::helper': 'src/w.rs' }) +}) + +test('buildRustTree names a macro by path only as the final segment of an invocation or at the crate root', (t) => { + const sources = new Map([ + ['src/lib.rs', ['mod macros;', 'mod span;', 'mod facade;', 'mod user;', 'pub use facade::*;', 'use log::debug;'].join('\n')], + ['src/macros.rs', '#[macro_export]\nmacro_rules! span { () => {} }\n#[macro_export]\nmacro_rules! helper { () => {} }\n#[macro_export]\nmacro_rules! log { () => {} }\n'], + ['src/span.rs', 'pub struct Span;\n'], + ['src/facade.rs', 'pub fn helper() {}\n'], + ['src/user.rs', 'fn f() { $crate::span!(); crate::span::Span::new(); $crate::helper!(); crate::helper(); }\nuse log::info;\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::span!': 'src/macros.rs', // `$crate::span!()`: the macro, not the module of that name + 'crate::span::Span::new': 'src/span.rs', + 'crate::helper!': 'src/macros.rs', // the invocation: the exported macro + 'crate::helper': 'src/facade.rs', // the path: the glob's re-export + }) + t.assert.deepEqual([...unresolvedCrates], ['log']) // `use log::debug` names the crate, not the crate's own `log!` macro +}) + +test('buildRustTree follows a `use` of a macro_rules! macro defined in the crate, not a crate of that name', (t) => { + const sources = new Map([ + ['src/lib.rs', 'macro_rules! cfg_if { () => {} }\npub(crate) use cfg_if;\nmod util;\nmod user;\n'], + ['src/util.rs', 'macro_rules! helper { () => {} }\npub(crate) use helper;\nfn f() { helper!(); }\n'], + ['src/user.rs', 'use crate::cfg_if;\nuse crate::util::helper;\nfn f() { cfg_if! {} helper!(); }\n'], + ['vendor/cfg-if/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod util': 'src/util.rs', 'mod user': 'src/user.rs' }) // `use cfg_if` is the macro above, not vendor/cfg-if + t.assert.deepEqual(edges(resolutions.get('src/util.rs')), {}) + // The bare `cfg_if!` call also sees lib.rs's `macro_rules!` textually (an ancestor file, defined + // before `mod user;`); `helper!` lives in a sibling file, and reaches this one through the `use`. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::cfg_if': 'src/lib.rs', 'crate::util::helper': 'src/util.rs', 'cfg_if!': 'src/lib.rs', 'helper!': 'src/util.rs' }) +}) + +test('buildRustTree answers through an import cycle the same whichever side is asked first', (t) => { + // a and b glob each other; each also names one item. Asking b for X walks b → a → (b again: + // cut) → a's `X`; the a-closure settled while b was still open must not be saved short of b, or + // the later `a::Y` (found only through b) would miss. + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\nmod d;\nmod first;\nmod second;\n'], + ['src/a.rs', 'pub use crate::b::*;\npub use crate::c::X;\n'], + ['src/b.rs', 'pub use crate::a::*;\npub use crate::d::Y;\n'], + ['src/c.rs', 'pub struct X;\n'], + ['src/d.rs', 'pub struct Y;\n'], + ['src/first.rs', 'use crate::b::X;\nuse crate::a::Y;\nuse crate::a::X;\nuse crate::b::Y;\n'], + ['src/second.rs', 'use crate::a::Y;\nuse crate::b::X;\n'], + ]) + // (`crate::a` / `crate::b`: the modules the paths went through, which nothing else here names.) + const want = { 'crate::b::X': 'src/c.rs', 'crate::a::Y': 'src/d.rs', 'crate::a::X': 'src/c.rs', 'crate::b::Y': 'src/d.rs', 'crate::b': 'src/b.rs', 'crate::a': 'src/a.rs' } + t.assert.deepEqual(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/first.rs')), want) + // The other order of first questions: swap which user file the walk meets first. + const swapped = new Map([...sources].map(([p, c]) => [p, p === 'src/lib.rs' ? c.replace('mod first;\nmod second;', 'mod second;\nmod first;') : c])) + t.assert.deepEqual(edges(buildRustTree(swapped, { roots: ['src/lib.rs'] }).resolutions.get('src/first.rs')), want) + // Three modules in a ring, the item two hops away. + const ring = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\nmod user;\n'], + ['src/a.rs', 'pub use crate::b::*;\npub struct A;\n'], + ['src/b.rs', 'pub use crate::c::*;\npub struct B;\n'], + ['src/c.rs', 'pub use crate::a::*;\npub struct C;\n'], + ['src/user.rs', 'use crate::b::A;\nuse crate::a::C;\nuse crate::c::B;\nuse crate::a::B;\nuse crate::b::C;\nuse crate::c::A;\n'], + ]) + t.assert.deepEqual(edges(buildRustTree(ring, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs')), { + 'crate::b::A': 'src/a.rs', 'crate::a::C': 'src/c.rs', 'crate::c::B': 'src/b.rs', 'crate::a::B': 'src/b.rs', 'crate::b::C': 'src/c.rs', 'crate::c::A': 'src/a.rs', + }) + // A name found only around the cycle: `a::X` through b, whose glob comes back through a. + const around = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\nmod user;\n'], + ['src/a.rs', 'pub use crate::b::*;\npub use crate::c::*;\n'], + ['src/b.rs', 'pub use crate::a::inner::*;\n'], + ['src/c.rs', 'pub mod inner { pub struct X; }\n'], + ['src/user.rs', 'use crate::a::X;\nuse crate::b::X as Y;\n'], + ]) + t.assert.deepEqual(edges(buildRustTree(around, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs')), { 'crate::a::X': 'src/c.rs', 'crate::b::X': 'src/c.rs', 'crate::a': 'src/a.rs', 'crate::b': 'src/b.rs' }) +}) + +test('buildRustTree survives a `use log;` or `use x::{self}` in a module spread over platform files', (t) => { + // A one-segment import in a file whose module has other (cfg-variant) files asks its own + // module for the name, which is that very import: no recursion. + const sources = new Map([ + ['src/lib.rs', '#[cfg_attr(unix, path = "u.rs")]\n#[cfg_attr(windows, path = "w.rs")]\nmod sys;\nmod x;\n'], + ['src/u.rs', 'use log;\nuse crate::x::{self, Y};\nfn f() { log::info!("x"); x::g(); }\n'], + ['src/w.rs', 'use crate::x::Y;\n'], + ['src/x.rs', 'pub struct Y;\npub fn g() {}\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/u.rs')), { 'crate::x': 'src/x.rs', 'crate::x::Y': 'src/x.rs', 'x::g': 'src/x.rs' }) + t.assert.deepEqual([...unresolvedCrates], ['log']) +}) + +test('buildRustTree keeps a path under one platform out of the other platforms\' files', (t) => { + // libc: each platform module is glob re-exported from the root in a `cfg_if!` branch. A file + // under `unix` asking `crate::c_int` means unix's, not fuchsia's or windows'; a platform-neutral + // file gets the first branch's. + const sources = new Map([ + ['src/lib.rs', ['mod user;', 'cfg_if! {', ' if #[cfg(windows)] { mod windows; pub use windows::*; }', ' else if #[cfg(target_os = "fuchsia")] { mod fuchsia; pub use fuchsia::*; }', ' else if #[cfg(unix)] { mod unix; pub use unix::*; }', '}'].join('\n')], + ['src/windows.rs', 'pub type c_int = i32;\npub struct sigset_t;\n'], + ['src/fuchsia.rs', 'pub type c_int = i32;\ns! { pub struct sigset_t { x: u8 } }\n'], + ['src/unix.rs', 'pub type c_int = i32;\nmod linux;\npub use self::linux::sigset_t;\nfn f() { crate::sigset_t::default(); }\n'], + ['src/unix/linux.rs', 'pub struct sigset_t;\nuse crate::c_int;\n'], + ['src/user.rs', 'use crate::c_int;\nuse crate::sigset_t;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/unix/linux.rs')), { 'crate::c_int': 'src/unix.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/unix.rs')), { 'mod linux': 'src/unix/linux.rs', 'self::linux::sigset_t': 'src/unix/linux.rs', 'crate::sigset_t::default': 'src/unix/linux.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::c_int': 'src/windows.rs', 'crate::sigset_t': 'src/windows.rs' }) + // mio: two `pub use … Waker` in one file, in exclusive `cfg_if!` branches; a file mounted in one + // branch means that branch's. + const mio = new Map([ + ['src/lib.rs', 'mod sys;\n'], + ['src/sys.rs', '#[cfg_attr(unix, path = "sys/unix/mod.rs")]\n#[cfg_attr(windows, path = "sys/windows/mod.rs")]\nmod imp;\npub use self::imp::*;\n'], + ['src/sys/unix/mod.rs', ['cfg_if! {', ' if #[cfg(mio_unsupported_force_poll_poll)] { mod selector_poll; pub use self::selector_poll::Waker; }', ' else { mod selector_epoll; mod waker; pub use self::waker::Waker; }', '}'].join('\n')], + ['src/sys/unix/selector_poll.rs', 'pub struct Waker;\n'], + ['src/sys/unix/selector_epoll.rs', 'use crate::sys::Waker;\n'], + ['src/sys/unix/waker.rs', 'pub struct Waker;\n'], + ['src/sys/windows/mod.rs', 'pub struct Waker;\n'], + ]) + t.assert.deepEqual(edges(buildRustTree(mio, { roots: ['src/lib.rs'] }).resolutions.get('src/sys/unix/selector_epoll.rs')), { 'crate::sys::Waker': 'src/sys/unix/waker.rs', 'crate::sys': 'src/sys.rs' }) + // tokio: a file under one macro-keyed variant of `mod imp` means that variant's items. + const tokio = new Map([ + ['src/lib.rs', 'mod atomic;\n'], + ['src/atomic.rs', 'cfg_has_atomic_u64! {\n #[path = "atomic_native.rs"]\n mod imp;\n}\ncfg_not_has_atomic_u64! {\n #[path = "atomic_as_mutex.rs"]\n mod imp;\n}\n'], + ['src/atomic_native.rs', 'pub struct AtomicU64;\n'], + ['src/atomic_as_mutex.rs', 'mod static_macro;\npub struct AtomicU64;\n'], + ['src/atomic_as_mutex/static_macro.rs', 'use super::AtomicU64;\nfn f() { crate::atomic::imp::AtomicU64::new(); }\n'], + ]) + t.assert.deepEqual(edges(buildRustTree(tokio, { roots: ['src/lib.rs'] }).resolutions.get('src/atomic_as_mutex/static_macro.rs')), { 'super::AtomicU64': 'src/atomic_as_mutex.rs', 'crate::atomic::imp::AtomicU64::new': 'src/atomic_as_mutex.rs' }) +}) + +test('buildRustTree takes a one-segment root path for an exported macro when no crate has the name', (t) => { + // anyhow: `pub use anyhow as format_err;` beside `#[macro_export] macro_rules! anyhow` in macros.rs. + const sources = new Map([ + ['src/lib.rs', 'mod macros;\npub use anyhow as format_err;\n'], + ['src/macros.rs', '#[macro_export]\nmacro_rules! anyhow { () => {} }\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod macros': 'src/macros.rs', anyhow: 'src/macros.rs' }) +}) + +test('scanRustItems leaves template items, `static ref`, fn-body items and template imports out of what a module defines', (t) => { + const items = scanRustItems([ + 'macro_rules! m { () => { pub struct FromTemplate; use $crate::a::*; } }', + 'lazy_static! { static ref TABLE: u8 = 1; }', + 'extern "C" fn cb() { struct Local; fn inner() {} }', + 'extern "C" { pub fn getpid() -> i32; }', + 'pub struct Real;', + ].join('\n')) + t.assert.deepEqual(items.defined.map((d) => d.name), ['TABLE', 'cb', 'getpid', 'Real']) + t.assert.deepEqual(items.imports.map((im) => [im.segments.join('::'), im.glob, im.macro]), [['crate::a', true, 'macro_rules']]) // recorded, but not as this module's (buildRustTree skips it) +}) + +test('scanRustItems skips a quote!-family body wherever it comes from, unless the file defines that macro itself', (t) => { + const own = scanRustItems('macro_rules! quote { ($($t:tt)*) => {} }\nfn f() { quote! { crate::a::b() } }\n') + t.assert.deepEqual(own.refs.map((r) => r.spec), ['crate::a::b']) + const ext = scanRustItems('fn f() { quote::quote! { crate::a::b() } ::quote::quote_spanned! { crate::c::d() } }\n') + t.assert.deepEqual(ext.refs.map((r) => r.spec), []) +}) + +test('buildRustTree flattens a mod whose every cfg names the same file', (t) => { + // libc's `mod primitives` under a dozen cfgs: one file, one edge. + const sources = new Map([ + ['src/lib.rs', '#[cfg(unix)] mod primitives;\n#[cfg(windows)] mod primitives;\n#[cfg(target_os = "fuchsia")] mod primitives;\n'], + ['src/primitives.rs', ''], + ]) + t.assert.deepEqual(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/lib.rs')), { 'mod primitives': 'src/primitives.rs' }) +}) + +test('scanRustItems takes a dead `let` or arm\'s else branches with it, and the branches after a cfg_if! branch that holds', (t) => { + const target = new Set(['unix', 'target_os="linux"', 'target_family="unix"']) + const items = scanRustItems([ + 'fn f() {', + ' #[cfg(feature = "x")] let v = if a { dead_a::f() } else { dead_b::g() };', + ' match k {', + ' #[cfg(feature = "x")] 1 => if a { dead_c::h() } else { dead_d::i() },', + ' _ => live::j(),', + ' }', + '}', + 'cfg_if! { if #[cfg(unix)] { mod u; use live_u::X; } else if #[cfg(windows)] { mod w; use dead_w::Y; } else { mod o; use dead_o::Z; } }', + ].join('\n'), { features: new Set(), target }) + t.assert.deepEqual(items.refs.map((r) => r.spec).toSorted(), ['live::j', 'live_u::X']) + t.assert.deepEqual(items.mods.map((m) => m.name), ['u']) + // Without a target the branches stay, each under what it means. + t.assert.deepEqual(scanRustItems('cfg_if! { if #[cfg(unix)] { mod u; } else if #[cfg(windows)] { mod w; } else { mod o; } }').mods.map((m) => [m.name, m.cfg]), [['u', 'unix'], ['w', 'all(not(unix), windows)'], ['o', 'all(not(unix), not(windows))']]) +}) + +test('buildRustTree gives a bare macro call its textual scope: definitions before the mounting mod, a #[macro_use] mod\'s within its module', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod early;\nmacro_rules! late { () => {} }\nmod a;\nmod b;\nmod later;\n'], + ['src/early.rs', 'fn f() { late!(); }\n'], + ['src/later.rs', 'fn f() { late!(); }\n'], + ['src/a.rs', '#[macro_use]\nmod inner_macros;\nmod uses;\nfn f() { nested!(); }\n'], + ['src/a/inner_macros.rs', 'macro_rules! nested { () => {} }\n'], + ['src/a/uses.rs', 'fn f() { nested!(); }\n'], + ['src/b.rs', 'fn f() { nested!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/early.rs')), {}) // `late!` is defined after `mod early;` + t.assert.deepEqual(edges(resolutions.get('src/later.rs')), { 'late!': 'src/lib.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/a.rs')), { 'mod inner_macros': 'src/a/inner_macros.rs', 'mod uses': 'src/a/uses.rs', 'nested!': 'src/a/inner_macros.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/a/uses.rs')), { 'nested!': 'src/a/inner_macros.rs' }) // mounted after the `#[macro_use] mod` + t.assert.deepEqual(edges(resolutions.get('src/b.rs')), {}) // a's `#[macro_use]` reaches a, not the crate +}) + +test('scanRustItems reads a manifest-relative include inside a doc attribute', (t) => { + const items = scanRustItems('#![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))]\n#[doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/docs/a.md"))]\npub struct S;\n') + t.assert.deepEqual(items.includes, [ + { kind: 'include_str', path: 'README.md', base: 'manifest', conditional: false }, + { kind: 'include_str', path: 'docs/a.md', base: 'manifest', conditional: false }, + ]) +}) + +test('buildRustTree honours restricted visibility on re-exports', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\n'], + ['src/a.rs', 'pub mod inner;\npub mod other;\npub(self) use crate::c::Priv;\npub(super) use crate::c::ToParent;\npub(in crate::a) use crate::c::InA;\nfn f() { Priv::new(); }\n'], + ['src/a/inner.rs', 'use super::InA;\nuse super::Priv;\nuse crate::a::ToParent;\n'], + ['src/a/other.rs', 'use super::InA;\n'], + ['src/b.rs', 'use crate::a::ToParent as T;\nuse crate::a::InA as I;\nuse crate::a::Priv as P;\n'], + ['src/c.rs', 'pub struct Priv;\npub struct ToParent;\npub struct InA;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + // Inside `a`: everything (and `super`, the module whose imports the paths went through). + t.assert.deepEqual(edges(resolutions.get('src/a/inner.rs')), { 'super::InA': 'src/c.rs', 'super::Priv': 'src/c.rs', 'crate::a::ToParent': 'src/c.rs', super: 'src/a.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/a/other.rs')), { 'super::InA': 'src/c.rs', super: 'src/a.rs' }) + // From b (a sibling of a, inside the crate root): only what `pub(super)` -- the root -- lets out; + // the other two names are a's own, as far as b can see. + t.assert.deepEqual(edges(resolutions.get('src/b.rs')), { 'crate::a::ToParent': 'src/c.rs', 'crate::a::InA': 'src/a.rs', 'crate::a::Priv': 'src/a.rs' }) +}) + +test('buildRustTree follows `extern crate … as` like a use, and `extern crate self` to the crate root', (t) => { + const sources = new Map([ + ['src/lib.rs', 'pub extern crate serde_json as json;\nextern crate self as me;\nmod util;\nmod user;\n'], + ['src/util.rs', 'pub fn f() {}\n'], + ['src/user.rs', 'extern crate serde_core as s;\nfn f() { s::Value::Null; crate::json::Value::Null; me::util::f(); }\n'], + ['vendor/serde_json/src/lib.rs', ''], + ['vendor/serde_core/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'use serde_core': 'vendor/serde_core/src/lib.rs', // `s::Value`, through the alias + 'use serde_json': 'vendor/serde_json/src/lib.rs', // `crate::json::Value`, through the root's `pub extern crate … as json` + }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'use serde_json': 'vendor/serde_json/src/lib.rs', 'mod util': 'src/util.rs', 'mod user': 'src/user.rs' }) + // `me::util::f` is `crate::util::f`: recorded under its lead once the crate root's alias is known. + t.assert.equal(resolutions.get('src/user.rs').get('me::util::f'), undefined) +}) + +test('buildRustTree records no via edge for the crate root, however the path reaches it', (t) => { + // quote's ext.rs: `use super::ToTokens;` with `pub use to_tokens::ToTokens;` in lib.rs. + const sources = new Map([ + ['src/lib.rs', 'mod to_tokens;\nmod ext;\nmod runtime;\npub use to_tokens::ToTokens;\npub mod __private { pub use crate::runtime::push_ident; }\n'], + ['src/to_tokens.rs', 'pub trait ToTokens {}\n'], + ['src/runtime.rs', 'pub fn push_ident() {}\n'], + ['src/ext.rs', 'use super::ToTokens;\nuse crate::__private::push_ident;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/ext.rs')), { 'super::ToTokens': 'src/to_tokens.rs', 'crate::__private::push_ident': 'src/runtime.rs' }) +}) + +test('buildRustTree anchors a path on the module\'s own imports: a `use`d name, or one a glob re-export provides', (t) => { + // typenum: `pub use crate::{gen::consts}; pub use consts::*;` -- libc: `pub use linux_uapi::*; + // pub use linux::types::*;` where `linux` is a module of linux_uapi. Neither `consts` nor + // `linux` is a crate, and nothing is reported as one. + const sources = new Map([ + ['src/lib.rs', ['mod gen;', 'mod uapi;', 'mod user;', 'pub use gen::consts;', 'pub use consts::*;', 'pub use uapi::*;', 'pub use linux::types::*;'].join('\n')], + ['src/gen.rs', 'pub mod consts;\n'], + ['src/gen/consts.rs', 'pub struct U1;\n'], + ['src/uapi.rs', 'pub(crate) mod linux;\n'], + ['src/uapi/linux.rs', 'pub mod types;\n'], + ['src/uapi/linux/types.rs', 'pub type Foo = u8;\n'], + ['src/user.rs', 'use crate::consts::U1;\nuse crate::linux::types::Foo;\nuse crate::U1 as Root;\n'], + ]) + const { resolutions, missing, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { + 'mod gen': 'src/gen.rs', + 'mod uapi': 'src/uapi.rs', + 'mod user': 'src/user.rs', + 'gen::consts': 'src/gen/consts.rs', + consts: 'src/gen/consts.rs', // through this module's own `pub use gen::consts` + uapi: 'src/uapi.rs', + 'linux::types': 'src/uapi/linux/types.rs', // `linux` through `pub use uapi::*` + }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::consts::U1': 'src/gen/consts.rs', + 'crate::linux::types::Foo': 'src/uapi/linux/types.rs', + 'crate::U1': 'src/gen/consts.rs', // an item behind a glob: the module defining it, through `pub use consts::*` + }) +}) + +test('buildRustTree resolves the submodules of a file mounted both through #[path] and a plain mod', (t) => { + // rustc mounts src/b.rs twice: as `crate::b` (its `mod c;` is src/b/c.rs) and as `crate::alias` + // (its `mod c;` would be src/c.rs). The loader keeps one resolution: whichever file exists. + const sources = new Map([ + ['src/lib.rs', 'mod b;\n#[path = "b.rs"]\nmod alias;\n'], + ['src/b.rs', 'mod c;\n'], + ['src/b/c.rs', ''], + ]) + const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(missing, []) + t.assert.equal(resolutions.get('src/b.rs').get('mod c'), 'src/b/c.rs') +}) + +test('buildRustTree rescans a file whose content changed in a reused sources map', (t) => { + const sources = new Map([['src/lib.rs', 'mod a;\n'], ['src/a.rs', ''], ['src/b.rs', '']]) + t.assert.deepEqual([...buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/lib.rs').keys()], ['mod a']) + sources.set('src/lib.rs', 'mod a;\nmod b;\n') + t.assert.deepEqual([...buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/lib.rs').keys()], ['mod a', 'mod b']) +}) + test('buildRustTree resolves super:: and self:: paths against the module tree', async (t) => { const sources = await collectRustFilesFromDisk(join(fixtures, 'use-groups'), ['src/main.rs']) const { resolutions } = buildRustTree(sources) @@ -758,3 +1788,314 @@ test('loadRust rejects an absolute entry path in the listing', async (t) => { /Entry path must not be absolute/, ) }) + +// --- fifth review: two platform files with the same import, namespaces, cfg precision, order --- + +test('buildRustTree follows the same `use log;` in two platform files of one module without looping', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod sys;\n'], + ['src/sys.rs', '#[cfg(unix)] #[path = "sys/u.rs"] mod imp;\n#[cfg(windows)] #[path = "sys/w.rs"] mod imp;\npub use self::imp::*;\n'], + ['src/sys/u.rs', 'use log;\npub fn u() { log::info!("u"); }\n'], + ['src/sys/w.rs', 'use log;\npub fn w() { log::info!("w"); }\n'], + ['vendor/log/src/lib.rs', ''], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/sys/u.rs')), { 'use log': 'vendor/log/src/lib.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/sys/w.rs')), { 'use log': 'vendor/log/src/lib.rs' }) + // The same through cfg_if! branches both importing the crate. + const chain = new Map([ + ['src/lib.rs', 'cfg_if! { if #[cfg(unix)] { use libc; mod u; } else { use libc; mod w; } }\n'], + ['src/u.rs', 'fn f() { libc::getpid(); }\n'], + ['src/w.rs', ''], + ['vendor/libc/src/lib.rs', ''], + ]) + const r = buildRustTree(chain, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(r.resolutions.get('src/u.rs')), { 'use libc': 'vendor/libc/src/lib.rs' }) +}) + +test('buildRustTree resolves a glob written in a platform file beside a glob of its parent', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod sys;\n'], + ['src/sys.rs', 'mod helpers;\n#[cfg(unix)] #[path = "sys/u.rs"] mod imp;\n#[cfg(windows)] #[path = "sys/w.rs"] mod imp;\npub use self::imp::*;\n'], + ['src/sys/helpers.rs', 'pub struct Thing;\n'], + ['src/sys/u.rs', 'use super::*;\npub use helpers::*;\npub fn u() -> helpers::Thing { helpers::Thing }\n'], + ['src/sys/w.rs', 'pub fn w() {}\n'], + ['src/user.rs', ''], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/sys/u.rs')), { super: 'src/sys.rs', helpers: 'src/sys/helpers.rs', 'helpers::Thing': 'src/sys/helpers.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/sys/w.rs')), {}) +}) + +test('buildRustTree never lets a value item lead a path: `fn log` and `*const libc::c_char` beside the crates', (t) => { + const items = scanRustItems('extern "C" { pub fn f(p: *const libc::c_char) -> *mut libc::c_void; }\nfn g(p: *const libc::c_char) {}\nfn log() {}\nstatic S: u8 = 0;\nconst C: u8 = 0;\nstruct T;\nuse log::info;\n') + t.assert.deepEqual(items.defined.map((d) => [d.name, d.ns]), [['f', 'value'], ['g', 'value'], ['log', 'value'], ['S', 'value'], ['C', 'value'], ['T', 'type']]) + const sources = new Map([ + ['src/lib.rs', 'mod net;\n'], + ['src/net.rs', 'use libc;\nfn g(p: *const libc::c_char) {}\nfn log() {}\nuse log::info;\n'], + ['vendor/libc/src/lib.rs', ''], + ['vendor/log/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/net.rs')), { 'use libc': 'vendor/libc/src/lib.rs', 'use log': 'vendor/log/src/lib.rs' }) + t.assert.deepEqual([...unresolvedCrates], []) + // A one-segment path may still name the fn (`log()` is `crate::log`). + const own = buildRustTree(new Map([['src/lib.rs', 'mod net;\nmod user;\n'], ['src/net.rs', 'pub fn log() {}\n'], ['src/user.rs', 'fn f() { crate::net::log(); }\n']]), { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(own.resolutions.get('src/user.rs')), { 'crate::net::log': 'src/net.rs' }) +}) + +test('buildRustTree gives the same answers whatever order the sources come in', (t) => { + const files = { + 'src/lib.rs': 'mod sys;\nmod user;\n', + 'src/sys.rs': '#[cfg(unix)] #[path = "sys/u.rs"] mod imp;\n#[cfg(windows)] #[path = "sys/w.rs"] mod imp;\npub use self::imp::*;\n', + 'src/sys/u.rs': 'pub struct Thing;\n', + 'src/sys/w.rs': 'pub struct Thing;\n', + 'src/user.rs': 'use crate::sys::Thing;\n', + } + const answers = new Set() + const keys = Object.keys(files) + for (let i = 0; i < keys.length; i++) { + const order = [...keys.slice(i), ...keys.slice(0, i)] + const { resolutions } = buildRustTree(new Map(order.map((k) => [k, files[k]])), { roots: ['src/lib.rs'] }) + answers.add(JSON.stringify([...resolutions].toSorted().map(([f, m]) => [f, edges(m)]))) + } + t.assert.equal(answers.size, 1) + const [only] = answers + t.assert.equal(JSON.parse(only).find(([f]) => f === 'src/user.rs')[1]['crate::sys::Thing'], 'src/sys/u.rs') // the first variant declared +}) + +test('buildRustTree keeps a file mounted under several cfgs under any of them, and an any(…) branch out of another target_os', (t) => { + // libc: `mod linux_l4re_shared` in both the linux and the l4re branch; aix's `crate::T` is aix's. + const sources = new Map([ + ['src/lib.rs', 'cfg_if! { if #[cfg(target_os = "linux")] { mod shared; pub use shared::*; } else if #[cfg(target_os = "l4re")] { mod shared; pub use shared::*; } else if #[cfg(target_os = "aix")] { mod aix; pub use aix::*; mod user_aix; } }\ncfg_if! { if #[cfg(any(target_os = "linux", target_os = "l4re"))] { mod either; pub use either::*; } else if #[cfg(target_os = "aix")] { mod user2; } }\n'], + ['src/shared.rs', 'pub struct T;\npub struct U;\n'], + ['src/either.rs', 'pub struct T;\n'], + ['src/aix.rs', 'pub struct T;\npub struct U;\n'], + ['src/user_aix.rs', 'fn f() { crate::T; crate::U; }\n'], + ['src/user2.rs', 'fn f() { crate::T; }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user_aix.rs')), { 'crate::T': 'src/aix.rs', 'crate::U': 'src/aix.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/user2.rs')), { 'crate::T': 'src/aix.rs' }) +}) + +test('buildRustTree keeps a module two glob paths reach under either path\'s cfg', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\nmod shared;\n#[cfg(target_os = "linux")] pub use a::*;\n#[cfg(target_os = "macos")] pub use b::*;\n#[cfg(target_os = "aix")] pub use c::*;\n#[cfg(target_os = "aix")] mod user;\n'], + ['src/a.rs', 'pub use crate::shared::*;\n'], + ['src/b.rs', 'pub use crate::shared::*;\n'], + ['src/c.rs', 'pub struct T;\n'], + ['src/shared.rs', 'pub struct T;\n'], + ['src/user.rs', 'fn f() { crate::T; }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::T': 'src/c.rs' }) +}) + +test('buildRustTree checks every hop of an import chain against the asking file\'s cfgs', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod mid;\nmod u;\nmod w;\npub use mid::X;\n#[cfg(unix)] mod user;\n'], + ['src/mid.rs', '#[cfg(windows)] pub use crate::w::X;\n#[cfg(unix)] pub use crate::u::X;\n'], + ['src/u.rs', 'pub struct X;\n'], + ['src/w.rs', 'pub struct X;\n'], + ['src/user.rs', 'fn f() { crate::X; }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::X': 'src/u.rs' }) +}) + +test('scanRustItems keeps a nested cfg_if! chain apart from the chain around it', (t) => { + const src = 'cfg_if! { if #[cfg(unix)] { cfg_if! { if #[cfg(target_os = "linux")] { mod l; } else { mod o; } } mod after; } else { mod w; } }\ncfg_if! { if #[cfg(a)] { mod x; } else { mod y; } }\n' + t.assert.deepEqual(scanRustItems(src).mods.map((m) => [m.name, m.cfg]), [ + ['l', 'all(unix, target_os = "linux")'], + ['o', 'all(unix, not(target_os = "linux"))'], + ['after', 'unix'], + ['w', 'not(unix)'], + ['x', 'a'], + ['y', 'not(a)'], + ]) +}) + +test('buildRustTree shadows macros in source order: a #[macro_use] mod and the file\'s own definitions alike', (t) => { + const first = new Map([ + ['src/lib.rs', 'macro_rules! tri { () => {} }\n#[macro_use]\nmod macros;\nmod user;\n'], + ['src/macros.rs', 'macro_rules! tri { () => {} }\n'], + ['src/user.rs', 'fn f() { tri!(); }\n'], + ]) + t.assert.deepEqual(edges(buildRustTree(first, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs')), { 'tri!': 'src/macros.rs' }) + const second = new Map([...first, ['src/lib.rs', '#[macro_use]\nmod macros;\nmacro_rules! tri { () => {} }\nmod user;\n']]) + t.assert.deepEqual(edges(buildRustTree(second, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs')), { 'tri!': 'src/lib.rs' }) +}) + +test('buildRustTree puts what a macro_rules! template declares where the macro is invoked', (t) => { + // serde: `crate_root! { … macro_rules! tri { … } … pub mod de; … }` invoked in lib.rs after `#[macro_use] mod m;`. + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod m;\nmod before;\nmacro_rules! root { () => { macro_rules! tri { () => {} } mod de; } }\nroot!();\nmod later;\n'], + ['src/m.rs', 'macro_rules! tri { () => {} }\n'], + ['src/before.rs', 'fn f() { tri!(); }\n'], + ['src/de.rs', 'fn f() { tri!(); }\n'], + ['src/later.rs', 'fn f() { tri!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/before.rs')), { 'tri!': 'src/m.rs' }) // mounted before the invocation + t.assert.deepEqual(edges(resolutions.get('src/de.rs')), { 'tri!': 'src/lib.rs' }) // `tri` stands before `mod de` in the template + t.assert.deepEqual(edges(resolutions.get('src/later.rs')), { 'tri!': 'src/lib.rs' }) + const items = scanRustItems(sources.get('src/lib.rs')) + const call = sources.get('src/lib.rs').indexOf('root!()') + t.assert.deepEqual(items.mods.filter((m) => m.name === 'de').map((m) => [m.template, m.offset === call]), [['root', true]]) + t.assert.deepEqual(items.macros.map((m) => [m.name, m.template, m.offset === call]), [['root', null, false], ['tri', 'root', true]]) +}) + +test('buildRustTree takes a one-segment path for an exported macro only at the crate root', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod macros;\nmod child;\npub use anyhow as format_err;\n'], + ['src/macros.rs', '#[macro_export]\nmacro_rules! anyhow { () => {} }\n'], + ['src/child.rs', 'use anyhow;\nuse crate::anyhow as also;\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod macros': 'src/macros.rs', 'mod child': 'src/child.rs', anyhow: 'src/macros.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/child.rs')), { 'crate::anyhow': 'src/macros.rs' }) // `use anyhow;` in a child names the crate + t.assert.deepEqual([...unresolvedCrates], ['anyhow']) +}) + +test('buildRustTree lets a glob into an in-tree crate explain the names of the module it names there', (t) => { + const sources = new Map([ + ['src/lib.rs', 'use tokio::sync::*;\nuse mpsc::channel;\nuse nothere::x;\nmod deep;\n'], + ['src/deep.rs', 'use tokio::*;\nuse oneshot::Sender;\n'], // through tokio's root's own glob + ['vendor/tokio/src/lib.rs', 'pub mod sync;\npub use sync::*;\n'], + ['vendor/tokio/src/sync/mod.rs', 'pub mod mpsc;\npub use inner::*;\nmod inner;\n'], + ['vendor/tokio/src/sync/mpsc.rs', 'pub fn channel() {}\n'], + ['vendor/tokio/src/sync/inner.rs', 'pub mod oneshot;\n'], + ['vendor/tokio/src/sync/inner/oneshot.rs', 'pub struct Sender;\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'use tokio': 'vendor/tokio/src/lib.rs', 'mod deep': 'src/deep.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/deep.rs')), { 'use tokio': 'vendor/tokio/src/lib.rs' }) + t.assert.deepEqual([...unresolvedCrates], ['nothere']) +}) + +test('buildRustTree reads a quote!-family body as macro input throughout a package that defines the macro itself', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod m;\nmod a;\nmod user;\n'], + ['src/m.rs', 'macro_rules! quote { ($($t:tt)*) => {} }\n'], + ['src/a.rs', 'pub fn b() {}\n'], + ['src/user.rs', 'fn f() { quote! { crate::a::b() } }\n'], + ['vendor/other/src/lib.rs', 'fn g() { quote! { crate::a::b() } }\n'], // another package: a template, skipped + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs', 'vendor/other/src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::a::b': 'src/a.rs', 'quote!': 'src/m.rs' }) + t.assert.deepEqual(edges(resolutions.get('vendor/other/src/lib.rs')), {}) + t.assert.deepEqual([...scanRustItems(sources.get('src/user.rs')).skipped], ['quote']) +}) + +// --- sixth review: unknown gates, cycles' cached answers, imported values, reached-again modules --- + +test('buildRustTree reads a #[cfg] on a macro invocation as gating its body, and a cfg_! body as a gate of its own', (t) => { + const items = scanRustItems('#[cfg(any(unix, target_os = "hermit"))]\ncfg_os_poll! {\n mod unix;\n pub use self::unix::*;\n}\n#[cfg(windows)]\ncfg_os_poll! {\n mod windows;\n}\ncfg_not_os_poll! {\n mod shell;\n pub(crate) use self::shell::*;\n}\n#[cfg(feature = "x")]\nmacro_rules! m { () => { mod inner; } }\n') + t.assert.deepEqual(items.mods.map((m) => [m.name, m.cfg, m.macro]), [['unix', 'any(unix, target_os = "hermit")', 'cfg_os_poll'], ['windows', 'windows', 'cfg_os_poll'], ['shell', null, 'cfg_not_os_poll'], ['inner', null, 'macro_rules']]) + t.assert.deepEqual(items.imports.map((im) => [im.segments.join('::'), im.cfg, im.macro]), [['self::unix', 'any(unix, target_os = "hermit")', 'cfg_os_poll'], ['self::shell', null, 'cfg_not_os_poll']]) + // mio: a unix waker's `crate::sys::Selector` is its own platform's selector, not the windows + // one (whose `#[cfg(windows)]` sits on the `cfg_os_poll!` invocation) nor the shell one (under + // `cfg_not_os_poll!`, the negation of the gate the waker sits behind -- a gate the loader can't + // see into is never "certainly compiled", so it can't beat a compatible candidate either). + const sources = new Map([ + ['src/lib.rs', 'mod sys;\n'], + ['src/sys/mod.rs', '#[cfg(any(unix, target_os = "hermit"))]\ncfg_os_poll! {\n mod unix;\n pub use self::unix::*;\n}\n#[cfg(windows)]\ncfg_os_poll! {\n mod windows;\n pub use self::windows::*;\n}\ncfg_not_os_poll! {\n mod shell;\n pub(crate) use self::shell::*;\n}\n'], + ['src/sys/unix/mod.rs', 'cfg_os_poll! {\n #[cfg_attr(target_os = "linux", path = "selector/epoll.rs")]\n #[cfg_attr(target_os = "macos", path = "selector/kqueue.rs")]\n mod selector;\n pub(crate) use self::selector::*;\n #[cfg_attr(target_os = "linux", path = "waker/eventfd.rs")]\n #[cfg_attr(target_os = "macos", path = "waker/kqueue.rs")]\n mod waker;\n pub(crate) use self::waker::Waker;\n}\n'], + ['src/sys/unix/selector/epoll.rs', 'pub struct Selector;\n'], + ['src/sys/unix/selector/kqueue.rs', 'pub struct Selector;\n'], + ['src/sys/unix/waker/eventfd.rs', 'use crate::sys::Selector;\npub struct Waker;\n'], + ['src/sys/unix/waker/kqueue.rs', 'use crate::sys::Selector;\npub struct Waker;\n'], + ['src/sys/windows/mod.rs', 'mod selector;\npub use selector::Selector;\n'], + ['src/sys/windows/selector.rs', 'pub struct Selector;\n'], + ['src/sys/shell/mod.rs', 'mod selector;\nmod waker;\npub use self::selector::Selector;\n'], + ['src/sys/shell/selector.rs', 'pub struct Selector;\n'], + ['src/sys/shell/waker.rs', 'use crate::sys::Selector;\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.equal(edges(resolutions.get('src/sys/unix/waker/eventfd.rs'))['crate::sys::Selector'], 'src/sys/unix/selector/epoll.rs') + t.assert.equal(edges(resolutions.get('src/sys/unix/waker/kqueue.rs'))['crate::sys::Selector'], 'src/sys/unix/selector/kqueue.rs') + t.assert.equal(edges(resolutions.get('src/sys/shell/waker.rs'))['crate::sys::Selector'], 'src/sys/shell/selector.rs') +}) + +test('buildRustTree drops the answers imports gave on top of a short closure when the cycle is resolved', (t) => { + // A1 through a named import at the root: `pub use child::run;` asked first, while child's glob + // closure was still short of the cycle through b. + const sources = new Map([ + ['src/lib.rs', 'mod child;\nmod b;\nmod c;\npub use child::run;\nmod user;\n'], + ['src/child.rs', 'pub use crate::b::*;\npub use crate::c::*;\n'], + ['src/b.rs', 'pub use crate::child::inner::*;\n'], + ['src/c.rs', 'pub mod inner { pub fn run() {} }\n'], + ['src/user.rs', 'fn f() { crate::run(); crate::child::run(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::run': 'src/c.rs', 'crate::child::run': 'src/c.rs', 'crate::child': 'src/child.rs' }) +}) + +test('buildRustTree lets a name imported as a value still lead a path to the crate of that name', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod util;\nmod user;\n'], + ['src/util.rs', 'pub fn log() {}\npub static TABLE: u8 = 0;\npub mod real { pub struct Formatter; }\n'], + ['src/user.rs', 'use crate::util::log;\nuse crate::util::TABLE;\nuse crate::util::real as fmt;\nfn f() { log::info!("x"); log(); fmt::Formatter; }\n'], + ['vendor/log/src/lib.rs', ''], + ]) + const { resolutions, unresolvedCrates, wantedRoots } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::util::log': 'src/util.rs', 'crate::util::TABLE': 'src/util.rs', 'crate::util::real': 'src/util.rs', 'use log': 'vendor/log/src/lib.rs', 'fmt::Formatter': 'src/util.rs' }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(wantedRoots, []) + // The crate's root not among the sources (the walk took `log` for the fn): asked for. + const without = new Map([...sources].filter(([f]) => f !== 'vendor/log/src/lib.rs')) + const r = buildRustTree(without, { roots: ['src/lib.rs'], baseDir: join(fixtures, 'use-crate') }) + t.assert.deepEqual(edges(r.resolutions.get('src/user.rs'))['use log'], undefined) +}) + +test('buildRustTree walks on from a module a second glob path reaches under another cfg', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod mid;\nmod deep;\n#[cfg(target_os = "linux")] pub use a::*;\n#[cfg(target_os = "aix")] pub use b::*;\n#[cfg(target_os = "aix")] mod user;\n'], + ['src/a.rs', 'pub use crate::mid::*;\n'], + ['src/b.rs', 'pub use crate::mid::*;\n'], + ['src/mid.rs', 'pub use crate::deep::*;\n'], + ['src/deep.rs', 'pub struct T;\n'], + ['src/user.rs', 'fn f() { crate::T; }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + // deep is reached through mid under linux first, then under aix: under either, so the aix user finds T there. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::T': 'src/deep.rs' }) +}) + +test('buildRustTree puts a mod a template declares at the invocation in another file, for textual macro scope', (t) => { + // serde: core/crate_root.rs's `crate_root! { … pub mod de; … }` is invoked in lib.rs after + // `#[macro_use] mod macros;`, so de sees `forward_to_deserialize_any!`. + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\n#[path = "core/crate_root.rs"]\nmod crate_root;\n#[macro_use]\n#[path = "core/macros.rs"]\nmod macros;\ncrate_root!();\n'], + ['src/core/crate_root.rs', 'macro_rules! crate_root { () => { pub mod de; } }\n'], + ['src/core/macros.rs', 'macro_rules! forward_to_deserialize_any { () => {} }\n'], + ['src/core/de.rs', 'fn f() { forward_to_deserialize_any!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/core/de.rs')), { 'forward_to_deserialize_any!': 'src/core/macros.rs' }) +}) + +test('buildRustTree decides a bare macro call\'s scope at the call: a later definition in the file does not shadow it', (t) => { + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod m;\nmod user;\n'], + ['src/m.rs', 'macro_rules! x { () => {} }\n'], + ['src/user.rs', 'fn f() { x!(); }\nmacro_rules! x { () => {} }\nfn g() { x!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'x!': 'src/m.rs' }) + t.assert.deepEqual([...scanRustItems(sources.get('src/user.rs')).calls], [['x', [9, 54]]]) +}) + +test('buildRustTree takes a candidate under a custom cfg only after one under none: `loom` is off in a default build', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod user;\n#[cfg(loom)] pub use a::X;\n#[cfg(not(loom))] pub use b::X;\n#[cfg(any(loom, target_os = "aix"))] pub use a::Y;\n#[cfg(unix)] pub use b::Y;\n'], + ['src/a.rs', 'pub struct X;\npub struct Y;\n'], + ['src/b.rs', 'pub struct X;\npub struct Y;\n'], + ['src/user.rs', 'fn f() { crate::X; crate::Y; }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + // `any(loom, target_os = "aix")` may hold on aix: no target, so it is only written order that + // puts a's Y first, and it stays. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::X': 'src/b.rs', 'crate::Y': 'src/a.rs' }) +}) From 2d80f0a79dd6d0d259778e2a54aad36ac1b0c71b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 12:42:04 +0000 Subject: [PATCH 02/10] fix(bundle): Rust loader review 7: compile units, honest ranking, template mods - Refuse a Rust source or include_str! file that isn't UTF-8 (and a non-UTF-8 manifest); carry an include_bytes! file as base64. - Compile units: code built for the host (build scripts, build-dependencies, proc-macro crates) is scanned against the host's cfgs and, under resolver 2, the host's own feature context; a crate built both ways is followed as both. Features are on, maybe (only an undecided target table enables them) or off. - --cargo-manifests carries a vendored crate's Cargo.toml and .cargo-checksum.json, and a project package's configs up to the bundle root; the vendored directory follows the replace-with chain whatever its name. - Custom cfgs a build script prints or rustflags set are not presumed off; cfg_x! gates take the cfg their macro_rules! definition writes. - Ambiguous path edges record a cfg-keyed map of their candidates instead of the first written; candidates the build rules out come last; a child module under a custom or dead cfg gives way to an import of its name. - Macro scope: a template's bare calls are made where it is invoked; a mod a template declares belongs to each in-package invoking module and is found beside it (libc's prelude!() -> src/types.rs), else beside the definition. - `extern crate self as x;` is in the extern prelude; a declared dependency is reported missing whatever a glob into a missing crate may provide. - Vendored versions come from the lock, else the one that fits, else a warning and no guess; out-of-root path/[patch] deps never bind to vendored copies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 144 ++++--- stasis/bin/stasis.js | 4 +- stasis/src/cmd/bundle.js | 36 +- stasis/src/loaders/cargo.js | 507 +++++++++++++++++++------ stasis/src/loaders/rust.js | 734 ++++++++++++++++++++++++++---------- tests/rust-features.test.js | 294 +++++++++++++-- tests/rust-loader.test.js | 172 +++++++-- 7 files changed, 1473 insertions(+), 418 deletions(-) diff --git a/doc/file-formats.md b/doc/file-formats.md index b694f686..e2d65798 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -500,7 +500,14 @@ or its own `#[path = "…"]` (solana-program's `#[path = ""] mod non_bpf_modules an `include!`d file is the including module's child, as rustc splices it, but its file is looked up beside the included file: `include!("gen/list.rs")` with `mod bar;` in it finds `gen/bar.rs`, whose own submodules sit under `gen/bar/` -(checked against rustc). When several +(checked against rustc). A `mod` a `macro_rules!` body declares is declared in +each module of its package that invokes the macro by bare name -- through the +macros whose bodies invoke it, the outermost call -- and found beside that +module's file, as rustc expands it (serde_core's `crate_root!`, defined in +crate_root.rs and invoked in lib.rs, declares lib.rs's `de`: `src/de/mod.rs`), +under the cfgs the invoking file mounts the definition's file with; one +nothing in its package invokes by bare name (`crate::m!()` only) is looked up +beside the definition. When several `#[path]` / `#[cfg_attr(, path = …)]` attributes sit on one `mod`, the first whose predicate holds decides the file and ends the list, and the default `name.rs` / `name/mod.rs` lookup is off; variants whose predicate is undecided @@ -512,8 +519,10 @@ the package's dev-dependencies take part in the feature resolution. A `use`/`ext that crate's root in: the package's own lib target (`use my_app::…` from `main.rs`), a Cargo `path` dependency (incl. `workspace = true` ones and `package = …` renames, honouring `[lib] path`), or a `cargo vendor`ed crate under -`vendor/` (or the directory `.cargo/config.toml` names under -`[source.vendored-sources]`). Registry dependencies live in `~/.cargo/registry`, outside the bundle +`vendor/` (or the `directory` of the source the project's cargo config replaces +crates.io with, through any chain of `replace-with`s, whatever its name). A +`path` or `[patch]` dependency outside the bundle root is not in-tree, and never +bound to a vendored crate of its name instead. Registry dependencies live in `~/.cargo/registry`, outside the bundle root, so they're never read: vendor them first (`cargo vendor`). When a bundle references crates it can't find and there is no `vendor/` dir, `stasis bundle` says so and suggests it. @@ -549,38 +558,61 @@ against `unix`, an `any(…)` none of whose alternatives can hold -- libc's `any(target_os = "linux", target_os = "l4re")` branch against a file under `target_os = "aix"`), and of the compatible ones the first whose cfgs its own entail (a candidate under the same `target_os` leaf it was mounted under, over -one under an `any(…)` nothing rules out), else the first in written order -under no custom cfg, else the first in written order at all: a positive cfg -that neither rustc nor cargo sets (`loom`, `docsrs`, `tokio_unstable`, mio's -`mio_unsupported_force_poll_poll`, an `any(…)` of such and of alternatives -the file rules out) is a `--cfg` a default build lacks, so a candidate under -one is taken only when nothing else fits, and one under its negation -(`not(loom)`) counts as certain. With `--cargo-target`, every file compiled +one under an `any(…)` nothing rules out). When none is entailed, every +compatible one under no custom cfg is the answer under its own cfgs, and the +edge is a cfg-keyed map of their files, the shape of a `mod`'s variants below +(a file under no platform cfg asking libc's `crate::sockaddr`, defined per +platform, maps each platform's file, not the first written) -- or that file, +when they agree; else the first under a custom cfg; and a candidate under a cfg +the build rules out (a feature that is off, another target) only when nothing +else fits -- unless the asking file is itself under such a cfg, when the build +says nothing. A positive cfg that neither rustc nor cargo sets (`loom`, +`docsrs`, `tokio_unstable`, mio's `mio_unsupported_force_poll_poll`, an +`any(…)` of such and of alternatives the file rules out) is a `--cfg` a default +build lacks, so a candidate under one is taken only after those, and one under +its negation (`not(loom)`) counts as certain -- unless the package's build may +set it: a name its build script prints as `cargo:rustc-cfg=…`, or that a +rustflags `--cfg` sets (the cargo config's `build`/`target.<…>` `rustflags`, +`RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, `CARGO_BUILD_RUSTFLAGS`), is neither +presumed off nor on, and a build script that formats a name or uses `autocfg` +makes every custom cfg of its package so. A child module under a custom cfg, +or one the build rules out, gives way to an import of its name in the same +module (serde's docsrs-only `mod de` beside the `pub use serde_core::de` of +every other build). With `--cargo-target`, every file compiled for the target is under the target's cfgs too, so a `windows` candidate is out for every file and a `target_os = "linux"` file takes the `any(android, linux)` branch over the `any(aix, solaris)` one (mio's `sys::Waker` is the -eventfd one under a Linux target; without one, written order decides). An +eventfd one under a Linux target; without one, the edge maps each branch's). An item a module defines in several of its files follows the same ranking, the module's own tree file first within a rank. A `cfg_if!` branch is under its own predicate and the negation of every branch before it, nested chains each on their own; a `#[cfg(…)]` on a macro invocation gates everything its body declares (mio's `#[cfg(unix)] cfg_os_poll! { mod unix; … }`); the body -of a `cfg_!` macro (tokio's `cfg_io_uring!`, mio's `cfg_os_poll!`) is a gate -the loader can't see into and `cfg_not_!` its negation, so what the two -declare is never taken together and neither counts as certainly compiled; a +of a `cfg_!` gate macro (tokio's `cfg_io_uring!`, mio's `cfg_os_poll!`) is +under the cfg its package's `macro_rules!` definition puts on each item it +wraps (`$( #[cfg(feature = "rt")] $item )*`, the same in every arm), so +`cfg_x!` and `cfg_not_x!` are told apart by their definitions, not their names; +one whose definition the loader can't read (none in the package, arms that +differ or write no cfg, two definitions that differ) is a gate it can't see +into, whose items are never certainly compiled nor ruled out; a file mounted by several declarations is under any of their cfgs; a module reached by several glob paths is under either's, and so is everything its own globs reach. A glob into the sysroot or into another crate brings in nothing the loader can see, so it claims nothing -- and since a glob into a crate that isn't in-tree may well provide a name, an unresolved lead in a file with such a glob (`use syn::*; … -punctuated::Punctuated`) is not reported as an unresolved crate; a glob into an +punctuated::Punctuated`) is not reported as an unresolved crate -- unless the +file's package declares a dependency of that name, which the lead then is +(rustc rejects a name both a glob and a crate provide): one missing from the +bundle is reported whatever the globs beside it; a glob into an in-tree crate explains only the names the module its path names there has, or its own globs bring in (tokio's `use tokio::sync::*;` explains `mpsc`; tracing's `use tracing_core::*;` does not hide its `tracing_attributes`), and a glob into the sysroot or an enum explains none. An `extern crate x as y;` at the crate root puts `y` in the extern prelude, so `y::…` and `::y::…` resolve -from every file of the crate. At the crate root the final segment may be a +from every file of the crate, and `extern crate self as y;` puts the crate +itself there (syn's `use syn::parse::ParseStream` in syn is syn's own +parse.rs). At the crate root the final segment may be a `#[macro_export]` macro (`$crate::name!`), which lives in the file defining it: an invocation (`crate::helper!()`, recorded as `crate::helper!`) names the macro before an import or module of that name, an ordinary path @@ -647,11 +679,24 @@ test code reaches for. Two kinds of cfg are decided: never count (sha2's `[dev-dependencies] digest = { features = ["dev"] }` doesn't turn on digest's `dev`); the entries' packages' dev-dependencies count under resolver 1 only, since resolver 2 (edition 2021+, or - `resolver = "2"`) keeps them out of a normal build. Target-specific - dependency tables count unless `--cargo-target` says they are for another - target (without it, an over-approximation: it only keeps files). + `resolver = "2"`) keeps them out of a normal build. Resolver 2 also resolves + what is built for the host -- build-dependencies, proc-macro crates and what + they depend on -- apart from what is built for the target: a feature a + build-dependency asks of a crate isn't on in that crate as a normal + dependency. And it decides target-specific dependency tables: one counts + when `--cargo-target` says it applies (a build-dependency's table against the + host), and not when it says it doesn't; without a target, what only such a + table enables -- a dependency, a feature -- is on *maybe*: its gated code is + kept, but a missing module behind it is not fatal, and a candidate under it + is never taken as certain. Resolver 1 unifies it all, every table included. + The resolver is the workspace's: its `resolver`, else its edition. The crate a versioned dependency resolves to comes from `Cargo.lock`, so two - vendored versions of one crate each get their own features and edges. A + vendored versions of one crate each get their own features and edges; + without a lock entry, it is the one vendored version the requirement allows. + When the locked version isn't vendored, or no vendored version fits, or + several do and no lock says which, the loader warns and doesn't follow the + dependency rather than guess: cargo would build none of them from what the + bundle holds. A package the resolved build doesn't pull in has unknown features, and its gated code is kept. The manifests, the lock and the cargo config are read with `@preventive/lockfile`'s strict TOML parser (so is a `foundry.toml`): a @@ -671,8 +716,12 @@ test code reaches for. Two kinds of cfg are decided: stay undecided: a build may add features (`-C target-cpu=native`), and both differ with the toolchain and its flags. Profile cfgs (`debug_assertions`, `panic = …`) and custom ones stay undecided too. Build - scripts (see `--cargo-manifests`) compile for the host, so no target cfg is - decided in them and every `[target.'cfg(…)'.build-dependencies]` table counts. + scripts (see `--cargo-manifests`), build-dependencies and proc-macro crates + compile for the host: with `--cargo-target=host` its cfgs decide them as they + decide the rest; with another target no platform cfg is decided in them, and + a `[target.'cfg(…)'.build-dependencies]` table is on maybe (resolver 2). A crate + compiled for both (a dependency that is also a build-dependency) is followed + as both. This runs rustc — `$RUSTC` when set, as cargo honours it, else `rustc` from `PATH` — from the user's home directory (the filesystem root when there is none, or when the home directory is the bundle root or lies inside it, real @@ -717,14 +766,17 @@ packages leaves off; the manifest replay describes that build. Set package, in either mode. `stasis bundle --cargo-manifests` also carries what describes each bundled -package's build, the way `--package-json` carries npm manifests: every bundled -package's `Cargo.toml` and the workspace `Cargo.toml` above it, and that -workspace's -- or, for a package outside any, its own -- `Cargo.lock` and cargo -config (`.cargo/config` before `.cargo/config.toml`, the order cargo reads -them; a workspace nested below the bundle root keeps its own pair), when they -lie inside the bundle root (a workspace's may sit above), all as `resource` -files in their package's bucket, or the workspace bucket for the root-level -ones. They are carried as written, as `--package-json` carries `package.json`: +package's build, the way `--package-json` carries npm manifests. For a package +of the project (the root package, a workspace member, a path dependency): its +`Cargo.toml` and the workspace `Cargo.toml` above it, that workspace's -- or, +for a package outside any, its own -- `Cargo.lock`, and the cargo config of +every directory from the package's up to the bundle root, each of which cargo +reads when run there (`.cargo/config` before `.cargo/config.toml`), those that +lie inside the bundle root (a workspace's may sit above). For a vendored crate: +its `Cargo.toml` and the `.cargo-checksum.json` cargo checks its files against +-- the lock and config it was published with play no part in a build that +depends on it. All are `resource` files in their package's bucket, or the +workspace bucket for the root-level ones. They are carried as written, as `--package-json` carries `package.json`: stasis doesn't edit them, so whatever they hold -- a `[registries]` token or `[http]` proxy in the cargo config, the credentials in a `git = …` URL or a lockfile's `source = "git+https://…"` -- is in the bundle too. Keep registry @@ -745,9 +797,7 @@ can't carry the project's secrets into a bundle; the project's own code may include anything under the bundle root. The check follows symlinks: a file of a vendored crate that really lies outside its package (a link to the project's `.env`, a linked directory of modules, a linked manifest) is refused too, and -so is a `[lib] path` outside the package. Credentials in the URLs of the -carried config tables (`registry = "sparse+https://user:token@…"`, a `[patch]` -`git = "https://token@…"`) are stripped. +so is a `[lib] path` outside the package. The root packages' features follow the same flags as `cargo build`, in either mode: `--cargo-features=a,b` (repeatable; `x/feat` is a feature of the entries' @@ -770,11 +820,17 @@ one `concat!` form the loader can read, relative to the package root, inside a under `concat!(env!("OUT_DIR"), …)` included, can't be followed and is warned about unless it is `OUT_DIR`. `include!` splices Rust source, carried and scanned as a file of its own; the other two carry an asset as `resource` / -`resource:base64`, each relative to the including file. A file one place names +`resource:base64`, each relative to the including file. Rust source -- a +module, an `include!`d file, a build script -- and an `include_str!` file are +UTF-8 text, as rustc requires: one that isn't stops the build, naming the +file, never carried with its bytes replaced; an `include_bytes!` file is any +bytes, carried as `resource:base64` when it isn't UTF-8. A file one place names with `mod` or `include!` and another with `include_str!` is Rust: scanned and followed, never a resource. An include written inside a `macro_rules!` body -resolves relative to each file that invokes the macro, as rustc expands it (the -edge is that file's, at the invocation). `!` marks a `macro_rules!` +resolves relative to each file that invokes the macro, as rustc expands it -- +through the macros whose bodies invoke it, the outermost call (the edge is +that file's, at the invocation); so are the bare macro calls a body makes +made and resolved where it is invoked. `!` marks a `macro_rules!` invoked by bare name and defined in another file the invocation can see -- in textual scope, as rustc has it: the files above it in the module tree, each up to the `mod` that mounts the next (a macro defined after `mod early;` is not @@ -784,11 +840,11 @@ early.rs's), what a `#[macro_use] mod` hands the file mounting it, from that a name shadowing the earlier in source order whichever kind each is (serde's `tri!`, defined by its root's `crate_root!()` after `#[macro_use] mod crate_root;`, is the root's) as far as each call (a `macro_rules!` written -after a call doesn't reach it), a `mod` or `macro_rules!` a macro's body -declares standing where that macro is first invoked -- in the file, else in -the first file of the crate that invokes it (serde's `crate_root!`, declared -in core/crate_root.rs and invoked in lib.rs after `#[macro_use] mod macros;`, -so `de` sees `forward_to_deserialize_any!`), and, in the crate +after a call doesn't reach it), a `macro_rules!` a macro's body declares +standing where that macro is first invoked in the file, and a `mod` it +declares at the invocation in each file hosting it (see above: serde's +`crate_root!`, declared in core/crate_root.rs and invoked in lib.rs after +`#[macro_use] mod macros;`, so `de` sees `forward_to_deserialize_any!`), and, in the crate root's file, whose items they are, the `#[macro_export]`ed ones -- or by path, through the module's imports (`use crate::combinator::dispatch;`, a `use super::*` from the root); a `macro_rules!` in a sibling file is out of scope @@ -804,9 +860,9 @@ not(windows))`) — records a `{ : file, …, "*": }` map, the same shape as a JS edge that diverges per Metro platform; when every predicate names one file (libc's `mod primitives` under a dozen) the edge is that file. A -declaration a macro emits under a cfg the loader can't see (tokio's -`cfg_has_atomic_u64! { mod imp; }` beside `cfg_not_has_atomic_u64! { mod imp; }`) -is keyed by the macro (`cfg_has_atomic_u64!`); a second file under an already +declaration a gate macro emits (tokio's `cfg_has_atomic_u64! { mod imp; }` +beside `cfg_not_has_atomic_u64! { mod imp; }`) is keyed by the macro +(`cfg_has_atomic_u64!`), whose definition's cfg decides between them; a second file under an already used key gets it numbered (`*#2`) rather than dropped. A `use` is an import only when a use tree follows it: the word in macro input (syn's `Token![use]`) or a tree interpolating a metavariable (`use #path as _serde;` in a `quote!`) is not diff --git a/stasis/bin/stasis.js b/stasis/bin/stasis.js index 35358c26..41b71523 100755 --- a/stasis/bin/stasis.js +++ b/stasis/bin/stasis.js @@ -48,8 +48,8 @@ function usage(prefix = '') { feature flags are cargo's --features / --no-default-features / --all-features for those packages; --cargo-target asks rustc for the target's cfgs so #[cfg(unix)]-style code for other targets stays out too, otherwise it is all kept; --cargo-manifests also bundles each bundled package's Cargo.toml - and build script, the workspace Cargo.toml, Cargo.lock and .cargo/config.toml, as written: tokens - and URL credentials in them included) + and build script (a vendored crate's .cargo-checksum.json too), the workspace Cargo.toml, Cargo.lock + and cargo configs, as written: tokens and URL credentials in them included) (writes to stasis.code.br by default; --output=- streams to stdout; --add merges into an existing bundle instead of replacing it (not with --output=-); --brotli-quality=0..11, default 9; --jsx parses JSX in .js/.cjs/.mjs files, e.g. React Native source (put JSX-in-TS in a .tsx file); diff --git a/stasis/src/cmd/bundle.js b/stasis/src/cmd/bundle.js index 199207fd..f95b1d77 100644 --- a/stasis/src/cmd/bundle.js +++ b/stasis/src/cmd/bundle.js @@ -27,7 +27,7 @@ import { import { decodeUtf8 } from '../loaders/solidity-ownership.js' import { buildBashTree, collectBashFilesFromDisk } from '../loaders/bash.js' import { buildRustTree, collectRustFilesFromDisk } from '../loaders/rust.js' -import { createCargoContext } from '../loaders/cargo.js' +import { TARGET_UNIT, createCargoContext } from '../loaders/cargo.js' import { bucketizePhpSources, buildPhpTree, @@ -467,32 +467,42 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa const formats = new Map() const sources = new Map() // The crate roots: the entries, then -- with `cargoManifests` -- the build script of every - // package the walk reaches, each a crate root of its own, until no package is new. Build scripts - // and what they reach compile for the host, not the target (`hostFiles`). + // package the walk reaches, each a crate root of its own, until no package is new. Each file is + // compiled as the units `units` records (cargo.js unitKey): a build script for the host, with + // its package's features, and so what it reaches; a proc-macro crate and what it depends on for + // the host too. const roots = [...normalized] - const hostFiles = new Set() + const units = new Map() + // Record `file` as compiled as each of `more` too; whether that is news. + const addUnits = (file, more) => { + const have = units.get(file) ?? units.set(file, new Set()).get(file) + if (more.isSubsetOf(have)) return false + for (const u of more) have.add(u) + return true + } const walk = async (wave) => { - await collectRustFilesFromDisk(baseDir, wave, { cargo: cargoCtx, formats, sources, hostFiles }) + await collectRustFilesFromDisk(baseDir, wave, { cargo: cargoCtx, formats, sources, units }) if (!cargoManifests) return const scripts = new Set() for (const path of sources.keys()) { const script = formats.has(path) ? null : cargoCtx.buildScriptOf(path) - if (script !== null && !roots.includes(script)) scripts.add(script) + if (script === null) continue + const as = new Set([...(units.get(path) ?? [TARGET_UNIT])].map((u) => cargoCtx.buildScriptUnit(u))) + if (addUnits(script, as)) scripts.add(script) } if (scripts.size === 0) return - roots.push(...scripts) - for (const s of scripts) hostFiles.add(s) + for (const s of scripts) if (!roots.includes(s)) roots.push(s) await walk([...scripts]) } await walk(normalized) // The tree pass may name in-tree crate roots the walk left out (a crate whose name a file also - // binds as a value, `use crate::util::log;` beside `log::info!`: buildRustTree's wantedRoots): - // load them and build again, until none is new. + // binds as a value, `use crate::util::log;` beside `log::info!`), or walked as fewer units than + // it is named as: buildRustTree's wantedRoots. Load them and build again, until none is new. const complete = async () => { - const built = buildRustTree(sources, { roots, baseDir, cargo: cargoCtx, formats, hostFiles }) - const wanted = built.wantedRoots.filter((r) => !roots.includes(r)) + const built = buildRustTree(sources, { roots, baseDir, cargo: cargoCtx, formats, units }) + const wanted = built.wantedRoots.filter((r) => addUnits(r, built.wantedUnits.get(r)) || !sources.has(r)) if (wanted.length === 0) return built - roots.push(...wanted) + for (const r of wanted) if (!roots.includes(r)) roots.push(r) await walk(wanted) return complete() } diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index 32ecbc8c..a2564bb6 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -83,7 +83,7 @@ export function parseCargoManifest(text, file = null) { const manifest = { package: null, // { name, version, versionFromWorkspace, edition, editionFromWorkspace, build } resolver: null, // "1" | "2" | "3" from [workspace] or [package] - lib: { name: null, path: null }, + lib: { name: null, path: null, procMacro: false }, // `procMacro`: `[lib] proc-macro = true`, a crate compiled for the host features: new Map(), // name -> implied entries (`other`, `dep:key`, `key/feat`, `key?/feat`) // key -> { key, name, version, path, package, workspace, kinds: Map }: // what identifies the crate is shared, what is asked of it is per dependency table -- sha2's @@ -149,7 +149,7 @@ export function parseCargoManifest(text, file = null) { const wsPackage = table(ws?.package) manifest.workspacePackage = { version: str(wsPackage?.version), edition: str(wsPackage?.edition) } const lib = table(doc.lib) - manifest.lib = { name: str(lib?.name), path: str(lib?.path) } + manifest.lib = { name: str(lib?.name), path: str(lib?.path), procMacro: lib?.['proc-macro'] === true || lib?.proc_macro === true } for (const [name, list] of Object.entries(table(doc.features) ?? {})) { if (Array.isArray(list)) manifest.features.set(name, list.filter((v) => typeof v === 'string')) } @@ -337,17 +337,24 @@ export const TARGET_CFG_KEYS = new Set(['unix', 'windows', 'target_abi', 'target // item it gates is dead code for the bundle), `true` when it always does, `null` when the loader // can't tell. `all`/`any`/`not` compose. `test` holds only in a test/bench target (`env.test`); // `doctest` and `doc` never do when a program is built; `feature = "x"` is decided against -// `env.features`, the crate's resolved feature set (see createCargoContext), and unknown without -// one; a target leaf (`unix`, `target_os = …`) against `env.target`, the target's cfg set as rustc -// prints it (rustcTargetCfgs), and unknown without one; every other leaf stays unknown. +// `env.features`, the crate's features on for certain (see createCargoContext) -- unknown for one +// of `env.maybeFeatures`, on only in some builds, and without either set; a target leaf (`unix`, +// `target_os = …`) against `env.target`, the platform's cfg set as rustc prints it +// (rustcTargetCfgs), and unknown without one; every other leaf stays unknown. `env.units`, a list +// of such envs, is code compiled several ways (a crate built for the target and, as a +// build-dependency, for the host): false only when false in each, true only when true in each. export function evalCfg(pred, env = {}) { + if (env.units) { + const each = env.units.map((u) => evalCfg(pred, u)) + return each.every((r) => r === false) ? false : (each.every((r) => r === true) ? true : null) + } const p = pred.trim() const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) if (!m) { if (p === 'test') return env.test === true if (p === 'doctest' || p === 'doc') return false const feature = FEATURE_CFG_RE.exec(p) - if (feature) return env.features ? env.features.has(feature[1]) : null + if (feature) return env.features ? (env.features.has(feature[1]) ? true : (env.maybeFeatures?.has(feature[1]) ? null : false)) : null const leaf = CFG_LEAF_RE.exec(p) if (leaf && env.target && TARGET_CFG_KEYS.has(leaf[1])) return env.target.has(leaf[2] === undefined ? leaf[1] : `${leaf[1]}="${leaf[2]}"`) return null @@ -485,18 +492,113 @@ function cargoConfigIn(baseDir, dir) { } const cargoConfigOf = (baseDir) => cargoConfigIn(baseDir, '.') -// The directory `cargo vendor` filled: what `[source.vendored-sources] directory = "…"` in the -// project's cargo config says (however the table is spelled out), else the default `vendor`. -// Throws a TomlError when the config isn't TOML. +// The directory `cargo vendor` filled, as the project's cargo config says: the `directory` of the +// source `[source.crates-io] replace-with` names, through any chain of replacements (`cargo +// vendor` suggests `vendored-sources`, but any name does), else of the one source some +// `replace-with` names; the default `vendor` when the config replaces nothing with a directory. +// A configured directory that doesn't exist is warned about (a stale `vendor/` beside it is not +// what cargo builds from). Throws a TomlError when the config isn't TOML. function vendorDirOf(baseDir) { const rel = cargoConfigOf(baseDir) const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) - const dir = text === null ? undefined : readToml(text, rel).source?.['vendored-sources']?.directory - return (typeof dir === 'string' ? normalizeRel('.', dir)?.replace(/\/+$/u, '') : null) || VENDOR_DIR + if (text === null) return VENDOR_DIR + // source name -> { replaceWith, directory } + const sources = new Map() + const config = readToml(text, rel) + for (const [name, fields] of Object.entries(isTomlTable(config.source) ? config.source : {})) { + if (!isTomlTable(fields)) continue + const str = (v) => (typeof v === 'string' ? v : null) + sources.set(name, { replaceWith: str(fields['replace-with']), directory: str(fields.directory) }) + } + const follow = (name) => { + const seen = new Set() + for (let s = sources.get(name); s && !seen.has(s); s = sources.get(s.replaceWith)) { + seen.add(s) + if (s.directory !== null) return s.directory + if (s.replaceWith === null) return null + } + return null + } + let dir = follow('crates-io') + if (dir === null) { + const replaced = new Set([...sources.values()].map((s) => (s.replaceWith === null ? null : follow(s.replaceWith))).filter((d) => d !== null)) + if (replaced.size === 1) [dir] = replaced + } + if (dir === null) return VENDOR_DIR + const norm = normalizeRel('.', dir)?.replace(/\/+$/u, '') || null + if (norm === null) { + console.warn(`[loader.cargo] ${rel} names a vendored source directory outside the bundle root: ${dir}`) + return VENDOR_DIR + } + if (!existsSync(join(baseDir, norm))) console.warn(`[loader.cargo] ${rel} names a vendored source directory that doesn't exist: ${norm}`) + return norm +} + +// The `--cfg` names in rustflags: `--cfg name`, `--cfg=name`, `--cfg 'name="value"'`. +function cfgFlags(flags) { + const out = [] + for (let i = 0; i < flags.length; i++) { + const flag = flags[i] + const arg = flag === '--cfg' ? flags[++i] : (flag.startsWith('--cfg=') ? flag.slice('--cfg='.length) : null) + const name = typeof arg === 'string' ? /^[A-Za-z_]\w*/u.exec(arg)?.[0] : undefined + if (name !== undefined) out.push(name) + } + return out +} +// The cfgs the rustflags of a build of `baseDir` may set: those of its cargo config's +// `build.rustflags` and `target.<…>.rustflags` (every target's: which applies is the build's +// business), and of the environment cargo reads them from (CARGO_ENCODED_RUSTFLAGS, RUSTFLAGS, +// CARGO_BUILD_RUSTFLAGS). Throws a TomlError when the config isn't TOML. +function rustflagsCfgsOf(baseDir, env = process.env) { + const names = new Set() + const add = (value) => { + const flags = Array.isArray(value) ? value.filter((f) => typeof f === 'string') : (typeof value === 'string' ? value.split(/\s+/u).filter(Boolean) : []) + for (const name of cfgFlags(flags)) names.add(name) + } + const rel = cargoConfigOf(baseDir) + const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) + // The `rustflags` of the `build` table and of every `target.<…>` one. + const config = text === null ? null : readToml(text, rel) + const table = (v) => (isTomlTable(v) ? v : null) + add(table(config?.build)?.rustflags) + for (const target of Object.values(table(config?.target) ?? {})) add(table(target)?.rustflags) + if (env.CARGO_ENCODED_RUSTFLAGS) add(env.CARGO_ENCODED_RUSTFLAGS.split('\u001F')) + for (const v of [env.RUSTFLAGS, env.CARGO_BUILD_RUSTFLAGS]) if (v) add(v) + return names +} +// What a build script prints as `cargo:rustc-cfg=` (or `cargo::`), among the texts of its +// files: the names it may set, and whether it may set one the loader can't read (a name it +// formats, `autocfg`'s probes). +const RUSTC_CFG_RE = /cargo::?rustc-cfg=([A-Za-z_]\w*)?/gu +function cfgsPrinted(texts) { + const names = new Set() + let any = false + for (const text of texts) { + for (const m of text.matchAll(RUSTC_CFG_RE)) { + if (m[1] === undefined) any = true + else names.add(m[1]) + } + if (/\bautocfg\b/u.test(text)) any = true + } + return { names, any } } // --- Context ---------------------------------------------------------------------------- +// No custom cfg a build sets (see cfgsSetFor). +export const NO_CFGS_SET = { names: new Set(), any: false } + +// A compile unit: the feature context a file's code sees (`target`: the build's; `host`: that of +// what is built for the host -- build-dependencies, proc-macro crates and their dependencies -- +// which resolver 2 keeps apart) and the platform it is compiled for, as `|`. +// A package's code is compiled as `target|target`, its build script as `target|host`, a +// build-dependency's or proc-macro's code as `host|host`; one file may be compiled as several. +export const unitKey = (featureCtx, platform) => `${featureCtx}|${platform}` +export const TARGET_UNIT = unitKey('target', 'target') +export const HOST_UNIT = unitKey('host', 'host') +const unitFeatureCtx = (unit) => unit.slice(0, unit.indexOf('|')) +export const unitPlatform = (unit) => unit.slice(unit.indexOf('|') + 1) + // Per-bundle Cargo state: manifest lookup (memoized per directory), crate-name resolution against // in-tree sources, and feature resolution for the packages owning `entries` (the crate roots being // bundled, like `cargo build -p …`). `features` / `noDefaultFeatures` / `allFeatures` mirror @@ -508,6 +610,8 @@ function vendorDirOf(baseDir) { // root; every path in and out is project-relative POSIX. export function createCargoContext(baseDir, { entries = [], features = [], noDefaultFeatures = false, allFeatures = false, cargo = false, target = null } = {}) { const targetInfo = target === null ? null : (typeof target === 'string' ? rustcTargetCfgs(target, baseDir) : target) + // The host's platform, known when the target is the host. + const hostInfo = target === 'host' ? targetInfo : null const manifests = new Map() const readManifest = (dir) => { if (!manifests.has(dir)) { @@ -636,11 +740,14 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // Dependency → package, memoized per (package, dependency): the fixed-point loop asks many times. const depTargets = new Map() // The in-tree package a dependency of `m` resolves to: a `path` dep (or a `[patch]` path - // override in the root manifest), else the vendored crate of that name -- the version Cargo.lock - // records for `m`, else the newest that satisfies the requirement. A package can depend on two - // versions of one crate (`borsh = "1"` beside `borsh0-9 = { package = "borsh", version = "0.9" }`): - // the lock then lists both under it, and the requirement tells which is which. Null when it - // isn't in-tree. + // override in the workspace root's manifest), else the vendored crate of that name -- the + // version Cargo.lock records for `m`, else the one vendored version that satisfies the + // requirement. A package can depend on two versions of one crate (`borsh = "1"` beside + // `borsh0-9 = { package = "borsh", version = "0.9" }`): the lock then lists both under it, and + // the requirement tells which is which. Null when it isn't in-tree -- a path outside the bundle + // root, or no vendored copy -- and, warned, when the vendored copies don't settle it: the locked + // version isn't among them, none satisfies the requirement, or several do and no lock chooses. + // Cargo would build none of those from what the bundle holds, so none is guessed. const resolveDep = (m, dep) => { const memo = `${m.dir}\0${dep.key}` if (!depTargets.has(memo)) depTargets.set(memo, resolveDepUncached(m, dep)) @@ -658,12 +765,15 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef if (!spec) return null if (spec.path) return asPackage(normalizeRel(spec.relTo, spec.path)) const crate = normName(spec.package ?? spec.key) - const patch = readManifest('.')?.patches.get(crate) - if (patch !== undefined) return asPackage(normalizeRel('.', patch)) + const root = workspaceFor(m.dir) ?? readManifest('.') + const patch = root?.patches.get(crate) + if (patch !== undefined) return asPackage(normalizeRel(root.dir, patch)) const candidates = vendored().byName.get(crate) ?? [] if (candidates.length === 0) return null const req = typeof spec.version === 'string' ? spec.version : null const fits = (ver) => req === null || satisfiesCargoReq(ver, req) + const who = `${m.package.name} ${version(m)}` + const vendoredList = candidates.map((c) => c.version).toSorted(compareVersionsDesc).join(', ') const lk = lockfile() if (lk) { const entry = lk.byId.get(`${normName(m.package.name)} ${version(m)}`) @@ -676,25 +786,60 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } else if (listed.length > 1) { want = listed.map((x) => x.version).find((ver) => ver !== null && fits(ver)) ?? null } - const hit = want === null ? undefined : candidates.find((c) => c.version === want) - if (hit) return readManifest(hit.dir) + if (want !== null) { + const hit = candidates.find((c) => c.version === want) + if (hit) return readManifest(hit.dir) + console.warn(`[loader.cargo] ${who} is locked to ${crate} ${want}, which isn't vendored (vendored: ${vendoredList})`) + return null + } } const fitting = candidates.filter((c) => fits(c.version)) - const pool = fitting.length > 0 ? fitting : candidates - return readManifest(pool.toSorted((a, b) => compareVersionsDesc(a.version, b.version))[0].dir) + if (fitting.length === 1) return readManifest(fitting[0].dir) + if (fitting.length === 0) console.warn(`[loader.cargo] No vendored version of ${crate} satisfies ${who}'s requirement ${req} (vendored: ${vendoredList})`) + else console.warn(`[loader.cargo] Several vendored versions of ${crate} satisfy ${who}'s requirement ${req ?? '*'} (${fitting.map((c) => c.version).join(', ')}) and no Cargo.lock says which`) + return null } // --- feature resolution // Cargo's feature resolver: v2 (edition 2021+, or `resolver = "2"`/`"3"`) leaves dev-dependencies - // out of a normal build's unification; v1 counts them. + // out of a normal build's unification, resolves what is built for the host apart from what is + // built for the target, and ignores the tables of platforms not being built; v1 unifies them all. + // It is the workspace's setting: its root manifest's `resolver`, else its edition -- the + // workspace of the entries' package (the package itself outside any), else the bundle root's. + let resolverMemo = null const resolverVersion = () => { - const root = readManifest('.') + if (resolverMemo !== null) return resolverMemo + const pkg = entries.map(packageFor).find(Boolean) + const root = (pkg ? workspaceFor(pkg.dir) ?? pkg : null) ?? readManifest('.') const explicit = Number(root?.resolver) - if (Number.isInteger(explicit) && explicit > 0) return explicit - const edition = root?.package?.editionFromWorkspace ? workspaceFor('.')?.workspacePackage.edition : root?.package?.edition - return Number(edition ?? 0) >= 2021 ? 2 : 1 + const edition = root?.package?.editionFromWorkspace ? workspaceFor(root.dir)?.workspacePackage.edition : root?.package?.edition + resolverMemo = Number.isInteger(explicit) && explicit > 0 ? explicit : (Number(edition ?? 0) >= 2021 ? 2 : 1) + return resolverMemo } + // The feature context a unit's code sees (resolver 2 keeps the host's apart; resolver 1 has one). + const featureCtx = (c) => (resolverVersion() === 1 ? 'target' : c) + const isProcMacroPkg = (m) => m?.lib.procMacro === true + // The platform a context compiles for, as far as the loader knows it: the target's triple and + // cfgs when `target` was given; the host's only when the target is the host (`host`), since the + // machine that builds need not be the one that bundles. + const platformInfo = (platform) => (platform === 'target' ? targetInfo : hostInfo) + // Whether a table applies in a build for `platform`: `yes`, `no`, or `maybe` when the platform's + // cfgs aren't known or don't decide it. A build-dependency table is about the host its build + // script runs on. Resolver 1 counts every platform's table, as cargo's v1 unification does. + const tableApplies = (request, platform) => { + const at = request.indexOf('@') + if (at === -1 || resolverVersion() === 1) return 'yes' + const info = platformInfo(kindOf(request) === 'build' ? 'host' : platform) + if (info === null) return 'maybe' + const spec = request.slice(at + 1) + const cfg = /^cfg\((.*)\)$/u.exec(spec) + const holds = cfg ? evalCfg(cfg[1], { target: info.cfgs }) : spec === info.triple + return holds === true ? 'yes' : (holds === false ? 'no' : 'maybe') + } + // The context a dependency of a package built in context `c` is built in: the host's for a + // build-dependency (its build script's) and for a proc-macro crate, else `c`'s. + const depCtx = (c, request, t) => featureCtx(kindOf(request) === 'build' || isProcMacroPkg(t) ? 'host' : c) // An optional dependency no `dep:` entry names gets an implicit feature of its own name -- as // the manifest spells it (`proc-macro-crate`), which is what `std = ["proc-macro-crate"]` and // `#[cfg(feature = "proc-macro-crate")]` refer to. @@ -711,46 +856,44 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } const isOptional = (d) => [...d.kinds.values()].some((r) => r.optional) const featureImplications = (m, f) => m.features.get(f) ?? implicitFeatures(m).get(f) ?? null + const nodeKey = (c, dir) => `${c}\0${dir}` - let enabled = null - // Enabled features per package dir, for every package the root packages' builds pull in: - // the roots start from `default` (or the flags), then features imply features, activate + // Enabled features per (context, package dir), for every package the root packages' builds pull + // in: the roots start from `default` (or the flags), then features imply features, activate // optional deps and request dependency features, and active deps get `default` plus what the - // dependent asks for, to a fixed point -- cargo's unification, over-approximating where the - // loader can't tell (a target-specific dependency table counts unless the target is known and - // it isn't for it). Dev-dependencies: a dependency's own are never built by anyone, so they - // never count; the root packages' count under resolver 1 only (resolver 2 keeps them out of a - // normal build). - const ensureResolved = () => { - if (enabled !== null) return enabled - if (metadata) { - enabled = metadata.enabled - return enabled - } - enabled = new Map() + // dependent asks for, to a fixed point -- cargo's unification. `includeMaybe`: whether the + // tables whose platform the loader can't decide count; resolved both ways, the features on + // without them are on for certain, the rest of those on with them only maybe (a feature can turn + // code off -- `#[cfg(not(feature = "std"))]` -- so counting a table that may not apply is no + // safe over-approximation). Dev-dependencies: a dependency's own are never built by anyone, so + // they never count; the root packages' count under resolver 1, and for a test/bench entry. + const resolveFeatures = (includeMaybe) => { + const enabled = new Map() // The entries' packages (manifests are memoized per dir, so identity dedupes them). const roots = [...new Set(entries.map(packageFor).filter(Boolean))] if (roots.length === 0) return enabled const resolver = resolverVersion() - const active = new Map() // dir -> Set (activated optional deps) + const active = new Map() // node -> Set (activated optional deps) let changed = false - const inGraph = (m) => { - if (!enabled.has(m.dir)) { - enabled.set(m.dir, new Set()) + const inGraph = (c, m) => { + const key = nodeKey(c, m.dir) + if (!enabled.has(key)) { + enabled.set(key, new Set()) changed = true } - return enabled.get(m.dir) + return enabled.get(key) } - const enable = (m, f) => { - const set = inGraph(m) + const enable = (c, m, f) => { + const set = inGraph(c, m) if (featureImplications(m, f) === null || set.has(f)) return // unknown feature: cargo would error set.add(f) changed = true } - const activate = (m, key) => { - if (!active.has(m.dir)) active.set(m.dir, new Set()) - if (active.get(m.dir).has(key)) return - active.get(m.dir).add(key) + const activate = (c, m, key) => { + const node = nodeKey(c, m.dir) + if (!active.has(node)) active.set(node, new Set()) + if (active.get(node).has(key)) return + active.get(node).add(key) changed = true } const rootDirs = new Set(roots.map((r) => r.dir)) @@ -758,26 +901,19 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // test/bench target (that build is `cargo test`'s, which links them). const testRootDirs = new Set(entries.filter((e) => isTestTarget(e)).map((e) => packageFor(e)?.dir)) const kindApplies = (m, request) => kindOf(request) !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) - // Whether a table's `[target..…]` is for the build's target: a `cfg(…)` decided against - // its cfg set (kept when undecidable), a triple compared; any table when no target is known, - // and any build-dependency table, which is about the host the build script runs on. - const targetApplies = (request) => { - const at = request.indexOf('@') - if (at === -1 || targetInfo === null || kindOf(request) === 'build') return true - const spec = request.slice(at + 1) - const cfg = /^cfg\((.*)\)$/u.exec(spec) - return cfg ? evalCfg(cfg[1], { target: targetInfo.cfgs }) !== false : spec === targetInfo.triple - } - // The tables of `d` that take part in the build (a target-specific one when it is for the - // target), with an optional one only once activated. - const activeRequests = (m, d) => [...d.kinds] - .filter(([request, r]) => kindApplies(m, request) && targetApplies(request) && (!r.optional || active.get(m.dir)?.has(d.key) === true)) - .map(([, r]) => r) + const applies = (request, c) => { + const a = tableApplies(request, c) + return a === 'yes' || (includeMaybe && a === 'maybe') + } + // The tables of `d` that take part in `m`'s build in context `c`, as [request, entry] pairs, + // an optional one only once activated. + const activeRequests = (c, m, d) => [...d.kinds] + .filter(([request, r]) => kindApplies(m, request) && applies(request, c) && (!r.optional || active.get(nodeKey(c, m.dir))?.has(d.key) === true)) // One entry of a feature's list: `other`, `dep:key`, `key/feat`, `key?/feat`. Returns whether it named anything. - const applyImplication = (m, imp) => { + const applyImplication = (c, m, imp) => { const explicitDep = DEP_IMPLICATION_RE.exec(imp) if (explicitDep) { - activate(m, normName(explicitDep[1])) + activate(c, m, normName(explicitDep[1])) return true } const depFeature = DEP_FEATURE_RE.exec(imp) @@ -786,27 +922,25 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef if (!d) return false // `dep/feat` enables an optional dep (and its implicit feature); `dep?/feat` only asks if it is already on. if (depFeature[2] !== '?' && isOptional(d)) { - activate(m, d.key) - if (implicitFeatures(m).has(d.name)) enable(m, d.name) - } - if (activeRequests(m, d).length > 0) { - const t = resolveDep(m, d) - if (t) enable(t, depFeature[3]) + activate(c, m, d.key) + if (implicitFeatures(m).has(d.name)) enable(c, m, d.name) } + const t = resolveDep(m, d) + if (t) for (const [request] of activeRequests(c, m, d)) enable(depCtx(c, request, t), t, depFeature[3]) return true } if (featureImplications(m, imp) === null) return false - enable(m, imp) + enable(c, m, imp) return true } for (const m of roots) { - inGraph(m) + inGraph('target', m) if (allFeatures) { - for (const f of m.features.keys()) enable(m, f) - for (const f of implicitFeatures(m).keys()) enable(m, f) + for (const f of m.features.keys()) enable('target', m, f) + for (const f of implicitFeatures(m).keys()) enable('target', m, f) } else if (!noDefaultFeatures) { - enable(m, 'default') + enable('target', m, 'default') } } // `--features a,b,pkg/c`: a bare name is a feature of every root package; `x/c` is a feature of @@ -820,7 +954,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const targets = scoped ? roots.filter((m) => normName(m.package.name) === pkg) : roots const imp = scoped ? flag.slice(slash + 1) : flag // Whether a name is known doesn't depend on what is active yet, so the check happens once. - if (targets.filter((m) => applyImplication(m, imp)).length === 0) { + if (targets.filter((m) => applyImplication('target', m, imp)).length === 0 && includeMaybe) { console.warn(`[stasis] --cargo-features: '${flag}' names no feature of the entries' packages, nor a dependency of theirs`) } requested.push({ targets, imp }) @@ -830,33 +964,92 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef changed = false // The requested features re-apply on every pass like the manifest's entries do: a weak // `dep?/x` asked for on the command line takes effect once `default` has activated `dep`. - for (const { targets, imp } of requested) for (const m of targets) applyImplication(m, imp) + for (const { targets, imp } of requested) for (const m of targets) applyImplication('target', m, imp) // Map/Set iteration is live: packages and features added mid-pass are visited in this pass. - for (const dir of enabled.keys()) { - const m = readManifest(dir) - for (const f of enabled.get(dir)) { - for (const imp of featureImplications(m, f) ?? []) applyImplication(m, imp) + for (const node of enabled.keys()) { + const cut = node.indexOf('\0') + const c = node.slice(0, cut) + const m = readManifest(node.slice(cut + 1)) + for (const f of enabled.get(node)) { + for (const imp of featureImplications(m, f) ?? []) applyImplication(c, m, imp) } for (const d of m.deps.values()) { - for (const request of activeRequests(m, d)) { - const spec = depSpec(m, d, request) + for (const [request, r] of activeRequests(c, m, d)) { + const spec = depSpec(m, d, r) const t = spec === null ? null : resolveDep(m, d) if (!t) continue - inGraph(t) - if (spec.defaultFeatures) enable(t, 'default') - for (const f of spec.features) enable(t, f) + const dc = depCtx(c, request, t) + inGraph(dc, t) + if (spec.defaultFeatures) enable(dc, t, 'default') + for (const f of spec.features) enable(dc, t, f) } } } } while (changed) return enabled } + // Both resolutions, per (context, package dir): `sure` without the undecided tables, `all` with + // them. `--cargo`: cargo metadata's features, the same in either context. + let resolved = null + const ensureResolved = () => { + if (resolved !== null) return resolved + if (metadata) { + const byNode = new Map([...metadata.enabled].flatMap(([dir, set]) => [[nodeKey('target', dir), set], [nodeKey('host', dir), set]])) + resolved = { sure: byNode, all: byNode } + } else { + resolved = { sure: resolveFeatures(false), all: resolveFeatures(true) } + } + return resolved + } + // Per node, the features on only maybe: in `all`, not in `sure` (one object per node). + const maybeByNode = new Map() + const maybeOf = (node) => { + if (!maybeByNode.has(node)) { + const { sure, all } = ensureResolved() + const on = sure.get(node) + const maybe = new Set([...(all.get(node) ?? [])].filter((f) => !on?.has(f))) + maybeByNode.set(node, maybe.size === 0 ? null : maybe) + } + return maybeByNode.get(node) + } + // A package in the `all` resolution but not the `sure` one is built only maybe: nothing of its + // is on for certain. + const NO_FEATURES = new Set() + + // --- cfgs a build may set + let rustflagCfgs = null + const cfgsSetMemo = new Map() + // A build script's text and that of the modules it declares (`mod probe;`, beside it), a few + // levels down: where its `cargo:rustc-cfg=` lines are. + const buildScriptTexts = (script) => { + const texts = [] + const queue = [[script, 0]] + const seen = new Set() + for (let qi = 0; qi < queue.length; qi++) { + const [rel, depth] = queue[qi] + if (seen.has(rel)) continue + seen.add(rel) + const text = readFileOrNull(join(baseDir, rel)) + if (text === null) continue + texts.push(text) + if (depth >= 3) continue + const dir = posix.dirname(rel) + const sub = qi === 0 || posix.basename(rel) === 'mod.rs' ? dir : posix.join(dir, posix.basename(rel, '.rs')) + for (const m of text.matchAll(/\bmod\s+([A-Za-z_]\w*)\s*;/gu)) { + for (const f of [posix.join(sub, `${m[1]}.rs`), posix.join(sub, m[1], 'mod.rs')]) if (isFile(join(baseDir, f))) queue.push([f, depth + 1]) + } + } + return texts + } // The in-tree crate root a name resolves to from package `m` (null: no owning package), leaving // out `m`'s own lib (which depends on the asking file): the package its dependency of that name // resolves to (path dep, `[patch]`, or the vendored version Cargo.lock says); a dependency whose - // lib is named that (`md-5` is used as `md5`); or -- for a name no manifest declares -- a - // vendored crate of that lib or package name. Memoized per (package, name): every file of a + // lib is named that (`md-5` is used as `md5`); or -- for a file no manifest claims -- the one + // vendored crate of that lib or package name. A package's own name for a crate is its manifest's + // word: a declared dependency that doesn't resolve in-tree (a path outside the bundle root, a + // version no vendored copy has) never falls back to some vendored copy of that name, and a + // name it doesn't declare is no crate of its. Memoized per (package, name): every file of a // package asks for the same few crates. const crateTargets = new Map() const depCrate = (m, norm) => { @@ -869,25 +1062,23 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const d = m.deps.get(norm) if (d) { const t = resolveDep(m, d) - const lib = t ? libPath(t) : null - if (lib) return lib + return t ? libPath(t) : null } for (const other of m.deps.values()) { - if (other.key === norm || other.package !== null) continue // a rename is used by its key, not its lib name + if (other.package !== null) continue // a rename is used by its key, not its lib name const t = resolveDep(m, other) - if (t && libName(t) === norm) { - const lib = libPath(t) - if (lib) return lib - } + if (t && libName(t) === norm) return libPath(t) } + return null } const { byLib, byName } = vendored() - for (const index of [byLib, byName]) { - for (const c of (index.get(norm) ?? []).toSorted((a, b) => compareVersionsDesc(a.version, b.version))) { - const lib = libPath(readManifest(c.dir)) - if (lib) return lib - } + const found = new Set() + for (const index of [byLib, byName]) for (const c of index.get(norm) ?? []) found.add(c.dir) + if (found.size > 1) { + console.warn(`[loader.cargo] Several vendored crates are named ${norm} (${[...found].join(', ')}) and no manifest says which`) + return null } + for (const dir of found) return libPath(readManifest(dir)) return null } @@ -921,25 +1112,43 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return depCrate(m, norm) }, isTestTarget, + // Whether `fromFile`'s package declares a dependency of that name (in any table, by the key + // code uses): then the name is that crate -- rustc refuses a `use` path whose lead a glob + // import also provides, as ambiguous (E0659) -- in-tree or not. + declaresCrate(name, fromFile) { + return packageFor(fromFile)?.deps.has(normName(name)) === true + }, // The build's target: its triple and cfg set (`unix`, `target_os="linux"`, … as rustc prints // them), or null for both when none was given. targetTriple: targetInfo?.triple ?? null, targetCfgs: targetInfo?.cfgs ?? null, // The files that describe the build of `fileRel`'s package, project-relative, as `{ path, - // kind }`: its own Cargo.toml and, inside the bundle root, the workspace's above it - // (`manifest`); that workspace's -- or, without one, the package's own -- Cargo.lock (`lock`) - // and cargo config (`config`) when on disk. Empty for a file no manifest claims. + // kind }`, those on disk. A vendored package's: its Cargo.toml (`manifest`) and the + // `.cargo-checksum.json` cargo checks its files against (`checksum`) -- the lock and config a + // registry crate was published with play no part in a build that depends on it. Any other + // package's: its own Cargo.toml and, inside the bundle root, the workspace's above it + // (`manifest`); that workspace's -- or, without one, the package's own -- Cargo.lock (`lock`); + // and the cargo config of every directory from the package's up to the bundle root + // (`config`), each of which cargo reads when run there. Empty for a file no manifest claims. buildFilesFor(fileRel) { const m = packageFor(fileRel) if (!m) return [] const out = [{ path: posix.join(m.dir, 'Cargo.toml'), kind: 'manifest' }] + if (isVendoredDir(m.dir)) { + const checksum = posix.join(m.dir, '.cargo-checksum.json') + if (isFile(join(baseDir, checksum))) out.push({ path: checksum, kind: 'checksum' }) + return out + } const ws = workspaceFor(m.dir) if (ws && ws.dir !== m.dir) out.push({ path: posix.join(ws.dir, 'Cargo.toml'), kind: 'manifest' }) const top = ws?.dir ?? m.dir const lockPath = posix.join(top, 'Cargo.lock') if (isFile(join(baseDir, lockPath))) out.push({ path: lockPath, kind: 'lock' }) - const config = cargoConfigIn(baseDir, top) - if (config !== null) out.push({ path: config, kind: 'config' }) + for (let dir = m.dir; ; dir = posix.dirname(dir)) { + const config = cargoConfigIn(baseDir, dir) + if (config !== null) out.push({ path: config, kind: 'config' }) + if (dir === '.') break + } return out }, // Whether `fileRel` belongs to a vendored package (a registry crate `cargo vendor` copied in). @@ -961,17 +1170,77 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return rel }, - // The features enabled for the package owning `fileRel` in the build of the root packages, or - // null when that is unknown: no owning manifest, no root package to resolve from, or a package - // the resolved build doesn't pull in (its gated code is then kept, not dropped). - featuresFor(fileRel) { + // The custom cfgs the build of `fileRel`'s package may set (a `--cfg` a default build lacks is + // otherwise presumed off): `{ names, any }` -- the names its build script prints as + // `cargo:rustc-cfg=…` and those the rustflags set, and whether the script may set one the + // loader can't read. One object per package (and one for every package that sets none). + cfgsSetFor(fileRel) { + const m = packageFor(fileRel) + const memo = m?.dir ?? '\0' + if (!cfgsSetMemo.has(memo)) { + rustflagCfgs ??= rustflagsCfgsOf(baseDir) + const script = m ? this.buildScriptOf(fileRel) : null + const printed = script === null ? { names: new Set(), any: false } : cfgsPrinted(buildScriptTexts(script)) + const names = new Set([...rustflagCfgs, ...printed.names]) + cfgsSetMemo.set(memo, names.size === 0 && !printed.any ? NO_CFGS_SET : { names, any: printed.any }) + } + return cfgsSetMemo.get(memo) + }, + // The features on for certain for the package owning `fileRel`, compiled as `unit` (see + // unitOfCrate), in the build of the root packages; null when that is unknown: no owning + // manifest, no root package to resolve from, or a package the resolved build doesn't pull in + // (its gated code is then kept, not dropped). + featuresFor(fileRel, unit = TARGET_UNIT) { const m = packageFor(fileRel) if (!m) return null - return ensureResolved().get(m.dir) ?? null + const node = nodeKey(featureCtx(unitFeatureCtx(unit)), m.dir) + const { sure, all } = ensureResolved() + return sure.get(node) ?? (all.has(node) ? NO_FEATURES : null) + }, + // The features on only in some of the builds the loader can't tell apart (a target-specific + // table it can't decide, or what only such a table requests), for the same package and unit; + // null when there are none. + maybeFeaturesFor(fileRel, unit = TARGET_UNIT) { + const m = packageFor(fileRel) + if (!m) return null + return maybeOf(nodeKey(featureCtx(unitFeatureCtx(unit)), m.dir)) + }, + // Every package of a context the build may pull in: dir -> { on, maybe }, the features on for + // certain and those on only maybe (a package only an undecided table pulls in has every + // feature of its maybe). For diagnostics and tests. + featureResolution(context = 'target') { + const out = new Map() + for (const node of ensureResolved().all.keys()) { + const cut = node.indexOf('\0') + if (node.slice(0, cut) !== featureCtx(context)) continue + out.set(node.slice(cut + 1), { on: ensureResolved().sure.get(node) ?? NO_FEATURES, maybe: maybeOf(node) ?? NO_FEATURES }) + } + return out + }, + // Every resolved package of a context: dir -> Set. For diagnostics and tests. + resolvedFeatures(context = 'target') { + const out = new Map() + for (const [node, set] of ensureResolved().sure) { + const cut = node.indexOf('\0') + if (node.slice(0, cut) === featureCtx(context)) out.set(node.slice(cut + 1), set) + } + return out + }, + // The compile unit of a crate root `libFile` named from code compiled as `fromUnit`: a proc-macro + // crate, and anything code compiled for the host names (a build script's build-dependencies, + // a proc-macro's dependencies), is built for the host in the host's feature context; else the + // namer's unit. + unitOfCrate(libFile, fromUnit = TARGET_UNIT) { + return isProcMacroPkg(packageFor(libFile)) || unitPlatform(fromUnit) === 'host' ? HOST_UNIT : fromUnit + }, + // The unit of a build script of a package compiled as `pkgUnit`: for the host, with the + // package's features. + buildScriptUnit(pkgUnit = TARGET_UNIT) { + return unitKey(unitFeatureCtx(pkgUnit), 'host') }, - // Every resolved package: dir -> Set. For diagnostics and tests. - resolvedFeatures() { - return ensureResolved() + // The cfgs and triple of the platform `unit` compiles for, as far as known (see platformInfo). + platformOf(unit = TARGET_UNIT) { + return platformInfo(unitPlatform(unit)) }, } } diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index bb33a66e..dc7e31da 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -14,7 +14,7 @@ import { readFile } from 'node:fs/promises' import { dirname, extname, isAbsolute, join, posix, relative, resolve } from 'node:path' import { assertRealPathWithinBase } from '@exodus/stasis-core/util' -import { TARGET_CFG_KEYS, VENDOR_DIR, createCargoContext, evalCfg, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' +import { TARGET_CFG_KEYS, TARGET_UNIT, VENDOR_DIR, createCargoContext, evalCfg, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' import { matchClose, splitTopLevel } from './toml.js' // Leads of the expression-position paths anchored on the module tree rather than on a name. @@ -390,16 +390,18 @@ export function parseUseTree(body) { // a::b`), what a path from another module may continue along, i.e. a re-export; // `cfg` gates it (see mods), `macro` is the invocation body it sits in, if any (one // in a `macro_rules!` template is the invoking module's, not this one's); -// macros: `macro_rules!` definitions `{ name, exported, includes, offset, template }` -- an +// macros: `macro_rules!` definitions `{ name, exported, includes, calls, offset, template, gate }` -- an // exported one is `$crate::name!` from anywhere in the crate, and lives in this // file; `includes` are the include macros in its body, which rustc resolves // relative to each file that invokes the macro; one written in another // `macro_rules!`'s body (`template`, its name) is defined where that one is invoked // (`offset`: the first bare invocation in this file, Infinity for none), as is a // `mod` there; -// invocations: the names of the macros invoked by bare name (`ready!(…)`) -- for an edge to the -// file of a `macro_rules!` defined elsewhere in the crate; `calls` has each name's -// invocation offsets, in order (textual scope is decided at the call); +// invocations: the names of the macros invoked by bare name (`ready!(…)`) outside `macro_rules!` +// templates -- for an edge to the file of a `macro_rules!` defined elsewhere in +// the crate; `calls` has each name's invocation offsets, in order (textual scope +// is decided at the call); a template's own bare calls are its definition's +// `calls`, made wherever it is invoked; // includes: `include!` / `include_str!` / `include_bytes!` invocations outside macro // definitions, `{ kind, path, base, conditional }` -- a file to carry, named // relative to this one (`base` `file`) or to the package root (`manifest`, for @@ -410,9 +412,11 @@ export function parseUseTree(body) { // skipped: the template macros (see `templates` below) whose bodies were skipped. // A file whose inner `#![cfg()]` can never hold is compiled empty: the scan is empty too. // `templates` names the macros whose bodies are token templates for another crate's code (quote!) -// and are skipped whole: TEMPLATE_MACROS, less the ones the file's package defines itself. -export function scanRustItems(content, { features = null, test = false, target = null, templates = TEMPLATE_MACROS } = {}) { - const env = { features, test, target } +// and are skipped whole: TEMPLATE_MACROS, less the ones the file's package defines itself. The +// build (see evalCfg): `features` on for certain and `maybeFeatures` on maybe, `test`, the +// platform's cfg set `target`; or `units`, one such build per way the file is compiled. +export function scanRustItems(content, { features = null, maybeFeatures = null, test = false, target = null, units = null, templates = TEMPLATE_MACROS } = {}) { + const env = units === null ? { features, maybeFeatures, test, target } : { units: units.map((u) => ({ ...u, test })) } const { code, masked } = lexRust(content) const n = masked.length const mods = [] @@ -692,7 +696,8 @@ export function scanRustItems(content, { features = null, test = false, target = const name = readWord(open) if (name !== null) { open = skipWs(open + name.length) - macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], offset: i, at: i, template: i < macroUntil && macroName === 'macro_rules' ? templateName : null }) + const gate = GATE_MACRO_RE.test(name) && masked[open] === '{' ? gatePredicate(code, masked, open, matchClose(masked, open)) : undefined + macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], calls: new Set(), offset: i, at: i, template: i < macroUntil && macroName === 'macro_rules' ? templateName : null, gate }) if (i >= macroUntil) templateName = name } } @@ -712,10 +717,16 @@ export function scanRustItems(content, { features = null, test = false, target = // declares wherever it is invoked -- an approximation kept from before.) if (own !== null && masked[open] === '{' && word !== 'macro_rules') macroBodyCfg = own if (word !== 'macro_rules') { - // Invoked by bare name; `$crate::name!` / `serde_json::json!` is a path (a ref). + // Invoked by bare name; `$crate::name!` / `serde_json::json!` is a path (a ref). One in a + // `macro_rules!` template is the template's (`calls` on the definition): rustc resolves + // it where the macro is invoked, not where it is written -- and a recursive arm's + // `m!(…)` is no invocation of this file's. if (!pathQualified) { - invocations.add(word) - ;(calls.get(word) ?? calls.set(word, []).get(word)).push(i) + if (i < macroUntil && macroName === 'macro_rules' && macros.length > 0) macros.at(-1).calls.add(word) + else { + invocations.add(word) + ;(calls.get(word) ?? calls.set(word, []).get(word)).push(i) + } } if (INCLUDE_MACROS.has(word)) { const text = code.slice(open + 1, close) @@ -1077,16 +1088,17 @@ export function crateRoots(sources, explicit = [], cargo = null) { // entry also carries `leaves`, the cfg leaves (cfgLeaves) of every `mod` on the way down to it -- // the declaration's own cfg and the gate macro it sits in (`mountCfgs`: file → spec → target // file → its declarations' `{ cfg, macro }`; a file declared under several is compiled under any -// one of them: an `any` leaf, see cfgLeaves and gateLeaves) and, for a `mod` with several files, a `variant` leaf naming which (two -// variants of one `mod` never both apply) -- and `parent`, the file that mounted it. -export function buildModuleTrees(sources, resolutions, roots, inlineModules = null, mountCfgs = null) { +// one of them: an `any` leaf, see cfgLeaves and gateLeaves; `gatesOf`: file → the gate macros +// of its package, see gateLeaves) and, for a `mod` with several files, a `variant` leaf naming +// which (two variants of one `mod` never both apply) -- and `parent`, the file that mounted it. +export function buildModuleTrees(sources, resolutions, roots, inlineModules = null, mountCfgs = null, gatesOf = null) { const trees = new Map() const files = new Map() const variantsOf = (target) => (target instanceof Map ? [...target] : [['*', target]]) - const mountLeaves = (gates) => (gates === undefined ? [] : anyOf(gates.map(({ cfg, macro }) => [...cfgLeaves(cfg), ...gateLeaves(macro)]))) + const mountLeaves = (decls, path) => (decls === undefined ? [] : anyOf(decls.map(({ cfg, macro, via }) => [...cfgLeaves(cfg), ...gateLeaves(macro, gatesOf?.(path)), ...mountLeaves(via, path)]))) const mounted = (path, spec, sub, key, file, several, leaves) => [ ...leaves, - ...mountLeaves(mountCfgs?.get(path)?.get(spec)?.get(file)), + ...mountLeaves(mountCfgs?.get(path)?.get(spec)?.get(file), path), ...(several ? [{ key: 'variant', site: sub, value: key.replace(/#\d+$/u, ''), neg: false }] : []), ] for (const root of roots) { @@ -1247,14 +1259,44 @@ const negated = (leaves) => { return l.alts === undefined ? [{ ...l, neg: !l.neg }] : l.alts.flatMap(negated) } -// The leaf of a cfg gate the loader can't see into: the body of a `cfg_!` macro (tokio's -// `cfg_io_uring! { … }`, mio's `cfg_os_poll! { … }`), whose `cfg_not_!` is its negation -- -// so what the two declare is never taken together, and neither counts as certainly compiled -// (`cfg_if!` is read branch by branch; any other macro's body is no gate). -const GATE_MACRO_RE = /^cfg_(not_)?(?!if$)(\w+)$/u -function gateLeaves(macro) { - const m = macro === null || macro === undefined ? null : GATE_MACRO_RE.exec(macro) - return m === null ? [] : [{ key: `cfg_${m[2]}!`, value: null, neg: m[1] !== undefined }] +// A cfg gate macro: `cfg_!` (tokio's `cfg_io_uring! { … }`, mio's `cfg_os_poll! { … }`) +// wraps each item of its body in the cfg its definition writes -- `($($item:item)*) => { $( +// #[cfg(feature = "rt")] $item )* }` -- read from that definition (gatePredicate); `cfg_if!` is +// read branch by branch, any other macro's body is no gate. +const GATE_MACRO_RE = /^cfg_(?!if$)\w+$/u +// The cfg a gate macro's definition puts on what it wraps: the `#[cfg(…)]` attributes of its +// transcribers (their conjunction), when every arm writes the same ones; null when an arm writes +// none or the arms differ -- a gate the loader can't see into. `code` and `masked` are the lexed +// views, `open`/`close` the braces of the `macro_rules!` body. +function gatePredicate(code, masked, open, close) { + let seen = null + for (let i = masked.indexOf('=>', open); i !== -1 && i < close; i = masked.indexOf('=>', i + 2)) { + let start = i + 2 + while (/\s/u.test(masked[start] ?? '')) start++ + if (!'{(['.includes(masked[start] ?? ' ')) continue + const end = matchClose(masked, start) + const preds = [] + for (const m of masked.slice(start, end).matchAll(/#\s*\[\s*cfg\s*\(/gu)) { + const paren = start + m.index + m[0].length - 1 + preds.push(normalizeCfg(code.slice(paren + 1, matchClose(masked, paren)))) + } + if (preds.length === 0) return null + const uniq = [...new Set(preds)].toSorted() + const pred = uniq.length === 1 ? uniq[0] : `all(${uniq.join(', ')})` + if (seen !== null && seen !== pred) return null + seen = pred + i = end + } + return seen +} +// The leaves of what a gate macro's body declares: its definition's cfg (`gates`: macro name → +// predicate, null when unreadable, see gatePredicate), else a leaf of its own name -- which holds +// along with anything and is never certain: `cfg_x!` and `cfg_not_x!` are not taken for each +// other's negation by their names alone. +function gateLeaves(macro, gates = null) { + if (macro === null || macro === undefined || !GATE_MACRO_RE.test(macro)) return [] + const pred = gates?.get(macro) + return typeof pred === 'string' ? cfgLeaves(pred) : [{ key: `${macro}!`, value: null, neg: false }] } // Keys a target has one value of: `target_os = "linux"` and `target_os = "macos"` never both hold @@ -1360,42 +1402,81 @@ const compatible = (asker, set) => { } return verdict } +// Whether a leaf list can't hold in the asker's build (scanRustItems' build, see buildOf): one of +// its leaves the build decides is false -- a feature that is off, a platform the build isn't -- +// or an `any` leaf each of whose alternatives is. What a gate macro's cfg wraps (tokio's +// `cfg_rt! { … }` with `rt` off) is then taken last (`dead`, see pick): after any candidate the +// build may compile, and only when there is none -- a path written behind the same gate, whose +// code the build doesn't compile either, means what it would mean where it compiles. For an +// asker whose own leaves can't hold (`deadHere`) the build says nothing. +const EVALUABLE_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doc', 'doctest']) +const leafFalse = (l, build) => { + if (l.alts !== undefined) return l.alts.every((alt) => alt.some((m) => leafFalse(m, build))) + if (!EVALUABLE_CFG_KEYS.has(l.key)) return false + const pred = l.value === null ? l.key : `${l.key} = "${l.value}"` + return evalCfg(l.neg ? `not(${pred})` : pred, build) === false +} +const deadUnder = (leaves, build) => build !== undefined && leaves.some((l) => leafFalse(l, build)) +const deadFor = (asker, set) => { + if (asker === undefined || asker.deadHere || set.key === '') return false + let verdict = asker.dead.get(set.key) + if (verdict === undefined) { + verdict = deadUnder(set.leaves, asker.build) + asker.dead.set(set.key, verdict) + } + return verdict +} // Whether every leaf of a candidate's set is among the asker's own (`keys`, by leafKey): the two // are certainly compiled together, not merely possibly (compatible). libc's aix/mod.rs, mounted // under `target_os = "aix"`, asks `crate::fsid_t`: the aix glob's, under that same leaf, is the // answer, rather than the first platform's whose cfg is an `any(…)` nothing rules out. Kept per // pair in `sure`. An `any` leaf holds when one of its alternatives does, and so does the negation -// of a custom cfg (`not(loom)`: presumably off, see doubtful). An asker under no cfg has nothing -// to prefer by: for it, written order stands (everything not doubtful is as sure). -const holds = (l, keys) => keys.has(leafKey(l)) || (l.alts === undefined ? l.neg && customKey(l.key) : l.alts.some((alt) => alt.every((m) => holds(m, keys)))) +// of a custom cfg (`not(loom)`: presumably off, see doubtful). An asker under no cfg holds nothing +// else: for it, candidates under cfgs of their own are each only maybe the answer, and all of +// them are (see withAlternatives), not the first in written order. +const holds = (l, keys, custom) => keys.has(leafKey(l)) || (l.alts === undefined ? l.neg && custom(l.key) : l.alts.some((alt) => alt.every((m) => holds(m, keys, custom)))) const entailed = (asker, set) => { - if (asker === undefined || asker.set.key === '' || set.key === '') return true + if (asker === undefined || set.key === '') return true let verdict = asker.sure.get(set.key) if (verdict === undefined) { - verdict = set.leaves.every((l) => holds(l, asker.keys)) + verdict = set.leaves.every((l) => holds(l, asker.keys, asker.custom)) asker.sure.set(set.key, verdict) } return verdict } // The cfgs rustc and cargo set: anything else in a positive leaf is a custom `--cfg` (`loom`, // `docsrs`, `tokio_unstable`, mio's `mio_unsupported_force_poll_poll`), off in a default build -// unless a build script or RUSTFLAGS sets it -- so a candidate under one is `doubtful`: taken +// unless a build script or the rustflags set it -- so a candidate under one is `doubtful`: taken // after a compatible candidate under none (never ruled out: the loader doesn't know the build's -// flags). An `any` leaf is doubtful when each alternative is, or can't hold with the asker. A -// gate macro's leaf (`cfg_x!`) and a `mod` variant's are neither, nor is a leaf the asker holds -// itself (a file under `#[cfg(loom)]` takes the loom candidates). +// flags). One the asker's package may set (its build script prints `cargo:rustc-cfg=`, a +// rustflags `--cfg` names it: cargo.js cfgsSetFor) is no custom cfg for it: neither doubtful nor +// presumed off; nor is any when its build script may print one the loader can't read (`asker. +// custom`, see customFor). An `any` leaf is doubtful when each alternative is, or can't hold with +// the asker. A gate macro's leaf (`cfg_x!`) and a `mod` variant's are neither, nor is a leaf the +// asker holds itself (a file under `#[cfg(loom)]` takes the loom candidates). const KNOWN_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doctest', 'doc', 'debug_assertions', 'overflow_checks', 'panic', 'proc_macro', 'miri', 'sanitize', 'target_feature', 'target_thread_local', 'ub_checks', 'relocation_model', 'fmt_debug', 'clippy', 'rustfmt', 'variant']) const customKey = (key) => !KNOWN_CFG_KEYS.has(key) && !key.endsWith('!') -const doubtfulLeaf = (l, askerLeaves) => (l.alts === undefined ? !l.neg && customKey(l.key) : l.alts.every((alt) => cfgExclusive(alt, askerLeaves) || alt.some((m) => doubtfulLeaf(m, askerLeaves)))) -// Whether a leaf list has any custom cfg at all -- else no asker finds it doubtful. -const hasCustom = (leaves) => leaves.some((l) => (l.alts === undefined ? !l.neg && customKey(l.key) : l.alts.some(hasCustom))) +// The custom-cfg test of an asker whose build may set `settable` (`{ names, any }`), one per object. +const customMemo = new WeakMap() +const NO_CUSTOM = () => false +const customFor = (settable) => { + if (settable === undefined || settable === null) return customKey + if (settable.any) return NO_CUSTOM + if (settable.names.size === 0) return customKey + if (!customMemo.has(settable)) customMemo.set(settable, (key) => customKey(key) && !settable.names.has(key)) + return customMemo.get(settable) +} +const doubtfulLeaf = (l, askerLeaves, custom) => (l.alts === undefined ? !l.neg && custom(l.key) : l.alts.every((alt) => cfgExclusive(alt, askerLeaves) || alt.some((m) => doubtfulLeaf(m, askerLeaves, custom)))) +// Whether a leaf list has any custom cfg at all -- else no asker finds it doubtful (kept on the +// set for the plain test, which most askers share). +const hasCustom = (leaves, custom) => leaves.some((l) => (l.alts === undefined ? !l.neg && custom(l.key) : l.alts.some((alt) => hasCustom(alt, custom)))) const doubtful = (asker, set) => { if (asker === undefined || set.key === '') return false - if (set.custom === undefined) set.custom = hasCustom(set.leaves) + if (set.custom === undefined) set.custom = hasCustom(set.leaves, customKey) if (!set.custom) return false let verdict = asker.doubt.get(set.key) if (verdict === undefined) { - verdict = set.leaves.some((l) => !holds(l, asker.keys) && doubtfulLeaf(l, asker.set.leaves)) + verdict = set.leaves.some((l) => !holds(l, asker.keys, asker.custom) && doubtfulLeaf(l, asker.set.leaves, asker.custom)) asker.doubt.set(set.key, verdict) } return verdict @@ -1529,8 +1610,9 @@ function provided(root, at, name, from, ctx, asker, typesOnly = false) { // is VALUE_ONLY rather than nothing: the name is taken, but not in this namespace. function pick(candidates, root, ctx, asker, typesOnly = false) { let unseen = null - let maybe = null // the first compatible answer under no doubt, for when none is entailed + const maybes = [] // the compatible answers under no doubt, for when none is entailed (see withAlternatives) let doubted = null // the first compatible answer under a doubtful cfg, for when there is nothing else + let lastResort = null // the first under a cfg the asker's build can't compile (see deadUnder) let valueOnly = false // a candidate passed over for being a value where a type is wanted for (let i = 0; ; i++) { const c = candidates.get(i) @@ -1540,10 +1622,11 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { valueOnly = true continue } - const doubt = doubtful(asker, c.leaves) - const sure = !doubt && entailed(asker, c.leaves) - if (!sure && maybe !== null) continue - if (doubt && doubted !== null) continue + const dead = deadFor(asker, c.leaves) + const doubt = !dead && doubtful(asker, c.leaves) + const sure = !dead && !doubt && entailed(asker, c.leaves) + if ((doubt || dead) && (maybes.length > 0 || doubted !== null)) continue + if (dead && lastResort !== null) continue let r if (c.import === undefined) { r = c.answer @@ -1574,15 +1657,60 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { } } if (sure) return { ...r, through: c.through } - if (doubt) doubted = { ...r, through: c.through } - else maybe = { ...r, through: c.through } + if (dead) lastResort = { ...r, through: c.through } + else if (doubt) doubted = { ...r, through: c.through } + else maybes.push({ answer: { ...r, through: c.through }, leaves: c.leaves }) } - if (maybe !== null) return maybe + if (maybes.length > 0) return withAlternatives(maybes, asker) if (doubted !== null) return doubted + if (lastResort !== null) return lastResort if (unseen !== null) return { kind: 'item', file: unseen.import.file, through: unseen.through } return valueOnly ? VALUE_ONLY : null } +// A leaf list as cfg text, for a key: `all(unix, feature = "std")`, `not(loom)`, `any(…)`, a gate +// macro by name (`cfg_x!`); a `mod` variant's leaf by the key of the variant (its cfg, or its +// macro), and only when nothing else tells the candidates apart. Null for no leaves. +const leafText = (l) => { + if (l.alts !== undefined) return `any(${l.alts.map((alt) => cfgTextOf(alt) ?? '*').join(', ')})` + const plain = l.value === null ? l.key : `${l.key} = "${l.value}"` + return l.neg ? `not(${plain})` : plain +} +function cfgTextOf(leaves) { + const own = leaves.filter((l) => l.key !== 'variant') + // What a variant is, rather than all it isn't: its positive leaves (a `cfg_if!` branch's own + // cfg, not the negations of the branches before it), else its `any(…)` of positive ones. + const positive = own.filter((l) => l.alts === undefined && !l.neg) + const anyPositive = own.filter((l) => l.alts !== undefined && l.alts.every((alt) => alt.every((m) => m.alts === undefined && !m.neg))) + const shown = positive.length + anyPositive.length > 0 ? [...positive, ...anyPositive] : (own.length > 0 ? own : leaves) + if (shown.length === 0) return null + const texts = [...new Set(shown.map((l) => (l.key === 'variant' ? l.value : leafText(l))))].toSorted() + return texts.length === 1 ? texts[0] : `all(${texts.join(', ')})` +} + +// The answer of several candidates none of which the asker's cfgs entail (pick, definedIn): each +// applies under cfgs of its own -- a file under no platform cfg asking libc's `crate::sockaddr`, +// defined per platform -- so the first in written order is no answer. The first, carrying +// `alternatives`: cfg key → file, each distinct file under the leaves of its first candidate +// that the asker doesn't hold itself (a module a path goes on through takes the first; only a +// path's target is recorded as the lot). One file, or a module first: that answer alone. +function withAlternatives(found, asker) { + const [first] = found + const files = new Set(found.map((f) => f.answer.file).filter((f) => f !== undefined)) + if (found.length === 1 || files.size < 2 || first.answer.kind === 'module') return first.answer + const alternatives = new Map() + const seen = new Set() + for (const { answer, leaves } of found) { + if (answer.file === undefined || seen.has(answer.file)) continue + seen.add(answer.file) + const base = cfgKey(cfgTextOf(leaves.leaves.filter((l) => asker === undefined || !asker.keys.has(leafKey(l))))) + let key = base + for (let k = 2; alternatives.has(key); k++) key = `${base}#${k}` + alternatives.set(key, answer.file) + } + return { ...first.answer, alternatives } +} + // Every candidate module `at`'s imports offer for `name` as seen `seeing` levels into it, in // written order -- `{ answer | import, through, leaves }`: what its `use` items bind (as imports // to follow, see pick), then what each module its globs reach has (hasAll), under the glob's cfg @@ -1696,18 +1824,24 @@ function hasAll(root, at, name, seeing, ctx) { const NONE = Object.freeze([]) // The file in which module `at` defines item `name` (a `struct`, `fn`, … visible `seeing` levels -// in, under cfgs the asker's allow; in the type namespace when `typesOnly`, see pick), or null. +// in, under cfgs the asker's allow; in the type namespace when `typesOnly`, see pick), or null -- +// or `{ file, alternatives }` when several files define it, none for certain (see +// withAlternatives; itemAt makes an answer of either). function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { const defs = ctx.defined.get(root)?.get(at)?.get(name) if (defs === undefined) return null const own = ctx.trees.get(root).get(at) - // Ranked as pick ranks candidates -- sure, then compatible under no custom cfg, then doubtful - // -- the module's own tree file first within a rank, else the first written. + // Ranked as pick ranks candidates -- sure, then compatible under no custom cfg, then doubtful, + // then dead in the asker's build -- the module's own tree file first within a rank, else the + // first written. + // Several of the middle rank, none sure: each applies under its own cfgs (see withAlternatives). let best = null let bestRank = Infinity + const maybes = [] for (const d of defs) { if (d.scopeDepth > seeing || !compatible(asker, d.leaves) || (typesOnly && d.ns === 'value')) continue - const tier = doubtful(asker, d.leaves) ? 2 : (entailed(asker, d.leaves) ? 0 : 1) + const tier = deadFor(asker, d.leaves) ? 3 : (doubtful(asker, d.leaves) ? 2 : (entailed(asker, d.leaves) ? 0 : 1)) + if (tier === 1) maybes.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves }) const rank = tier * 2 + (d.file === own ? 0 : 1) if (rank < bestRank) { best = d.file @@ -1715,6 +1849,10 @@ function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { if (rank === 0) break } } + if (bestRank >= 2 && bestRank < 4 && maybes.length > 1) { + const r = withAlternatives([...maybes.filter((m) => m.answer.file === best), ...maybes.filter((m) => m.answer.file !== best)], asker) + if (r.alternatives !== undefined) return { file: best, alternatives: r.alternatives } + } return best } @@ -1858,6 +1996,9 @@ function followImport(im, root, ctx, asker, typesOnly = false) { return target ? { kind: 'crate', name: head, file: target } : null } +// An item answer for what definedIn found. +const itemAt = (def) => (typeof def === 'string' ? { kind: 'item', file: def } : { kind: 'item', file: def.file, alternatives: def.alternatives }) + // walkPath's answer for a path naming a value-namespace item (a `fn`, `const` or `static`) where // a type or module is wanted: as a prefix (`log::info!` beside a `fn log`), or as the target of // an import followed as one (`use crate::util::log; log::info!`). Nothing here -- and not an @@ -1903,7 +2044,7 @@ function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, t if (p === null || p === VALUE_ONLY) { // An item of the module itself (`enum Kind { … } use Kind::*;`), or a macro of the file. const own = definedIn(root, from, head, Infinity, ctx, asker, typed) - if (own !== null) return { kind: 'item', file: own } + if (own !== null) return itemAt(own) if (typed && (p === VALUE_ONLY || onlyValues(root, from, head, ctx))) return VALUE_ONLY return segments.length === 1 && file !== undefined && ctx.fileMacros.get(file)?.has(head) === true ? { kind: 'item', file } : null } @@ -1924,17 +2065,24 @@ function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, t const macro = at === 'crate' && last ? ctx.macros.get(root)?.get(name) : undefined if (macro !== undefined && macroCall) return { kind: 'item', file: macro, via } const child = `${at}::${name}` - if (tree.has(child)) { - cur.push(name) - i++ - continue - } const typed = !last || typesOnly - const p = lookup(at, name, typed) + let p + if (tree.has(child)) { + // A child module the asker's build presumably doesn't compile -- under a cfg off in it, or + // a custom one (doubtful): serde's docsrs-only `mod de` beside the `pub use serde_core::de` + // of every other build -- gives way to what else the module has of the name, if anything. + const set = ctx.files.get(tree.get(child))?.leaves + p = set !== undefined && (deadFor(asker, set) || doubtful(asker, set)) ? lookup(at, name, typed) : null + if (p === null || p === VALUE_ONLY) { + cur.push(name) + i++ + continue + } + } else p = lookup(at, name, typed) if (p === null || p === VALUE_ONLY) { if (macro !== undefined) return { kind: 'item', file: macro, via } const def = definedIn(root, at, name, Infinity, ctx, asker, typed) - if (def !== null) return { kind: 'item', file: def, via } + if (def !== null) return { ...itemAt(def), via } if (typed && (p === VALUE_ONLY || onlyValues(root, at, name, ctx))) return VALUE_ONLY return { kind: 'item', file: tree.get(at), via } } @@ -2028,12 +2176,12 @@ function globMayProvide(root, at, lead, ctx) { // Resolve one path reference made in `file` to what it names in the bundle (see walkPath): a // module or item of the same crate, an in-tree crate (`{ kind: 'crate', name, file }`, through // `resolveCrate` when the lead names one, by its own name or as the crate root's `extern crate x -// as y;` aliases it, `::y::…` included), an exported macro named by a one-segment path written at -// the crate root, where such a macro is an item (`pub use anyhow as format_err;` in the anyhow -// crate's lib.rs; a child module reaches it as `crate::anyhow`), or `{ kind: 'unresolved', name }` -// when the lead could name a crate (crateLead) but nothing in-tree did; null for anything else -// (an item in scope, std, a path above the crate root, a name a glob into another crate may well -// provide). `bindings` are the names `file`'s imports bind -- in whichever namespace: one bound +// as y;` aliases it, `::y::…` included -- this crate's own root for `extern crate self as y;`), +// an exported macro named by a one-segment path written at the crate root, where such a macro is +// an item (`pub use anyhow as format_err;` in the anyhow crate's lib.rs; a child module reaches it +// as `crate::anyhow`), or `{ kind: 'unresolved', name }` when the lead could name a crate +// (crateLead) but nothing in-tree did; null for anything else (an item in scope, std, a path +// above the crate root, a name a glob into another crate may well provide). `bindings` are the names `file`'s imports bind -- in whichever namespace: one bound // only as a value (`use crate::util::log;` of a `fn log`) doesn't keep `log::info!` from naming // the crate. const NO_BINDINGS = new Set() @@ -2051,11 +2199,18 @@ function resolvePathRef(ref, file, bindings, ctx) { const lead = crateLead(ref, valueOnly ? NO_BINDINGS : bindings) if (lead === null) return null const name = ctx.externPrelude.get(here?.root)?.get(lead) ?? lead + if (name === 'self') { + const r = walkPath(['crate', ...segments.slice(1)], here.root, from, ctx, { macroCall: ref.macroCall, file, asker: here.asker }) + return r === VALUE_ONLY ? null : r + } const target = ctx.resolveCrate(name, file) if (target) return { kind: 'crate', name, file: target } const macro = segments.length === 1 && from === 'crate' ? ctx.macros.get(here.root)?.get(lead) : undefined if (macro !== undefined) return { kind: 'item', file: macro } - return here && globMayProvide(here.root, from, lead, ctx) ? null : { kind: 'unresolved', name } + // A glob into a crate that isn't in-tree (`use serde::*;`) may bring the lead in -- unless the + // package declares a dependency of that name, which the lead then is (see declaresCrate): one + // missing from the bundle is reported whatever the globs beside it. + return here && !ctx.declaresCrate(name, file) && globMayProvide(here.root, from, lead, ctx) ? null : { kind: 'unresolved', name } } // `map`: root → name → Set, noting `module` under (root, name). @@ -2071,24 +2226,58 @@ const cfgKey = (cfg) => (cfg ?? '*').replaceAll('/', '|') // `benches/*.rs` and their modules), judged against its package dir when a Cargo context knows it. const isTestTarget = (rel, ctx) => (ctx ? ctx.isTestTarget(rel) : isTestTargetPath('.', rel)) -// The build `rel` is scanned under (scanRustItems' options): its crate's resolved features, whether -// it is test code, and the target's cfg set, as far as the Cargo context `ctx` knows them -- none -// for a file compiled for the host (`hostFiles`: a build script and what it reaches), whose -// target the loader doesn't know. -const buildOf = (rel, ctx, hostFiles = null) => ({ features: ctx?.featuresFor(rel) ?? null, test: isTestTarget(rel, ctx), target: hostFiles?.has(rel) === true ? null : ctx?.targetCfgs ?? null }) +// The files a macro is in effect invoked from, by name: those invoking it by bare name +// (`invokers`: name → files), and -- since a bare call in a `macro_rules!` template is made where +// that macro is invoked (`templateCalls`: name → the names its definitions' templates call) -- +// those invoking a macro whose template calls it, on up: what rustc resolves the macro's include +// paths against, the outermost call site. +function effectiveInvokers(name, invokers, templateCalls, seen = new Set()) { + if (seen.has(name)) return new Set() + seen.add(name) + const out = new Set(invokers.get(name) ?? []) + for (const [caller, called] of templateCalls) if (called.has(name)) for (const f of effectiveInvokers(caller, invokers, templateCalls, seen)) out.add(f) + return out +} + +// The compile units a file is compiled as (see cargo.js unitKey): what `units` recorded, else +// the target's alone. +const DEFAULT_UNITS = new Set([TARGET_UNIT]) +const unitsOf = (units, rel) => units?.get(rel) ?? DEFAULT_UNITS + +// The build `rel` is scanned under (scanRustItems' options): per unit it is compiled as, its +// crate's features in that unit's feature context -- on for certain, and on maybe -- and the cfg +// set of the platform the unit compiles for, as far as the Cargo context `ctx` knows them; and +// whether it is test code. A build script is compiled for the host, whose cfgs the loader knows +// only when the target is the host; a crate built both for the target and, as a +// build-dependency, for the host is scanned under both. Interned per (package, test flag, +// units), so a scan of the same build is reused (cachedScan compares identity). +const buildMemo = new WeakMap() // ctx -> key -> build +const NO_CTX = {} +function buildOf(rel, ctx, units = null) { + const test = isTestTarget(rel, ctx) + const set = unitsOf(units, rel) + const key = `${ctx?.packageInfo(rel)?.dir ?? ''}\0${test}\0${[...set].toSorted().join()}` + const memo = buildMemo.get(ctx ?? NO_CTX) ?? buildMemo.set(ctx ?? NO_CTX, new Map()).get(ctx ?? NO_CTX) + if (!memo.has(key)) { + const one = (unit) => ({ features: ctx?.featuresFor(rel, unit) ?? null, maybeFeatures: ctx?.maybeFeaturesFor(rel, unit) ?? null, target: ctx?.platformOf(unit)?.cfgs ?? null }) + const each = [...set].map(one) + memo.set(key, each.length === 1 ? { ...each[0], test } : { units: each, test }) + } + return memo.get(key) +} // Scan results per `sources` map, so the tree pass reuses the walk's scan of each file when it -// ran under the same build (the same feature set object, test flag and target cfg set object) on -// the same content (a caller may edit a file in a map it reuses). +// ran under the same build (one interned object, see buildOf) on the same content (a caller may +// edit a file in a map it reuses). const scanCache = new WeakMap() -function cachedScan(sources, path, content, build) { +function cachedScan(sources, path, content, build, templates = undefined) { if (!scanCache.has(sources)) scanCache.set(sources, new Map()) const cache = scanCache.get(sources) - const key = build.templates === undefined ? path : `${path}\0${[...build.templates].join()}` // a rescan with fewer templates, kept beside the plain scan + const key = templates === undefined ? path : `${path}\0${[...templates].join()}` // a rescan with fewer templates, kept beside the plain scan const hit = cache.get(key) - if (hit && hit.content === content && hit.features === build.features && hit.test === build.test && hit.target === build.target) return hit.items - const items = scanRustItems(content, build) - cache.set(key, { ...build, content, items }) + if (hit && hit.content === content && hit.build === build) return hit.items + const items = scanRustItems(content, templates === undefined ? build : { ...build, templates }) + cache.set(key, { build, content, items }) return items } @@ -2107,8 +2296,9 @@ function cachedScan(sources, path, content, build) { // complete the bundle. A `cargo` context (cargo.js) may be passed in to share one // between the walk and this pass; it also carries each crate's resolved features, which decide // `#[cfg(feature = …)]`, and the build target's cfg set when one was given, which decides -// `#[cfg(unix)]` and the like. -export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null, formats = null, hostFiles = null } = {}) { +// `#[cfg(unix)]` and the like. `units`: file → the compile units it is compiled as (see +// collectRustFilesFromDisk); `wantedUnits` gives each wanted root's, for the walk. +export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null, formats = null, units = null } = {}) { const ctx = cargo ?? (baseDir ? createCargoContext(baseDir, { entries: roots }) : null) const rootSet = crateRoots(sources, roots, ctx) const resolutions = new Map() @@ -2116,13 +2306,14 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const scanned = new Map() for (const [path, content] of sources) { if (formats?.has(path)) continue // an `include_str!` / `include_bytes!` asset: carried, not Rust - scanned.set(path, cachedScan(sources, path, content, buildOf(path, ctx, hostFiles))) + scanned.set(path, cachedScan(sources, path, content, buildOf(path, ctx, units))) } // A package with a `macro_rules!` of a template macro's name (its own `quote!`, in whichever of // its files) invokes that one everywhere in the package: no template there, so the files of the // package that skipped a body of that name are scanned again with the name dropped from the set - // (the package: the Cargo context's, or the whole bundle without one). - const packageOf = (path) => ctx?.packageInfo(path)?.dir ?? '.' + // (the package: the Cargo context's; without one, a vendored crate's `vendor/`, else the + // bundle's). + const packageOf = (path) => (ctx ? ctx.packageInfo(path)?.dir ?? '.' : path.startsWith(`${VENDOR_DIR}/`) ? path.split('/', 2).join('/') : '.') const ownTemplates = new Map() // package → the template names it defines for (const [path, items] of scanned) { for (const m of items.macros) if (TEMPLATE_MACROS.has(m.name)) (ownTemplates.get(packageOf(path)) ?? ownTemplates.set(packageOf(path), new Set()).get(packageOf(path))).add(m.name) @@ -2132,8 +2323,36 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const own = items.skipped.size > 0 ? ownTemplates.get(packageOf(path)) : undefined if (own === undefined || own.isDisjointFrom(items.skipped)) continue const templates = reducedTemplates.get(own) ?? reducedTemplates.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) - scanned.set(path, cachedScan(sources, path, sources.get(path), { ...buildOf(path, ctx, hostFiles), templates })) + scanned.set(path, cachedScan(sources, path, sources.get(path), buildOf(path, ctx, units), templates)) } + // Bare macro invocations per file, for what a `macro_rules!` body holds: rustc expands it where + // the macro is invoked -- through the templates that call it, the outermost (effectiveInvokers). + const invokers = new Map() + const templateCalls = new Map() + for (const [path, items] of scanned) { + for (const name of items.invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(path) + for (const m of items.macros) for (const name of m.calls) (templateCalls.get(m.name) ?? templateCalls.set(m.name, new Set()).get(m.name)).add(name) + } + // A `mod` a `macro_rules!` template declares is declared in each module invoking the macro, and + // its file found from there (serde_core's src/crate_root.rs holds `crate_root! { … pub mod de; + // … }`, which lib.rs invokes: src/de/mod.rs). Only its own package's invocations count -- a + // macro of the same name elsewhere is likely another -- and one nothing there invokes by bare + // name (`crate::m!()` only) stays beside its definition. A hosted `mod` stands at the host's + // first invocation (`offset`), for textual macro scope, and keeps the file defining it + // (`definedIn`): the host's cfgs on mounting that file are its own too (serde's lib.rs mounts + // core/crate_root.rs under `#[cfg(docsrs)]`). + const hosted = new Map() // file → the template `mod`s it declares by invoking their macros + const hostedAway = new Set() // the template `mod`s that moved to their invokers + for (const [path, items] of scanned) { + for (const decl of items.mods) { + if (decl.template === null) continue + const hosts = [...effectiveInvokers(decl.template, invokers, templateCalls)].filter((f) => packageOf(f) === packageOf(path)) + if (hosts.length === 0) continue + hostedAway.add(decl) + for (const f of hosts) (hosted.get(f) ?? hosted.set(f, []).get(f)).push({ ...decl, offset: scanned.get(f).calls.get(decl.template)?.[0] ?? Infinity, definedIn: path }) + } + } + const modsOf = (path) => [...scanned.get(path).mods.filter((d) => !hostedAway.has(d)), ...(hosted.get(path) ?? [])] // An `include!`d file's items join the includer's module (buildModuleTrees), but a `mod` it // declares resolves beside the included file itself, as rustc does: like a `#[path]`-loaded // file, it owns its directory (pathLoaded). @@ -2152,8 +2371,8 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const declsOf = (path, decl) => resolveModDecl(decl, path, { ...modOpts, boundary: boundaryOf(path, ctx) }) for (let grew = true; grew;) { grew = false - for (const [path, items] of scanned) { - for (const decl of items.mods) { + for (const path of scanned.keys()) { + for (const decl of modsOf(path)) { if (decl.paths.length === 0) continue for (const t of declsOf(path, decl)) { if (t.explicit && !pathLoaded.has(t.file)) { @@ -2164,24 +2383,22 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul } } } - // Bare macro invocations per file, for the include macros a `macro_rules!` body holds: rustc - // resolves those relative to each invoking file (recorded there, as `include_str `). - const invokers = new Map() - for (const [path, items] of scanned) { - for (const name of items.invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(path) - } + // The include macros a `macro_rules!` body holds name files relative to each file in effect + // invoking the macro, recorded there, as `include_str `. Every definition of a name counts + // (two `macro_rules! m` in two files: both bodies' assets). const includeEdges = new Map() // file → [{ kind, path, base, conditional }] from macro templates it invokes for (const [, items] of scanned) { for (const m of items.macros) { if (m.includes.length === 0) continue - for (const f of invokers.get(m.name) ?? []) { + for (const f of effectiveInvokers(m.name, invokers, templateCalls)) { const list = includeEdges.get(f) ?? includeEdges.set(f, []).get(f) for (const inc of m.includes) list.push({ ...inc, conditional: true }) } } } // file → spec → target file → the declarations mounting it, each its cfg (null for an ungated - // one) and the macro body it sits in, for the cfg leaves each file carries (buildModuleTrees). + // one), the macro body it sits in and -- for a hosted template `mod` -- the declarations here + // mounting the file defining it (`via`), for the cfg leaves each file carries (buildModuleTrees). const mountCfgs = new Map() for (const [path, items] of scanned) { // spec -> Map: each declaration's files under its cfg predicate ('*' when @@ -2191,7 +2408,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // key wins; another file under the same key gets the key numbered (`*#2`) rather than lost. const byCfg = new Map() const cfgsHere = mountCfgs.set(path, new Map()).get(path) - for (const decl of items.mods) { + for (const decl of modsOf(path)) { const spec = `mod ${[...decl.inlinePath, decl.name].join('::')}` const targets = declsOf(path, decl) if (targets.length === 0) { @@ -2210,7 +2427,8 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul let key = base for (let k = 2; keyed.has(key) && keyed.get(key) !== t.file; k++) key = `${base}#${k}` if (!keyed.has(key)) keyed.set(key, t.file) - ;(byFile.get(t.file) ?? byFile.set(t.file, []).get(t.file)).push({ cfg, macro: decl.macro }) + const via = decl.definedIn === undefined ? undefined : [...cfgsHere.values()].flatMap((m) => m.get(decl.definedIn) ?? []) + ;(byFile.get(t.file) ?? byFile.set(t.file, []).get(t.file)).push({ cfg, macro: decl.macro, via }) } } // A single file is a flat string -- also when every cfg names the same file (libc's `mod @@ -2231,22 +2449,48 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul resolutions.set(path, specMap) } - const { trees, files } = buildModuleTrees(sources, resolutions, rootSet, new Map([...scanned].map(([p, items]) => [p, items.inlineModules])), mountCfgs) + // Each package's gate macros (`cfg_x!`), by name: the cfg its definition wraps items in, or null + // when that can't be read -- or when the package defines the name twice, differently. + const gatesByPackage = new Map() + for (const [path, items] of scanned) { + for (const m of items.macros) { + if (m.gate === undefined) continue + const gates = gatesByPackage.get(packageOf(path)) ?? gatesByPackage.set(packageOf(path), new Map()).get(packageOf(path)) + gates.set(m.name, gates.has(m.name) && gates.get(m.name) !== m.gate ? null : m.gate) + } + } + const gatesOf = (path) => gatesByPackage.get(packageOf(path)) ?? null + const { trees, files } = buildModuleTrees(sources, resolutions, rootSet, new Map([...scanned].map(([p, items]) => [p, items.inlineModules])), mountCfgs, gatesOf) // What each file asks as: itself, with the cfg leaves it was mounted under (see cfg // compatibility), as an interned leaf set (leafSets: every set of this tree, by key). const leafSets = new Map() - const askerSets = new Map() // per leaf set: what every asker of that set shares - // With the build's target known, its cfgs hold for every file compiled for it (not a build - // script's, compiled for the host): a `unix` asker never takes a `windows` candidate, a - // `target_os = "linux"` one prefers the `any(android, linux, …)` branch (see compatible). - const targetLeaves = leafSet([...(ctx?.targetCfgs ?? [])].flatMap((cfg) => { + const askerSets = new Map() // per leaf set, build and settable cfgs: what every asker of those shares + const ids = new WeakMap() + let nextId = 0 + const idOf = (object) => ids.get(object) ?? ids.set(object, nextId++).get(object) + // With a file's platform known, its cfgs hold for the file: a `unix` asker never takes a + // `windows` candidate, a `target_os = "linux"` one prefers the `any(android, linux, …)` branch + // (see compatible). The platform is the target's for code compiled for it, the host's for a + // build script (known only when the target is the host); a file compiled for both platforms + // asks under the cfgs they share, which the loader knows only when they are one. + const platformLeaves = new Map() // cfg set -> its leaf set + const leavesOfCfgs = (cfgs) => platformLeaves.get(cfgs) ?? platformLeaves.set(cfgs, leafSet([...cfgs].flatMap((cfg) => { const m = /^([\w-]+)(?:="([^"]*)")?$/u.exec(cfg) return m && TARGET_CFG_KEYS.has(m[1]) ? [{ key: m[1], value: m[2] ?? null, neg: false }] : [] - }), leafSets) + }), leafSets)).get(cfgs) + const platformCfgsOf = (path) => { + const each = new Set([...unitsOf(units, path)].map((u) => ctx?.platformOf(u)?.cfgs ?? null)) + return each.size === 1 ? [...each][0] : null + } for (const [path, f] of files) { f.leaves = leafSet(f.leaves, leafSets) - const set = hostFiles?.has(path) === true ? f.leaves : unionLeaves(f.leaves, targetLeaves, leafSets) - const shared = askerSets.get(set.key) ?? askerSets.set(set.key, { keys: new Set(set.leaves.map(leafKey)), compat: new Map(), sure: new Map(), doubt: new Map() }).get(set.key) + const cfgs = platformCfgsOf(path) + const set = cfgs === null ? f.leaves : unionLeaves(f.leaves, leavesOfCfgs(cfgs), leafSets) + // What the askers of one leaf set, build and settable cfgs share (one object each). + const build = buildOf(path, ctx, units) + const settable = ctx?.cfgsSetFor(path) ?? null + const sharedKey = `${set.key}\0${idOf(build)}\0${settable === null ? '' : idOf(settable)}` + const shared = askerSets.get(sharedKey) ?? askerSets.set(sharedKey, { keys: new Set(set.leaves.map(leafKey)), compat: new Map(), sure: new Map(), doubt: new Map(), dead: new Map(), build, deadHere: deadUnder(set.leaves, build), custom: customFor(settable) }).get(sharedKey) f.asker = { file: path, set, ...shared } } // Per crate root, each module's child modules by name (every module's parent is in its tree). @@ -2288,19 +2532,6 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const mounts = new Map() const macroUseMods = new Map() const filesOf = (target) => (target instanceof Map ? [...target.values()] : [target]) - // Where a `mod` stands for textual scope: in its file at its offset -- or, for one declared in - // a `macro_rules!` template this file never invokes (serde's core/crate_root.rs declares `pub - // mod de;` inside `crate_root!`, which lib.rs invokes), in the first file (tree order) invoking - // the macro, at that invocation: rustc emits the item there. - const callerMemo = new Map() - const firstCaller = (name) => { - if (!callerMemo.has(name)) callerMemo.set(name, files.keys().find((f) => scanned.get(f)?.calls.has(name) === true) ?? null) - return callerMemo.get(name) - } - const mountOf = (path, decl) => { - const caller = decl.template !== null && decl.offset === Infinity ? firstCaller(decl.template) : null - return caller === null ? { parent: path, offset: decl.offset, at: decl.at } : { parent: caller, offset: scanned.get(caller).calls.get(decl.template)[0], at: decl.at } - } // In tree order (`files`: each crate's, from its root down, the variants of a `mod` in the order // declared), whatever order the sources came in: among several files defining or importing a // name for one module, the first is the tree's own, and the answers are the same for any order @@ -2316,7 +2547,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const modulePath = [here.modulePath, ...im.inlinePath].join('::') const byModule = perRoot(imports, here.root) const of = byModule.get(modulePath) ?? byModule.set(modulePath, { named: new Map(), globs: [] }).get(modulePath) - const entry = { ...im, file: path, module: modulePath, scopeDepth: scopeDepthOf(im.vis, modulePath), leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(im.cfg), ...gateLeaves(im.macro)], leafSets), leafSets) } + const entry = { ...im, file: path, module: modulePath, scopeDepth: scopeDepthOf(im.vis, modulePath), leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(im.cfg), ...gateLeaves(im.macro, gatesOf(path))], leafSets), leafSets) } if (im.glob) of.globs.push(entry) else if (im.binding !== null) (of.named.get(im.binding) ?? of.named.set(im.binding, []).get(im.binding)).push(entry) } @@ -2324,18 +2555,18 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const modulePath = [here.modulePath, ...d.inlinePath].join('::') const byModule = perRoot(defined, here.root) const names = byModule.get(modulePath) ?? byModule.set(modulePath, new Map()).get(modulePath) - ;(names.get(d.name) ?? names.set(d.name, []).get(d.name)).push({ file: path, scopeDepth: scopeDepthOf(d.vis, modulePath), ns: d.ns, leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(d.cfg), ...gateLeaves(d.macro)], leafSets), leafSets) }) + ;(names.get(d.name) ?? names.set(d.name, []).get(d.name)).push({ file: path, scopeDepth: scopeDepthOf(d.vis, modulePath), ns: d.ns, leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(d.cfg), ...gateLeaves(d.macro, gatesOf(path))], leafSets), leafSets) }) } const scopes = perRoot(modScope, here.root) const specMap = resolutions.get(path) - for (const decl of items.mods) { + for (const decl of modsOf(path)) { const parent = [here.modulePath, ...decl.inlinePath].join('::') const modulePath = `${parent}::${decl.name}` if (!scopes.has(modulePath)) scopes.set(modulePath, scopeDepthOf(decl.vis, parent)) const target = specMap.get(`mod ${[...decl.inlinePath, decl.name].join('::')}`) if (target === undefined) continue const mountedFiles = filesOf(target).filter((f) => f !== path) - const mount = mountOf(path, decl) + const mount = { parent: path, offset: decl.offset, at: decl.at } for (const f of mountedFiles) if (!mounts.has(f)) mounts.set(f, mount) if (decl.macroUse && mountedFiles.length > 0) (macroUseMods.get(mount.parent) ?? macroUseMods.set(mount.parent, []).get(mount.parent)).push({ offset: mount.offset, at: mount.at, files: mountedFiles }) } @@ -2355,7 +2586,9 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul if (here.modulePath === 'crate') { const prelude = perRoot(externPrelude, here.root) for (const e of items.externCrates) { - if (e.inlinePath.length === 0 && !NON_CRATE_LEADS.has(e.name) && !prelude.has(e.alias ?? e.name)) prelude.set(e.alias ?? e.name, e.name) + // `extern crate self as x;` puts this crate's root in the prelude as `x` (kept as `self`). + const named = e.name === 'self' ? e.alias !== null : !NON_CRATE_LEADS.has(e.name) + if (e.inlinePath.length === 0 && named && !prelude.has(e.alias ?? e.name)) prelude.set(e.alias ?? e.name, e.name) } } } @@ -2441,13 +2674,14 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul for (const [root, byModule] of defined) for (const [module, names] of byModule) for (const name of names.keys()) note(having, root, name, module) for (const [root, byModule] of imports) for (const [module, of] of byModule) for (const name of of.named.keys()) note(having, root, name, module) for (const [root, byName] of macros) for (const name of byName.keys()) note(having, root, name, 'crate') - const pathCtx = { trees, children, files, resolveCrate, imports, defined, modScope, macros, fileMacros, externPrelude, leafSets, having, provided: new Map(), closures: new Map(), opaque: new Map(), opaqueGlobs: new Map(), walking: 0, hits: new Set() } + const declaresCrate = (name, from) => ctx?.declaresCrate?.(name, from) === true + const pathCtx = { trees, children, files, resolveCrate, declaresCrate, imports, defined, modScope, macros, fileMacros, externPrelude, leafSets, having, provided: new Map(), closures: new Map(), opaque: new Map(), opaqueGlobs: new Map(), walking: 0, hits: new Set() } // Every path of every file, on top of the `mod` / include edges (`resolutions`): once, and once // more when the first pass left glob closures short of an import cycle (stabilizeClosures). const resolveAll = () => { const out = new Map() const unresolved = new Set() - const wanted = new Set() + const wanted = new Map() // root -> the units it is wanted as for (const [path, items] of scanned) { const specMap = new Map(resolutions.get(path)) out.set(path, specMap) @@ -2456,14 +2690,31 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul return { out, unresolved, wanted } } const resolveFile = (path, items, specMap, unresolved, wanted) => { - // Only edges to bundled files, never to self, first spelling wins. - const add = (spec, target) => { - if (target && target !== path && sources.has(target) && !specMap.has(spec)) specMap.set(spec, target) + // Only edges to bundled files, never to self, first spelling wins. A target with + // `alternatives` (withAlternatives) is recorded as the cfg-keyed map of them -- as a `mod` + // with cfg variants is -- less the asking file and files not bundled; one file left is a plain edge. + const add = (spec, target, alternatives) => { + if (specMap.has(spec)) return + if (alternatives !== undefined) { + const keyed = new Map([...alternatives].filter(([, f]) => f !== path && sources.has(f))) + const distinct = new Set(keyed.values()) + if (distinct.size > 1) { + specMap.set(spec, keyed) + return + } + if (distinct.size === 1) [target] = distinct + } + if (target && target !== path && sources.has(target)) specMap.set(spec, target) } // A crate resolved to a root the walk didn't load (its name was bound in the walking file as - // a value, see resolvePathRef): the caller may load it and build again. + // a value, see resolvePathRef), or loaded as fewer units than this file names it as: the + // caller may load it and build again. const addCrate = (name, target) => { - if (target && !sources.has(target)) wanted.add(target) + if (target && ctx) { + const want = new Set([...unitsOf(units, path)].map((u) => ctx.unitOfCrate(target, u))) + const have = sources.has(target) ? unitsOf(units, target) : new Set() + if (!want.isSubsetOf(have)) for (const u of want) (wanted.get(target) ?? wanted.set(target, new Set()).get(target)).add(u) + } else if (target && !sources.has(target)) wanted.set(target, DEFAULT_UNITS) add(`use ${name}`, target) } // A crate name nothing in-tree satisfied -- sysroot crates aside, and an uppercase lead is an @@ -2481,7 +2732,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // Only a `use` reliably says its lead names a crate (see scanRustItems' `fromUse`). if (ref.fromUse) noteUnresolved(r.name) continue - } else add(ref.macroCall ? `${ref.spec}!` : ref.spec, r.file) // `crate::helper!()` beside `crate::helper()`: two edges + } else add(ref.macroCall ? `${ref.spec}!` : ref.spec, r.file, r.alternatives) // `crate::helper!()` beside `crate::helper()`: two edges // The module an import was followed from, when the path's own segments name it (`super`, // `crate::util`) and its file is neither this one nor the crate root, which every file of // the crate hangs off anyway. @@ -2506,19 +2757,35 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul if (here !== undefined) { const above = inherited(path) const exported = here.modulePath === 'crate' ? macros.get(here.root) : undefined + // The file defining the macro `name` a bare call at `offset` names. + const macroFileAt = (name, offset) => { + const inScope = macroAt(path, name, offset, offset, above) ?? exported?.get(name) + if (inScope !== undefined) return inScope + const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { macroCall: true, file: path, asker: here.asker })?.file + return byPath !== undefined && fileMacros.get(byPath)?.has(name) === true ? byPath : undefined // a file that defines a macro of that name, not a `fn write` beside `write!` + } for (const [name, offsets] of items.calls) { // Scope is decided at each call: a `macro_rules!` later in the file doesn't shadow what an - // earlier `#[macro_use] mod` handed an earlier call. + // earlier `#[macro_use] mod` handed an earlier call. The bare calls in the invoked + // macro's template, and in theirs on down, are made here too -- resolved in this file's + // scope at this call, as rustc expands them. const found = new Set() for (const offset of offsets) { - let macroFile = macroAt(path, name, offset, offset, above) ?? exported?.get(name) - if (macroFile === undefined) { - const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { macroCall: true, file: path, asker: here.asker })?.file - if (byPath !== undefined && fileMacros.get(byPath)?.has(name) === true) macroFile = byPath // a file that defines a macro of that name, not a `fn write` beside `write!` + const seen = new Set() + const expand = (called) => { + if (seen.has(called)) return + seen.add(called) + const macroFile = macroFileAt(called, offset) + if (macroFile === undefined) return + found.add(`${called}!\0${macroFile}`) + for (const m of scanned.get(macroFile)?.macros ?? []) if (m.name === called) for (const inner of m.calls) expand(inner) } - if (macroFile !== undefined) found.add(macroFile) + expand(name) + } + for (const edge of found) { + const cut = edge.indexOf('\0') + add(edge.slice(0, cut), edge.slice(cut + 1)) } - for (const macroFile of found) add(`${name}!`, macroFile) } } // A module whose import a path went through is a dependency of the file too, whatever the @@ -2532,7 +2799,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul } let { out, unresolved, wanted } = resolveAll() if (stabilizeClosures(pathCtx)) ({ out, unresolved, wanted } = resolveAll()) - return { sources, resolutions: out, missing, unresolvedCrates: unresolved, wantedRoots: [...wanted] } + return { sources, resolutions: out, missing, unresolvedCrates: unresolved, wantedRoots: [...wanted.keys()], wantedUnits: wanted } } // --- Disk walk -------------------------------------------------------------------------- @@ -2541,8 +2808,13 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // crates, reading each reachable file once; a missing file warns and is skipped without aborting // the walk (buildRustTree decides what is fatal). `cargo` (cargo.js) resolves crate names and // carries each crate's features; one is created for `entries` when not passed in. Pass `sources` -// to walk further roots into an earlier walk's map (files already there are not read again). -export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, formats = null, sources = new Map(), hostFiles = new Set() } = {}) { +// to walk further roots into an earlier walk's map (files already there are not read again, but +// are gone through again when they are now compiled as more units). `units`: file → the compile +// units it is compiled as (cargo.js unitKey), filled in as the walk goes -- an entry the map +// doesn't name is the target's; what a file declares or includes is compiled as that file is, +// and a crate it names as the Cargo context says (unitOfCrate: a proc-macro crate, or a crate +// named from code built for the host, is built for the host). +export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, formats = null, sources = new Map(), units = new Map() } = {}) { const realBase = realpathSync(baseDir) const roots = new Set(entries) const pathLoaded = new Set() // files a `#[path]` named: their submodules sit beside them @@ -2554,8 +2826,12 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, // vendored crate's files stay in its package, symlinks included. const boundaries = new Map(entries.map((e) => [e, boundaryOf(e, ctx)])) // An include macro in a `macro_rules!` body names a file relative to each file invoking the - // macro: definitions and invocations arrive in any order, so each side is kept and paired. - const templates = new Map() // macro name → its body's includes + // macro -- through the templates that call it, the outermost (effectiveInvokers): definitions + // and invocations arrive in any order, so each side is kept and paired. Every definition of a + // name counts (two `macro_rules! m` in two files: both bodies' assets). + const templateIncludes = new Map() // macro name → its bodies' includes + const templateMods = new Map() // macro name → its bodies' `mod`s, each `{ decl, from }` (see buildRustTree) + const templateCalls = new Map() // macro name → the names its bodies call by bare name const invokers = new Map() // macro name → the files invoking it by bare name // A package defining a `macro_rules!` of a template macro's name (its own `quote!`) invokes that // one everywhere in the package (see buildRustTree): its files are scanned without the name in @@ -2566,15 +2842,96 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, const reduced = new Map() // per package's own set: TEMPLATE_MACROS less those, one object const scanFor = (relPath, content) => { const own = ownTemplates.get(packageOf(relPath)) - const build = buildOf(relPath, ctx, hostFiles) + const build = buildOf(relPath, ctx, units) if (own === undefined) return cachedScan(sources, relPath, content, build) const set = reduced.get(own) ?? reduced.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) - return cachedScan(sources, relPath, content, { ...build, templates: set }) + return cachedScan(sources, relPath, content, build, set) + } + // Record `file` as compiled as each of `more` too; whether that is news. + const addUnits = (file, more) => { + const have = units.get(file) + if (have === undefined) { + units.set(file, new Set(more)) + return true + } + if (more.isSubsetOf(have)) return false + for (const u of more) have.add(u) + return true + } + for (const e of entries) addUnits(e, unitsOf(units, e)) + // Files already walked that are now compiled as more units: gone through again (their build + // changed, and so may what they name, and as what). + const revisit = new Set(entries.filter((e) => sources.has(e) && !formats?.has(e))) + + // What a Rust file names, queued for the next wave: compiled as the naming file is -- as its + // units, a crate as unitOfCrate says (`as`) -- and, for a vendored crate's, within its package + // (`within`). + let next = [] + const queue = (from, file, { rust, within = boundaryOf(from, ctx), as = unitsOf(units, from) }) => { + if (rust) { + rustFiles.add(file) + // Named as Rust source after being carried as an asset: read again, as source. + if (assets.delete(file) && sources.has(file)) { + sources.delete(file) + formats?.delete(file) + } + } + if (!boundaries.has(file)) boundaries.set(file, within) + const grew = addUnits(file, as) + if (!sources.has(file)) next.push(file) + else if (grew && rust && !formats?.has(file)) revisit.add(file) + } + // An include macro's file, relative to `from` (the file it is written in, or one invoking the + // macro whose body it is in). + const includeFrom = (inc, from) => { + const file = includeTarget(inc, from, ctx) + if (file === null || file === from) return + // A FIFO or device named would hang the read; a missing file is the tree pass's to report. + if (!sources.has(file) && !isFile(join(baseDir, file))) return + if (inc.kind === 'include') { + pathLoaded.add(file) // its own `mod`s resolve beside it, as rustc has it + queue(from, file, { rust: true }) + } else if (!rustFiles.has(file) && !sources.has(file)) { + assets.set(file, inc.kind) + queue(from, file, { rust: false }) + } + } + // A `mod` declaration's files, relative to `from` (the file it is written in, or one invoking + // the macro whose body it is in). + const modFrom = (decl, from) => { + for (const { file, explicit } of resolveModDecl(decl, from, { ...modOpts, boundary: boundaryOf(from, ctx) })) { + if (explicit) pathLoaded.add(file) + queue(from, file, { rust: true }) + } + } + // The files of `from`'s package in effect invoking macro `name`: where its template's `mod`s + // are declared (see buildRustTree). + const hostsOf = (name, from) => [...effectiveInvokers(name, invokers, templateCalls)].filter((f) => packageOf(f) === packageOf(from)) + // The include macros of every template, relative to every file in effect invoking it; its + // `mod`s relative to those of its own package. + const expandTemplates = () => { + for (const [name, incs] of templateIncludes) { + for (const f of effectiveInvokers(name, invokers, templateCalls)) for (const inc of incs) includeFrom(inc, f) + } + for (const [name, decls] of templateMods) for (const { decl, from } of decls.values()) for (const f of hostsOf(name, from)) modFrom(decl, f) + } + + // A template's `mod`s that nothing in its package invokes by bare name stand beside its + // definition (see buildRustTree): walked once nothing else is left to, and what they name after. + const placed = new Set() + const placeUninvoked = () => { + for (const [name, decls] of templateMods) { + for (const { decl, from } of decls.values()) { + if (placed.has(decl) || hostsOf(name, from).length > 0) continue + placed.add(decl) + modFrom(decl, from) + } + } } const processWave = async (wave) => { const toLoad = [...new Set(wave)].filter((p) => !sources.has(p)) - if (toLoad.length === 0) return + if (toLoad.length === 0 && revisit.size === 0) return const reads = await Promise.all( toLoad.map(async (relPath) => { try { @@ -2589,11 +2946,14 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, return null } } + // Rust source (a module, a build script, an `include!`d file) and an `include_str!` asset + // are UTF-8 text, or rustc rejects them: one that isn't is refused, never carried with + // its bytes replaced. An `include_bytes!` asset is any bytes, carried as base64 if need be. const kind = assets.get(relPath) - if (kind === undefined) return [relPath, await readFile(join(baseDir, relPath), 'utf8'), false] - // `include_str!` embeds UTF-8 text (rustc rejects anything else); `include_bytes!` any bytes. const buf = await readFile(join(baseDir, relPath)) - const utf8 = kind === 'include_str' || isUtf8(buf) + const utf8 = isUtf8(buf) + if (!utf8 && kind !== 'include_bytes') throw new Error(`${kind === undefined ? 'Rust source' : 'include_str! file'} is not valid UTF-8: ${relPath}`) + if (kind === undefined) return [relPath, buf.toString('utf8'), false] formats?.set(relPath, utf8 ? 'resource' : 'resource:base64') return [relPath, utf8 ? buf.toString('utf8') : buf.toString('base64'), true] } catch (err) { @@ -2610,41 +2970,11 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, } }) ) - const next = [] - // What a Rust file names, queued for the next wave. Run again for a file whose package turned - // out to define a template macro (below): the file's `quote!` bodies are its own then. + next = [] + // What a Rust file names, queued (see queue). Run again for a file whose package turned out + // to define a template macro (below): the file's `quote!` bodies are its own then. const handle = (relPath, content) => { - // What this file names is compiled as it is: for the host when it is a build script's, and - // -- for a vendored crate's -- within its package (`boundary`). - const host = hostFiles.has(relPath) - const boundary = boundaryOf(relPath, ctx) - const queue = (file, { rust, within = boundary }) => { - if (rust) { - rustFiles.add(file) - // Named as Rust source after being carried as an asset: read again, as source. - if (assets.delete(file) && sources.has(file)) { - sources.delete(file) - formats?.delete(file) - } - } - if (!boundaries.has(file)) boundaries.set(file, within) - if (sources.has(file)) return - if (host) hostFiles.add(file) - next.push(file) - } - const includeOf = (inc, from) => { - const file = includeTarget(inc, from, ctx) - if (file === null || file === from) return - // A FIFO or device named would hang the read; a missing file is the tree pass's to report. - if (!sources.has(file) && !isFile(join(baseDir, file))) return - if (inc.kind === 'include') { - pathLoaded.add(file) // its own `mod`s resolve beside it, as rustc has it - queue(file, { rust: true }) - } else if (!rustFiles.has(file) && !sources.has(file)) { - assets.set(file, inc.kind) - queue(file, { rust: false }) - } - } + const here = unitsOf(units, relPath) const items = scanFor(relPath, content) const { mods, refs, externCrates, bindings, includes, macros, invocations } = items const pkg = packageOf(relPath) @@ -2658,25 +2988,24 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, } // An `include!`d file's `mod` declarations resolve beside that file (it is in `pathLoaded` // from when it was queued), within its package. + // A template's `mod`s wait for the files invoking it (expandTemplates). for (const decl of mods) { - for (const { file, explicit } of resolveModDecl(decl, relPath, { ...modOpts, boundary })) { - if (explicit) pathLoaded.add(file) - queue(file, { rust: true }) - } + if (decl.template === null) modFrom(decl, relPath) + else (templateMods.get(decl.template) ?? templateMods.set(decl.template, new Map()).get(decl.template)).set(`${relPath}\0${decl.at}`, { decl, from: relPath }) } // `include!("x.rs")` splices Rust source (scanned next wave, under this module); the asset // macros name a file to carry as it is. A build-output path (`concat!(env!("OUT_DIR"), …)`) - // never gets here. - for (const inc of includes) includeOf(inc, relPath) + // never gets here. A template's include macros are resolved once the wave is in + // (expandTemplates), against the files invoking it. + for (const inc of includes) includeFrom(inc, relPath) for (const m of macros) { - if (m.includes.length === 0) continue - templates.set(m.name, m.includes) - for (const f of invokers.get(m.name) ?? []) for (const inc of m.includes) includeOf(inc, f) - } - for (const name of invocations) { - ;(invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(relPath) - for (const inc of templates.get(name) ?? []) includeOf(inc, relPath) + if (m.includes.length > 0) { + const list = templateIncludes.get(m.name) ?? templateIncludes.set(m.name, []).get(m.name) + for (const inc of m.includes) if (!list.some((x) => x.kind === inc.kind && x.path === inc.path && x.base === inc.base)) list.push(inc) + } + for (const name of m.calls) (templateCalls.get(m.name) ?? templateCalls.set(m.name, new Set()).get(m.name)).add(name) } + for (const name of invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(relPath) // A reference to an in-tree crate pulls its root in (a crate root by role, whatever its // name -- `[lib] path` may point anywhere); the root's own `mod` edges follow next wave. const leads = new Set() @@ -2689,7 +3018,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, const lib = ctx.resolveCrate(name, relPath) if (!lib) continue roots.add(lib) - queue(lib, { rust: true, within: boundaryOf(lib, ctx) }) + queue(relPath, lib, { rust: true, within: boundaryOf(lib, ctx), as: new Set([...here].map((u) => ctx.unitOfCrate(lib, u))) }) } } for (const entry of reads) { @@ -2698,7 +3027,16 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, sources.set(relPath, content) if (!asset) handle(relPath, content) } - await processWave(next) + for (;;) { + expandTemplates() + if (revisit.size === 0) break + const again = [...revisit] + revisit.clear() + for (const relPath of again) handle(relPath, sources.get(relPath)) + } + if (next.length === 0) placeUninvoked() + const following = next + await processWave(following) } await processWave(entries) diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index f4332814..df999737 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -18,13 +18,15 @@ import { satisfiesCargoReq, } from '../stasis/src/loaders/cargo.js' import { buildRustBundle } from '../stasis/src/cmd/bundle.js' -import { buildRustTree } from '../stasis/src/loaders/rust.js' +import { buildRustTree, collectRustFilesFromDisk } from '../stasis/src/loaders/rust.js' const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'rust-bundle') const featuresFixture = join(fixtures, 'features') const sorted = (iter) => [...iter].toSorted() -const enabledOf = (cargo) => Object.fromEntries([...cargo.resolvedFeatures()].map(([dir, set]) => [dir, sorted(set)]).toSorted()) +const enabledOf = (cargo, context) => Object.fromEntries([...cargo.resolvedFeatures(context)].map(([dir, set]) => [dir, sorted(set)]).toSorted()) +// The packages built only maybe, or with features on only maybe: dir -> those features. +const maybeOf = (cargo, context) => Object.fromEntries([...cargo.featureResolution(context)].filter(([dir, r]) => r.maybe.size > 0 || !cargo.resolvedFeatures(context).has(dir)).map(([dir, r]) => [dir, sorted(r.maybe)]).toSorted()) // --- Cargo.toml / Cargo.lock --- @@ -328,9 +330,11 @@ test('createCargoContext keeps a target-specific dependency table apart from the 'vendor/win/src/lib.rs': '', }, (tmp) => { // The plain `dep = "1"` keeps its defaults and stays in the graph; the unix table's optional, - // default-less request is a second one. A target-only dependency counts (target cfgs are unknown). + // default-less request is a second one. A target-only dependency is built only maybe (the + // target's cfgs are unknown): its features are on only maybe, neither on nor off. const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) - t.assert.deepEqual(enabledOf(cargo), { '.': [], 'vendor/dep': ['default', 'std'], 'vendor/win': ['api', 'default'] }) + t.assert.deepEqual(enabledOf(cargo), { '.': [], 'vendor/dep': ['default', 'std'] }) + t.assert.deepEqual(maybeOf(cargo), { 'vendor/win': ['api', 'default'] }) }) }) @@ -365,9 +369,12 @@ test('createCargoContext applies a weak `dep?/feat` once the dependency is activ withProject(files(['[dependencies]', 'dep = { version = "1", default-features = false }']), (tmp) => { t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) }) - // required only on a target: the target table counts + // required only on a target: without one, the target table makes the dependency's `std` a maybe withProject(files(["[target.'cfg(unix)'.dependencies]", 'dep = { version = "1", default-features = false }']), (tmp) => { - t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.deepEqual(enabledOf(cargo), { '.': ['default', 'std'] }) + t.assert.deepEqual(maybeOf(cargo), { 'vendor/dep': ['std'] }) + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) }) }) @@ -514,7 +521,7 @@ test('parseCargoManifest reads [package] build', (t) => { t.assert.equal(parseCargoManifest('[package]\nname = "a"\nversion = "0.1.0"\n').package.build, null) }) -test('createCargoContext names each package\'s manifests, build script, and the root\'s lockfile and cargo config', (t) => { +test('createCargoContext names each package\'s manifests, build script, and the lockfile and cargo configs that apply', (t) => { withProject({ 'Cargo.toml': '[workspace]\nmembers = ["crates/app", "crates/nobuild"]\n', 'Cargo.lock': 'version = 3\n', @@ -526,6 +533,9 @@ test('createCargoContext names each package\'s manifests, build script, and the 'crates/nobuild/build.rs': 'fn main() {}\n', 'crates/nobuild/src/lib.rs': '', 'vendor/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "1.0.0"\nbuild = "scripts/build.rs"\n', + 'vendor/dep/.cargo-checksum.json': '{"files":{},"package":"0"}', + 'vendor/dep/Cargo.lock': 'version = 3\n', // as published: not the build's + 'vendor/dep/.cargo/config.toml': '[build]\n', 'vendor/dep/scripts/build.rs': 'fn main() {}\n', 'vendor/dep/src/lib.rs': '', 'vendor/plain/Cargo.toml': '[package]\nname = "plain"\nversion = "1.0.0"\n', @@ -535,7 +545,10 @@ test('createCargoContext names each package\'s manifests, build script, and the const cargo = createCargoContext(tmp, { entries: ['crates/app/src/main.rs'] }) const paths = (file) => cargo.buildFilesFor(file).map((f) => `${f.kind}:${f.path}`) t.assert.deepEqual(paths('crates/app/src/main.rs'), ['manifest:crates/app/Cargo.toml', 'manifest:Cargo.toml', 'lock:Cargo.lock', 'config:.cargo/config.toml']) - t.assert.deepEqual(paths('vendor/dep/src/lib.rs'), ['manifest:vendor/dep/Cargo.toml', 'manifest:Cargo.toml', 'lock:Cargo.lock', 'config:.cargo/config.toml']) // the root workspace is above it too + // A vendored crate: its manifest and the checksums cargo checks it against, never the lock or + // config it was published with (the build's are the workspace's, carried with its packages). + t.assert.deepEqual(paths('vendor/dep/src/lib.rs'), ['manifest:vendor/dep/Cargo.toml', 'checksum:vendor/dep/.cargo-checksum.json']) + t.assert.deepEqual(paths('vendor/plain/src/lib.rs'), ['manifest:vendor/plain/Cargo.toml']) t.assert.deepEqual(paths('loose.rs'), []) // no [package] claims it: the root manifest is a bare [workspace] t.assert.equal(cargo.buildScriptOf('crates/app/src/main.rs'), 'crates/app/build.rs') t.assert.equal(cargo.buildScriptOf('crates/nobuild/src/lib.rs'), null) // `build = false` @@ -548,8 +561,10 @@ test('createCargoContext names each package\'s manifests, build script, and the const member = createCargoContext(join(tmp, 'crates/app'), { entries: ['src/main.rs'] }) t.assert.deepEqual(member.buildFilesFor('src/main.rs'), [{ path: 'Cargo.toml', kind: 'manifest' }]) }) - // A workspace nested below the bundle root: its lockfile and config, not the root's (absent). + // A workspace nested below the bundle root: its lockfile, and the configs of its directory and + // of every one above it up to the bundle root, which cargo reads too. withProject({ + '.cargo/config.toml': '[net]\n', 'rust/Cargo.toml': '[workspace]\nmembers = ["app"]\n', 'rust/Cargo.lock': 'version = 3\n', 'rust/.cargo/config': '[build]\n', // the extensionless file, which cargo prefers when both exist @@ -558,7 +573,7 @@ test('createCargoContext names each package\'s manifests, build script, and the 'rust/app/src/main.rs': '', }, (tmp) => { const cargo = createCargoContext(tmp, { entries: ['rust/app/src/main.rs'] }) - t.assert.deepEqual(cargo.buildFilesFor('rust/app/src/main.rs').map((f) => f.path), ['rust/app/Cargo.toml', 'rust/Cargo.toml', 'rust/Cargo.lock', 'rust/.cargo/config']) + t.assert.deepEqual(cargo.buildFilesFor('rust/app/src/main.rs').map((f) => f.path), ['rust/app/Cargo.toml', 'rust/Cargo.toml', 'rust/Cargo.lock', 'rust/.cargo/config', '.cargo/config.toml']) }) }) @@ -767,19 +782,194 @@ test('createCargoContext decides target-specific dependency tables when the targ files[`vendor/${name}/src/lib.rs`] = '' } withProject(files, (tmp) => { - // Without a target every table counts. - t.assert.deepEqual(Object.keys(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] }))).toSorted(), ['.', 'vendor/gnu', 'vendor/loom', 'vendor/mac', 'vendor/nix', 'vendor/win']) + // Without a target every table's dependency is built only maybe. + const none = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.deepEqual(Object.keys(enabledOf(none)), ['.']) + t.assert.deepEqual(Object.keys(maybeOf(none)), ['vendor/gnu', 'vendor/loom', 'vendor/mac', 'vendor/nix', 'vendor/win']) const linux = createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }) t.assert.equal(linux.targetTriple, 'x86_64-unknown-linux-gnu') t.assert.ok(linux.targetCfgs.has('unix')) - // `cfg(loom)` is a custom cfg the loader can't decide: kept. - t.assert.deepEqual(Object.keys(enabledOf(linux)).toSorted(), ['.', 'vendor/gnu', 'vendor/loom', 'vendor/nix']) + // `cfg(loom)` is a custom cfg the loader can't decide: its dependency is built only maybe. + t.assert.deepEqual(Object.keys(enabledOf(linux)), ['.', 'vendor/gnu', 'vendor/nix']) + t.assert.deepEqual(Object.keys(maybeOf(linux)), ['vendor/loom']) const plain = createCargoContext(tmp, { entries: ['src/main.rs'] }) t.assert.equal(plain.targetTriple, null) t.assert.equal(plain.targetCfgs, null) }) }) +// A crate whose lib declares one module per platform and one per state of its `std` feature. +const platformCrate = (name, extra = '') => ({ + [`vendor/${name}/Cargo.toml`]: `[package]\nname = "${name}"\nversion = "1.0.0"\n${extra}[features]\ndefault = ["std"]\nstd = []\n`, + [`vendor/${name}/src/lib.rs`]: '#[cfg(windows)]\nmod win;\n#[cfg(unix)]\nmod nix;\n#[cfg(feature = "std")]\nmod with_std;\n#[cfg(not(feature = "std"))]\nmod no_std;\n', + [`vendor/${name}/src/win.rs`]: '', + [`vendor/${name}/src/nix.rs`]: '', + [`vendor/${name}/src/with_std.rs`]: '', + [`vendor/${name}/src/no_std.rs`]: '', +}) +const bundled = (bundle, name) => [...bundle.sources.keys()].filter((p) => p.startsWith(`vendor/${name}/src/`)).map((p) => p.slice(`vendor/${name}/src/`.length)).toSorted() + +test('buildRustBundle compiles proc-macro crates and what they depend on for the host, not the target', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\npm = "1"\nlib = "1"\n', + 'src/lib.rs': 'pub use pm::Derive;\npub use lib::f;\n', + 'vendor/pm/Cargo.toml': '[package]\nname = "pm"\nversion = "1.0.0"\n[lib]\nproc-macro = true\n[dependencies]\nhelper = "1"\n', + 'vendor/pm/src/lib.rs': '#[cfg(windows)]\nmod win;\n#[cfg(unix)]\nmod nix;\nuse helper::x;\n', + 'vendor/pm/src/win.rs': '', + 'vendor/pm/src/nix.rs': '', + ...platformCrate('helper'), + ...platformCrate('lib'), + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX }) + // The target's cfgs decide the target's code only; the host (a Linux target says nothing of it) keeps every platform. + t.assert.deepEqual(bundled(bundle, 'lib'), ['lib.rs', 'nix.rs', 'with_std.rs']) + t.assert.deepEqual(bundled(bundle, 'pm'), ['lib.rs', 'nix.rs', 'win.rs']) + t.assert.deepEqual(bundled(bundle, 'helper'), ['lib.rs', 'nix.rs', 'win.rs', 'with_std.rs']) + // `--cargo-target=host` says the host is the target: then its cfgs decide the host's code too. + const cargo = createCargoContext(tmp, { entries: ['src/lib.rs'], target: 'host' }) + t.assert.equal(cargo.platformOf('host|host'), cargo.platformOf('target|target')) + }) +}) + +test('buildRustBundle keeps what a crate built both for the target and for a build script compiles as either', async (t) => { + const files = (edition) => ({ + 'Cargo.toml': `[package]\nname = "app"\nversion = "0.1.0"\nedition = "${edition}"\n[dependencies]\nshared = { version = "1", default-features = false }\n[build-dependencies]\nshared = "1"\n`, + 'build.rs': 'use shared::f;\nfn main() {}\n', + 'src/lib.rs': 'pub use shared::f;\n', + ...platformCrate('shared'), + }) + // Resolver 2: the build script's `shared` has `std`, the target's hasn't; the host's platform is unknown. + await withProjectAsync(files('2021'), async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX, cargoManifests: true }) + t.assert.deepEqual(bundled(bundle, 'shared'), ['lib.rs', 'nix.rs', 'no_std.rs', 'win.rs', 'with_std.rs']) + const cargo = createCargoContext(tmp, { entries: ['src/lib.rs'] }) + t.assert.deepEqual([enabledOf(cargo)['vendor/shared'], enabledOf(cargo, 'host')['vendor/shared']], [[], ['default', 'std']]) + // Without the build script in the bundle, `shared` is the target's alone: `no_std` is what it compiles. + const lib = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX }) + t.assert.deepEqual(bundled(lib, 'shared'), ['lib.rs', 'nix.rs', 'no_std.rs']) + }) + // Resolver 1 unifies the two: `std` is on for both. + await withProjectAsync(files('2018'), async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX }) + t.assert.deepEqual(bundled(bundle, 'shared'), ['lib.rs', 'nix.rs', 'with_std.rs']) + }) +}) + +test('buildRustBundle walks a crate root the tree asks for as what the asking code is compiled as', async (t) => { + // The build script binds `log` as a fn, so the walk doesn't load the crate; the tree finds + // `log::info!` names it and asks for it -- from code compiled for the host. + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[build-dependencies]\nlog = "1"\n', + 'build.rs': 'mod util;\nuse crate::util::log;\nfn main() { log(); log::info!("x"); }\n', + 'util.rs': 'pub fn log() {}\n', + 'src/lib.rs': '', + ...platformCrate('log'), + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX, cargoManifests: true }) + t.assert.deepEqual(bundled(bundle, 'log'), ['lib.rs', 'nix.rs', 'win.rs', 'with_std.rs']) + }) +}) + +test('buildRustBundle does not presume off a custom cfg the build script or the rustflags may set', async (t) => { + const project = (extra) => ({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': 'mod fast;\nmod slow;\nmod user;\n#[cfg(fast)] pub use fast::F;\n#[cfg(not(fast))] pub use slow::F;\n', + 'src/fast.rs': 'pub struct F;\n', + 'src/slow.rs': 'pub struct F;\n', + 'src/user.rs': 'fn f() { crate::F; }\n', + ...extra, + }) + const target = async (extra) => withProjectAsync(project(extra), async (tmp) => (await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] })).imports.get('rust').get('src/user.rs').get('crate::F')) + // Nothing sets `fast`: a default build lacks it, so `not(fast)`'s is the answer. + t.assert.equal(await target({}), 'src/slow.rs') + t.assert.equal(await target({ 'build.rs': 'fn main() { println!("cargo:rustc-check-cfg=cfg(fast)"); }\n' }), 'src/slow.rs') // declared, not set + // The build script may set it (literally, through a module of its own, or under a name it + // formats), or the rustflags do: neither is presumed, and the edge is both, each under its cfg. + const either = new Map([['fast', 'src/fast.rs'], ['not(fast)', 'src/slow.rs']]) + t.assert.deepEqual(await target({ 'build.rs': 'fn main() { println!("cargo:rustc-cfg=fast"); }\n' }), either) + t.assert.deepEqual(await target({ 'build.rs': 'mod probe;\nfn main() { probe::run() }\n', 'probe.rs': 'pub fn run() { println!("cargo::rustc-cfg=fast"); }\n' }), either) + t.assert.deepEqual(await target({ 'build.rs': 'fn main() { let n = "fast"; println!("cargo:rustc-cfg={n}"); }\n' }), either) + t.assert.deepEqual(await target({ '.cargo/config.toml': '[build]\nrustflags = ["--cfg", "fast"]\n' }), either) + t.assert.deepEqual(await target({ '.cargo/config.toml': "[target.'cfg(unix)']\nrustflags = \"--cfg=fast\"\n" }), either) +}) + +test('buildRustBundle resolves a macro\'s include against the outermost call, and keeps every same-named macro\'s assets', async (t) => { + // `outer!()` in sub/mod.rs expands `inner!()`, whose `include_str!` rustc resolves beside the + // outermost call site -- sub/data.txt -- not beside m.rs, where `inner!` is written. + await withProjectAsync({ + 'src/lib.rs': '#[macro_use]\nmod m;\nmod sub;\n', + 'src/m.rs': 'macro_rules! outer { () => { inner!(); } }\nmacro_rules! inner { () => { const X: &str = include_str!("data.txt"); } }\n', + 'src/sub/mod.rs': 'outer!();\n', + 'src/sub/data.txt': 'right\n', + 'src/data.txt': 'wrong\n', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.ok(bundle.sources.has('src/sub/data.txt')) + t.assert.ok(!bundle.sources.has('src/data.txt')) + t.assert.equal(bundle.imports.get('rust').get('src/sub/mod.rs').get('include_str data.txt'), 'src/sub/data.txt') + }) + // Two `macro_rules! asset`, one per platform: both bodies' assets are carried. + await withProjectAsync({ + 'src/lib.rs': '#[cfg(unix)]\n#[macro_use]\nmod a;\n#[cfg(windows)]\n#[macro_use]\nmod b;\nmod user;\n', + 'src/a.rs': 'macro_rules! asset { () => { include_str!("a.txt") } }\n', + 'src/b.rs': 'macro_rules! asset { () => { include_str!("b.txt") } }\n', + 'src/user.rs': 'fn f() -> &\'static str { asset!() }\n', + 'src/a.txt': 'a\n', + 'src/b.txt': 'b\n', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.ok(bundle.sources.has('src/a.txt') && bundle.sources.has('src/b.txt')) + }) +}) + +test('buildRustBundle refuses a Rust source or include_str! file that isn\'t UTF-8, and carries an include_bytes! one as base64', async (t) => { + const bad = Buffer.from([0x66, 0x6e, 0x20, 0xff, 0x0a]) + await withProjectAsync({ 'src/lib.rs': 'mod m;\n' }, async (tmp) => { + writeFileSync(join(tmp, 'src/m.rs'), bad) + await t.assert.rejects(() => buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }), { message: 'Rust source is not valid UTF-8: src/m.rs' }) + }) + await withProjectAsync({ 'src/lib.rs': 'const S: &str = include_str!("s.txt");\nconst B: &[u8] = include_bytes!("b.bin");\n' }, async (tmp) => { + writeFileSync(join(tmp, 'src/b.bin'), bad) + writeFileSync(join(tmp, 'src/s.txt'), 'ok\n') + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.deepEqual([bundle.formats.get('src/b.bin'), bundle.sources.get('src/b.bin')], ['resource:base64', bad.toString('base64')]) + writeFileSync(join(tmp, 'src/s.txt'), bad) + await t.assert.rejects(() => buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }), { message: 'include_str! file is not valid UTF-8: src/s.txt' }) + }) +}) + +test('buildRustTree reports a missing dependency the manifest declares, whatever a glob into a missing crate may bring in', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nserde = "1"\ntokio = "1"\n', + 'src/lib.rs': 'use serde::*;\nuse tokio::runtime;\nuse de::Error;\n', + }, async (tmp) => { + const entries = ['src/lib.rs'] + const cargo = createCargoContext(tmp, { entries }) + const sources = await collectRustFilesFromDisk(tmp, entries, { cargo }) + const tree = buildRustTree(sources, { roots: entries, baseDir: tmp, cargo }) + // `tokio` is a declared dependency, so the crate (a `use` path whose lead a glob also + // provides is ambiguous to rustc); `de` is declared nowhere: serde's glob may well bring it. + t.assert.deepEqual(sorted(tree.unresolvedCrates), ['serde', 'tokio']) + }) +}) + +test('createCargoContext counts a feature only a target-specific table enables as maybe, until the target decides it', async (t) => { + await withProjectAsync({ + 'Cargo.toml': ['[package]', 'name = "app"', 'version = "0.1.0"', 'edition = "2021"', + '[dependencies]', 'dep = { version = "1", default-features = false }', + "[target.'cfg(windows)'.dependencies]", 'dep = { version = "1", features = ["std"] }'].join('\n'), + 'src/lib.rs': 'pub use dep::f;\n', + ...platformCrate('dep'), + }, async (tmp) => { + // No target: `std` may be on (a Windows build) or off -- both modules stay. + const cargo = createCargoContext(tmp, { entries: ['src/lib.rs'] }) + t.assert.deepEqual([enabledOf(cargo)['vendor/dep'], maybeOf(cargo)['vendor/dep']], [[], ['default', 'std']]) // that table keeps the defaults + t.assert.deepEqual(bundled(await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }), 'dep'), ['lib.rs', 'nix.rs', 'no_std.rs', 'win.rs', 'with_std.rs']) + // A Linux target: the Windows table doesn't apply, `std` is off. + t.assert.deepEqual(bundled(await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'], cargoTarget: LINUX }), 'dep'), ['lib.rs', 'nix.rs', 'no_std.rs']) + }) +}) + test('buildRustBundle with a target leaves out the other targets\' modules and path variants', async (t) => { await withProjectAsync({ 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', @@ -891,10 +1081,35 @@ test('createCargoContext finds vendored crates in the directory .cargo/config.to t.assert.deepEqual(enabledOf(cargo), { '.': [], 'third_party/crates/dep': [] }) }) t.assert.equal(createCargoContext(featuresFixture, { entries: ['src/main.rs'] }).vendorDir, 'vendor') - // however the table is spelled out + // Whatever the source is named and however the tables are spelled out, through a chain of + // replacements; a stale `vendor/` beside the configured directory is not what cargo builds from. const manifest = '[package]\nname = "app"\nversion = "0.1.0"\n' - withProject({ 'Cargo.toml': manifest, '.cargo/config': '[source]\nvendored-sources = { directory = "deps/" }\n' }, (tmp) => { - t.assert.equal(createCargoContext(tmp).vendorDir, 'deps') + const depFiles = (dir) => ({ [`${dir}/dep/Cargo.toml`]: '[package]\nname = "dep"\nversion = "1.0.0"\n', [`${dir}/dep/src/lib.rs`]: '' }) + withProject({ + 'Cargo.toml': `${manifest}[dependencies]\ndep = "1"\n`, + '.cargo/config': '[source]\ncrates-io = { replace-with = "mirror" }\nmirror = { replace-with = "local" }\nlocal = { directory = "deps/" }\n', + 'src/main.rs': '', + ...depFiles('deps'), + ...depFiles('vendor'), + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.equal(cargo.vendorDir, 'deps') + t.assert.equal(cargo.resolveCrate('dep', 'src/main.rs'), 'deps/dep/src/lib.rs') + }) + // A source no `replace-with` names is not the build's; a configured directory that is missing says so. + withProject({ 'Cargo.toml': manifest, '.cargo/config.toml': '[source.other]\ndirectory = "elsewhere"\n' }, (tmp) => { + t.assert.equal(createCargoContext(tmp).vendorDir, 'vendor') + }) + withProject({ 'Cargo.toml': manifest, '.cargo/config.toml': '[source.crates-io]\nreplace-with = "v"\n[source.v]\ndirectory = "gone"\n' }, (tmp) => { + const warnings = [] + const warn = console.warn + console.warn = (m) => warnings.push(m) + try { + t.assert.equal(createCargoContext(tmp).vendorDir, 'gone') + } finally { + console.warn = warn + } + t.assert.deepEqual(warnings, ['[loader.cargo] .cargo/config.toml names a vendored source directory that doesn\'t exist: gone']) }) // A cargo config, Cargo.toml or Cargo.lock that exists but isn't TOML stops the build, naming it. withProject({ 'Cargo.toml': manifest, '.cargo/config.toml': '[source.vendored-sources]\ndirectory = third_party\n' }, (tmp) => { @@ -1033,12 +1248,49 @@ test('buildRustTree with a target holds every file to the target\'s cfgs: mio\'s const sources = new Map(Object.entries(files).filter(([f]) => f.endsWith('.rs'))) const plain = buildRustTree(sources, { roots: entries, baseDir: tmp, cargo: createCargoContext(tmp, { entries }) }) // Without a target nothing rules either waker file out (`any(mio_unsupported_force_waker_pipe, - // aix, solaris)` may hold on aix): the first variant declared. With a Linux target the aix and - // solaris alternatives are out, the custom cfg is presumably off, and the eventfd branch's - // `any(android, linux)` holds: that one. - t.assert.equal(Object.fromEntries(plain.resolutions.get('src/waker.rs'))['sys::Waker::new'], 'src/sys/unix/waker/pipe.rs') + // aix, solaris)` may hold on aix): the edge is both, each under its cfg. With a Linux target the + // aix and solaris alternatives are out, the custom cfg is presumably off, and the eventfd + // branch's `any(android, linux)` holds: that one. + t.assert.deepEqual(Object.fromEntries(plain.resolutions.get('src/waker.rs'))['sys::Waker::new'], new Map([ + ['all(cfg_os_poll!, unix)', 'src/sys/unix/waker/pipe.rs'], + ['all(any(mio_unsupported_force_poll_poll, target_os = "aix", target_os = "solaris"), cfg_os_poll!, unix)', 'src/sys/unix/selector/poll.rs'], + ])) const linux = buildRustTree(sources, { roots: entries, baseDir: tmp, cargo: createCargoContext(tmp, { entries, target: LINUX }) }) t.assert.equal(Object.fromEntries(linux.resolutions.get('src/waker.rs'))['sys::Waker::new'], 'src/sys/unix/waker/eventfd.rs') t.assert.equal(Object.fromEntries(linux.resolutions.get('src/waker.rs'))['sys::Selector::new'], 'src/sys/unix/selector/epoll.rs') }) }) + +test('buildRustBundle declares a template\'s mod in each module of its package invoking the macro: no false "crate not found"', async (t) => { + // `decl!` is defined in lib.rs and invoked in a.rs: its `pub mod gm1;` is a's, src/a/gm1.rs + // (src/gm1.rs, beside the definition, is no module rustc looks for), and `use a::*;` brings it + // into the root; b reaches it through the root's `extern crate self as app;`. vendor/other + // invokes a `decl!` too -- another package's, which declares nothing here. `lone!` is invoked + // only by path: its `mod solo;` is walked beside the definition. + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': 'extern crate self as app;\nmacro_rules! decl { () => { pub mod gm1; } }\npub mod a;\nmod b;\nuse a::*;\nuse gm1::X;\n#[macro_export]\nmacro_rules! lone { () => { mod solo; } }\ncrate::lone!();\n', + 'src/solo.rs': 'mod deeper;\n', + 'src/solo/deeper.rs': '', + 'src/a.rs': 'decl!();\n', + 'src/a/gm1.rs': 'pub struct X;\n', + 'src/gm1.rs': 'pub struct X;\n', + 'src/b.rs': 'use app::a::gm1::X;\nfn f() { app::a::gm1::X; }\n', + 'vendor/other/Cargo.toml': '[package]\nname = "other"\nversion = "1.0.0"\n', + 'vendor/other/src/lib.rs': 'decl!();\n', + 'vendor/other/src/gm1.rs': '', + }, async (tmp) => { + const entries = ['src/lib.rs', 'vendor/other/src/lib.rs'] + const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries })) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/a.rs', 'src/a/gm1.rs', 'src/b.rs', 'src/lib.rs', 'src/solo.rs', 'src/solo/deeper.rs', 'vendor/other/src/lib.rs']) + const imports = bundle.imports.get('rust') + t.assert.deepEqual(Object.fromEntries(imports.get('src/a.rs')), { 'decl!': 'src/lib.rs', 'mod gm1': 'src/a/gm1.rs' }) + t.assert.equal(imports.get('src/lib.rs').get('gm1::X'), 'src/a/gm1.rs') + t.assert.equal(imports.get('src/lib.rs').get('mod solo'), 'src/solo.rs') + t.assert.equal(imports.get('src/b.rs').get('app::a::gm1::X'), 'src/a/gm1.rs') + t.assert.deepEqual(warnings, []) + const cargo = createCargoContext(tmp, { entries }) + const sources = await collectRustFilesFromDisk(tmp, entries, { cargo }) + t.assert.deepEqual(sorted(buildRustTree(sources, { roots: entries, baseDir: tmp, cargo }).unresolvedCrates), []) + }) +}) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 01f5a011..817fcd87 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -174,14 +174,15 @@ test('buildRustTree keeps every file of same-name mods a cfg_if! or cfg macros d ]) const { resolutions, missing, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.deepEqual([missing, [...unresolvedCrates]], [[], []]) // `static_macro` is a module, not a crate + // No definition of the gate macros here: which variant a path means is each's, under its gate. t.assert.deepEqual(edges(resolutions.get('src/atomic.rs')), { 'mod imp': { 'cfg_has_atomic_u64!': 'src/atomic_native.rs', 'cfg_not_has_atomic_u64!': 'src/atomic_as_mutex.rs' }, - 'imp::AtomicU64': 'src/atomic_native.rs', + 'imp::AtomicU64': { 'cfg_has_atomic_u64!': 'src/atomic_native.rs', 'cfg_not_has_atomic_u64!': 'src/atomic_as_mutex.rs' }, }) // The second variant's own declarations and paths resolve: it is in the module tree. t.assert.deepEqual(edges(resolutions.get('src/atomic_as_mutex.rs')), { 'mod static_macro': { 'cfg_has_const_mutex_new!': 'src/static_const_new.rs', 'cfg_not_has_const_mutex_new!': 'src/static_once_cell.rs' }, - 'static_macro::StaticAtomicU64': 'src/static_const_new.rs', + 'static_macro::StaticAtomicU64': { 'cfg_has_const_mutex_new!': 'src/static_const_new.rs', 'cfg_not_has_const_mutex_new!': 'src/static_once_cell.rs' }, }) // The `else` branch is keyed by what it means: the earlier cfg not holding. t.assert.deepEqual(edges(resolutions.get('src/sys.rs')), { 'mod imp': { unix: 'src/sys/unix.rs', 'not(unix)': 'src/sys/other.rs' }, imp: 'src/sys/unix.rs' }) @@ -752,7 +753,7 @@ test('parseCargoManifest reads package, lib, dependencies in every shape, and wo '[workspace.dependencies]', 'shared = { path = "crates/shared" }', ].join('\n')) t.assert.deepEqual(m.package, { name: 'my-app', version: null, versionFromWorkspace: true, edition: '2021', editionFromWorkspace: false, build: null }) - t.assert.deepEqual(m.lib, { name: 'myapp_lib', path: 'src/the_lib.rs' }) + t.assert.deepEqual(m.lib, { name: 'myapp_lib', path: 'src/the_lib.rs', procMacro: false }) const dep = (k) => { const d = m.deps.get(k) return { path: d.path, package: d.package, workspace: d.workspace, kinds: [...d.kinds.keys()].toSorted() } @@ -944,7 +945,7 @@ test('buildModuleTrees handles a pathologically deep mod chain without overflowi test('buildRustTree records mod edges and crate:: use edges', async (t) => { const sources = await collectRustFilesFromDisk(join(fixtures, 'use-crate'), ['src/main.rs']) const tree = buildRustTree(sources) - t.assert.deepEqual(Object.keys(tree).toSorted(), ['missing', 'resolutions', 'sources', 'unresolvedCrates', 'wantedRoots']) + t.assert.deepEqual(Object.keys(tree).toSorted(), ['missing', 'resolutions', 'sources', 'unresolvedCrates', 'wantedRoots', 'wantedUnits']) t.assert.deepEqual(tree.wantedRoots, []) t.assert.deepEqual(tree.missing, []) t.assert.deepEqual([...tree.unresolvedCrates], []) @@ -1241,7 +1242,8 @@ test('buildRustTree prefers a mod file to an inline module of the same name what ]) const { resolutions, missing } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.deepEqual(missing, []) - t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::imp::f': 'src/imp.rs' }) + // The file is the module in the tree; the path means either, each under its cfg. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::imp::f': { unix: 'src/imp.rs', 'not(unix)': 'src/lib.rs' } }) }) test('scanRustItems skips the else branches of a dead `if`, and takes a bare macro call after a single `:` for one', (t) => { @@ -1287,8 +1289,8 @@ test('buildRustTree prefers a mod file to an inline module of the same name, and const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.deepEqual(edges(resolutions.get('src/lib.rs'))['mod sys'], { unix: 'src/sys/unix.rs', windows: 'src/sys/windows.rs' }) t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { - 'crate::imp::f': 'src/imp.rs', - 'crate::sys::Handle': 'src/u.rs', // the first variant's import + 'crate::imp::f': { unix: 'src/imp.rs', 'not(unix)': 'src/lib.rs' }, + 'crate::sys::Handle': { unix: 'src/u.rs', windows: 'src/w.rs' }, // each variant's import, under its cfg 'crate::sys': 'src/sys/unix.rs', // the module the paths went through, in the file whose import was followed first 'crate::sys::Other': 'src/w.rs', // only windows.rs binds `Other`: its import is the one followed }) @@ -1401,7 +1403,11 @@ test('buildRustTree keeps a path under one platform out of the other platforms\' const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.deepEqual(edges(resolutions.get('src/unix/linux.rs')), { 'crate::c_int': 'src/unix.rs' }) t.assert.deepEqual(edges(resolutions.get('src/unix.rs')), { 'mod linux': 'src/unix/linux.rs', 'self::linux::sigset_t': 'src/unix/linux.rs', 'crate::sigset_t::default': 'src/unix/linux.rs' }) - t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::c_int': 'src/windows.rs', 'crate::sigset_t': 'src/windows.rs' }) + // A file under no platform cfg: every platform's, each under its cfg -- not the first written. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::c_int': { windows: 'src/windows.rs', 'target_os = "fuchsia"': 'src/fuchsia.rs', unix: 'src/unix.rs' }, + 'crate::sigset_t': { windows: 'src/windows.rs', 'target_os = "fuchsia"': 'src/fuchsia.rs', unix: 'src/unix/linux.rs' }, + }) // mio: two `pub use … Waker` in one file, in exclusive `cfg_if!` branches; a file mounted in one // branch means that branch's. const mio = new Map([ @@ -1539,10 +1545,9 @@ test('buildRustTree follows `extern crate … as` like a use, and `extern crate t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'use serde_core': 'vendor/serde_core/src/lib.rs', // `s::Value`, through the alias 'use serde_json': 'vendor/serde_json/src/lib.rs', // `crate::json::Value`, through the root's `pub extern crate … as json` + 'me::util::f': 'src/util.rs', // `crate::util::f`: the root's `extern crate self as me` is in the extern prelude }) t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'use serde_json': 'vendor/serde_json/src/lib.rs', 'mod util': 'src/util.rs', 'mod user': 'src/user.rs' }) - // `me::util::f` is `crate::util::f`: recorded under its lead once the crate root's alias is known. - t.assert.equal(resolutions.get('src/user.rs').get('me::util::f'), undefined) }) test('buildRustTree records no via edge for the crate root, however the path reaches it', (t) => { @@ -1708,7 +1713,7 @@ test('buildRustTree skips test/doc-only code: no files, no edges, no dev-dep pul 'mod sys': { unix: 'src/sys/unix.rs', windows: 'src/sys/windows.rs' }, // same-name cfg-exclusive declarations, merged 'mod backend': 'src/backend.rs', // the `test` path variant is dropped 'real::go': 'src/real.rs', - 'sys::name': 'src/sys/unix.rs', + 'sys::name': { unix: 'src/sys/unix.rs', windows: 'src/sys/windows.rs' }, 'backend::b': 'src/backend.rs', }) }) @@ -1861,7 +1866,7 @@ test('buildRustTree gives the same answers whatever order the sources come in', } t.assert.equal(answers.size, 1) const [only] = answers - t.assert.equal(JSON.parse(only).find(([f]) => f === 'src/user.rs')[1]['crate::sys::Thing'], 'src/sys/u.rs') // the first variant declared + t.assert.deepEqual(JSON.parse(only).find(([f]) => f === 'src/user.rs')[1]['crate::sys::Thing'], { unix: 'src/sys/u.rs', windows: 'src/sys/w.rs' }) // each variant, under its cfg }) test('buildRustTree keeps a file mounted under several cfgs under any of them, and an any(…) branch out of another target_os', (t) => { @@ -1996,10 +2001,10 @@ test('buildRustTree reads a #[cfg] on a macro invocation as gating its body, and t.assert.deepEqual(items.imports.map((im) => [im.segments.join('::'), im.cfg, im.macro]), [['self::unix', 'any(unix, target_os = "hermit")', 'cfg_os_poll'], ['self::shell', null, 'cfg_not_os_poll']]) // mio: a unix waker's `crate::sys::Selector` is its own platform's selector, not the windows // one (whose `#[cfg(windows)]` sits on the `cfg_os_poll!` invocation) nor the shell one (under - // `cfg_not_os_poll!`, the negation of the gate the waker sits behind -- a gate the loader can't - // see into is never "certainly compiled", so it can't beat a compatible candidate either). + // `cfg_not_os_poll!`, whose definition wraps items in the negation of the waker's gate). const sources = new Map([ - ['src/lib.rs', 'mod sys;\n'], + ['src/lib.rs', '#[macro_use]\nmod macros;\nmod sys;\n'], + ['src/macros.rs', 'macro_rules! cfg_os_poll { ($($i:item)*) => { $( #[cfg(feature = "os-poll")] $i )* } }\nmacro_rules! cfg_not_os_poll { ($($i:item)*) => { $( #[cfg(not(feature = "os-poll"))] $i )* } }\n'], ['src/sys/mod.rs', '#[cfg(any(unix, target_os = "hermit"))]\ncfg_os_poll! {\n mod unix;\n pub use self::unix::*;\n}\n#[cfg(windows)]\ncfg_os_poll! {\n mod windows;\n pub use self::windows::*;\n}\ncfg_not_os_poll! {\n mod shell;\n pub(crate) use self::shell::*;\n}\n'], ['src/sys/unix/mod.rs', 'cfg_os_poll! {\n #[cfg_attr(target_os = "linux", path = "selector/epoll.rs")]\n #[cfg_attr(target_os = "macos", path = "selector/kqueue.rs")]\n mod selector;\n pub(crate) use self::selector::*;\n #[cfg_attr(target_os = "linux", path = "waker/eventfd.rs")]\n #[cfg_attr(target_os = "macos", path = "waker/kqueue.rs")]\n mod waker;\n pub(crate) use self::waker::Waker;\n}\n'], ['src/sys/unix/selector/epoll.rs', 'pub struct Selector;\n'], @@ -2065,15 +2070,19 @@ test('buildRustTree walks on from a module a second glob path reaches under anot test('buildRustTree puts a mod a template declares at the invocation in another file, for textual macro scope', (t) => { // serde: core/crate_root.rs's `crate_root! { … pub mod de; … }` is invoked in lib.rs after - // `#[macro_use] mod macros;`, so de sees `forward_to_deserialize_any!`. + // `#[macro_use] mod macros;`, so de sees `forward_to_deserialize_any!` -- and is lib.rs's + // module, found beside lib.rs (src/de.rs), not beside crate_root.rs (src/core/de.rs). const sources = new Map([ ['src/lib.rs', '#[macro_use]\n#[path = "core/crate_root.rs"]\nmod crate_root;\n#[macro_use]\n#[path = "core/macros.rs"]\nmod macros;\ncrate_root!();\n'], ['src/core/crate_root.rs', 'macro_rules! crate_root { () => { pub mod de; } }\n'], ['src/core/macros.rs', 'macro_rules! forward_to_deserialize_any { () => {} }\n'], - ['src/core/de.rs', 'fn f() { forward_to_deserialize_any!(); }\n'], + ['src/de.rs', 'fn f() { forward_to_deserialize_any!(); }\n'], + ['src/core/de.rs', ''], ]) const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) - t.assert.deepEqual(edges(resolutions.get('src/core/de.rs')), { 'forward_to_deserialize_any!': 'src/core/macros.rs' }) + t.assert.equal(resolutions.get('src/lib.rs').get('mod de'), 'src/de.rs') + t.assert.equal(resolutions.get('src/core/crate_root.rs').get('mod de'), undefined) + t.assert.deepEqual(edges(resolutions.get('src/de.rs')), { 'forward_to_deserialize_any!': 'src/core/macros.rs' }) }) test('buildRustTree decides a bare macro call\'s scope at the call: a later definition in the file does not shadow it', (t) => { @@ -2095,7 +2104,128 @@ test('buildRustTree takes a candidate under a custom cfg only after one under no ['src/user.rs', 'fn f() { crate::X; crate::Y; }\n'], ]) const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) - // `any(loom, target_os = "aix")` may hold on aix: no target, so it is only written order that - // puts a's Y first, and it stays. - t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::X': 'src/b.rs', 'crate::Y': 'src/a.rs' }) + // `any(loom, target_os = "aix")` may hold on aix: no target, so a's Y is one of two. + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::X': 'src/b.rs', 'crate::Y': { 'any(loom, target_os = "aix")': 'src/a.rs', unix: 'src/b.rs' } }) +}) + +test('scanRustItems reads the cfg a gate macro\'s definition wraps items in; buildRustTree ranks by it', (t) => { + const macros = [ + 'macro_rules! cfg_rt { ($($item:item)*) => { $( #[cfg(feature = "rt")] #[cfg_attr(docsrs, doc(cfg(feature = "rt")))] $item )* } }', + 'macro_rules! cfg_not_rt { ($($item:item)*) => { $( #[cfg(not(feature = "rt"))] $item )* } }', + 'macro_rules! cfg_both { ($($item:item)*) => { $( #[cfg(unix)] #[cfg(feature = "rt")] $item )* } }', + 'macro_rules! cfg_plain { ($($item:item)*) => { $( $item )* } }', // wraps nothing in a cfg: unreadable + 'macro_rules! cfg_not_plain { ($($item:item)*) => { $( $item )* } }', + ].join('\n') + t.assert.deepEqual(scanRustItems(macros).macros.map((m) => [m.name, m.gate]), [ + ['cfg_rt', 'feature = "rt"'], ['cfg_not_rt', 'not(feature = "rt")'], ['cfg_both', 'all(feature = "rt", unix)'], ['cfg_plain', null], ['cfg_not_plain', null], + ]) + const sources = new Map([ + ['src/lib.rs', `#[macro_use]\nmod macros;\nmod a;\nmod b;\nmod user;\ncfg_rt! { pub use a::X; }\ncfg_not_rt! { pub use b::X; }\ncfg_plain! { pub use a::Y; }\ncfg_not_plain! { pub use b::Y; }\n`], + ['src/macros.rs', macros], + ['src/a.rs', 'pub struct X;\npub struct Y;\n'], + ['src/b.rs', 'pub struct X;\npub struct Y;\n'], + ['src/user.rs', 'fn f() { crate::X; crate::Y; }\n'], + ]) + // No features known: either, each under its gate's cfg; `rt` off: `cfg_not_rt!`'s is what the + // build compiles, and it is taken though written second. + t.assert.deepEqual(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['crate::X'], { 'feature = "rt"': 'src/a.rs', 'not(feature = "rt")': 'src/b.rs' }) + const features = new Set() + const cargo = { packageInfo: () => ({ dir: '.' }), isTestTarget: () => false, isVendored: () => false, featuresFor: () => features, maybeFeaturesFor: () => null, platformOf: () => null, cfgsSetFor: () => null, resolveCrate: () => null, isLibRoot: () => false, unitOfCrate: (_, u) => u } + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo }).resolutions.get('src/user.rs'))['crate::X'], 'src/b.rs') + features.add('rt') + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo }).resolutions.get('src/user.rs'))['crate::X'], 'src/a.rs') +}) + +test('buildRustTree takes `cfg_x!` and `cfg_not_x!` for each other\'s negation only when their definitions say so', (t) => { + // A file under `cfg_x!` asks for a name only `cfg_not_x!`'s body provides: exclusive by name, it + // was never taken; the definitions say `feature = "x"` and `feature = "y"`, which may hold together. + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\ncfg_x! { mod user; }\ncfg_not_x! { mod b; pub use b::Z; }\n'], + ['src/macros.rs', 'macro_rules! cfg_x { ($($i:item)*) => { $( #[cfg(feature = "x")] $i )* } }\nmacro_rules! cfg_not_x { ($($i:item)*) => { $( #[cfg(feature = "y")] $i )* } }\n'], + ['src/b.rs', 'pub struct Z;\n'], + ['src/user.rs', 'fn f() { crate::Z; }\n'], + ]) + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['crate::Z'], 'src/b.rs') + // Written as the negation: exclusive, as the two cfgs are -- the path stays on the module it reached. + sources.set('src/macros.rs', 'macro_rules! cfg_x { ($($i:item)*) => { $( #[cfg(feature = "x")] $i )* } }\nmacro_rules! cfg_not_x { ($($i:item)*) => { $( #[cfg(not(feature = "x"))] $i )* } }\n') + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['crate::Z'], 'src/lib.rs') +}) + +test('buildRustTree resolves a template\'s bare calls where the macro is invoked, and a recursive arm is no invocation', (t) => { + // `outer!(a)` in user.rs expands `outer!(b)`, then `inner!()`: both resolved in user.rs's scope + // at that call, where m2's `inner` is in scope -- not in m1.rs, where the template is written. + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod m1;\n#[macro_use]\nmod m2;\nmod user;\n'], + ['src/m1.rs', 'macro_rules! outer { (a) => { outer!(b); }; (b) => { inner!(); }; }\n'], + ['src/m2.rs', 'macro_rules! inner { () => {}; }\n'], + ['src/user.rs', 'fn f() { outer!(a); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'outer!': 'src/m1.rs', 'inner!': 'src/m2.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/m1.rs')), {}) + // What a template emits stands at the file's real invocation, not at a recursive arm's call + // inside the definition: `made` is in scope in late.rs, not in early.rs. + const made = new Map([ + ['src/lib.rs', 'macro_rules! make { () => { make!(@go); }; (@go) => { macro_rules! made { () => {} } }; }\nmod early;\nmake!();\nmod late;\n'], + ['src/early.rs', 'fn f() { made!(); }\n'], + ['src/late.rs', 'fn f() { made!(); }\n'], + ]) + const tree = buildRustTree(made, { roots: ['src/lib.rs'] }).resolutions + t.assert.deepEqual([edges(tree.get('src/early.rs'))['made!'], edges(tree.get('src/late.rs'))['made!']], [undefined, 'src/lib.rs']) +}) + +test('buildRustTree resolves a template\'s mod beside the files invoking the macro, else beside its definition', (t) => { + // lib.rs defines `decl!`, a.rs invokes it: `mod gm1` is a's (src/a/gm1.rs), and what `use + // a::*;` brings the root. `lone!` is invoked nowhere by bare name (`crate::lone!()` only): its + // `mod solo;` stays beside the definition. + const sources = new Map([ + ['src/lib.rs', '#[macro_export]\nmacro_rules! lone { () => { mod solo; } }\nmacro_rules! decl { () => { pub mod gm1; } }\nmod a;\nuse a::*;\nuse gm1::X;\ncrate::lone!();\n'], + ['src/a.rs', 'decl!();\n'], + ['src/a/gm1.rs', 'pub struct X;\n'], + ['src/gm1.rs', ''], + ['src/solo.rs', ''], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/a.rs')), { 'decl!': 'src/lib.rs', 'mod gm1': 'src/a/gm1.rs' }) + t.assert.equal(resolutions.get('src/lib.rs').get('gm1::X'), 'src/a/gm1.rs') + t.assert.equal(resolutions.get('src/lib.rs').get('mod gm1'), undefined) + t.assert.equal(resolutions.get('src/lib.rs').get('mod solo'), 'src/solo.rs') +}) + +test('buildRustTree lets a child module under a custom cfg give way to an import of its name, but not for an asker under that cfg', (t) => { + // serde: `mod de` only under docsrs (its crate_root.rs), `pub use serde_core::de` otherwise. + const sources = new Map([ + ['src/lib.rs', '#[cfg(docsrs)]\npub mod de;\n#[cfg(not(docsrs))]\npub use serde_core::de;\nmod private;\n#[cfg(docsrs)]\nmod docs;\n'], + ['src/de.rs', 'pub trait Error {}\n'], + ['src/private.rs', 'use crate::de::Error;\n'], + ['src/docs.rs', 'use crate::de::Error;\n'], + ['vendor/serde_core/src/lib.rs', 'pub mod de { pub trait Error {} }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/private.rs')), { 'use serde_core': 'vendor/serde_core/src/lib.rs' }) + t.assert.deepEqual(edges(resolutions.get('src/docs.rs')), { 'crate::de::Error': 'src/de.rs' }) +}) + +test('buildRustTree reads `extern crate self as x;` as the crate root from every module: no crate `x` to report', (t) => { + const sources = new Map([ + ['src/lib.rs', 'extern crate self as gm1;\nmod b;\npub struct X;\n'], + ['src/b.rs', 'use gm1::X;\nmod inner { use gm1::X; }\n'], + ]) + const { resolutions, unresolvedCrates } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual([...unresolvedCrates], []) + t.assert.deepEqual(edges(resolutions.get('src/b.rs')), { 'gm1::X': 'src/lib.rs' }) +}) + +test('buildRustTree places a template\'s mod at an invocation in its own crate, never in another crate\'s file', (t) => { + // Only the app invokes `a`'s `decl!`, with a `helper` of its own in scope: the `x` resolved + // beside a's lib.rs is still `a`'s module, where `a`'s helper is in scope -- not one standing in + // the app's file. + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nextern crate a;\nmacro_rules! helper { () => {} }\ndecl!();\n'], + ['vendor/a/src/lib.rs', 'macro_rules! helper { () => {} }\n#[macro_export]\nmacro_rules! decl { () => { mod x; } }\n'], + ['vendor/a/src/x.rs', 'fn f() { helper!(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.equal(edges(resolutions.get('vendor/a/src/x.rs'))['helper!'], 'vendor/a/src/lib.rs') }) From 2c336abf43b61e30a301d55de644a3123d2a941d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 12:54:28 +0000 Subject: [PATCH 03/10] fix(bundle): decide a cfg an unknown leaf repeats in when it can't vary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit evalCfg is three-valued per leaf, so an item's cfg joined with its enclosing blocks' -- zerocopy's `#[cfg(any(test, kani))] mod tests { #[cfg(not(kani))] mod compatibility { use rand::…; } }`, i.e. all(any(test, kani), not(kani)) -- stayed undecided and its `use rand` was reported as a missing crate. When an unknown leaf occurs more than once (up to six of them), the predicate is evaluated under each assignment and decided when every one agrees. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 4 ++++ stasis/src/loaders/cargo.js | 41 ++++++++++++++++++++++++++++++------- tests/rust-loader.test.js | 12 +++++++++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/doc/file-formats.md b/doc/file-formats.md index e2d65798..eff4abd9 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -743,6 +743,10 @@ test code reaches for. Two kinds of cfg are decided: `all(…)`/`any(…)`/`not(…)` compose; a predicate that reduces to true (`not(test)`, an enabled feature) is as firm as no cfg, so a missing module behind it is fatal. +One an undecided leaf occurs in more than once is decided when it comes out the +same whatever that leaf is: an item's cfg joined with its enclosing blocks' +`all(any(test, kani), not(kani))` never holds (zerocopy's test-only `use +rand::…`), and `any(unix, not(unix))` always does. Without a target, target cfgs stay undecided and their code is kept. A `cfg_attr` that applies a non-cfg attribute (`#[cfg_attr(docsrs, doc(cfg(…)))]`) gates nothing. diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index a2564bb6..d388a247 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -343,27 +343,54 @@ export const TARGET_CFG_KEYS = new Set(['unix', 'windows', 'target_abi', 'target // (rustcTargetCfgs), and unknown without one; every other leaf stays unknown. `env.units`, a list // of such envs, is code compiled several ways (a crate built for the target and, as a // build-dependency, for the host): false only when false in each, true only when true in each. +// A predicate an unknown leaf occurs in more than once is decided when it comes out the same +// whatever the unknown leaves are (up to six of them): `all(any(test, kani), not(kani))` -- zerocopy's +// `#[cfg(any(test, kani))] mod tests { #[cfg(not(kani))] mod compatibility { … } }` -- never holds. +const MAX_FREE_CFG_LEAVES = 6 export function evalCfg(pred, env = {}) { if (env.units) { const each = env.units.map((u) => evalCfg(pred, u)) return each.every((r) => r === false) ? false : (each.every((r) => r === true) ? true : null) } + const free = { counts: null } // unknown leaf → how often it occurs, once one is met + const r = evalCfgWith(pred, env, null, free) + if (r !== null || free.counts === null || free.counts.size > MAX_FREE_CFG_LEAVES || [...free.counts.values()].every((n) => n === 1)) return r + const leaves = [...free.counts.keys()] + let out + for (let bits = 0; bits < 1 << leaves.length; bits++) { + const v = evalCfgWith(pred, env, new Map(leaves.map((l, k) => [l, ((bits >> k) & 1) === 1]))) + if (out === undefined) out = v + else if (v !== out) return null + } + return out +} +// evalCfg's three-valued pass: an unknown leaf takes its value from `assume` (leaf → value) when +// that has it, else is counted in `free.counts`. +function evalCfgWith(pred, env, assume, free = null) { const p = pred.trim() const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) if (!m) { - if (p === 'test') return env.test === true - if (p === 'doctest' || p === 'doc') return false - const feature = FEATURE_CFG_RE.exec(p) - if (feature) return env.features ? (env.features.has(feature[1]) ? true : (env.maybeFeatures?.has(feature[1]) ? null : false)) : null - const leaf = CFG_LEAF_RE.exec(p) - if (leaf && env.target && TARGET_CFG_KEYS.has(leaf[1])) return env.target.has(leaf[2] === undefined ? leaf[1] : `${leaf[1]}="${leaf[2]}"`) + const known = evalCfgLeaf(p, env) + if (known !== null) return known + const leaf = p.replaceAll(/\s+/gu, '') + if (assume?.has(leaf)) return assume.get(leaf) + if (free !== null) (free.counts ??= new Map()).set(leaf, (free.counts.get(leaf) ?? 0) + 1) return null } - const args = splitTopLevel(m[2]).map((a) => a.trim()).filter(Boolean).map((a) => evalCfg(a, env)) + const args = splitTopLevel(m[2]).map((a) => a.trim()).filter(Boolean).map((a) => evalCfgWith(a, env, assume, free)) if (m[1] === 'not') return args.length === 1 && args[0] !== null ? !args[0] : null if (m[1] === 'all') return args.includes(false) ? false : (args.every((a) => a === true) ? true : null) return args.includes(true) ? true : (args.every((a) => a === false) ? false : null) } +function evalCfgLeaf(p, env) { + if (p === 'test') return env.test === true + if (p === 'doctest' || p === 'doc') return false + const feature = FEATURE_CFG_RE.exec(p) + if (feature) return env.features ? (env.features.has(feature[1]) ? true : (env.maybeFeatures?.has(feature[1]) ? null : false)) : null + const leaf = CFG_LEAF_RE.exec(p) + if (leaf && env.target && TARGET_CFG_KEYS.has(leaf[1])) return env.target.has(leaf[2] === undefined ? leaf[1] : `${leaf[1]}="${leaf[2]}"`) + return null +} // `rustc --print cfg` output → the set of cfg leaves as printed, one per line (`unix`, // `target_os="linux"`). diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 817fcd87..cffa75de 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -349,6 +349,18 @@ test('evalCfg decides test/doc-only predicates and leaves target/feature ones un t.assert.equal(evalCfg('any()'), false) }) +test('evalCfg decides a predicate an unknown leaf repeats in when it holds, or fails, whatever that leaf is', (t) => { + // zerocopy: `#[cfg(any(test, kani))] mod tests { #[cfg(not(kani))] mod compatibility { use rand::…; } }` + t.assert.equal(evalCfg('all(any(test, kani), not(kani))'), false) + t.assert.equal(evalCfg('any(unix, not(unix))'), true) + t.assert.equal(evalCfg('all(target_os = "linux", not(target_os="linux"))'), false) + t.assert.equal(evalCfg('any(all(a, b), all(a, not(b)))'), null) // `a`: not decided + t.assert.equal(evalCfg('all(unix, windows)'), null) // two leaves, each once: left to the ranking (cfgExclusive) + t.assert.equal(evalCfg('all(feature = "x", not(feature = "x"), kani)', { features: null }), false) + const { refs } = scanRustItems('#[cfg(any(test, kani))]\nmod tests {\n #[cfg(not(kani))]\n mod compatibility {\n pub(super) use rand::Rng;\n }\n use proptest::prelude::*;\n}\n') + t.assert.deepEqual(refs.map((r) => r.spec), ['proptest::prelude']) // under kani, which may be set +}) + test('evalCfg decides target predicates against a target cfg set, profile and custom ones never', (t) => { const target = new Set(['unix', 'target_os="linux"', 'target_family="unix"', 'target_arch="x86_64"', 'target_pointer_width="64"', 'target_feature="sse2"', 'target_has_atomic="64"', 'debug_assertions', 'panic="unwind"']) t.assert.equal(evalCfg('unix', { target }), true) From dd799ebf3a12e8cf0f97734f21af3ae254d2ca75 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 13:03:36 +0000 Subject: [PATCH 04/10] fix(bundle): rank a module's own items with its imports, and imports that lead out A path's segment was looked up in the module's imports first and its own definitions only when no import answered, so an import leading out of the bundle (tokio's `pub(crate) use std::sync::atomic::AtomicU64;` in one `imp` variant) hid the mutex-based `AtomicU64` the other variant defines, and a dead import (`cfg_taskdump!`'s) beat the live fn under `cfg_not_taskdump!`. The module's items are now candidates right after its named imports, ahead of globs, and an import leading out of the bundle is an answer under its cfgs rather than a last resort: `imp::AtomicU64` maps both variant files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 8 +++++++- stasis/src/loaders/rust.js | 26 +++++++++++++++++++------- tests/rust-loader.test.js | 28 ++++++++++++++++++++++++++-- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/doc/file-formats.md b/doc/file-formats.md index eff4abd9..9cb8080b 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -627,7 +627,13 @@ leads to another in-tree crate records that crate (`use serde_core`); one that leads out of the bundle, to the sysroot (`pub use core::result::Result`) or to a crate that isn't vendored, binds the name to its own file as far as the bundle knows (serde's `$crate::__private::Result` -records the file of the `lib` module that re-exports `core::result`). The +records the file of the `lib` module that re-exports `core::result`) -- a +candidate like any other, under its cfgs: tokio's `imp::AtomicU64` is std's, +re-exported in one variant file, or the mutex-based one the other defines. A +module's own items rank with its named imports, ahead of what its globs bring +in, which never shadows them (tokio's `crate::trace::trace_leaf` is the fn +defined under `cfg_not_taskdump!`, not the import under the `cfg_taskdump!` +that is off). The module an import was first followed from is a dependency too: when nothing else in the file points at its file, the path's prefix naming it is recorded (`crate::__private` → private/mod.rs) -- unless that file is the crate root, diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index dc7e31da..9a9561c7 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -1603,13 +1603,12 @@ function provided(root, at, name, from, ctx, asker, typesOnly = false) { // kept on the import per asker -- unless the answer came out of an import cycle still being // resolved (see memoized). An import being followed that is met again leads back to itself (`use // log;` in every platform file of a module asks the module for `log`, which is these imports): -// nothing here. An import that leads nowhere the bundle can see is passed over, and -- when it is -// a path, not a crate named outright (`use serde;`) -- stands in as the last resort: the name is -// an item here, as far as the bundle knows. A value where a type is wanted (`typesOnly`: a `fn` +// nothing here. An import that leads nowhere the bundle can see -- when it is a path, not a crate +// named outright (`use serde;`, passed over) -- answers where it is: the name is an item of its +// file, as far as the bundle knows. A value where a type is wanted (`typesOnly`: a `fn` // defined here, an import of one) is passed over too, and when nothing else answers, the answer // is VALUE_ONLY rather than nothing: the name is taken, but not in this namespace. function pick(candidates, root, ctx, asker, typesOnly = false) { - let unseen = null const maybes = [] // the compatible answers under no doubt, for when none is entailed (see withAlternatives) let doubted = null // the first compatible answer under a doubtful cfg, for when there is nothing else let lastResort = null // the first under a cfg the asker's build can't compile (see deadUnder) @@ -1647,9 +1646,13 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { ctx.hits.delete(depth) if (minHit(ctx) > depth) (im.answers ??= new Map()).set(key, r) } + // An import leading out of the bundle (`pub(crate) use std::sync::atomic::AtomicU64;`, a + // crate that isn't vendored) binds the name all the same: an item of its own file, as far + // as the bundle knows, under its cfgs. One naming a crate outright (`use serde;`) answers + // nothing: the lead is the crate's to report. if (r === null) { - if (im.segments.length > 1) unseen ??= c - continue + if (im.segments.length === 1) continue + r = { kind: 'item', file: im.file } } if (r === VALUE_ONLY) { valueOnly = true @@ -1664,7 +1667,6 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { if (maybes.length > 0) return withAlternatives(maybes, asker) if (doubted !== null) return doubted if (lastResort !== null) return lastResort - if (unseen !== null) return { kind: 'item', file: unseen.import.file, through: unseen.through } return valueOnly ? VALUE_ONLY : null } @@ -1729,6 +1731,16 @@ function providedAll(root, at, name, seeing, ctx) { ctx.walking-- list = new Lazy(function* () { yield* importsOf(root, at, name, seeing, ctx) + // The module's own items rank with its named imports -- two explicit bindings of one name + // are rustc's error unless their cfgs differ (tokio's `imp` re-exports std's `AtomicU64` in + // one variant file and defines its own in the other) -- ahead of what globs bring in, which + // never shadows them. The module's tree file first; visibility is the path's to break. + const defs = ctx.defined.get(root)?.get(at)?.get(name) + if (defs !== undefined) { + const own = ctx.trees.get(root).get(at) + for (const d of defs) if (d.file === own) yield { answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns } + for (const d of defs) if (d.file !== own) yield { answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns } + } // Of the closure's modules (hundreds, in libc), only the ones with something of the name // (`having`), in closure order. const having = ctx.having.get(root)?.get(name) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index cffa75de..3ed575ef 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -1271,7 +1271,7 @@ test('scanRustItems skips the else branches of a dead `if`, and takes a bare mac t.assert.deepEqual([...items.invocations].toSorted(), ['helper', 'other']) }) -test('buildRustTree resolves through several globs of one module, and the first import that leads somewhere', (t) => { +test('buildRustTree resolves through several globs of one module, and through each import under its own cfg', (t) => { const sources = new Map([ ['src/lib.rs', ['mod foo;', 'mod b;', 'mod user;', 'pub use foo::a::*;', 'pub use foo::b::*;', '#[cfg(unix)]', 'pub use notvendored::X;', '#[cfg(windows)]', 'pub use crate::b::X;'].join('\n')], ['src/foo.rs', 'pub mod a;\npub mod b;\n'], @@ -1284,7 +1284,9 @@ test('buildRustTree resolves through several globs of one module, and the first const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'crate::deep::D': 'src/foo/b/deep.rs', // `deep` through the second glob - 'crate::X': 'src/b.rs', // the `#[cfg(windows)]` import: the `#[cfg(unix)]` one leads to a crate that isn't in-tree + // Under unix the `pub use notvendored::X` binds it -- a crate that isn't in-tree, so the name + // is lib.rs's as far as the bundle knows -- under windows `crate::b::X`. + 'crate::X': { unix: 'src/lib.rs', windows: 'src/b.rs' }, }) }) @@ -2219,6 +2221,28 @@ test('buildRustTree lets a child module under a custom cfg give way to an import t.assert.deepEqual(edges(resolutions.get('src/docs.rs')), { 'crate::de::Error': 'src/de.rs' }) }) +test('buildRustTree ranks a module\'s own items with its imports, one leading out of the bundle included', (t) => { + const gate = (name, cfg) => `macro_rules! ${name} {\n ($($item:item)*) => {\n $(\n #[cfg(${cfg})]\n $item\n )*\n }\n}\n` + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\nmod atomic_u64;\nmod trace;\nmod user;\n'], + ['src/macros.rs', gate('cfg_has_atomic_u64', 'target_has_atomic = "64"') + gate('cfg_not_has_atomic_u64', 'not(target_has_atomic = "64")') + gate('cfg_taskdump', 'test') + gate('cfg_not_taskdump', 'not(test)')], + // tokio: std's `AtomicU64` re-exported in one variant file, a mutex-based one defined in the other. + ['src/atomic_u64.rs', 'cfg_has_atomic_u64! {\n #[path = "native.rs"]\n mod imp;\n}\ncfg_not_has_atomic_u64! {\n #[path = "as_mutex.rs"]\n mod imp;\n}\npub(crate) use imp::AtomicU64;\n'], + ['src/native.rs', 'pub(crate) use std::sync::atomic::{AtomicU64, Ordering};\n'], + ['src/as_mutex.rs', 'pub(crate) struct AtomicU64;\n'], + // tokio's `mod trace`: an import under a gate that is off (`taskdump`; here `test`), the fn + // defined under its negation. + ['src/trace.rs', 'cfg_taskdump! {\n pub(crate) use crate::user::trace_leaf;\n}\ncfg_not_taskdump! {\n pub(crate) fn trace_leaf() {}\n}\n'], + ['src/user.rs', 'use crate::atomic_u64::AtomicU64;\npub(crate) fn trace_leaf() {}\nfn f() { crate::trace::trace_leaf(); }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { + 'crate::atomic_u64::AtomicU64': { 'target_has_atomic = "64"': 'src/native.rs', 'not(target_has_atomic = "64")': 'src/as_mutex.rs' }, + 'crate::trace::trace_leaf': 'src/trace.rs', + 'crate::atomic_u64': 'src/atomic_u64.rs', + }) +}) + test('buildRustTree reads `extern crate self as x;` as the crate root from every module: no crate `x` to report', (t) => { const sources = new Map([ ['src/lib.rs', 'extern crate self as gm1;\nmod b;\npub struct X;\n'], From 41526ca8fd9eba00b0db4537f59a75abc561251f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 13:14:17 +0000 Subject: [PATCH 05/10] perf(bundle): cache leaf verdicts, alternative keys and package lookups in the tree pass The cfg-keyed alternatives and the build-deadness check added in review 7 rebuilt the same cfg texts and re-parsed the same leaf predicates for every candidate: libc's tree pass went from 227ms to 351ms. A leaf's verdict is kept per interned build, an alternative's cfg text per asker set, and a file's package per tree pass (asked for every import and item). libc 282ms, tokio 492ms -> 464ms (589afce: 444ms); output unchanged on the corpus. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- stasis/src/loaders/rust.js | 30 ++++++++++++++++++++++++++---- tests/rust-loader.test.js | 9 ++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index 9a9561c7..ba517016 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -1410,11 +1410,19 @@ const compatible = (asker, set) => { // code the build doesn't compile either, means what it would mean where it compiles. For an // asker whose own leaves can't hold (`deadHere`) the build says nothing. const EVALUABLE_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doc', 'doctest']) +const leafFalseMemo = new WeakMap() // build → leaf → verdict (builds are interned per settled context, see buildOf) const leafFalse = (l, build) => { if (l.alts !== undefined) return l.alts.every((alt) => alt.some((m) => leafFalse(m, build))) if (!EVALUABLE_CFG_KEYS.has(l.key)) return false + const memo = leafFalseMemo.get(build) ?? leafFalseMemo.set(build, new Map()).get(build) const pred = l.value === null ? l.key : `${l.key} = "${l.value}"` - return evalCfg(l.neg ? `not(${pred})` : pred, build) === false + const text = l.neg ? `not(${pred})` : pred + let verdict = memo.get(text) + if (verdict === undefined) { + verdict = evalCfg(text, build) === false + memo.set(text, verdict) + } + return verdict } const deadUnder = (leaves, build) => build !== undefined && leaves.some((l) => leafFalse(l, build)) const deadFor = (asker, set) => { @@ -1705,7 +1713,13 @@ function withAlternatives(found, asker) { for (const { answer, leaves } of found) { if (answer.file === undefined || seen.has(answer.file)) continue seen.add(answer.file) - const base = cfgKey(cfgTextOf(leaves.leaves.filter((l) => asker === undefined || !asker.keys.has(leafKey(l))))) + // Kept per asker set (the text leaves out what the asker holds itself). + let text = asker?.texts.get(leaves.key) + if (text === undefined) { + text = cfgTextOf(leaves.leaves.filter((l) => asker === undefined || !asker.keys.has(leafKey(l)))) + asker?.texts.set(leaves.key, text) + } + const base = cfgKey(text) let key = base for (let k = 2; alternatives.has(key); k++) key = `${base}#${k}` alternatives.set(key, answer.file) @@ -2325,7 +2339,15 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // package that skipped a body of that name are scanned again with the name dropped from the set // (the package: the Cargo context's; without one, a vendored crate's `vendor/`, else the // bundle's). - const packageOf = (path) => (ctx ? ctx.packageInfo(path)?.dir ?? '.' : path.startsWith(`${VENDOR_DIR}/`) ? path.split('/', 2).join('/') : '.') + const packages = new Map() // path → its package, asked for every import and item (gatesOf) + const packageOf = (path) => { + let dir = packages.get(path) + if (dir === undefined) { + dir = ctx ? ctx.packageInfo(path)?.dir ?? '.' : path.startsWith(`${VENDOR_DIR}/`) ? path.split('/', 2).join('/') : '.' + packages.set(path, dir) + } + return dir + } const ownTemplates = new Map() // package → the template names it defines for (const [path, items] of scanned) { for (const m of items.macros) if (TEMPLATE_MACROS.has(m.name)) (ownTemplates.get(packageOf(path)) ?? ownTemplates.set(packageOf(path), new Set()).get(packageOf(path))).add(m.name) @@ -2502,7 +2524,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const build = buildOf(path, ctx, units) const settable = ctx?.cfgsSetFor(path) ?? null const sharedKey = `${set.key}\0${idOf(build)}\0${settable === null ? '' : idOf(settable)}` - const shared = askerSets.get(sharedKey) ?? askerSets.set(sharedKey, { keys: new Set(set.leaves.map(leafKey)), compat: new Map(), sure: new Map(), doubt: new Map(), dead: new Map(), build, deadHere: deadUnder(set.leaves, build), custom: customFor(settable) }).get(sharedKey) + const shared = askerSets.get(sharedKey) ?? askerSets.set(sharedKey, { keys: new Set(set.leaves.map(leafKey)), compat: new Map(), sure: new Map(), doubt: new Map(), dead: new Map(), texts: new Map(), build, deadHere: deadUnder(set.leaves, build), custom: customFor(settable) }).get(sharedKey) f.asker = { file: path, set, ...shared } } // Per crate root, each module's child modules by name (every module's parent is in its tree). diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 3ed575ef..92250a55 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -2143,11 +2143,10 @@ test('scanRustItems reads the cfg a gate macro\'s definition wraps items in; bui // No features known: either, each under its gate's cfg; `rt` off: `cfg_not_rt!`'s is what the // build compiles, and it is taken though written second. t.assert.deepEqual(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['crate::X'], { 'feature = "rt"': 'src/a.rs', 'not(feature = "rt")': 'src/b.rs' }) - const features = new Set() - const cargo = { packageInfo: () => ({ dir: '.' }), isTestTarget: () => false, isVendored: () => false, featuresFor: () => features, maybeFeaturesFor: () => null, platformOf: () => null, cfgsSetFor: () => null, resolveCrate: () => null, isLibRoot: () => false, unitOfCrate: (_, u) => u } - t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo }).resolutions.get('src/user.rs'))['crate::X'], 'src/b.rs') - features.add('rt') - t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo }).resolutions.get('src/user.rs'))['crate::X'], 'src/a.rs') + // A context's features are settled once resolved (builds are interned per context): one each. + const cargoWith = (features) => ({ packageInfo: () => ({ dir: '.' }), isTestTarget: () => false, isVendored: () => false, featuresFor: () => features, maybeFeaturesFor: () => null, platformOf: () => null, cfgsSetFor: () => null, resolveCrate: () => null, isLibRoot: () => false, unitOfCrate: (_, u) => u }) + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo: cargoWith(new Set()) }).resolutions.get('src/user.rs'))['crate::X'], 'src/b.rs') + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'], cargo: cargoWith(new Set(['rt'])) }).resolutions.get('src/user.rs'))['crate::X'], 'src/a.rs') }) test('buildRustTree takes `cfg_x!` and `cfg_not_x!` for each other\'s negation only when their definitions say so', (t) => { From f1bb174dbf1704e0d8900fe21cefffd3ce6e35fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 11:40:32 +0000 Subject: [PATCH 06/10] fix(bundle): Rust loader review 8: refused roots, missing crates, per-table deps, resolver regressions - buildRustBundle: a crate root the tree pass wants but the walk refused is asked for once, so the walk/tree loop ends; every crate the bundle lacks is reported (refused roots included) whether or not a vendor dir exists, the `cargo vendor` hint only without one. - Cargo: each dependency table is its own dependency (version, path, package, workspace per table), and a file follows the one its target uses (build script, test target, normal); a Cargo.lock pin is taken only when the requirement allows it, else the lock is reported out of date; a [patch] is used only when its version fits, and [patch] in .cargo/config.toml (nearest first, ahead of the manifest's) is read; a path dependency or patch outside the bundle root, or naming no package, is reported. - Resolver: a local module under a custom cfg gives way only to the module's own `use` or item, never to a glob; a macro call is looked up among macros (a `fn m`, `mod m` or an import of a module/crate is no `m!`); a template's calls and includes stay with it past a nested `macro_rules!`; cfg values keep their spacing inside quotes, so `"a b"` and `"ab"` are not merged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 51 ++++++--- stasis/src/cmd/bundle.js | 26 +++-- stasis/src/loaders/cargo.js | 214 ++++++++++++++++++++++++------------ stasis/src/loaders/rust.js | 116 +++++++++++-------- tests/bundle-cmd.test.js | 6 +- tests/rust-features.test.js | 139 +++++++++++++++++++---- tests/rust-loader.test.js | 53 ++++++++- 7 files changed, 441 insertions(+), 164 deletions(-) diff --git a/doc/file-formats.md b/doc/file-formats.md index 9cb8080b..988addfb 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -522,10 +522,14 @@ that crate's root in: the package's own lib target (`use my_app::…` from `vendor/` (or the `directory` of the source the project's cargo config replaces crates.io with, through any chain of `replace-with`s, whatever its name). A `path` or `[patch]` dependency outside the bundle root is not in-tree, and never -bound to a vendored crate of its name instead. Registry dependencies live in `~/.cargo/registry`, outside the bundle -root, so they're never read: vendor them first (`cargo vendor`). When a bundle -references crates it can't find and there is no `vendor/` dir, `stasis bundle` -says so and suggests it. +bound to a vendored crate of its name instead: the loader says so, and so it +does of a `path` naming a directory with no `Cargo.toml` holding a `[package]`. +Registry dependencies live in `~/.cargo/registry`, outside the bundle +root, so they're never read: vendor them first (`cargo vendor`). Whenever a +bundle references crates it lacks -- none in-tree, or one whose root the walk +refused (a symlink leading out of the bundle root) -- `stasis bundle` lists +them, `vendor/` dir or not; with no `vendor/` dir it also suggests `cargo +vendor`. A path edge (`crate::a::b::Item`, `super::x`, a relative `child::y`) resolves module by module along the crate's tree, inline `mod x { … }` blocks included @@ -575,10 +579,11 @@ set it: a name its build script prints as `cargo:rustc-cfg=…`, or that a rustflags `--cfg` sets (the cargo config's `build`/`target.<…>` `rustflags`, `RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, `CARGO_BUILD_RUSTFLAGS`), is neither presumed off nor on, and a build script that formats a name or uses `autocfg` -makes every custom cfg of its package so. A child module under a custom cfg, -or one the build rules out, gives way to an import of its name in the same -module (serde's docsrs-only `mod de` beside the `pub use serde_core::de` of -every other build). With `--cargo-target`, every file compiled +makes every custom cfg of its package so. A child module the build rules out +gives way to whatever else its module has of the name; one under a custom cfg +only to an import or item of its name in the same module (serde's docsrs-only +`mod de` beside the `pub use serde_core::de` of every other build) -- what a +glob brings in never shadows a module that is there. With `--cargo-target`, every file compiled for the target is under the target's cfgs too, so a `windows` candidate is out for every file and a `target_os = "linux"` file takes the `any(android, linux)` branch over the `any(aix, solaris)` one (mio's `sys::Waker` is the @@ -616,7 +621,10 @@ parse.rs). At the crate root the final segment may be a `#[macro_export]` macro (`$crate::name!`), which lives in the file defining it: an invocation (`crate::helper!()`, recorded as `crate::helper!`) names the macro before an import or module of that name, an ordinary path -(`crate::helper()`) the other way round, and a one-segment path written at the +(`crate::helper()`) the other way round. An invocation is looked up among +macros throughout: an item a module defines (`fn m`), a module, or an import +of a module or crate is no `m!`, so a `fn m` beside `use crate::macros::*` leaves +the glob's `m!` in place. A one-segment path written at the crate root names it when no crate has the name (anyhow's `pub use anyhow as format_err;` in lib.rs; a child module's `use x;` does not reach it). The answers are the same whatever order the files are listed in: each module's @@ -696,9 +704,23 @@ test code reaches for. Two kinds of cfg are decided: kept, but a missing module behind it is not fatal, and a candidate under it is never taken as certain. Resolver 1 unifies it all, every table included. The resolver is the workspace's: its `resolver`, else its edition. - The crate a versioned dependency resolves to comes from `Cargo.lock`, so two - vendored versions of one crate each get their own features and edges; - without a lock entry, it is the one vendored version the requirement allows. + Each dependency table is a dependency of its own, as cargo has it: `rand = + "0.7"` under `[dependencies]` beside `rand = "0.8"` under + `[build-dependencies]` (or a `[dev-dependencies] fake = { package = "rand", + version = "0.8" }`) is two crates, and a file is followed into the one its + target uses -- the build script into the build-dependency, a test target into + the normal or dev one, anything else into the normal one. + The crate a versioned dependency resolves to comes from `Cargo.lock`: a + version it lists for the crate that the requirement allows, so two + vendored versions of one crate each get their own features and edges; a lock + that lists only versions the requirement doesn't allow is out of date, and the + loader says so and goes by the requirement, as when there is no lock entry: + the one vendored version the requirement allows. A `[patch]` -- in the root + manifest, or in the cargo config of a directory from the package's up to the + bundle root (nearest first, ahead of the manifest's, its `path` relative to + the directory holding `.cargo`) -- replaces a crate only where its version + fits the dependent's requirement, as cargo applies it; one that doesn't is + reported and not used. When the locked version isn't vendored, or no vendored version fits, or several do and no lock says which, the loader warns and doesn't follow the dependency rather than guess: cargo would build none of them from what the @@ -749,6 +771,8 @@ test code reaches for. Two kinds of cfg are decided: `all(…)`/`any(…)`/`not(…)` compose; a predicate that reduces to true (`not(test)`, an enabled feature) is as firm as no cfg, so a missing module behind it is fatal. +A leaf's value is taken as written, spacing outside its quotes aside: `my = "a +b"` and `my = "ab"` (or `"a b"`) are different values. One an undecided leaf occurs in more than once is decided when it comes out the same whatever that leaf is: an item's cfg joined with its enclosing blocks' `all(any(test, kani), not(kani))` never holds (zerocopy's test-only `use @@ -840,7 +864,8 @@ followed, never a resource. An include written inside a `macro_rules!` body resolves relative to each file that invokes the macro, as rustc expands it -- through the macros whose bodies invoke it, the outermost call (the edge is that file's, at the invocation); so are the bare macro calls a body makes -made and resolved where it is invoked. `!` marks a `macro_rules!` +made and resolved where it is invoked -- the body's own, even past a +`macro_rules!` it defines inside. `!` marks a `macro_rules!` invoked by bare name and defined in another file the invocation can see -- in textual scope, as rustc has it: the files above it in the module tree, each up to the `mod` that mounts the next (a macro defined after `mod early;` is not diff --git a/stasis/src/cmd/bundle.js b/stasis/src/cmd/bundle.js index f95b1d77..57d51e3d 100644 --- a/stasis/src/cmd/bundle.js +++ b/stasis/src/cmd/bundle.js @@ -497,12 +497,18 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa await walk(normalized) // The tree pass may name in-tree crate roots the walk left out (a crate whose name a file also // binds as a value, `use crate::util::log;` beside `log::info!`), or walked as fewer units than - // it is named as: buildRustTree's wantedRoots. Load them and build again, until none is new. + // it is named as: buildRustTree's wantedRoots. Load them and build again, until none is new. A + // root the walk was asked for and didn't load -- refused (a link out of its package) or gone -- + // is not asked for again: the walk has said why. + const tried = new Set() const complete = async () => { const built = buildRustTree(sources, { roots, baseDir, cargo: cargoCtx, formats, units }) - const wanted = built.wantedRoots.filter((r) => addUnits(r, built.wantedUnits.get(r)) || !sources.has(r)) + const wanted = built.wantedRoots.filter((r) => addUnits(r, built.wantedUnits.get(r)) || (!sources.has(r) && !tried.has(r))) if (wanted.length === 0) return built - for (const r of wanted) if (!roots.includes(r)) roots.push(r) + for (const r of wanted) { + tried.add(r) + if (!roots.includes(r)) roots.push(r) + } await walk(wanted) return complete() } @@ -547,12 +553,16 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa } } - // Deps live outside the bundle root unless vendored; with no vendor dir the fix is one command. - if (unresolvedCrates.size > 0 && !existsSync(join(baseDir, cargoCtx.vendorDir))) { - const names = [...unresolvedCrates].toSorted() + // A crate the code names that the bundle doesn't hold: a registry dependency that isn't vendored + // (or not in a version its requirement allows), a path or patch outside the bundle root (the + // loader warned of those), or a crate root the walk refused. Reported whatever the vendor dir + // holds; with none, the fix is one command. + const notLoaded = tree.wantedRoots.filter((r) => !sources.has(r)).map((r) => `${cargoCtx.packageInfo(r)?.name ?? r} (${r})`) + if (unresolvedCrates.size > 0 || notLoaded.length > 0) { + const names = [...unresolvedCrates, ...notLoaded].toSorted() const shown = names.slice(0, 10).join(', ') + (names.length > 10 ? `, ... and ${names.length - 10} more` : '') - console.warn(`[stasis] ${names.length} crate${names.length === 1 ? '' : 's'} referenced but not found in the bundle root: ${shown}`) - console.warn('[stasis] Registry dependencies are bundled only when vendored in-tree: run `cargo vendor` first.') + console.warn(`[stasis] ${names.length} crate${names.length === 1 ? '' : 's'} referenced but not in the bundle: ${shown}`) + if (!existsSync(join(baseDir, cargoCtx.vendorDir))) console.warn('[stasis] Registry dependencies are bundled only when vendored in-tree: run `cargo vendor` first.') } return assembleCodeBundle({ diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index d388a247..5295b97e 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -54,9 +54,12 @@ const kindOf = (request) => request.split('@')[0] const DEP_IMPLICATION_RE = /^dep:(.+)$/u const DEP_FEATURE_RE = /^([^/?]+)(\?)?\/(.+)$/u -// What one dependency table asks of a crate. `defaultFeatures` is tri-state: null until the table -// says (an inherited `workspace = true` entry can only turn defaults on, not off). -const newRequest = () => ({ optional: false, defaultFeatures: null, features: [] }) +// What one dependency table says of a crate: which crate (`version`, `path`, `package`, or +// `workspace = true` to inherit those), and what it asks of it. Each table is a dependency of its +// own: `[dependencies] rand = "0.7"` beside `[build-dependencies] rand = "0.8"` are two crates. +// `defaultFeatures` is tri-state: null until the table says (an inherited `workspace = true` entry +// can only turn defaults on, not off). +const newRequest = () => ({ version: null, path: null, package: null, workspace: false, optional: false, defaultFeatures: null, features: [] }) // `--features a,b pkg/c` (cargo's syntax: repeatable, comma- or space-separated) → the list of names. export function parseFeatureList(values) { @@ -85,8 +88,8 @@ export function parseCargoManifest(text, file = null) { resolver: null, // "1" | "2" | "3" from [workspace] or [package] lib: { name: null, path: null, procMacro: false }, // `procMacro`: `[lib] proc-macro = true`, a crate compiled for the host features: new Map(), // name -> implied entries (`other`, `dep:key`, `key/feat`, `key?/feat`) - // key -> { key, name, version, path, package, workspace, kinds: Map }: - // what identifies the crate is shared, what is asked of it is per dependency table -- sha2's + // key -> { key, name, kinds: Map }: + // each dependency table is a request of its own, for the crate it names -- sha2's // `[dependencies] digest = "0.10"` and `[dev-dependencies] digest = { features = ["dev"] }` are // two requests, and only the first is part of a build of sha2's dependents. A request is the // table's kind (`normal`, `dev`, `build`), with `@` appended for a target-specific table: @@ -103,23 +106,22 @@ export function parseCargoManifest(text, file = null) { const depOf = (map, name, { flat }) => { const key = normName(name) if (!map.has(key)) { - map.set(key, { key, name, version: null, path: null, package: null, workspace: false, ...(flat ? newRequest() : { kinds: new Map() }) }) + map.set(key, { key, name, ...(flat ? newRequest() : { kinds: new Map() }) }) } return map.get(key) } - // Apply one dependency's spec to its record: identity fields on the record, request fields on - // the entry for `kind` (or on the record itself for a flat one). + // Apply one dependency's spec to the entry for `kind` (or to the record itself for a flat one). const setDepFields = (dep, spec, kind) => { const request = kind === null ? dep : (dep.kinds.get(kind) ?? dep.kinds.set(kind, newRequest()).get(kind)) if (typeof spec === 'string') { - dep.version = spec // `foo = "1.2"`: a registry dep + request.version = spec // `foo = "1.2"`: a registry dep return } if (!isTomlTable(spec)) return - if (typeof spec.version === 'string') dep.version = spec.version - if (typeof spec.path === 'string') dep.path = spec.path - if (typeof spec.package === 'string') dep.package = spec.package - if (spec.workspace === true) dep.workspace = true + if (typeof spec.version === 'string') request.version = spec.version + if (typeof spec.path === 'string') request.path = spec.path + if (typeof spec.package === 'string') request.package = spec.package + if (spec.workspace === true) request.workspace = true if (spec.optional === true) request.optional = true const defaults = spec['default-features'] ?? spec.default_features if (defaults === true || defaults === false) request.defaultFeatures = defaults @@ -320,7 +322,9 @@ export function findCargoLock(baseDir) { // --- cfg predicates --------------------------------------------------------------------- -export const normalizeCfg = (pred) => pred.replaceAll(/\s+/gu, ' ').trim() +// A cfg predicate with its whitespace collapsed outside string literals: `my = "a b"` and +// `my = "a b"` are two values. +export const normalizeCfg = (pred) => pred.replaceAll(/"(?:[^"\\]|\\.)*"|\s+/gu, (m) => (m.startsWith('"') ? m : ' ')).trim() const FEATURE_CFG_RE = /^feature\s*=\s*"([^"]*)"$/u // Any other leaf: `unix`, `target_os = "linux"`. @@ -372,7 +376,8 @@ function evalCfgWith(pred, env, assume, free = null) { if (!m) { const known = evalCfgLeaf(p, env) if (known !== null) return known - const leaf = p.replaceAll(/\s+/gu, '') + const kv = CFG_LEAF_RE.exec(p) // the same leaf however it is spaced, its value as written + const leaf = kv === null ? p : (kv[2] === undefined ? kv[1] : `${kv[1]}="${kv[2]}"`) if (assume?.has(leaf)) return assume.get(leaf) if (free !== null) (free.counts ??= new Map()).set(leaf, (free.counts.get(leaf) ?? 0) + 1) return null @@ -741,15 +746,14 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef }) : null // A dependency as the package sees it through one of its tables (`request`, an entry of - // `dep.kinds`; omit it for the identity alone): a `workspace = true` entry merged with the - // workspace's -- features add up, the path is relative to the workspace root, and defaults are - // the workspace's call (a member's `default-features = false` is ignored, with a cargo warning, - // unless the workspace entry disables them too; a member's `true` turns them back on). Null when - // it can't be resolved. `defaultFeatures` comes out boolean. - const depSpec = (m, dep, request = null) => { - const own = request ?? newRequest() - if (!dep.workspace) { - return { key: dep.key, version: dep.version, path: dep.path, package: dep.package, defaultFeatures: own.defaultFeatures !== false, features: own.features, relTo: m.dir } + // `dep.kinds`): a `workspace = true` entry merged with the workspace's -- features add up, the + // path is relative to the workspace root, and defaults are the workspace's call (a member's + // `default-features = false` is ignored, with a cargo warning, unless the workspace entry + // disables them too; a member's `true` turns them back on). Null when it can't be resolved. + // `defaultFeatures` comes out boolean. + const depSpec = (m, dep, request) => { + if (!request.workspace) { + return { key: dep.key, version: request.version, path: request.path, package: request.package, defaultFeatures: request.defaultFeatures !== false, features: request.features, relTo: m.dir } } const ws = workspaceFor(m.dir) const base = ws?.workspaceDeps.get(dep.key) @@ -759,28 +763,61 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef version: base.version, path: base.path, package: base.package, - defaultFeatures: base.defaultFeatures !== false || own.defaultFeatures === true, - features: [...new Set([...base.features, ...own.features])], + defaultFeatures: base.defaultFeatures !== false || request.defaultFeatures === true, + features: [...new Set([...base.features, ...request.features])], relTo: ws.dir, } } - // Dependency → package, memoized per (package, dependency): the fixed-point loop asks many times. + // The `[patch]` path for crate `crate` in a build rooted at `root` (the workspace root's manifest, + // else the bundle root's), as `{ dir, from }`: the cargo config's `[patch.]` tables + // first -- of every directory from the root up to the bundle root, the nearest first, each path + // relative to the directory holding its `.cargo` -- then the root manifest's, relative to it. + // `dir` is null for a path outside the bundle root. Undefined when nothing patches the crate. + const configPatchMemo = new Map() + const configPatches = (dir) => { + if (!configPatchMemo.has(dir)) { + const out = new Map() + for (let d = dir; ; d = posix.dirname(d)) { + const rel = cargoConfigIn(baseDir, d) + const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) + const patch = text === null ? undefined : readToml(text, rel).patch + for (const registry of Object.values(isTomlTable(patch) ? patch : {})) { + for (const [crate, spec] of Object.entries(isTomlTable(registry) ? registry : {})) { + const key = normName(crate) + if (!out.has(key) && isTomlTable(spec) && typeof spec.path === 'string') out.set(key, { dir: normalizeRel(d, spec.path), from: rel }) + } + } + if (d === '.') break + } + configPatchMemo.set(dir, out) + } + return configPatchMemo.get(dir) + } + const patchFor = (root, crate) => { + const fromConfig = configPatches(root?.dir ?? '.').get(crate) + if (fromConfig !== undefined) return fromConfig + const path = root?.patches.get(crate) + return path === undefined ? undefined : { dir: normalizeRel(root.dir, path), from: posix.join(root.dir, 'Cargo.toml') } + } + // Dependency → package, memoized per (package, request): the fixed-point loop asks many times. const depTargets = new Map() - // The in-tree package a dependency of `m` resolves to: a `path` dep (or a `[patch]` path - // override in the workspace root's manifest), else the vendored crate of that name -- the - // version Cargo.lock records for `m`, else the one vendored version that satisfies the - // requirement. A package can depend on two versions of one crate (`borsh = "1"` beside - // `borsh0-9 = { package = "borsh", version = "0.9" }`): the lock then lists both under it, and - // the requirement tells which is which. Null when it isn't in-tree -- a path outside the bundle - // root, or no vendored copy -- and, warned, when the vendored copies don't settle it: the locked - // version isn't among them, none satisfies the requirement, or several do and no lock chooses. - // Cargo would build none of those from what the bundle holds, so none is guessed. - const resolveDep = (m, dep) => { - const memo = `${m.dir}\0${dep.key}` - if (!depTargets.has(memo)) depTargets.set(memo, resolveDepUncached(m, dep)) - return depTargets.get(memo) + // The in-tree package a dependency of `m` resolves to through one of its tables (`request`): a + // `path` dep, else a `[patch]` path override (patchFor) whose version satisfies the requirement, + // else the vendored crate of that name -- the version Cargo.lock records for `m` when it + // satisfies the requirement, else the one vendored version that does. A package can depend on + // two versions of one crate (`borsh = "1"` beside `borsh0-9 = { package = "borsh", version = + // "0.9" }`, or one per table): the lock then lists both under it, and the requirement tells which + // is which. Null when it isn't in-tree -- a path (or patch) outside the bundle root, or no + // vendored copy -- and, warned, when the vendored copies don't settle it: the locked version + // isn't among them, none satisfies the requirement, or several do and no lock chooses. Cargo + // would build none of those from what the bundle holds, so none is guessed. + const resolveDep = (m, dep, request) => { + let byRequest = depTargets.get(m.dir) + if (byRequest === undefined) depTargets.set(m.dir, byRequest = new Map()) + if (!byRequest.has(request)) byRequest.set(request, resolveDepUncached(m, dep, request)) + return byRequest.get(request) } - const resolveDepUncached = (m, dep) => { + const resolveDepUncached = (m, dep, request) => { const asPackage = (dir) => { const t = dir === null ? null : readManifest(dir) return t?.package ? t : null @@ -788,31 +825,47 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // cargo metadata knows exactly which package each dependency edge points at. const known = metadata?.deps.get(m.dir)?.get(dep.key) if (known !== undefined) return asPackage(known) - const spec = depSpec(m, dep) + const spec = depSpec(m, dep, request) if (!spec) return null - if (spec.path) return asPackage(normalizeRel(spec.relTo, spec.path)) + const who = `${m.package.name} ${version(m)}` + if (spec.path) { + const dir = normalizeRel(spec.relTo, spec.path) + const t = asPackage(dir) + if (dir === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} is a path outside the bundle root: ${spec.path}`) + else if (t === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} names ${dir}, which holds no Cargo.toml with a [package]`) + return t + } const crate = normName(spec.package ?? spec.key) - const root = workspaceFor(m.dir) ?? readManifest('.') - const patch = root?.patches.get(crate) - if (patch !== undefined) return asPackage(normalizeRel(root.dir, patch)) - const candidates = vendored().byName.get(crate) ?? [] - if (candidates.length === 0) return null const req = typeof spec.version === 'string' ? spec.version : null const fits = (ver) => req === null || satisfiesCargoReq(ver, req) - const who = `${m.package.name} ${version(m)}` + const patch = patchFor(workspaceFor(m.dir) ?? readManifest('.'), crate) + if (patch !== undefined) { + if (patch.dir === null) { + console.warn(`[loader.cargo] ${patch.from} patches ${crate} with a path outside the bundle root`) + return null + } + const t = asPackage(patch.dir) + // cargo uses a patch only where its version satisfies the requirement ("patch … was not used"). + if (t !== null && fits(version(t))) return t + console.warn(t === null + ? `[loader.cargo] ${patch.from} patches ${crate} with ${patch.dir}, which holds no Cargo.toml with a [package]` + : `[loader.cargo] ${patch.from}'s patch of ${crate} (${version(t)}) doesn't satisfy ${who}'s requirement ${req}: not used`) + } + const candidates = vendored().byName.get(crate) ?? [] + if (candidates.length === 0) return null const vendoredList = candidates.map((c) => c.version).toSorted(compareVersionsDesc).join(', ') const lk = lockfile() if (lk) { const entry = lk.byId.get(`${normName(m.package.name)} ${version(m)}`) - // The lock names a dependency with its version only when several versions of that crate are locked. + // The lock names a dependency with its version only when several versions of that crate are + // locked: the one this requirement allows is this table's. const listed = (entry?.deps ?? []).filter((x) => x.name === crate) - let want = null - if (listed.length === 1) { - const only = lk.byName.get(crate) - want = listed[0].version ?? (only?.length === 1 ? only[0].version : null) - } else if (listed.length > 1) { - want = listed.map((x) => x.version).find((ver) => ver !== null && fits(ver)) ?? null - } + const only = lk.byName.get(crate) + const pins = listed.map((x) => x.version ?? (only?.length === 1 ? only[0].version : null)).filter((ver) => ver !== null) + const want = pins.find(fits) ?? null + // A lock whose pin the requirement no longer allows is out of date: cargo would resolve + // again, so the requirement decides. + if (want === null && pins.length > 0) console.warn(`[loader.cargo] Cargo.lock pins ${who} to ${crate} ${pins.join(', ')}, which ${req} doesn't allow: the lock is out of date`) if (want !== null) { const hit = candidates.find((c) => c.version === want) if (hit) return readManifest(hit.dir) @@ -952,8 +1005,10 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef activate(c, m, d.key) if (implicitFeatures(m).has(d.name)) enable(c, m, d.name) } - const t = resolveDep(m, d) - if (t) for (const [request] of activeRequests(c, m, d)) enable(depCtx(c, request, t), t, depFeature[3]) + for (const [request, r] of activeRequests(c, m, d)) { + const t = resolveDep(m, d, r) + if (t) enable(depCtx(c, request, t), t, depFeature[3]) + } return true } if (featureImplications(m, imp) === null) return false @@ -1003,7 +1058,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const d of m.deps.values()) { for (const [request, r] of activeRequests(c, m, d)) { const spec = depSpec(m, d, r) - const t = spec === null ? null : resolveDep(m, d) + const t = spec === null ? null : resolveDep(m, d, r) if (!t) continue const dc = depCtx(c, request, t) inGraph(dc, t) @@ -1076,25 +1131,39 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // vendored crate of that lib or package name. A package's own name for a crate is its manifest's // word: a declared dependency that doesn't resolve in-tree (a path outside the bundle root, a // version no vendored copy has) never falls back to some vendored copy of that name, and a - // name it doesn't declare is no crate of its. Memoized per (package, name): every file of a - // package asks for the same few crates. + // name it doesn't declare is no crate of its. Which of the dependency's tables names the crate + // is the asking file's `role`: a build script's are the `[build-dependencies]`, a test or bench + // target's the `[dependencies]` and `[dev-dependencies]`, other code's the `[dependencies]` (any + // table when none of those has it). Memoized per (package, name, role): every file of a package + // asks for the same few crates. const crateTargets = new Map() - const depCrate = (m, norm) => { - const memo = `${m?.dir ?? ''}\0${norm}` - if (!crateTargets.has(memo)) crateTargets.set(memo, depCrateUncached(m, norm)) + const depCrate = (m, norm, role) => { + const memo = `${m?.dir ?? ''}\0${norm}\0${role}` + if (!crateTargets.has(memo)) crateTargets.set(memo, depCrateUncached(m, norm, role)) return crateTargets.get(memo) } - const depCrateUncached = (m, norm) => { + const ROLE_KINDS = { build: ['build'], test: ['normal', 'dev'], normal: ['normal'] } + const requestsFor = (d, role) => { + const all = [...d.kinds] + const own = all.filter(([request]) => ROLE_KINDS[role].includes(kindOf(request))) + return (own.length > 0 ? own : all).map(([, r]) => r) + } + const depCrateUncached = (m, norm, role) => { if (m) { const d = m.deps.get(norm) if (d) { - const t = resolveDep(m, d) - return t ? libPath(t) : null + for (const r of requestsFor(d, role)) { + const t = resolveDep(m, d, r) + if (t) return libPath(t) + } + return null } for (const other of m.deps.values()) { - if (other.package !== null) continue // a rename is used by its key, not its lib name - const t = resolveDep(m, other) - if (t && libName(t) === norm) return libPath(t) + for (const r of requestsFor(other, role)) { + if (depSpec(m, other, r)?.package) continue // a rename is used by its key, not its lib name + const t = resolveDep(m, other, r) + if (t && libName(t) === norm) return libPath(t) + } } return null } @@ -1136,7 +1205,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const lib = libPath(m) if (lib && lib !== fromFile) return lib } - return depCrate(m, norm) + const script = m === null || m.package.build === false ? null : normalizeRel(m.dir, m.package.build ?? 'build.rs') + return depCrate(m, norm, fromFile === script ? 'build' : (isTestTarget(fromFile) ? 'test' : 'normal')) }, isTestTarget, // Whether `fromFile`'s package declares a dependency of that name (in any table, by the key diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index ba517016..5a90a718 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -501,6 +501,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // `crate_root! { … mod de; … }`) appears where the macro is invoked, not where it is defined, so // such a `mod` takes the offset of the macro's first bare invocation in this file (`calls`). let templateName = null + let templateMacro = null // the top-level `macro_rules!` whose body the scan is in: what its body calls and includes is its own, nested definitions' too const inlinePath = () => stack.map((s) => s.name) const closeTo = (targetDepth, at) => { @@ -698,7 +699,10 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, open = skipWs(open + name.length) const gate = GATE_MACRO_RE.test(name) && masked[open] === '{' ? gatePredicate(code, masked, open, matchClose(masked, open)) : undefined macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], calls: new Set(), offset: i, at: i, template: i < macroUntil && macroName === 'macro_rules' ? templateName : null, gate }) - if (i >= macroUntil) templateName = name + if (i >= macroUntil) { + templateName = name + templateMacro = macros.at(-1) + } } } if (masked[open] === '{' || masked[open] === '(' || masked[open] === '[') { @@ -722,7 +726,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // it where the macro is invoked, not where it is written -- and a recursive arm's // `m!(…)` is no invocation of this file's. if (!pathQualified) { - if (i < macroUntil && macroName === 'macro_rules' && macros.length > 0) macros.at(-1).calls.add(word) + if (i < macroUntil && macroName === 'macro_rules' && templateMacro !== null) templateMacro.calls.add(word) else { invocations.add(word) ;(calls.get(word) ?? calls.set(word, []).get(word)).push(i) @@ -737,7 +741,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // (`concat!(env!("OUT_DIR"), …)`) is nobody's to follow; any other unreadable argument // is counted, for a warning. if (arg === null) unfollowed += /\bOUT_DIR\b/u.test(text) ? 0 : 1 - else if (i < macroUntil && macroName === 'macro_rules' && macros.length > 0) macros.at(-1).includes.push({ kind: word, ...arg }) + else if (i < macroUntil && macroName === 'macro_rules' && templateMacro !== null) templateMacro.includes.push({ kind: word, ...arg }) else includes.push({ kind: word, ...arg, conditional }) } } @@ -1598,14 +1602,23 @@ function commonDepth(a, b) { // the file the path was written in, with its cfg leaves (see cfg compatibility): an import, item // or glob source exclusive with them is passed over. The candidates are listed once per (at, // name, what `from` may see) -- providedAll -- and the asker picks among them. -function provided(root, at, name, from, ctx, asker, typesOnly = false) { - return pick(providedAll(root, at, name, commonDepth(from, at), ctx), root, ctx, asker, typesOnly) // scopes are ancestors of `at`: `from` is inside those this deep or shallower +function provided(root, at, name, from, ctx, asker, ns = null) { + return pick(providedAll(root, at, name, commonDepth(from, at), ctx), root, ctx, asker, ns) // scopes are ancestors of `at`: `from` is inside those this deep or shallower +} + +// What module `at` binds `name` to itself, as seen from `from` -- a `use` of the name or an item +// it defines (see providedAll), not what its globs bring in -- or null. +function bound(root, at, name, from, ctx, asker, ns = null) { + const list = [...importsOf(root, at, name, commonDepth(from, at), ctx)] + for (const d of ctx.defined.get(root)?.get(at)?.get(name) ?? NONE) list.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns }) + return pick({ get: (i) => list[i] }, root, ctx, asker, ns) } // The answer of the first of `candidates` (see providedAll: in written order) whose cfgs the // asker's entail (entailed) -- else the first compatible with them under no custom cfg, else the -// first compatible at all (doubtful) -- and, for a path's lead or -// prefix (`typesOnly`), in the type namespace: a `fn log` can't lead `log::info`. A child module, +// first compatible at all (doubtful) -- in the namespace `ns` asks for: for a path's lead or +// prefix (`type`), a `fn log` can't lead `log::info`; for a macro (`macro`), an item the module +// defines (`fn m`), a module, or an import of a module or crate is no `m!`. A child module, // item or macro answers as it is; an import answers what it leads to (followImport, on the // asker's behalf, so its cfgs hold along the whole chain), resolved when first asked and then // kept on the import per asker -- unless the answer came out of an import cycle still being @@ -1613,10 +1626,10 @@ function provided(root, at, name, from, ctx, asker, typesOnly = false) { // log;` in every platform file of a module asks the module for `log`, which is these imports): // nothing here. An import that leads nowhere the bundle can see -- when it is a path, not a crate // named outright (`use serde;`, passed over) -- answers where it is: the name is an item of its -// file, as far as the bundle knows. A value where a type is wanted (`typesOnly`: a `fn` -// defined here, an import of one) is passed over too, and when nothing else answers, the answer -// is VALUE_ONLY rather than nothing: the name is taken, but not in this namespace. -function pick(candidates, root, ctx, asker, typesOnly = false) { +// file, as far as the bundle knows. A value where a type is wanted (a `fn` defined here, an +// import of one) is passed over too, and when nothing else answers, the answer is VALUE_ONLY +// rather than nothing: the name is taken, but not in this namespace. +function pick(candidates, root, ctx, asker, ns = null) { const maybes = [] // the compatible answers under no doubt, for when none is entailed (see withAlternatives) let doubted = null // the first compatible answer under a doubtful cfg, for when there is nothing else let lastResort = null // the first under a cfg the asker's build can't compile (see deadUnder) @@ -1625,10 +1638,11 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { const c = candidates.get(i) if (c === undefined) break if (!compatible(asker, c.leaves)) continue - if (typesOnly && c.ns === 'value') { + if (ns === 'type' && c.ns === 'value') { valueOnly = true continue } + if (ns === 'macro' && (c.ns !== undefined || c.answer?.kind === 'module')) continue const dead = deadFor(asker, c.leaves) const doubt = !dead && doubtful(asker, c.leaves) const sure = !dead && !doubt && entailed(asker, c.leaves) @@ -1643,12 +1657,12 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { ctx.hits.add(im.following) continue } - const key = `${asker?.set.key ?? ''}\0${typesOnly}` + const key = `${asker?.set.key ?? ''}\0${ns}` r = im.answers?.get(key) if (r === undefined) { const depth = ctx.walking++ im.following = depth - r = followImport(im, root, ctx, asker, typesOnly) + r = followImport(im, root, ctx, asker, ns) im.following = undefined ctx.walking-- ctx.hits.delete(depth) @@ -1666,6 +1680,7 @@ function pick(candidates, root, ctx, asker, typesOnly = false) { valueOnly = true continue } + if (ns === 'macro' && (r.kind === 'module' || r.kind === 'crate')) continue } if (sure) return { ...r, through: c.through } if (dead) lastResort = { ...r, through: c.through } @@ -1850,10 +1865,10 @@ function hasAll(root, at, name, seeing, ctx) { const NONE = Object.freeze([]) // The file in which module `at` defines item `name` (a `struct`, `fn`, … visible `seeing` levels -// in, under cfgs the asker's allow; in the type namespace when `typesOnly`, see pick), or null -- +// in, under cfgs the asker's allow; in the type namespace when `ns` is `type`, see pick), or null -- // or `{ file, alternatives }` when several files define it, none for certain (see // withAlternatives; itemAt makes an answer of either). -function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { +function definedIn(root, at, name, seeing, ctx, asker, ns = null) { const defs = ctx.defined.get(root)?.get(at)?.get(name) if (defs === undefined) return null const own = ctx.trees.get(root).get(at) @@ -1865,7 +1880,7 @@ function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { let bestRank = Infinity const maybes = [] for (const d of defs) { - if (d.scopeDepth > seeing || !compatible(asker, d.leaves) || (typesOnly && d.ns === 'value')) continue + if (d.scopeDepth > seeing || !compatible(asker, d.leaves) || (ns === 'type' && d.ns === 'value')) continue const tier = deadFor(asker, d.leaves) ? 3 : (doubtful(asker, d.leaves) ? 2 : (entailed(asker, d.leaves) ? 0 : 1)) if (tier === 1) maybes.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves }) const rank = tier * 2 + (d.file === own ? 0 : 1) @@ -1887,10 +1902,10 @@ function definedIn(root, at, name, seeing, ctx, asker, typesOnly = false) { // name, then its own globs; only then the module's imports at large (providedAll). For a module // held in one file the two agree, so the shortcut is taken only when they may not. Memoized like // providedAll: a `use log;` in such a file asks for `log` in its own module, which is this import. -function providedFrom(root, at, name, ctx, file, asker, typesOnly = false) { +function providedFrom(root, at, name, ctx, file, asker, ns = null) { const of = ctx.imports.get(root)?.get(at) const several = of !== undefined && [...of.globs, ...[...of.named.values()].flat()].some((im) => im.file !== file) - if (!several) return provided(root, at, name, at, ctx, asker, typesOnly) + if (!several) return provided(root, at, name, at, ctx, asker, ns) const memo = ctx.provided.get(root) ?? ctx.provided.set(root, new Map()).get(root) const seeing = at.split('::').length const key = `${at}\0${name}\0@${file}` @@ -1923,7 +1938,7 @@ function providedFrom(root, at, name, ctx, file, asker, typesOnly = false) { ctx.hits.add(list.depth) return null } - return pick(list, root, ctx, asker, typesOnly) + return pick(list, root, ctx, asker, ns) } // The `use` items of module `at` binding `name`, visible to one seeing `seeing` levels into it, @@ -2011,9 +2026,9 @@ function globSource(im, root, ctx) { // What an import leads to: its path resolved as written in its module, else the crate its lead // names -- the sysroot's (`use core::fmt`: an item, living here as far as the bundle knows), an // in-tree one, or null for one that isn't (nor anything the tree can model). -function followImport(im, root, ctx, asker, typesOnly = false) { +function followImport(im, root, ctx, asker, ns = null) { if (!im.absolute) { - const r = walkPath(im.segments, root, im.module, ctx, { file: im.file, asker, typesOnly }) + const r = walkPath(im.segments, root, im.module, ctx, { file: im.file, asker, ns }) if (r !== null) return r } const head = im.segments[0] @@ -2045,17 +2060,20 @@ const onlyValues = (root, at, name, ctx) => { // walk goes on from a module that provides, or ends at whatever else it provides. A segment // nothing provides is an item of the module reached, in the file defining it when that is known. // At the crate root a final segment may name a `#[macro_export]` macro, which lives in the file -// defining it: `$crate::name!` names the macro before anything else (`macroCall`), any other path -// the module or import of that name first. In the module the path is written in, `file`'s own -// imports come first: the module's other files, if any, are cfg variants (providedFrom). `asker` -// (see cfg compatibility) rules out what can't be compiled together with the path. -function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, typesOnly = false } = {}) { +// defining it: `$crate::name!` names the macro before anything else (`ns` is `macro`: a path +// invoked), any other path the module or import of that name first. In the module the path is +// written in, `file`'s own imports come first: the module's other files, if any, are cfg variants +// (providedFrom). `asker` (see cfg compatibility) rules out what can't be compiled together with +// the path. +function walkPath(segments, root, from, ctx, { ns = null, file, asker } = {}) { const tree = ctx.trees.get(root) const head = segments[0] // A segment with more after it names a module or type-namespace item: a `fn log` doesn't lead - // `log::info!`, the crate does (`typed`, see pick) -- the last one too when the caller wants a - // type (`typesOnly`: an import followed as a path's prefix). - const lookup = (at, name, typed) => (at === from && file !== undefined ? providedFrom(root, at, name, ctx, file, asker, typed) : provided(root, at, name, from, ctx, asker, typed)) + // `log::info!`, the crate does (`type`, see pick); the last one is in the namespace the caller + // wants (`type` for an import followed as a path's prefix, `macro` for one invoked). + const nsOf = (last) => (last ? ns : 'type') + const isModule = (path, last) => tree.has(path) && !(last && ns === 'macro') // a `mod m` is no `m!` + const lookup = (at, name, want) => (at === from && file !== undefined ? providedFrom(root, at, name, ctx, file, asker, want) : provided(root, at, name, from, ctx, asker, want)) let cur = from.split('::') let i = 0 let via @@ -2064,14 +2082,14 @@ function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, t i = 1 } else if (head === 'self') { i = 1 - } else if (head !== 'super' && !tree.has(`${from}::${head}`)) { - const typed = segments.length > 1 || typesOnly - const p = lookup(from, head, typed) + } else if (head !== 'super' && !isModule(`${from}::${head}`, segments.length === 1)) { + const want = nsOf(segments.length === 1) + const p = lookup(from, head, want) if (p === null || p === VALUE_ONLY) { // An item of the module itself (`enum Kind { … } use Kind::*;`), or a macro of the file. - const own = definedIn(root, from, head, Infinity, ctx, asker, typed) + const own = definedIn(root, from, head, Infinity, ctx, asker, want) if (own !== null) return itemAt(own) - if (typed && (p === VALUE_ONLY || onlyValues(root, from, head, ctx))) return VALUE_ONLY + if (want === 'type' && (p === VALUE_ONLY || onlyValues(root, from, head, ctx))) return VALUE_ONLY return segments.length === 1 && file !== undefined && ctx.fileMacros.get(file)?.has(head) === true ? { kind: 'item', file } : null } via = { modulePath: from, consumed: 0, through: p.through } @@ -2089,27 +2107,29 @@ function walkPath(segments, root, from, ctx, { macroCall = false, file, asker, t const name = segments[i] const last = i === segments.length - 1 const macro = at === 'crate' && last ? ctx.macros.get(root)?.get(name) : undefined - if (macro !== undefined && macroCall) return { kind: 'item', file: macro, via } + if (macro !== undefined && ns === 'macro') return { kind: 'item', file: macro, via } const child = `${at}::${name}` - const typed = !last || typesOnly + const want = nsOf(last) let p - if (tree.has(child)) { - // A child module the asker's build presumably doesn't compile -- under a cfg off in it, or - // a custom one (doubtful): serde's docsrs-only `mod de` beside the `pub use serde_core::de` - // of every other build -- gives way to what else the module has of the name, if anything. + if (isModule(child, last)) { + // A child module the asker's build doesn't compile (under a cfg off in it) gives way to what + // else the module has of the name, if anything; one it presumably doesn't (under a custom + // cfg, doubtful: serde's docsrs-only `mod de` beside the `pub use serde_core::de` of every + // other build) only to what the module binds the name to itself, which can stand beside the + // module only under other cfgs. What a glob brings in never shadows a module that is there. const set = ctx.files.get(tree.get(child))?.leaves - p = set !== undefined && (deadFor(asker, set) || doubtful(asker, set)) ? lookup(at, name, typed) : null + p = set === undefined ? null : (deadFor(asker, set) ? lookup(at, name, want) : (doubtful(asker, set) ? bound(root, at, name, from, ctx, asker, want) : null)) if (p === null || p === VALUE_ONLY) { cur.push(name) i++ continue } - } else p = lookup(at, name, typed) + } else p = lookup(at, name, want) if (p === null || p === VALUE_ONLY) { if (macro !== undefined) return { kind: 'item', file: macro, via } - const def = definedIn(root, at, name, Infinity, ctx, asker, typed) + const def = definedIn(root, at, name, Infinity, ctx, asker, want) if (def !== null) return { ...itemAt(def), via } - if (typed && (p === VALUE_ONLY || onlyValues(root, at, name, ctx))) return VALUE_ONLY + if (want === 'type' && (p === VALUE_ONLY || onlyValues(root, at, name, ctx))) return VALUE_ONLY return { kind: 'item', file: tree.get(at), via } } via ??= { modulePath: at, consumed: i, through: p.through } @@ -2218,7 +2238,7 @@ function resolvePathRef(ref, file, bindings, ctx) { const from = here ? [here.modulePath, ...inlinePath].join('::') : null let valueOnly = false if (!absolute && here) { - const r = walkPath(segments, here.root, from, ctx, { macroCall: ref.macroCall, file, asker: here.asker }) + const r = walkPath(segments, here.root, from, ctx, { ns: ref.macroCall ? 'macro' : null, file, asker: here.asker }) if (r === VALUE_ONLY) valueOnly = true // the lead is bound here, but as a value: `log::…` names the crate else if (r !== null) return r } @@ -2226,7 +2246,7 @@ function resolvePathRef(ref, file, bindings, ctx) { if (lead === null) return null const name = ctx.externPrelude.get(here?.root)?.get(lead) ?? lead if (name === 'self') { - const r = walkPath(['crate', ...segments.slice(1)], here.root, from, ctx, { macroCall: ref.macroCall, file, asker: here.asker }) + const r = walkPath(['crate', ...segments.slice(1)], here.root, from, ctx, { ns: ref.macroCall ? 'macro' : null, file, asker: here.asker }) return r === VALUE_ONLY ? null : r } const target = ctx.resolveCrate(name, file) @@ -2795,7 +2815,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const macroFileAt = (name, offset) => { const inScope = macroAt(path, name, offset, offset, above) ?? exported?.get(name) if (inScope !== undefined) return inScope - const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { macroCall: true, file: path, asker: here.asker })?.file + const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { ns: 'macro', file: path, asker: here.asker })?.file return byPath !== undefined && fileMacros.get(byPath)?.has(name) === true ? byPath : undefined // a file that defines a macro of that name, not a `fn write` beside `write!` } for (const [name, offsets] of items.calls) { diff --git a/tests/bundle-cmd.test.js b/tests/bundle-cmd.test.js index 29ed11a0..a0cf65ad 100644 --- a/tests/bundle-cmd.test.js +++ b/tests/bundle-cmd.test.js @@ -4090,8 +4090,8 @@ test('buildRustBundle bundles what is in-tree and hints at `cargo vendor` when d const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: join(rustFixtures, 'no-vendor'), entries: ['src/main.rs'] })) t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/a.rs', 'src/main.rs']) // Two short lines: the crates, then the remedy (no absolute path). - const listed = warnings.find((w) => w.includes('not found in the bundle root')) - t.assert.equal(listed, '[stasis] 2 crates referenced but not found in the bundle root: serde, syn') // not std, not the local module `a` + const listed = warnings.find((w) => w.includes('referenced but not in the bundle')) + t.assert.equal(listed, '[stasis] 2 crates referenced but not in the bundle: serde, syn') // not std, not the local module `a` const hint = warnings.find((w) => w.includes('cargo vendor')) t.assert.equal(hint, '[stasis] Registry dependencies are bundled only when vendored in-tree: run `cargo vendor` first.') }) @@ -4100,7 +4100,7 @@ test('CLI: bundle (rust) prints the `cargo vendor` hint to stderr and still exit const outPath = join(tmp, 'out.stasis.code.br') const r = runCli(['bundle', '-o', outPath, 'src/main.rs'], { cwd: join(rustFixtures, 'no-vendor') }) t.assert.equal(r.status, 0, r.stderr) - t.assert.match(r.stderr, /^\[stasis\] 2 crates referenced but not found in the bundle root: serde, syn$/mu) + t.assert.match(r.stderr, /^\[stasis\] 2 crates referenced but not in the bundle: serde, syn$/mu) t.assert.match(r.stderr, /^\[stasis\] Registry dependencies are bundled only when vendored in-tree: run `cargo vendor` first\.$/mu) t.assert.match(r.stderr, /Bundled 2 files in 1 package/u) })) diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index df999737..2b31a0de 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -45,21 +45,17 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin t.assert.deepEqual(m.package, { name: 'app', version: '0.1.0', versionFromWorkspace: false, edition: '2021', editionFromWorkspace: false, build: null }) t.assert.equal(m.resolver, '2') t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']]]) - const dep = (k) => { - const d = m.deps.get(k) - return { version: d.version, kinds: Object.fromEntries([...d.kinds].toSorted()) } - } - // Each dependency table is its own request: a dev-dependency's features stay out of the normal one. - // `defaultFeatures` stays null until a table says (an inherited entry may only turn defaults on). - t.assert.deepEqual(dep('plain'), { version: '1', kinds: { - dev: { optional: false, defaultFeatures: null, features: ['dev-only'] }, - normal: { optional: false, defaultFeatures: null, features: [] }, - } }) - t.assert.deepEqual(dep('opt'), { version: '1', kinds: { normal: { optional: true, defaultFeatures: false, features: ['a'] } } }) - t.assert.deepEqual(dep('sub'), { version: '2', kinds: { normal: { optional: false, defaultFeatures: null, features: ['one', 'two'] } } }) - t.assert.deepEqual(Object.keys(dep('cc').kinds), ['build']) + const dep = (k) => Object.fromEntries([...m.deps.get(k).kinds].toSorted()) + const ask = (version, more) => ({ version, path: null, package: null, workspace: false, optional: false, defaultFeatures: null, features: [], ...more }) + // Each dependency table is its own request, for the crate it names: a dev-dependency's features + // stay out of the normal one. `defaultFeatures` stays null until a table says (an inherited + // entry may only turn defaults on). + t.assert.deepEqual(dep('plain'), { dev: ask('1', { features: ['dev-only'] }), normal: ask('1') }) + t.assert.deepEqual(dep('opt'), { normal: ask('1', { optional: true, defaultFeatures: false, features: ['a'] }) }) + t.assert.deepEqual(dep('sub'), { normal: ask('2', { features: ['one', 'two'] }) }) + t.assert.deepEqual(Object.keys(dep('cc')), ['build']) // A target-specific table is a request of its own, beside the plain one. - t.assert.deepEqual(Object.keys(dep('nix').kinds), ['normal@cfg(unix)']) + t.assert.deepEqual(Object.keys(dep('nix')), ['normal@cfg(unix)']) // the key is the `use` spelling, the name the manifest's (an optional dep's implicit feature name) t.assert.deepEqual([m.deps.get('pm_crate').key, m.deps.get('pm_crate').name, m.deps.get('pm_crate').kinds.get('normal').optional], ['pm_crate', 'pm-crate', true]) // `[patch.]` entries in every spelling: inline table, dotted key, sub-table. @@ -81,9 +77,9 @@ test('parseCargoManifest splits dotted dependency keys and survives multi-line s ].join('\n')) t.assert.equal(m.package.version, '0.4.0') t.assert.deepEqual([...m.deps.keys()], ['util', 'serde']) - t.assert.equal(m.deps.get('util').workspace, true) + t.assert.equal(m.deps.get('util').kinds.get('normal').workspace, true) t.assert.deepEqual(m.deps.get('util').kinds.get('normal').features, ['extra']) - t.assert.equal(m.deps.get('serde').version, '1') + t.assert.equal(m.deps.get('serde').kinds.get('normal').version, '1') t.assert.deepEqual(m.deps.get('serde').kinds.get('normal').features, ['derive']) }) @@ -95,8 +91,8 @@ test('parseCargoManifest reads a pair by the table it lands in, whichever way th '[dependencies]\nfoo = { version = "1", features = ["x"] }\n', ] for (const text of spellings) { - const dep = parseCargoManifest(text).deps.get('foo') - t.assert.deepEqual([dep.version, dep.kinds.get('normal').features], ['1', ['x']], text) + const normal = parseCargoManifest(text).deps.get('foo').kinds.get('normal') + t.assert.deepEqual([normal.version, normal.features], ['1', ['x']], text) } const m = parseCargoManifest([ '[package]', 'name = "app"', 'version = { workspace = true }', @@ -106,7 +102,7 @@ test('parseCargoManifest reads a pair by the table it lands in, whichever way th '[workspace.package]', 'version = "0.9.0"', // a [workspace.*] table alone makes this a workspace root ].join('\n')) t.assert.deepEqual(m.package, { name: 'app', version: null, versionFromWorkspace: true, edition: null, editionFromWorkspace: false, build: null }) - t.assert.deepEqual([m.deps.get('winapi').version, [...m.deps.get('winapi').kinds.keys()]], ['0.3', ['dev@cfg(windows)']]) + t.assert.deepEqual([...m.deps.get('winapi').kinds].map(([request, r]) => [request, r.version]), [['dev@cfg(windows)', '0.3']]) t.assert.deepEqual([...m.patches], [['plain', 'patches/plain']]) t.assert.deepEqual([m.isWorkspace, m.workspacePackage.version], [true, '0.9.0']) }) @@ -1294,3 +1290,108 @@ test('buildRustBundle declares a template\'s mod in each module of its package i t.assert.deepEqual(sorted(buildRustTree(sources, { roots: entries, baseDir: tmp, cargo }).unresolvedCrates), []) }) }) + +// Warnings `fn` prints, and its result. +const captureWarningsSync = (fn) => { + const original = console.warn + const warnings = [] + console.warn = (...args) => warnings.push(args.join(' ')) + try { + return { result: fn(), warnings } + } finally { + console.warn = original + } +} +const vendoredPackage = (name, version) => ({ + [`vendor/${name}-${version}/Cargo.toml`]: `[package]\nname = "${name}"\nversion = "${version}"\n`, + [`vendor/${name}-${version}/src/lib.rs`]: '', +}) + +test('createCargoContext resolves each dependency table on its own: one version per table, the asking file\'s table', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nrand = "0.7"\n[build-dependencies]\nrand = "0.8"\n[dev-dependencies]\nfake = { package = "rand", version = "0.8" }\n', + 'src/main.rs': '', 'build.rs': '', 'tests/it.rs': '', + ...vendoredPackage('rand', '0.7.3'), ...vendoredPackage('rand', '0.8.5'), + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + const { result, warnings } = captureWarningsSync(() => [ + cargo.resolveCrate('rand', 'src/main.rs'), cargo.resolveCrate('rand', 'build.rs'), cargo.resolveCrate('rand', 'tests/it.rs'), cargo.resolveCrate('fake', 'tests/it.rs'), + ]) + t.assert.deepEqual(result, ['vendor/rand-0.7.3/src/lib.rs', 'vendor/rand-0.8.5/src/lib.rs', 'vendor/rand-0.7.3/src/lib.rs', 'vendor/rand-0.8.5/src/lib.rs']) + t.assert.deepEqual(warnings, []) + // both are in the build: the normal one for the target, the build-dependency for the host + t.assert.deepEqual([...cargo.featureResolution('target').keys(), ...cargo.featureResolution('host').keys()].toSorted(), ['.', 'vendor/rand-0.7.3', 'vendor/rand-0.8.5']) + }) +}) + +test('createCargoContext takes a Cargo.lock pin only when the requirement allows it', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nrand = "0.8"\n', + // out of date: the manifest moved on to 0.8, the lock still says 0.7.3 + 'Cargo.lock': 'version = 3\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["rand"]\n\n[[package]]\nname = "rand"\nversion = "0.7.3"\n', + 'src/main.rs': '', + ...vendoredPackage('rand', '0.7.3'), ...vendoredPackage('rand', '0.8.5'), + }, (tmp) => { + const { result, warnings } = captureWarningsSync(() => createCargoContext(tmp, { entries: ['src/main.rs'] }).resolveCrate('rand', 'src/main.rs')) + t.assert.equal(result, 'vendor/rand-0.8.5/src/lib.rs') + t.assert.deepEqual(warnings, ['[loader.cargo] Cargo.lock pins app 0.1.0 to rand 0.7.3, which 0.8 doesn\'t allow: the lock is out of date']) + }) +}) + +test('createCargoContext applies a [patch] only where it fits, from the manifest or the cargo config', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nfoo = "1"\nbar = "1"\nbaz = "1"\n[patch.crates-io]\nfoo = { path = "patches/foo" }\nbaz = { path = "../outside/baz" }\n', + '.cargo/config.toml': '[patch.crates-io]\nbar = { path = "patches/bar" }\n', + 'src/main.rs': '', + 'patches/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "2.0.0"\n', 'patches/foo/src/lib.rs': '', // doesn't satisfy `1` + 'patches/bar/Cargo.toml': '[package]\nname = "bar"\nversion = "1.5.0"\n', 'patches/bar/src/lib.rs': '', + ...vendoredPackage('foo', '1.0.0'), ...vendoredPackage('bar', '1.0.0'), ...vendoredPackage('baz', '1.0.0'), + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + const { result, warnings } = captureWarningsSync(() => ['foo', 'bar', 'baz'].map((name) => cargo.resolveCrate(name, 'src/main.rs'))) + // foo: the patch isn't used, the registry's (vendored) 1.0.0 is; bar: the config's patch; baz: + // patched with a crate outside the bundle, which is not some vendored copy of that name + t.assert.deepEqual(result, ['vendor/foo-1.0.0/src/lib.rs', 'patches/bar/src/lib.rs', null]) + t.assert.deepEqual(warnings, [ + '[loader.cargo] Cargo.toml\'s patch of foo (2.0.0) doesn\'t satisfy app 0.1.0\'s requirement 1: not used', + '[loader.cargo] Cargo.toml patches baz with a path outside the bundle root', + ]) + }) +}) + +test('createCargoContext says when a path dependency lies outside the bundle root or names no package', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nfar = { path = "../far" }\nempty = { path = "crates/empty" }\n', + 'src/main.rs': '', 'crates/empty/README.md': '', + ...vendoredPackage('far', '1.0.0'), + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + const { result, warnings } = captureWarningsSync(() => [cargo.resolveCrate('far', 'src/main.rs'), cargo.resolveCrate('empty', 'src/main.rs')]) + t.assert.deepEqual(result, [null, null]) + t.assert.deepEqual(warnings, [ + '[loader.cargo] app 0.1.0\'s dependency far is a path outside the bundle root: ../far', + '[loader.cargo] app 0.1.0\'s dependency empty names crates/empty, which holds no Cargo.toml with a [package]', + ]) + }) +}) + +test('buildRustBundle ends when a crate root it wants is refused, and reports every crate it lacks, vendor dir or not', async (t) => { + if (process.platform === 'win32') return t.skip('symlinks') + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nlinked = "1"\nmissing = "1"\nfar = { path = "../far" }\n', + 'src/main.rs': 'use linked::X;\nuse missing::Y;\nuse far::Z;\nfn main() {}\n', + 'vendor/linked/Cargo.toml': '[package]\nname = "linked"\nversion = "1.0.0"\n', + 'secret.rs': 'pub struct X;\n', + }, async (tmp) => { + // a vendored crate whose root is a link out of its package: the walk refuses it, the tree pass asks for it + mkdirSync(join(tmp, 'vendor', 'linked', 'src')) + symlinkSync(join('..', '..', '..', 'secret.rs'), join(tmp, 'vendor', 'linked', 'src', 'lib.rs')) + const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'] })) + t.assert.deepEqual([...bundle.sources.keys()], ['src/main.rs']) + t.assert.deepEqual([...new Set(warnings)], [ + '[loader.cargo] app 0.1.0\'s dependency far is a path outside the bundle root: ../far', + '[loader.rust] Refusing file outside its package: vendor/linked/src/lib.rs (a link out of vendor/linked)', + '[stasis] 3 crates referenced but not in the bundle: far, linked (vendor/linked/src/lib.rs), missing', + ]) + }) +}) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 92250a55..a3b7730d 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -768,7 +768,8 @@ test('parseCargoManifest reads package, lib, dependencies in every shape, and wo t.assert.deepEqual(m.lib, { name: 'myapp_lib', path: 'src/the_lib.rs', procMacro: false }) const dep = (k) => { const d = m.deps.get(k) - return { path: d.path, package: d.package, workspace: d.workspace, kinds: [...d.kinds.keys()].toSorted() } + const [r] = d.kinds.values() // one table each here: a dependency's identity is its table's + return { path: r.path, package: r.package, workspace: r.workspace, kinds: [...d.kinds.keys()].toSorted() } } t.assert.deepEqual([...m.deps.keys()].toSorted(), ['inline_sub', 'nix', 'serde', 'shared', 'tempfile', 'tools', 'util']) t.assert.deepEqual(dep('inline_sub'), { path: '../sub', package: null, workspace: false, kinds: ['normal'] }) @@ -2264,3 +2265,53 @@ test('buildRustTree places a template\'s mod at an invocation in its own crate, const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) t.assert.equal(edges(resolutions.get('vendor/a/src/x.rs'))['helper!'], 'vendor/a/src/lib.rs') }) + +// --- eighth review: nested macro_rules!, glob-provided names against local ones, cfg strings --- + +test('buildRustTree keeps a template\'s calls and includes with the template when it defines a macro_rules! of its own', (t) => { + const sources = new Map([ + ['src/lib.rs', 'macro_rules! helper { () => {} }\nmacro_rules! outer { () => { macro_rules! inner { () => {} } fn f() -> &\'static str { helper!(); include_str!("data.txt") } } }\nmod user;\n'], + ['src/user.rs', 'outer!();\n'], + ['src/data.txt', 'x'], + ]) + const items = scanRustItems(sources.get('src/lib.rs')) + t.assert.deepEqual(items.macros.map((m) => [m.name, m.includes.map((i) => i.path), [...m.calls]]), [['helper', [], []], ['outer', ['data.txt'], ['helper', 'include_str']], ['inner', [], []]]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'outer!': 'src/lib.rs', 'include_str data.txt': 'src/data.txt', 'helper!': 'src/lib.rs' }) +}) + +test('buildRustTree lets a local module under a custom cfg stand against a glob, giving way only to the module\'s own binding', (t) => { + const base = [ + ['src/lib.rs', 'mod other;\n#[cfg(docsrs)]\nmod util;\nuse other::*;\nfn f() { util::g(); }\n'], + ['src/other.rs', 'pub mod util;\n'], + ['src/other/util.rs', 'pub fn g() {}\n'], + ['src/util.rs', 'pub fn g() {}\n'], + ] + const r = (sources) => edges(buildRustTree(new Map(sources), { roots: ['src/lib.rs'] }).resolutions.get('src/lib.rs'))['util::g'] + t.assert.equal(r(base), 'src/util.rs') // a glob never shadows a module that is there + // serde: the docsrs-only module beside the `pub use` of every other build. + t.assert.equal(r([...base, ['src/lib.rs', 'mod other;\n#[cfg(docsrs)]\nmod util;\n#[cfg(not(docsrs))]\nuse other::util;\nfn f() { util::g(); }\n']]), 'src/other/util.rs') +}) + +test('buildRustTree looks a macro call up among macros: a fn or module of the name is no `m!`', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod macros;\nmod user;\nmod m {}\n'], + ['src/macros.rs', 'macro_rules! m { () => {} }\npub(crate) use m;\n'], + ['src/user.rs', 'use crate::macros::*;\nfn m() {}\nfn f() { m!(); }\n'], + ]) + t.assert.equal(edges(buildRustTree(sources, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['m!'], 'src/macros.rs') + const viaRoot = new Map([...sources, ['src/user.rs', 'use crate::*;\nfn f() { m!(); }\n'], ['src/lib.rs', 'mod macros;\nmod user;\nmod m {}\npub(crate) use macros::*;\n']]) + t.assert.equal(edges(buildRustTree(viaRoot, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['m!'], 'src/macros.rs') + const besideModule = new Map([...sources, ['src/user.rs', 'use crate::macros::*;\nmod m;\nfn f() { m!(); }\n'], ['src/user/m.rs', '']]) + t.assert.equal(edges(buildRustTree(besideModule, { roots: ['src/lib.rs'] }).resolutions.get('src/user.rs'))['m!'], 'src/macros.rs') +}) + +test('evalCfg and buildRustTree keep cfg values that differ only in spaces apart', (t) => { + t.assert.equal(evalCfg('all(my = "a b", not(my = "ab"))'), null) + t.assert.equal(evalCfg('all(my = "a b", not(my = "a b"))'), null) + t.assert.equal(evalCfg('all(my="a b", not(my = "a b"))'), false) // the same leaf, spaced differently + for (const cfg of ['all(my = "a b", not(my = "ab"))', 'all(my = "a b", not(my = "a b"))']) { + const { resolutions } = buildRustTree(new Map([['src/lib.rs', `#[cfg(${cfg})]\nmod x;\n`], ['src/x.rs', '']]), { roots: ['src/lib.rs'] }) + t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod x': 'src/x.rs' }) + } +}) From e4461de541258f7512d9fe4477776323578e3599 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 19:33:49 +0000 Subject: [PATCH 07/10] feat(bundle): read Cargo through @preventive/lockfile, and resolve with cargo's resolver where it can - Cargo.toml, Cargo.lock and the cargo config's [patch] are read by @preventive/lockfile 1.0.0-alpha.3 (parseCargoManifest, parseCargoLock, parseCargoConfig): what cargo refuses, or what the reader can't tell how cargo reads, stops the build naming the file; a Cargo.lock older than version 3 is refused. Version requirements are matched by its rust-semver.js. The loader's own manifest and lock readers and semver matcher are gone. - With a lockfile, --cargo-target and every locked package in-tree (path packages inside the bundle root, the rest vendored with checksums), features come from cargo's resolver: linkCargo over the manifests, then resolveCargoFeatures for `cargo build -p `; a lockfile out of date or a vendored copy whose checksum isn't the lockfile's stops the build. An unknown host is resolved both ways (its target tables maybe). - Otherwise the manifests are replayed as before, with fixes: a git dependency takes a git checkout's vendored copy and a registry one a registry's (by .cargo-checksum.json and the lockfile's sources); `dep/feat` turns on the package's feature of that name, written or implicit; a proc-macro entry is resolved and scanned for the host; prereleases follow the semver crate; the lockfile is the entries' workspace's, below the bundle root too. - cfg(true), cfg(false) and raw identifiers in cfgs. - Tests: cargo's recorded builds of @preventive/lockfile's workspace fixture (PreventiveMeasures/libraries#56, MIT) through createCargoContext, the refusals, the fallback, and each fix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 136 +-- stasis/bin/stasis.js | 4 +- stasis/src/loaders/cargo.js | 839 ++++++++++-------- stasis/src/loaders/rust.js | 19 +- tests/bundle-cmd.test.js | 4 +- .../rust-bundle/cargo-recorded/Cargo.lock | 79 ++ .../rust-bundle/cargo-recorded/Cargo.toml | 42 + .../cargo-recorded/crates/extra/Cargo.toml | 9 + .../cargo-recorded/crates/lib/Cargo.toml | 15 + .../cargo-recorded/crates/macros/Cargo.toml | 10 + .../cargo-recorded/patched/cfg-if/Cargo.toml | 4 + .../rust-bundle/cargo-recorded/recorded.json | 500 +++++++++++ .../vendor/autocfg/.cargo-checksum.json | 1 + .../cargo-recorded/vendor/autocfg/Cargo.toml | 75 ++ .../vendor/itoa-0.4.8/.cargo-checksum.json | 1 + .../vendor/itoa-0.4.8/Cargo.toml | 29 + .../vendor/itoa-1.0.0/.cargo-checksum.json | 1 + .../vendor/itoa-1.0.0/Cargo.toml | 48 + .../vendor/itoa/.cargo-checksum.json | 1 + .../cargo-recorded/vendor/itoa/Cargo.toml | 66 ++ .../vendor/memchr/.cargo-checksum.json | 1 + .../cargo-recorded/vendor/memchr/Cargo.toml | 89 ++ .../vendor/paste/.cargo-checksum.json | 1 + .../cargo-recorded/vendor/paste/Cargo.toml | 72 ++ tests/rust-features.test.js | 494 +++++++---- tests/rust-loader.test.js | 31 +- 26 files changed, 1999 insertions(+), 572 deletions(-) create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/Cargo.lock create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/extra/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/lib/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/macros/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/patched/cfg-if/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/recorded.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json create mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml diff --git a/doc/file-formats.md b/doc/file-formats.md index 988addfb..ec0a1e43 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -682,56 +682,85 @@ vendored crates' test modules stay out, and with them the dev-dependencies only test code reaches for. Two kinds of cfg are decided: - `test`, `doctest`, `doc` and `#[test]` are never on when a program is built. -- `feature = "…"` is decided per crate from **Cargo feature resolution**: the - loader reads the `Cargo.toml` of every package in-tree (the root package or - workspace, `path` dependencies, `vendor/`) plus `Cargo.lock`, and replays what - `cargo build` of the entries' packages does — the roots start from their - `default` feature, features imply features (`std = ["alloc", "dep:serde", - "serde?/std"]`), enable optional dependencies and request dependency features, - and every active dependency gets `default` plus what its dependents ask for, - to a fixed point. A dependency's own dev-dependencies are nobody's build and - never count (sha2's `[dev-dependencies] digest = { features = ["dev"] }` - doesn't turn on digest's `dev`); the entries' packages' dev-dependencies - count under resolver 1 only, since resolver 2 (edition 2021+, or - `resolver = "2"`) keeps them out of a normal build. Resolver 2 also resolves - what is built for the host -- build-dependencies, proc-macro crates and what - they depend on -- apart from what is built for the target: a feature a - build-dependency asks of a crate isn't on in that crate as a normal - dependency. And it decides target-specific dependency tables: one counts - when `--cargo-target` says it applies (a build-dependency's table against the - host), and not when it says it doesn't; without a target, what only such a - table enables -- a dependency, a feature -- is on *maybe*: its gated code is - kept, but a missing module behind it is not fatal, and a candidate under it - is never taken as certain. Resolver 1 unifies it all, every table included. - The resolver is the workspace's: its `resolver`, else its edition. - Each dependency table is a dependency of its own, as cargo has it: `rand = - "0.7"` under `[dependencies]` beside `rand = "0.8"` under - `[build-dependencies]` (or a `[dev-dependencies] fake = { package = "rand", - version = "0.8" }`) is two crates, and a file is followed into the one its - target uses -- the build script into the build-dependency, a test target into - the normal or dev one, anything else into the normal one. - The crate a versioned dependency resolves to comes from `Cargo.lock`: a - version it lists for the crate that the requirement allows, so two - vendored versions of one crate each get their own features and edges; a lock - that lists only versions the requirement doesn't allow is out of date, and the - loader says so and goes by the requirement, as when there is no lock entry: - the one vendored version the requirement allows. A `[patch]` -- in the root - manifest, or in the cargo config of a directory from the package's up to the - bundle root (nearest first, ahead of the manifest's, its `path` relative to - the directory holding `.cargo`) -- replaces a crate only where its version - fits the dependent's requirement, as cargo applies it; one that doesn't is - reported and not used. - When the locked version isn't vendored, or no vendored version fits, or - several do and no lock says which, the loader warns and doesn't follow the - dependency rather than guess: cargo would build none of them from what the - bundle holds. A - package the resolved build doesn't pull in has unknown features, and its gated - code is kept. The manifests, the lock and the cargo config are read with - `@preventive/lockfile`'s strict TOML parser (so is a `foundry.toml`): a - `Cargo.toml`, `Cargo.lock` or cargo config the loader reads that exists but - isn't TOML stops the build, naming the file and line, and so does TOML those - files are never written in (a local date, a byte order mark, U+FFFD where - bytes weren't UTF-8). +- `feature = "…"` is decided per crate from **Cargo feature resolution**: what + `cargo build` of the entries' packages turns on. The loader reads the + `Cargo.toml` of every package in-tree (the root package or workspace, `path` + dependencies, `vendor/`), the `Cargo.lock` of the entries' workspace (beside + its root manifest, below the bundle root too; a vendored crate's own published + lock plays no part) and the `[patch]` of the cargo config with + `@preventive/lockfile`'s Cargo reader, which reads them as cargo does and + refuses what cargo would refuse or what it can't tell how cargo reads: a + `Cargo.toml` with a key cargo doesn't know, a feature naming nothing, a + `dep?/x` of a dependency no table makes optional, `[replace]`; a `Cargo.lock` + older than version 3 (no `version`), or one that could be read two ways. Any + of those stops the build, naming the file -- and so does text that isn't TOML, + naming the line, or TOML those files are never written in (a local date, a + byte order mark, U+FFFD where bytes weren't UTF-8); so is a `foundry.toml`. + - **Cargo's resolver**, where the loader holds all it takes: the lockfile, + `--cargo-target`, every package the lockfile has in-tree (each path package + inside the bundle root, every other one vendored with its + `.cargo-checksum.json`) and entries whose packages are workspace members. + `@preventive/lockfile` lays the lockfile's graph over the manifests and + turns on the features the build does, by cargo's rules throughout: each + dependency table its own source (a git dependency is the git checkout, a + registry one the registry's copy, whatever their versions), a `[patch]` from + the root manifest or the cargo config, proc-macro crates and + build-dependencies built for the host, `--cargo-features` handed out among + the packages as cargo hands them out. A lockfile out of date with the + manifests, a vendored copy whose checksum isn't the lockfile's, a feature + asked of a package that hasn't it: each stops the build. A host the loader + doesn't know (a target that isn't `host`) is resolved both ways: what the + host's target-specific tables turn on is on *maybe*. + - **The replay**, otherwise -- no lockfile, no target, a locked package not + vendored: the roots start from their `default` feature (a proc-macro crate + is resolved for the host, where cargo builds it), features imply features + (`std = ["alloc", "dep:serde", "serde?/std"]`; `serde/derive` turns on the + package's feature `serde` too, written or the one an optional dependency + gets), enable optional dependencies and request dependency features, and + every active dependency gets `default` plus what its dependents ask for, to + a fixed point. A dependency's own dev-dependencies are nobody's build and + never count (sha2's `[dev-dependencies] digest = { features = ["dev"] }` + doesn't turn on digest's `dev`); the entries' packages' dev-dependencies + count under resolver 1 only, since resolver 2 (edition 2021+, or `resolver = + "2"`) keeps them out of a normal build. Resolver 2 also resolves what is + built for the host -- build-dependencies, proc-macro crates and what they + depend on -- apart from what is built for the target: a feature a + build-dependency asks of a crate isn't on in that crate as a normal + dependency. And it decides target-specific dependency tables: one counts + when `--cargo-target` says it applies (a build-dependency's table against + the host), and not when it says it doesn't; without a target, what only such + a table enables -- a dependency, a feature -- is on *maybe*: its gated code + is kept, but a missing module behind it is not fatal, and a candidate under + it is never taken as certain. Resolver 1 unifies it all, every table + included. The resolver is the workspace's: its `resolver`, else its + edition. Each dependency table is a dependency of its own, as cargo has it: + `rand = "0.7"` under `[dependencies]` beside `rand = "0.8"` under + `[build-dependencies]` (or a `[dev-dependencies] fake = { package = "rand", + version = "0.8" }`) is two crates, and a file is followed into the one its + target uses -- the build script into the build-dependency, a test target + into the normal or dev one, anything else into the normal one. The vendored + copy a dependency resolves to is from where the dependency says: a git + dependency's (or one a git `[patch]` replaces) a git checkout's -- a copy + whose `.cargo-checksum.json` has no package checksum -- any other's a + registry's; of those, the version `Cargo.lock` lists for the crate from that + source that the requirement allows, so two vendored versions of one crate + each get their own features and edges; a lock that lists only versions the + requirement doesn't allow is out of date, and the loader says so and goes by + the requirement, as when there is no lock entry: the one vendored version + the requirement allows. Requirements are read by the semver crate's rules + (`@preventive/lockfile`'s `rust-semver.js`): `1` takes no prerelease, `=2.0.0-rc.1` + takes that one. A `[patch]` -- in the root manifest, or in the cargo config + of a directory from the package's up to the bundle root (nearest first, + ahead of the manifest's, its `path` relative to the directory holding + `.cargo`) -- replaces a crate only where its version fits the dependent's + requirement, as cargo applies it; one that doesn't is reported and not used. + When the locked version isn't vendored, or no vendored version fits, or + several do and no lock says which, the loader warns and doesn't follow the + dependency rather than guess: cargo would build none of them from what the + bundle holds. + + A package the resolved build doesn't pull in has unknown features, and its + gated code is kept. - With `--cargo-target=`, **target cfgs** are decided too: the loader asks `rustc --print cfg --target ` (`host`: the running @@ -769,7 +798,8 @@ test code reaches for. Two kinds of cfg are decided: rustup one. With `--cargo`, the same triple goes to `cargo metadata --filter-platform`. -`all(…)`/`any(…)`/`not(…)` compose; a predicate that reduces to true (`not(test)`, +`all(…)`/`any(…)`/`not(…)` compose, and `true` and `false` are what they say (a +raw `r#name` is the name); a predicate that reduces to true (`not(test)`, an enabled feature) is as firm as no cfg, so a missing module behind it is fatal. A leaf's value is taken as written, spacing outside its quotes aside: `my = "a b"` and `my = "ab"` (or `"a b"`) are different values. @@ -782,7 +812,7 @@ Without a target, target cfgs stay undecided and their code is kept. A gates nothing. `stasis bundle --cargo` takes the dependency graph and features from -`cargo metadata` instead of replaying the manifests. It is opt-in because it runs +`cargo metadata` instead of resolving them itself. It is opt-in because it runs cargo: nothing is compiled and no build script runs, but cargo reads the project's `.cargo/config.toml` (which can point `build.rustc` or a wrapper at any executable), may refresh the registry index, and writes `Cargo.lock` when there is @@ -795,7 +825,7 @@ feature set per package — the union across normal, dev and build dependency ki and across platforms (resolver-1-style unification). So `--cargo` describes everything cargo would ever compile for the workspace, tests included, and can enable features (and so bundle modules) that a plain `cargo build` of the entries' -packages leaves off; the manifest replay describes that build. Set +packages leaves off; the loader's own resolution describes that build. Set `EXODUS_STASIS_DEBUG=1` to have `stasis bundle` print the resolved features per package, in either mode. diff --git a/stasis/bin/stasis.js b/stasis/bin/stasis.js index 41b71523..5fdaf4b6 100755 --- a/stasis/bin/stasis.js +++ b/stasis/bin/stasis.js @@ -43,7 +43,9 @@ function usage(prefix = '') { stasis bundle [--add] [--output=(path|-)] path/to/file.(sh|bash) ... stasis bundle [--cargo] [--cargo-features=a,b,pkg/c] [--cargo-no-default-features] [--cargo-all-features] [--cargo-target=(triple|host)] [--cargo-manifests] [--add] [--output=(path|-)] path/to/file.rs ... (Rust: each crate's Cargo features are resolved from Cargo.toml/Cargo.lock like "cargo build" of the - entries' packages, so #[cfg(feature = ...)] code that is off stays out; --cargo takes the resolution + entries' packages, so #[cfg(feature = ...)] code that is off stays out -- by cargo's own resolver + when there is a Cargo.lock (version 3 or 4; an older one stops the build), --cargo-target and every + locked package in-tree, by replaying the manifests otherwise; --cargo takes the resolution from "cargo metadata" instead -- it runs cargo, so only on a project you trust; the --cargo-* feature flags are cargo's --features / --no-default-features / --all-features for those packages; --cargo-target asks rustc for the target's cfgs so #[cfg(unix)]-style code for other targets stays diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index 5295b97e..f2540c2b 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -1,7 +1,9 @@ -// Cargo manifests for the Rust loader: a Cargo.toml / Cargo.lock reader (over the TOML reader in -// toml.js), per-bundle package lookup, dependency resolution among in-tree crates (the package's -// own lib, workspace `path` deps, `cargo vendor`ed registry crates) and feature resolution done the -// way `cargo build` does it, so `#[cfg(feature = "…")]` can be decided per crate. +// Cargo manifests for the Rust loader: Cargo.toml, Cargo.lock and cargo's configuration read by +// @preventive/lockfile, per-bundle package lookup, dependency resolution among in-tree crates (the +// package's own lib, workspace `path` deps, `cargo vendor`ed registry crates) and feature +// resolution done the way `cargo build` does it, so `#[cfg(feature = "…")]` can be decided per +// crate: @preventive/lockfile's, cargo's own resolver, where the build's lockfile and target are +// known; a replay of the manifests otherwise. import { spawnSync } from 'node:child_process' import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' @@ -10,7 +12,10 @@ import { dirname, isAbsolute, join, parse, posix, relative, resolve, sep } from import { toPosix } from '@exodus/stasis-core/util' -import { isTomlTable, readToml, splitTopLevel } from './toml.js' +import { LockfileError, linkCargo, parseCargoConfig, parseCargoLock, parseCargoManifest as readCargoManifest, readCargoVendor, resolveCargoFeatures } from '@preventive/lockfile/cargo.js' +import { matches, parseVersion, parseVersionReq } from '@preventive/lockfile/rust-semver.js' + +import { TomlError, isTomlTable, readToml, splitTopLevel } from './toml.js' // `cargo vendor` copies registry crates in-tree under this dir. export const VENDOR_DIR = 'vendor' @@ -46,7 +51,6 @@ export function normalizeRel(dir, sub) { // --- Cargo.toml ----------------------------------------------------------------------- -const DEP_KINDS = new Map([['dependencies', 'normal'], ['dev-dependencies', 'dev'], ['build-dependencies', 'build']]) // A request's kind (`normal`/`dev`/`build`), a target-specific table's `@` suffix dropped. const kindOf = (request) => request.split('@')[0] @@ -54,13 +58,6 @@ const kindOf = (request) => request.split('@')[0] const DEP_IMPLICATION_RE = /^dep:(.+)$/u const DEP_FEATURE_RE = /^([^/?]+)(\?)?\/(.+)$/u -// What one dependency table says of a crate: which crate (`version`, `path`, `package`, or -// `workspace = true` to inherit those), and what it asks of it. Each table is a dependency of its -// own: `[dependencies] rand = "0.7"` beside `[build-dependencies] rand = "0.8"` are two crates. -// `defaultFeatures` is tri-state: null until the table says (an inherited `workspace = true` entry -// can only turn defaults on, not off). -const newRequest = () => ({ version: null, path: null, package: null, workspace: false, optional: false, defaultFeatures: null, features: [] }) - // `--features a,b pkg/c` (cargo's syntax: repeatable, comma- or space-separated) → the list of names. export function parseFeatureList(values) { return [...new Set(values.flatMap((s) => s.split(/[\s,]+/u)).map((s) => s.trim()).filter(Boolean))] @@ -74,227 +71,79 @@ export function isTestTargetPath(pkgDir, fileRel) { return first === 'tests' || first === 'benches' } -// Minimal Cargo.toml reader covering what crate and feature resolution and bucketing need: the -// package identity (name, version, edition, resolver), the lib target, every dependency table -// (kind, `path`/`version`/`package`/`workspace`, `optional`, `default-features`, `features`), -// `[features]`, `[patch.*]` path overrides and the workspace tables members inherit from. -// Dependency keys are normalized to the `use` spelling (`-` → `_`); feature names keep theirs. -// It reads the parsed table tree, so `[dependencies.foo] features = […]`, `[dependencies] -// foo.features = […]` and `foo = { features = […] }` are one thing. Throws a TomlError naming -// `file` on text that isn't TOML. -export function parseCargoManifest(text, file = null) { - const manifest = { - package: null, // { name, version, versionFromWorkspace, edition, editionFromWorkspace, build } - resolver: null, // "1" | "2" | "3" from [workspace] or [package] - lib: { name: null, path: null, procMacro: false }, // `procMacro`: `[lib] proc-macro = true`, a crate compiled for the host - features: new Map(), // name -> implied entries (`other`, `dep:key`, `key/feat`, `key?/feat`) - // key -> { key, name, kinds: Map }: - // each dependency table is a request of its own, for the crate it names -- sha2's - // `[dependencies] digest = "0.10"` and `[dev-dependencies] digest = { features = ["dev"] }` are - // two requests, and only the first is part of a build of sha2's dependents. A request is the - // table's kind (`normal`, `dev`, `build`), with `@` appended for a target-specific table: - // `[target.'cfg(unix)'.dependencies] foo = { optional = true }` beside `[dependencies] foo = "1"` - // is a second request of `foo`, not a change to the first. - deps: new Map(), - workspaceDeps: new Map(), // key -> { key, name, version, path, package, optional, defaultFeatures, features } - workspacePackage: { version: null, edition: null }, - patches: new Map(), // crate -> path - isWorkspace: false, - } - // `key` is the `use` spelling (`-` → `_`); `name` the manifest's, which is also the implicit - // feature an optional dependency defines (`#[cfg(feature = "proc-macro-crate")]`). - const depOf = (map, name, { flat }) => { - const key = normName(name) - if (!map.has(key)) { - map.set(key, { key, name, ...(flat ? newRequest() : { kinds: new Map() }) }) +// `read()`, with a TomlError or LockfileError it throws naming `file`: what @preventive/lockfile +// refuses stops the build, saying which file. +function readNamed(file, read) { + try { + return read() + } catch (err) { + if (file === null) throw err + if (err instanceof TomlError) { + const named = new TomlError(`${file}: ${err.message}`) + named.line = err.line + throw named } - return map.get(key) - } - // Apply one dependency's spec to the entry for `kind` (or to the record itself for a flat one). - const setDepFields = (dep, spec, kind) => { - const request = kind === null ? dep : (dep.kinds.get(kind) ?? dep.kinds.set(kind, newRequest()).get(kind)) - if (typeof spec === 'string') { - request.version = spec // `foo = "1.2"`: a registry dep - return + if (err instanceof LockfileError) { + const named = new LockfileError(`${file}: ${err.message}`) + named.where = err.where + throw named } - if (!isTomlTable(spec)) return - if (typeof spec.version === 'string') request.version = spec.version - if (typeof spec.path === 'string') request.path = spec.path - if (typeof spec.package === 'string') request.package = spec.package - if (spec.workspace === true) request.workspace = true - if (spec.optional === true) request.optional = true - const defaults = spec['default-features'] ?? spec.default_features - if (defaults === true || defaults === false) request.defaultFeatures = defaults - if (Array.isArray(spec.features)) request.features = [...new Set([...request.features, ...spec.features.filter((f) => typeof f === 'string')])] + throw err } +} + +// A Cargo.toml as the loader reads it: @preventive/lockfile's reading (`cargo`) -- the package, its +// features (with the one each optional dependency no `dep:` names turns on) and every dependency +// table, what a member inherits from its workspace applied, refused where cargo would refuse it or +// the reader can't tell how cargo reads it -- plus what that reading leaves out: the lib target +// (`[lib] name`, `path`) and the build script (`[package] build`). `workspace` is the workspace +// root's, read before, for a member that inherits from it. Dependency keys are the `use` spelling +// (`-` → `_`); each table is a request of its own, `` or `@` (`normal`, +// `build`, `dev@cfg(windows)`): `[dependencies] rand = "0.7"` beside `[build-dependencies] rand = +// "0.8"` are two crates, and sha2's dev-dependency on digest asks digest for nothing a build of +// sha2's dependents sees. Throws a TomlError or LockfileError naming `file`. +export function parseCargoManifest(text, file = null, workspace = null) { + const root = workspace?.cargo.workspace === undefined ? undefined : workspace.cargo const doc = readToml(text, file) - const str = (v) => (typeof v === 'string' ? v : null) + const cargo = readNamed(file, () => readCargoManifest(text, isTomlTable(doc.workspace) ? undefined : root)) + const pkg = cargo.package const table = (v) => (isTomlTable(v) ? v : null) - // `version = { workspace = true }` or `version.workspace = true`: inherited, as `edition` may be. - const inherited = (v) => table(v)?.workspace === true - const pkg = table(doc.package) - if (pkg !== null && typeof pkg.name === 'string') { - manifest.package = { - name: pkg.name, - version: str(pkg.version), - versionFromWorkspace: inherited(pkg.version), - edition: str(pkg.edition), - editionFromWorkspace: inherited(pkg.edition), - // the build script: a path, `false` for none, null to look for `build.rs` - build: typeof pkg.build === 'string' || pkg.build === false ? pkg.build : null, - } - } - const ws = table(doc.workspace) - // A `[workspace]` table, however it is spelled out, makes this a workspace root. - manifest.isWorkspace = 'workspace' in doc - manifest.resolver = str(ws?.resolver) ?? str(pkg?.resolver) - const wsPackage = table(ws?.package) - manifest.workspacePackage = { version: str(wsPackage?.version), edition: str(wsPackage?.edition) } const lib = table(doc.lib) - manifest.lib = { name: str(lib?.name), path: str(lib?.path), procMacro: lib?.['proc-macro'] === true || lib?.proc_macro === true } - for (const [name, list] of Object.entries(table(doc.features) ?? {})) { - if (Array.isArray(list)) manifest.features.set(name, list.filter((v) => typeof v === 'string')) - } - // `[patch.] crate = { path = "…" }`, however it is spelled out. - for (const registry of Object.values(table(doc.patch) ?? {})) { - for (const [crate, spec] of Object.entries(table(registry) ?? {})) { - const patch = str(table(spec)?.path) - if (patch !== null) manifest.patches.set(normName(crate), patch) - } - } - // `[dependencies]`, `[dev-dependencies]`, `[build-dependencies]`, under `target.` (a - // request of its own, `@`) or `workspace`, in the order the manifest has them: each - // dependency a version string or a table. - const depTables = (scope, into, flat, cfg = null) => { - for (const [key, value] of Object.entries(scope)) { - if (key === 'target' && cfg === null) { - for (const [platform, tables] of Object.entries(table(value) ?? {})) depTables(table(tables) ?? {}, into, flat, platform) - continue - } - const kind = DEP_KINDS.get(key) - if (kind === undefined) continue - const request = flat ? null : (cfg === null ? kind : `${kind}@${cfg}`) - for (const [depName, spec] of Object.entries(table(value) ?? {})) setDepFields(depOf(into, depName, { flat }), spec, request) - } - } - depTables(doc, manifest.deps, false) - if (ws !== null) depTables(ws, manifest.workspaceDeps, true) - return manifest -} - -// --- Cargo.lock ----------------------------------------------------------------------- - -// `Cargo.lock` → `{ byId: Map<"name version", { name, version, deps: [{ name, version }] }>, -// byName: Map }`, or null for no text. The lock is what says which of several -// vendored versions of a crate a given package depends on. Throws a TomlError naming `file` on -// text that isn't TOML. -export function parseCargoLock(text, file = null) { - if (text === null) return null - // A dependency is `"name"`, or `"name version"` when several versions of it are locked. - const dep = (s) => { - const [name, version] = s.split(' ') - return { name: normName(name), version: version ?? null } - } - const packages = [] - const locked = readToml(text, file).package - for (const p of Array.isArray(locked) ? locked : []) { - if (!isTomlTable(p)) continue - packages.push({ - name: typeof p.name === 'string' ? normName(p.name) : null, - version: typeof p.version === 'string' ? p.version : null, - deps: Array.isArray(p.dependencies) ? p.dependencies.filter((d) => typeof d === 'string').map(dep) : [], + const build = table(doc.package ?? doc.project)?.build + const deps = new Map() + for (const d of pkg?.dependencies ?? []) { + const key = normName(d.name) + if (!deps.has(key)) deps.set(key, { key, name: d.name, kinds: new Map() }) + deps.get(key).kinds.set(d.target === undefined ? d.kind : `${d.kind}@${d.target}`, { + version: d.version ?? null, + path: d.source.type === 'path' ? d.source.path : null, + source: d.source.type, + package: d.package, + renamed: d.package !== d.name, + inherited: d.inherited, + optional: d.optional, + defaultFeatures: d.defaultFeatures, + features: d.features, }) } - const byId = new Map() - const byName = new Map() - for (const p of packages) { - if (!p.name || !p.version) continue - byId.set(`${p.name} ${p.version}`, p) - if (!byName.has(p.name)) byName.set(p.name, []) - byName.get(p.name).push(p) - } - return { byId, byName } -} - -// --- Versions ------------------------------------------------------------------------- - -// `1.2.3-beta.1+build` → { parts: [1, 2, 3], pre: 'beta.1' }; a partial `1.2` keeps null for the -// missing parts (a requirement's precision matters). Null when it isn't a version. -function parseVersion(text) { - const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/u.exec(text.trim()) - if (!m) return null - return { parts: [Number(m[1]), m[2] === undefined ? null : Number(m[2]), m[3] === undefined ? null : Number(m[3])], pre: m[4] ?? null } -} - -// Semver order; a prerelease sorts below its release. -function compareVersions(a, b) { - for (let i = 0; i < 3; i++) { - const d = (a.parts[i] ?? 0) - (b.parts[i] ?? 0) - if (d !== 0) return d - } - if (a.pre === b.pre) return 0 - if (a.pre === null) return 1 - if (b.pre === null) return -1 - return a.pre < b.pre ? -1 : 1 -} - -// Descending, for picking the newest of several versions; unparsable strings sort last. -function compareVersionsDesc(a, b) { - const va = parseVersion(a) - const vb = parseVersion(b) - if (va && vb) return compareVersions(vb, va) - if (va) return -1 - if (vb) return 1 - return a < b ? 1 : (a > b ? -1 : 0) -} - -// One comparator of a Cargo requirement against a version: caret (the default -- `1.2` is -// `>=1.2.0, <2.0.0`; `0.9` is `>=0.9.0, <0.10.0`; `0.0.3` is `<0.0.4`), tilde, wildcard (`1.*`), -// `=` (partial `=1.2` covers the minor), and the comparison operators (a partial `>1.2` means -// `>=1.3.0`, `<=1.2` means `<1.3.0`: the whole minor is inside). -function satisfiesComparator(v, comparator) { - const c = comparator.trim() - if (c === '*' || c === '') return true - const m = /^(\^|~|=|>=|>|<=|<)?\s*(.+)$/u.exec(c) - let op = m[1] ?? '^' - let text = m[2] - const wild = /^(\d+)(?:\.(\d+))?\.[*xX]$/u.exec(text) - if (wild) { - // A bare wildcard is a tilde range; with an operator the `*` just omits the component (`>=1.*` is `>=1`). - if (m[1] === undefined) op = '~' - text = wild[2] === undefined ? wild[1] : `${wild[1]}.${wild[2]}` - } - const r = parseVersion(text) - if (!r) return false - const [ma, mi, pa] = r.parts - const lower = { parts: [ma, mi ?? 0, pa ?? 0], pre: r.pre } - const upper = (a, b, p) => ({ parts: [a, b, p], pre: null }) - const within = (hi) => compareVersions(v, lower) >= 0 && compareVersions(v, hi) < 0 - // The version just past a partial requirement's range (`1.2` → 1.3.0, `1` → 2.0.0); null when full. - const past = pa !== null ? null : (mi === null ? upper(ma + 1, 0, 0) : upper(ma, mi + 1, 0)) - switch (op) { - case '=': - if (past === null) return compareVersions(v, lower) === 0 - return within(past) - case '>': return past === null ? compareVersions(v, lower) > 0 : compareVersions(v, past) >= 0 - case '>=': return compareVersions(v, lower) >= 0 - case '<': return compareVersions(v, lower) < 0 - case '<=': return past === null ? compareVersions(v, lower) <= 0 : compareVersions(v, past) < 0 - case '~': return within(mi === null ? upper(ma + 1, 0, 0) : upper(ma, mi + 1, 0)) - default: // caret: the leftmost non-zero part may not change - if (ma > 0) return within(upper(ma + 1, 0, 0)) - if (mi === null) return within(upper(1, 0, 0)) - if (mi > 0 || pa === null) return within(upper(0, mi + 1, 0)) - return within(upper(0, 0, pa + 1)) + return { + cargo, + package: pkg === undefined ? null : { name: pkg.name, version: pkg.version, edition: pkg.edition, build: typeof build === 'string' || build === false ? build : null }, + resolver: cargo.workspace?.resolver ?? pkg?.resolver ?? null, + isWorkspace: cargo.workspace !== undefined, + lib: { name: typeof lib?.name === 'string' ? lib.name : null, path: typeof lib?.path === 'string' ? lib.path : null, procMacro: pkg?.procMacro === true }, + features: new Map(Object.entries(pkg?.features ?? {})), + deps, } } -// Whether `version` satisfies a Cargo version requirement (`"0.9"`, `"^1.2"`, `"~1.2.3"`, -// `"=1.0.0"`, `">=1, <2"`, `"1.*"`, `"*"`). -export function satisfiesCargoReq(version, req) { +// Whether `version` satisfies the Cargo requirement `req`, by the semver crate's rules +// (@preventive/lockfile's rust-semver.js): a prerelease only where the requirement names one of +// its major, minor and patch. +const satisfies = (version, req) => { const v = parseVersion(version) - if (!v) return false - return req.split(',').every((c) => satisfiesComparator(v, c)) + const comparators = parseVersionReq(req) + return v !== undefined && comparators !== undefined && matches(comparators, v) } // --- cargo metadata ------------------------------------------------------------------- @@ -308,7 +157,7 @@ export function findCargoLock(baseDir) { for (let dir = baseDir; ;) { const file = join(dir, 'Cargo.toml') const text = readFileOrNull(file) - if (text !== null && parseCargoManifest(text, file).isWorkspace) { + if (text !== null && 'workspace' in readToml(text, file)) { root = dir break } @@ -326,9 +175,9 @@ export function findCargoLock(baseDir) { // `my = "a b"` are two values. export const normalizeCfg = (pred) => pred.replaceAll(/"(?:[^"\\]|\\.)*"|\s+/gu, (m) => (m.startsWith('"') ? m : ' ')).trim() -const FEATURE_CFG_RE = /^feature\s*=\s*"([^"]*)"$/u -// Any other leaf: `unix`, `target_os = "linux"`. -const CFG_LEAF_RE = /^([A-Za-z_]\w*)\s*(?:=\s*"([^"]*)")?$/u +const FEATURE_CFG_RE = /^(?:r#)?feature\s*=\s*"([^"]*)"$/u +// Any other leaf: `unix`, `target_os = "linux"`, a raw identifier's `r#` dropped. +const CFG_LEAF_RE = /^(?:r#)?([A-Za-z_]\w*)\s*(?:=\s*"([^"]*)")?$/u // The cfg keys rustc sets from the target alone (`rustc --print cfg --target `), decided by // membership in that set when the target is known. Not among them: `debug_assertions`, `panic`, @@ -388,8 +237,10 @@ function evalCfgWith(pred, env, assume, free = null) { return args.includes(true) ? true : (args.every((a) => a === false) ? false : null) } function evalCfgLeaf(p, env) { - if (p === 'test') return env.test === true - if (p === 'doctest' || p === 'doc') return false + if (p === 'true' || p === 'false') return p === 'true' // the literals (rustc 1.88); `r#true` is a name + const name = p.startsWith('r#') ? p.slice(2) : p + if (name === 'test') return env.test === true + if (name === 'doctest' || name === 'doc') return false const feature = FEATURE_CFG_RE.exec(p) if (feature) return env.features ? (env.features.has(feature[1]) ? true : (env.maybeFeatures?.has(feature[1]) ? null : false)) : null const leaf = CFG_LEAF_RE.exec(p) @@ -615,6 +466,30 @@ function cfgsPrinted(texts) { return { names, any } } +// --- An undecided host ---------------------------------------------------------------- + +// The stand-in platforms an undecided host is resolved against (hostUndecided): the target's cfgs +// carry a mark of the target and of its name, the host's only a mark of the host. +const MARK_TARGET = '__stasis_target' +const MARK_HOST = '__stasis_host' +const MARK_NAME = '__stasis_target_name' +// `graph` (linkCargo's) with every target-specific dependency's platform holding on the target as +// written and on the host as `on` says -- nowhere, for the features on for certain, or everywhere, +// for all that may be: cargo's resolver takes a host it knows, and the machine that builds need not +// be the one that bundles. +function hostUndecided(graph, on) { + const packages = Object.create(null) + for (const [key, pkg] of Object.entries(graph.packages)) { + const dependencies = pkg.dependencies.map((d) => { + if (d.target === undefined) return d + const onTarget = /^cfg\((.*)\)$/su.exec(d.target)?.[1] ?? `${MARK_NAME} = "${d.target}"` + return { ...d, target: `cfg(any(all(${MARK_TARGET}, ${onTarget}), all(${MARK_HOST}, ${on})))` } + }) + packages[key] = { ...pkg, dependencies } + } + return { ...graph, packages } +} + // --- Context ---------------------------------------------------------------------------- // No custom cfg a build sets (see cfgsSetFor). @@ -635,21 +510,47 @@ export const unitPlatform = (unit) => unit.slice(unit.indexOf('|') + 1) // in-tree sources, and feature resolution for the packages owning `entries` (the crate roots being // bundled, like `cargo build -p …`). `features` / `noDefaultFeatures` / `allFeatures` mirror // cargo's flags for those root packages (`pkg/feat` targets one of them). `cargo: true` takes the -// dependency graph and features from `cargo metadata` (see runCargoMetadata) instead of replaying -// the manifests. `target` names the build's target -- a triple or `host`, asked of rustc -// (rustcTargetCfgs), or its `{ triple, cfgs }` outright -- so target-specific dependency tables -// and `#[cfg(unix)]`-style code are decided; without one they are kept. `baseDir` is the bundle -// root; every path in and out is project-relative POSIX. -export function createCargoContext(baseDir, { entries = [], features = [], noDefaultFeatures = false, allFeatures = false, cargo = false, target = null } = {}) { +// dependency graph and features from `cargo metadata` (see runCargoMetadata) instead. `target` +// names the build's target -- a triple or `host`, asked of rustc (rustcTargetCfgs), or its `{ +// triple, cfgs }` outright -- so target-specific dependency tables and `#[cfg(unix)]`-style code +// are decided; without one they are kept. The host's platform is known when the target is the +// host, or given as `host` (`{ triple, cfgs }`). `baseDir` is the bundle root; every path in and +// out is project-relative POSIX. +export function createCargoContext(baseDir, { entries = [], features = [], noDefaultFeatures = false, allFeatures = false, cargo = false, target = null, host = null } = {}) { const targetInfo = target === null ? null : (typeof target === 'string' ? rustcTargetCfgs(target, baseDir) : target) - // The host's platform, known when the target is the host. - const hostInfo = target === 'host' ? targetInfo : null + const hostInfo = target === 'host' ? targetInfo : host const manifests = new Map() - const readManifest = (dir) => { - if (!manifests.has(dir)) { + const tables = new Map() // dir -> the manifest's table tree, or null (rootOf's look for [workspace]) + const tableOf = (dir) => { + if (!tables.has(dir)) { const file = posix.join(dir, 'Cargo.toml') const text = readFileOrNull(join(baseDir, file)) - manifests.set(dir, text === null ? null : { dir, ...parseCargoManifest(text, file) }) + tables.set(dir, text === null ? null : { file, text, doc: readToml(text, file) }) + } + return tables.get(dir) + } + // The workspace root a manifest inherits from: its `[package] workspace` path, else the nearest + // manifest above it with a [workspace] table, inside the bundle root -- null for a root, a + // vendored crate (a published manifest inherits nothing) or a package outside any workspace. + const rootOf = (dir, doc) => { + if (isTomlTable(doc.workspace) || isVendoredDir(dir)) return null + const named = (isTomlTable(doc.package) ? doc.package : null)?.workspace + if (typeof named === 'string') { + const rel = normalizeRel(dir, named) + return rel === null ? null : rel + } + for (let d = dir; d !== '.' && d !== '';) { + d = posix.dirname(d) + if (isTomlTable(tableOf(d)?.doc.workspace)) return d + } + return null + } + const readManifest = (dir) => { + if (!manifests.has(dir)) { + const raw = tableOf(dir) + const root = raw === null ? null : rootOf(dir, raw.doc) + const workspace = root === null ? null : readManifest(root) + manifests.set(dir, raw === null ? null : { dir, ...parseCargoManifest(raw.text, raw.file, workspace) }) } return manifests.get(dir) } @@ -699,17 +600,16 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const libName = (m) => normName(m.lib.name ?? m.package?.name ?? '') // Whether `fileRel` belongs to a test or bench target of its package (compiled with `cfg(test)`). const isTestTarget = (fileRel) => isTestTargetPath(packageFor(fileRel)?.dir ?? '.', fileRel) - const version = (m) => { - if (m.package.versionFromWorkspace) return workspaceFor(m.dir)?.workspacePackage.version ?? '0.0.0' - return m.package.version ?? '0.0.0' - } + const version = (m) => m.package.version // --- dependency resolution - // The vendored crates, `//`, as `[{ version, dir }]`, indexed `byName` (normalized - // package name; the dir may hyphenate a snake_case name, and an older duplicate version lives in - // `-/`) and `byLib` (lib name, for the crates whose `[lib] name` differs: - // `md-5` → `md5`). + // The vendored crates, `//`, as `[{ version, dir, git }]`, indexed `byName` + // (normalized package name; the dir may hyphenate a snake_case name, and an older duplicate + // version lives in `-/`) and `byLib` (lib name, for the crates whose `[lib] name` + // differs: `md-5` → `md5`). `git`: whether the copy is of a git repository -- its + // `.cargo-checksum.json` has no package checksum, which a registry's always does -- or null + // without that file. const vendorDir = vendorDirOf(baseDir) // Whether a package directory is a vendored crate's (a registry crate `cargo vendor` copied in). const isVendoredDir = (dir) => dir !== vendorDir && dir.startsWith(`${vendorDir}/`) @@ -722,9 +622,11 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef dirs = readdirSync(join(baseDir, vendorDir)) } catch { /* no vendor dir */ } for (const d of dirs) { + if (d.startsWith('.')) continue // a directory source reads none of these const m = readManifest(`${vendorDir}/${d}`) if (!m?.package) continue - const entry = { version: version(m), dir: m.dir } + const sums = checksumOf(m.dir) + const entry = { version: version(m), dir: m.dir, git: sums === null ? null : sums.package === null } for (const [index, key] of [[vendorIndex.byName, normName(m.package.name)], [vendorIndex.byLib, libName(m)]]) { if (!index.has(key)) index.set(key, []) index.get(key).push(entry) @@ -733,9 +635,53 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return vendorIndex } + // A vendored package's `.cargo-checksum.json`, parsed, or null when it has none. + const checksums = new Map() + const checksumOf = (dir) => { + if (!checksums.has(dir)) { + const file = posix.join(dir, '.cargo-checksum.json') + const text = readFileOrNull(join(baseDir, file)) + let value = null + if (text !== null) { + try { + value = JSON.parse(text) + } catch (err) { + throw new Error(`${file}: not JSON: ${err.message}`, { cause: err }) + } + } + checksums.set(dir, value === null ? null : { text, package: value?.package ?? null }) + } + return checksums.get(dir) + } + // The build's workspace root: that of the first entry's package that isn't vendored (else the + // package itself, outside any workspace), else the bundle root's manifest; its Cargo.lock is the + // build's, as cargo finds it. A vendored crate's own published lock plays no part. + const buildRoot = () => { + const pkg = entries.map(packageFor).find((m) => m && !isVendoredDir(m.dir)) + return pkg ? (workspaceFor(pkg.dir) ?? pkg) : readManifest('.') + } + // The build's Cargo.lock, read by @preventive/lockfile (`version = 3` or `4`; an older one, or a + // lock that isn't what cargo writes or could be read two ways, stops the build), as `{ file, + // text, lock, byId }` (`byId`: `name version` → the keys of that version's packages); null when + // there is none. let lock const lockfile = () => { - if (lock === undefined) lock = parseCargoLock(readFileOrNull(join(baseDir, 'Cargo.lock')), 'Cargo.lock') + if (lock === undefined) { + const root = buildRoot() + const file = posix.join(root?.dir ?? '.', 'Cargo.lock') + const text = readFileOrNull(join(baseDir, file)) + lock = null + if (text !== null) { + const parsed = readNamed(file, () => parseCargoLock(text)) + const byId = new Map() + for (const [key, p] of Object.entries(parsed.packages)) { + const id = `${normName(p.name)} ${p.version}` + if (!byId.has(id)) byId.set(id, []) + byId.get(id).push(key) + } + lock = { file, lock: parsed, byId } + } + } return lock } // `--cargo`: the graph and features as cargo resolved them, with registry packages it read from @@ -746,71 +692,82 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef }) : null // A dependency as the package sees it through one of its tables (`request`, an entry of - // `dep.kinds`): a `workspace = true` entry merged with the workspace's -- features add up, the - // path is relative to the workspace root, and defaults are the workspace's call (a member's - // `default-features = false` is ignored, with a cargo warning, unless the workspace entry - // disables them too; a member's `true` turns them back on). Null when it can't be resolved. - // `defaultFeatures` comes out boolean. - const depSpec = (m, dep, request) => { - if (!request.workspace) { - return { key: dep.key, version: request.version, path: request.path, package: request.package, defaultFeatures: request.defaultFeatures !== false, features: request.features, relTo: m.dir } - } - const ws = workspaceFor(m.dir) - const base = ws?.workspaceDeps.get(dep.key) - if (!base) return null - return { - key: dep.key, - version: base.version, - path: base.path, - package: base.package, - defaultFeatures: base.defaultFeatures !== false || request.defaultFeatures === true, - features: [...new Set([...base.features, ...request.features])], - relTo: ws.dir, - } - } - // The `[patch]` path for crate `crate` in a build rooted at `root` (the workspace root's manifest, - // else the bundle root's), as `{ dir, from }`: the cargo config's `[patch.]` tables - // first -- of every directory from the root up to the bundle root, the nearest first, each path - // relative to the directory holding its `.cargo` -- then the root manifest's, relative to it. - // `dir` is null for a path outside the bundle root. Undefined when nothing patches the crate. - const configPatchMemo = new Map() - const configPatches = (dir) => { - if (!configPatchMemo.has(dir)) { - const out = new Map() + // `dep.kinds`; what a member inherits from its workspace is already in it): its path relative to + // the manifest's directory, or the workspace root's where the entry is inherited. + const depSpec = (m, dep, request) => ({ + key: dep.key, + version: request.version, + path: request.path, + source: request.source, + package: request.package, + renamed: request.renamed, + defaultFeatures: request.defaultFeatures, + features: request.features, + relTo: request.inherited ? (workspaceFor(m.dir)?.dir ?? m.dir) : m.dir, + }) + // The `[patch]` tables of the cargo config of every directory from `dir` up to the bundle root, + // the nearest first, as @preventive/lockfile reads them, each with the directory its paths are + // relative to (the one holding its `.cargo`); the config's text too, for the reading as cargo + // merges them (the exact resolution). Throws a TomlError or LockfileError naming the config. + const configsMemo = new Map() + const configsFrom = (dir) => { + if (!configsMemo.has(dir)) { + const out = [] for (let d = dir; ; d = posix.dirname(d)) { const rel = cargoConfigIn(baseDir, d) const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) - const patch = text === null ? undefined : readToml(text, rel).patch - for (const registry of Object.values(isTomlTable(patch) ? patch : {})) { - for (const [crate, spec] of Object.entries(isTomlTable(registry) ? registry : {})) { - const key = normName(crate) - if (!out.has(key) && isTomlTable(spec) && typeof spec.path === 'string') out.set(key, { dir: normalizeRel(d, spec.path), from: rel }) - } - } - if (d === '.') break + if (text !== null) out.push({ dir: d, file: rel, text, patch: readNamed(rel, () => parseCargoConfig([text])).patch }) + if (d === '.' || d === '') break } - configPatchMemo.set(dir, out) + configsMemo.set(dir, out) } - return configPatchMemo.get(dir) + return configsMemo.get(dir) } + // The `[patch]` of crate `crate` in a build rooted at `root` (the workspace root's manifest, else + // the bundle root's), as `{ spec, dir, from }`: the cargo config's first, then the root + // manifest's -- `spec` as @preventive/lockfile reads it, `dir` the patch's directory for a path + // (null outside the bundle root), `from` the file saying so. Undefined when nothing patches it. const patchFor = (root, crate) => { - const fromConfig = configPatches(root?.dir ?? '.').get(crate) - if (fromConfig !== undefined) return fromConfig - const path = root?.patches.get(crate) - return path === undefined ? undefined : { dir: normalizeRel(root.dir, path), from: posix.join(root.dir, 'Cargo.toml') } + const patches = [ + ...configsFrom(root?.dir ?? '.').map((c) => ({ patch: c.patch, relTo: c.dir, from: c.file })), + ...(root ? [{ patch: root.cargo.patch, relTo: root.dir, from: posix.join(root.dir, 'Cargo.toml') }] : []), + ] + for (const { patch, relTo, from } of patches) { + for (const specs of Object.values(patch)) { + for (const [name, spec] of Object.entries(specs)) { + if (normName(name) !== crate) continue + return { spec, dir: spec.source.type === 'path' ? normalizeRel(relTo, spec.source.path) : null, from } + } + } + } + return undefined + } + // Where a lockfile's package comes from: `path`, `git` or `registry` (sparse or not). + const sourceKind = (source) => (source === undefined ? 'path' : (source.startsWith('git+') ? 'git' : 'registry')) + // The keys of the build's lockfile that are package `m`: its path package, or for a vendored copy + // the locked package of its name and version from where the copy came from (a git checkout or a + // registry, see vendored). + const lockKeysOf = (lk, m) => { + const keys = lk.byId.get(`${normName(m.package.name)} ${version(m)}`) ?? [] + if (!isVendoredDir(m.dir)) return keys.filter((k) => lk.lock.packages[k].source === undefined) + const sums = checksumOf(m.dir) + const kind = sums === null ? null : (sums.package === null ? 'git' : 'registry') + return keys.filter((k) => lk.lock.packages[k].source !== undefined && (kind === null || sourceKind(lk.lock.packages[k].source) === kind)) } // Dependency → package, memoized per (package, request): the fixed-point loop asks many times. const depTargets = new Map() // The in-tree package a dependency of `m` resolves to through one of its tables (`request`): a // `path` dep, else a `[patch]` path override (patchFor) whose version satisfies the requirement, - // else the vendored crate of that name -- the version Cargo.lock records for `m` when it - // satisfies the requirement, else the one vendored version that does. A package can depend on - // two versions of one crate (`borsh = "1"` beside `borsh0-9 = { package = "borsh", version = - // "0.9" }`, or one per table): the lock then lists both under it, and the requirement tells which - // is which. Null when it isn't in-tree -- a path (or patch) outside the bundle root, or no - // vendored copy -- and, warned, when the vendored copies don't settle it: the locked version - // isn't among them, none satisfies the requirement, or several do and no lock chooses. Cargo - // would build none of those from what the bundle holds, so none is guessed. + // else the vendored crate of that name from where the dependency says (a git dependency, or one a + // git `[patch]` replaces, takes a copy of a git checkout; any other a registry's) -- the version + // Cargo.lock records for `m` from that source when it satisfies the requirement, else the one + // vendored version that does. A package can depend on two versions of one crate (`borsh = "1"` + // beside `borsh0-9 = { package = "borsh", version = "0.9" }`, or one per table): the lock then + // lists both under it, and the requirement tells which is which. Null when it isn't in-tree -- a + // path (or patch) outside the bundle root, or no vendored copy -- and, warned, when the vendored + // copies don't settle it: the locked version isn't among them, none satisfies the requirement, + // or several do and no lock chooses. Cargo would build none of those from what the bundle holds, + // so none is guessed. const resolveDep = (m, dep, request) => { let byRequest = depTargets.get(m.dir) if (byRequest === undefined) depTargets.set(m.dir, byRequest = new Map()) @@ -826,7 +783,6 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const known = metadata?.deps.get(m.dir)?.get(dep.key) if (known !== undefined) return asPackage(known) const spec = depSpec(m, dep, request) - if (!spec) return null const who = `${m.package.name} ${version(m)}` if (spec.path) { const dir = normalizeRel(spec.relTo, spec.path) @@ -835,11 +791,12 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef else if (t === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} names ${dir}, which holds no Cargo.toml with a [package]`) return t } - const crate = normName(spec.package ?? spec.key) - const req = typeof spec.version === 'string' ? spec.version : null - const fits = (ver) => req === null || satisfiesCargoReq(ver, req) + const crate = normName(spec.package) + const req = spec.version + const fits = (ver) => req === null || satisfies(ver, req) + let kind = spec.source const patch = patchFor(workspaceFor(m.dir) ?? readManifest('.'), crate) - if (patch !== undefined) { + if (patch?.spec.source.type === 'path') { if (patch.dir === null) { console.warn(`[loader.cargo] ${patch.from} patches ${crate} with a path outside the bundle root`) return null @@ -850,22 +807,24 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef console.warn(t === null ? `[loader.cargo] ${patch.from} patches ${crate} with ${patch.dir}, which holds no Cargo.toml with a [package]` : `[loader.cargo] ${patch.from}'s patch of ${crate} (${version(t)}) doesn't satisfy ${who}'s requirement ${req}: not used`) + } else if (patch?.spec.source.type === 'git') { + kind = 'git' } - const candidates = vendored().byName.get(crate) ?? [] + // A copy of where the dependency comes from (a copy without `.cargo-checksum.json` could be either). + const candidates = (vendored().byName.get(crate) ?? []).filter((c) => c.git === null || c.git === (kind === 'git')) if (candidates.length === 0) return null - const vendoredList = candidates.map((c) => c.version).toSorted(compareVersionsDesc).join(', ') + const vendoredList = candidates.map((c) => c.version).toSorted((a, b) => b.localeCompare(a, 'en', { numeric: true })).join(', ') const lk = lockfile() if (lk) { - const entry = lk.byId.get(`${normName(m.package.name)} ${version(m)}`) - // The lock names a dependency with its version only when several versions of that crate are - // locked: the one this requirement allows is this table's. - const listed = (entry?.deps ?? []).filter((x) => x.name === crate) - const only = lk.byName.get(crate) - const pins = listed.map((x) => x.version ?? (only?.length === 1 ? only[0].version : null)).filter((ver) => ver !== null) + const pins = lockKeysOf(lk, m) + .flatMap((k) => lk.lock.packages[k].dependencies) + .map((k) => lk.lock.packages[k]) + .filter((p) => normName(p.name) === crate && sourceKind(p.source) === kind) + .map((p) => p.version) const want = pins.find(fits) ?? null // A lock whose pin the requirement no longer allows is out of date: cargo would resolve // again, so the requirement decides. - if (want === null && pins.length > 0) console.warn(`[loader.cargo] Cargo.lock pins ${who} to ${crate} ${pins.join(', ')}, which ${req} doesn't allow: the lock is out of date`) + if (want === null && pins.length > 0) console.warn(`[loader.cargo] ${lk.file} pins ${who} to ${crate} ${pins.join(', ')}, which ${req} doesn't allow: the lock is out of date`) if (want !== null) { const hit = candidates.find((c) => c.version === want) if (hit) return readManifest(hit.dir) @@ -892,9 +851,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef if (resolverMemo !== null) return resolverMemo const pkg = entries.map(packageFor).find(Boolean) const root = (pkg ? workspaceFor(pkg.dir) ?? pkg : null) ?? readManifest('.') - const explicit = Number(root?.resolver) - const edition = root?.package?.editionFromWorkspace ? workspaceFor(root.dir)?.workspacePackage.edition : root?.package?.edition - resolverMemo = Number.isInteger(explicit) && explicit > 0 ? explicit : (Number(edition ?? 0) >= 2021 ? 2 : 1) + resolverMemo = root?.resolver ?? (Number(root?.package?.edition ?? 0) >= 2021 ? 2 : 1) return resolverMemo } // The feature context a unit's code sees (resolver 2 keeps the host's apart; resolver 1 has one). @@ -920,22 +877,13 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // The context a dependency of a package built in context `c` is built in: the host's for a // build-dependency (its build script's) and for a proc-macro crate, else `c`'s. const depCtx = (c, request, t) => featureCtx(kindOf(request) === 'build' || isProcMacroPkg(t) ? 'host' : c) - // An optional dependency no `dep:` entry names gets an implicit feature of its own name -- as - // the manifest spells it (`proc-macro-crate`), which is what `std = ["proc-macro-crate"]` and - // `#[cfg(feature = "proc-macro-crate")]` refer to. - const implicitFeatures = (m) => { - if (m.implicit === undefined) { - const referenced = new Set() - for (const imps of m.features.values()) for (const s of imps) if (s.startsWith('dep:')) referenced.add(normName(s.slice(4))) - m.implicit = new Map() - for (const d of m.deps.values()) { - if (isOptional(d) && !referenced.has(d.key)) m.implicit.set(d.name, [`dep:${d.key}`]) - } - } - return m.implicit - } const isOptional = (d) => [...d.kinds.values()].some((r) => r.optional) - const featureImplications = (m, f) => m.features.get(f) ?? implicitFeatures(m).get(f) ?? null + // A feature's list, from cargo's feature map (an optional dependency no `dep:` names has a + // feature of its own name, as the manifest spells it: `proc-macro-crate`), or null for none. + const featureImplications = (m, f) => m.features.get(f) ?? null + // The feature context a root package's own code is resolved in: the host's for a proc-macro + // crate, which cargo builds for the host; the target's for any other. + const rootCtx = (m) => featureCtx(isProcMacroPkg(m) ? 'host' : 'target') const nodeKey = (c, dir) => `${c}\0${dir}` // Enabled features per (context, package dir), for every package the root packages' builds pull @@ -1000,10 +948,12 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef if (depFeature) { const d = m.deps.get(normName(depFeature[1])) if (!d) return false - // `dep/feat` enables an optional dep (and its implicit feature); `dep?/feat` only asks if it is already on. + // `dep/feat` enables an optional dep, and the package's feature of the dependency's name + // where it has one, implicit or written (`serde = ["dep:serde", "extra"]` beside `full = + // ["serde/derive"]` turns `serde` and `extra` on); `dep?/feat` only asks if it is already on. if (depFeature[2] !== '?' && isOptional(d)) { activate(c, m, d.key) - if (implicitFeatures(m).has(d.name)) enable(c, m, d.name) + if (featureImplications(m, d.name) !== null) enable(c, m, d.name) } for (const [request, r] of activeRequests(c, m, d)) { const t = resolveDep(m, d, r) @@ -1017,12 +967,11 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } for (const m of roots) { - inGraph('target', m) + inGraph(rootCtx(m), m) if (allFeatures) { - for (const f of m.features.keys()) enable('target', m, f) - for (const f of implicitFeatures(m).keys()) enable('target', m, f) + for (const f of m.features.keys()) enable(rootCtx(m), m, f) } else if (!noDefaultFeatures) { - enable('target', m, 'default') + enable(rootCtx(m), m, 'default') } } // `--features a,b,pkg/c`: a bare name is a feature of every root package; `x/c` is a feature of @@ -1036,7 +985,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const targets = scoped ? roots.filter((m) => normName(m.package.name) === pkg) : roots const imp = scoped ? flag.slice(slash + 1) : flag // Whether a name is known doesn't depend on what is active yet, so the check happens once. - if (targets.filter((m) => applyImplication('target', m, imp)).length === 0 && includeMaybe) { + if (targets.filter((m) => applyImplication(rootCtx(m), m, imp)).length === 0 && includeMaybe) { console.warn(`[stasis] --cargo-features: '${flag}' names no feature of the entries' packages, nor a dependency of theirs`) } requested.push({ targets, imp }) @@ -1046,7 +995,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef changed = false // The requested features re-apply on every pass like the manifest's entries do: a weak // `dep?/x` asked for on the command line takes effect once `default` has activated `dep`. - for (const { targets, imp } of requested) for (const m of targets) applyImplication('target', m, imp) + for (const { targets, imp } of requested) for (const m of targets) applyImplication(rootCtx(m), m, imp) // Map/Set iteration is live: packages and features added mid-pass are visited in this pass. for (const node of enabled.keys()) { const cut = node.indexOf('\0') @@ -1058,7 +1007,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const d of m.deps.values()) { for (const [request, r] of activeRequests(c, m, d)) { const spec = depSpec(m, d, r) - const t = spec === null ? null : resolveDep(m, d, r) + const t = resolveDep(m, d, r) if (!t) continue const dc = depCtx(c, request, t) inGraph(dc, t) @@ -1070,8 +1019,188 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } while (changed) return enabled } + // --- the exact resolution: cargo's resolver, by @preventive/lockfile + + // The directories a workspace `members` pattern names, project-relative (glob's syntax, as cargo + // takes it: `*`, `?` and `[…]` within a segment, `**` for any depth); null when it leaves the + // bundle root. + const subdirs = (dir) => { + try { + return readdirSync(join(baseDir, dir), { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => (dir === '.' ? e.name : `${dir}/${e.name}`)) + } catch { + return [] + } + } + const globDirs = (rootDir, pattern) => { + let dirs = [rootDir] + for (const seg of pattern.split('/').filter((x) => x !== '' && x !== '.')) { + const next = [] + for (const dir of dirs) { + if (seg === '**') { + const all = [dir] + for (let i = 0; i < all.length; i++) all.push(...subdirs(all[i]).filter((d) => !posix.basename(d).startsWith('.'))) + next.push(...all) + } else if (/[*?[]/u.test(seg)) { + const re = new RegExp(`^${seg.replaceAll(/[.+^${}()|\\]/gu, '\\$&').replaceAll('[!', '[^').replaceAll('*', '[^/]*').replaceAll('?', '[^/]')}$`, 'u') + next.push(...subdirs(dir).filter((d) => re.test(posix.basename(d)))) + } else { + const rel = normalizeRel(dir, seg) + if (rel === null) return null + next.push(rel) + } + } + dirs = next + } + return dirs + } + // The workspace's members, project-relative, as cargo finds them: the root package, the + // packages its `members` patterns name, and the path dependencies of members inside the root's + // directory, less what `exclude` names; null when a pattern leaves the bundle root. + const membersOf = (root) => { + const ws = root.cargo.workspace + if (ws === undefined) return [root.dir] + const excluded = ws.exclude.map((e) => normalizeRel(root.dir, e)).filter((e) => e !== null) + const isExcluded = (dir) => excluded.some((e) => dir === e || dir.startsWith(`${e}/`)) + const inRoot = (dir) => root.dir === '.' || dir === root.dir || dir.startsWith(`${root.dir}/`) + const out = new Set(root.package ? [root.dir] : []) + for (const pattern of ws.members) { + const dirs = globDirs(root.dir, pattern) + if (dirs === null) return null + for (const dir of dirs) if (!isExcluded(dir) && readManifest(dir)?.package) out.add(dir) + } + // Set iteration is live: a path dependency added is walked in turn. + for (const dir of out) { + for (const d of readManifest(dir).deps.values()) { + for (const r of d.kinds.values()) { + if (r.path === null) continue + const sub = normalizeRel(r.inherited ? root.dir : dir, r.path) + if (sub !== null && inRoot(sub) && !isExcluded(sub) && readManifest(sub)?.package) out.add(sub) + } + } + } + return [...out] + } + // The build as cargo resolves it, where the loader holds all that takes -- the build's lockfile + // and target, a manifest for every package the lockfile has (each path package inside the + // bundle root, every other one vendored, `.cargo-checksum.json` and all) and entries whose + // packages are workspace members: @preventive/lockfile lays the lockfile's graph over the + // manifests (linkCargo) and turns on the features `cargo build -p ` does + // (resolveCargoFeatures), cargo's rules throughout -- each table's own source, a [patch] from the + // root manifest or the cargo config, proc-macros and build-dependencies built for the host, the + // command line's features handed out as cargo hands them out. What either refuses -- a lockfile + // out of date with the manifests, a vendored copy whose checksum isn't the lockfile's, a feature + // asked of a package that hasn't it -- stops the build. A host the loader doesn't know (a + // target that isn't `host`) is resolved both ways: the host's target-specific tables off for the + // features on for certain, on for all that may be (hostUndecided). `{ graph, dirOf, keyOf, + // resolved }` -- package key ↔ dir, and the features per (context, dir) as resolveFeatures + // gives them -- or null when something it takes is missing, for the replay to decide: no + // lockfile, no target, a package not vendored, a path outside the bundle root. + let exactMemo + const exact = () => { + if (exactMemo === undefined) exactMemo = exactUncached() + return exactMemo + } + const exactUncached = () => { + if (metadata || targetInfo === null) return null + const lk = lockfile() + const root = buildRoot() + if (lk === null || root === null) return null + const memberDirs = membersOf(root) + if (memberDirs === null) return null + // The path packages: the members, the path dependencies of each, the path [patch]es. + const keyOf = new Map() // dir -> package key + const dirOf = new Map() // package key -> dir + const queue = [...memberDirs] + for (const { patch, relTo } of [...configsFrom(root.dir).map((c) => ({ patch: c.patch, relTo: c.dir })), { patch: root.cargo.patch, relTo: root.dir }]) { + for (const specs of Object.values(patch)) { + for (const spec of Object.values(specs)) { + if (spec.source.type !== 'path') continue + const dir = normalizeRel(relTo, spec.source.path) + if (dir === null) return null + queue.push(dir) + } + } + } + for (const dir of queue) { + if (keyOf.has(dir)) continue + const m = readManifest(dir) + if (!m?.package) return null + const key = `${m.package.name} ${m.package.version}` + keyOf.set(dir, key) + dirOf.set(key, dir) + for (const d of m.deps.values()) { + for (const r of d.kinds.values()) { + if (r.path === null) continue + const sub = normalizeRel(r.inherited ? (workspaceFor(dir)?.dir ?? dir) : dir, r.path) + if (sub === null) return null + queue.push(sub) + } + } + } + // The vendor directory as a directory source reads it, and every package of the lockfile there. + const vendor = Object.create(null) + const held = new Set() + for (const sub of subdirs(vendorDir)) { + const name = posix.basename(sub) + const m = name.startsWith('.') ? null : readManifest(sub) + if (!m?.package) continue + const sums = checksumOf(sub) + if (sums === null) return null + vendor[name] = { manifest: tableOf(sub).text, checksum: sums.text } + held.add(`${m.package.name} ${m.package.version}`) + } + for (const [key, p] of Object.entries(lk.lock.packages)) { + if (p.source === undefined ? !dirOf.has(key) : !held.has(`${p.name} ${p.version}`)) return null + } + const copies = readNamed(lk.file, () => readCargoVendor(lk.lock, vendor)) + const manifestsByKey = Object.create(null) + for (const [key, dir] of dirOf) manifestsByKey[key] = readManifest(dir).cargo + for (const [key, { directory }] of Object.entries(copies)) { + const dir = `${vendorDir}/${directory}` + keyOf.set(dir, key) + dirOf.set(key, dir) + manifestsByKey[key] = readManifest(dir).cargo + } + const memberKeys = memberDirs.map((dir) => keyOf.get(dir)) + const entryKeys = [...new Set(entries.map(packageFor).filter(Boolean).map((m) => keyOf.get(m.dir)))] + if (entryKeys.length === 0 || entryKeys.some((k) => !memberKeys.includes(k))) return null + const configs = configsFrom(root.dir) + const config = readNamed(configs[0]?.file ?? null, () => parseCargoConfig(configs.map((c) => c.text))) + const graph = readNamed(lk.file, () => linkCargo(lk.lock, manifestsByKey, { workspace: root.cargo, members: memberKeys, config })) + const rootFile = posix.join(root.dir, 'Cargo.toml') + const build = { packages: entryKeys, features, allFeatures, noDefaultFeatures, dev: entries.some((e) => isTestTarget(e)) } + const platform = (info, marks = []) => ({ name: info.triple, cfg: [...info.cfgs, ...marks] }) + const featuresOf = (g, platforms) => readNamed(rootFile, () => resolveCargoFeatures(g, { ...build, ...platforms })) + let sure + let all + if (hostInfo === null) { + const platforms = { host: { name: MARK_HOST, cfg: [MARK_HOST] }, targets: [platform(targetInfo, [MARK_TARGET, `${MARK_NAME}="${targetInfo.triple}"`])] } + sure = featuresOf(hostUndecided(graph, false), platforms) + all = featuresOf(hostUndecided(graph, true), platforms) + } else { + sure = featuresOf(graph, { host: platform(hostInfo), targets: [platform(targetInfo)] }) + all = sure + } + // Per (context, dir), as resolveFeatures keys them: resolver 1's one set under `target`. + const byNode = (result) => { + const out = new Map() + for (const [key, { normal, host: onHost }] of Object.entries(result)) { + const dir = dirOf.get(key) + if (dir === undefined) continue + if (resolverVersion() === 1) out.set(nodeKey('target', dir), new Set(normal ?? onHost)) + else { + if (normal !== undefined) out.set(nodeKey('target', dir), new Set(normal)) + if (onHost !== undefined) out.set(nodeKey('host', dir), new Set(onHost)) + } + } + return out + } + return { graph, dirOf, keyOf, resolved: { sure: byNode(sure), all: byNode(all) } } + } + // Both resolutions, per (context, package dir): `sure` without the undecided tables, `all` with - // them. `--cargo`: cargo metadata's features, the same in either context. + // them -- the exact resolution's where there is one, the replay's otherwise. `--cargo`: cargo + // metadata's features, the same in either context. let resolved = null const ensureResolved = () => { if (resolved !== null) return resolved @@ -1079,7 +1208,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const byNode = new Map([...metadata.enabled].flatMap(([dir, set]) => [[nodeKey('target', dir), set], [nodeKey('host', dir), set]])) resolved = { sure: byNode, all: byNode } } else { - resolved = { sure: resolveFeatures(false), all: resolveFeatures(true) } + resolved = exact()?.resolved ?? { sure: resolveFeatures(false), all: resolveFeatures(true) } } return resolved } @@ -1148,7 +1277,25 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const own = all.filter(([request]) => ROLE_KINDS[role].includes(kindOf(request))) return (own.length > 0 ? own : all).map(([, r]) => r) } + // depCrate from the exact resolution: the package's dependency of that key, else the unrenamed + // one whose lib is named that, from the asking file's tables when it has one there, as the + // lockfile resolves it. + const exactCrate = (x, key, norm, role) => { + const deps = x.graph.packages[key].dependencies + const libOf = (d) => readManifest(x.dirOf.get(d.resolved)) + const pick = (list) => { + const own = list.filter((d) => ROLE_KINDS[role].includes(d.kind)) + const d = (own.length > 0 ? own : list).find((dep) => dep.resolved !== undefined) + return d === undefined ? null : libPath(libOf(d)) + } + const declared = deps.filter((d) => normName(d.name) === norm) + if (declared.length > 0) return pick(declared) + return pick(deps.filter((d) => d.name === d.package && d.resolved !== undefined && libName(libOf(d)) === norm)) + } const depCrateUncached = (m, norm, role) => { + const x = m ? exact() : null + const key = x?.keyOf.get(m.dir) + if (key !== undefined) return exactCrate(x, key, norm, role) if (m) { const d = m.deps.get(norm) if (d) { @@ -1160,7 +1307,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } for (const other of m.deps.values()) { for (const r of requestsFor(other, role)) { - if (depSpec(m, other, r)?.package) continue // a rename is used by its key, not its lib name + if (r.renamed) continue // a rename is used by its key, not its lib name const t = resolveDep(m, other, r) if (t && libName(t) === norm) return libPath(t) } diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index 5a90a718..c9725d83 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -1234,7 +1234,8 @@ function cfgLeaves(pred) { const p = pred.trim() const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) if (!m) { - const kv = /^([\w-]+)\s*(?:=\s*"([^"]*)")?$/u.exec(p) + if (p === 'true') return [] // holds: decides nothing (`false` is a leaf the build rules out, see leafFalse) + const kv = /^(?:r#)?([\w-]+)\s*(?:=\s*"([^"]*)")?$/u.exec(p) return kv ? [{ key: kv[1], value: kv[2] ?? null, neg: false }] : [] } const parts = splitTopLevel(m[2]).filter((part) => part.trim() !== '').map(cfgLeaves) @@ -1413,7 +1414,7 @@ const compatible = (asker, set) => { // build may compile, and only when there is none -- a path written behind the same gate, whose // code the build doesn't compile either, means what it would mean where it compiles. For an // asker whose own leaves can't hold (`deadHere`) the build says nothing. -const EVALUABLE_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doc', 'doctest']) +const EVALUABLE_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doc', 'doctest', 'false']) const leafFalseMemo = new WeakMap() // build → leaf → verdict (builds are interned per settled context, see buildOf) const leafFalse = (l, build) => { if (l.alts !== undefined) return l.alts.every((alt) => alt.some((m) => leafFalse(m, build))) @@ -1466,7 +1467,7 @@ const entailed = (asker, set) => { // custom`, see customFor). An `any` leaf is doubtful when each alternative is, or can't hold with // the asker. A gate macro's leaf (`cfg_x!`) and a `mod` variant's are neither, nor is a leaf the // asker holds itself (a file under `#[cfg(loom)]` takes the loom candidates). -const KNOWN_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doctest', 'doc', 'debug_assertions', 'overflow_checks', 'panic', 'proc_macro', 'miri', 'sanitize', 'target_feature', 'target_thread_local', 'ub_checks', 'relocation_model', 'fmt_debug', 'clippy', 'rustfmt', 'variant']) +const KNOWN_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'false', 'feature', 'test', 'doctest', 'doc', 'debug_assertions', 'overflow_checks', 'panic', 'proc_macro', 'miri', 'sanitize', 'target_feature', 'target_thread_local', 'ub_checks', 'relocation_model', 'fmt_debug', 'clippy', 'rustfmt', 'variant']) const customKey = (key) => !KNOWN_CFG_KEYS.has(key) && !key.endsWith('!') // The custom-cfg test of an asker whose build may set `settable` (`{ names, any }`), one per object. const customMemo = new WeakMap() @@ -2289,6 +2290,13 @@ function effectiveInvokers(name, invokers, templateCalls, seen = new Set()) { // the target's alone. const DEFAULT_UNITS = new Set([TARGET_UNIT]) const unitsOf = (units, rel) => units?.get(rel) ?? DEFAULT_UNITS +// `units` (a new map for none) with each entry it doesn't name compiled as its package is +// (unitOfCrate): a proc-macro crate's for the host, as cargo builds it, any other's for the target. +function withEntryUnits(units, entries, ctx) { + const out = units ?? new Map() + for (const e of entries) if (!out.has(e)) out.set(e, new Set([ctx?.unitOfCrate(e, TARGET_UNIT) ?? TARGET_UNIT])) + return out +} // The build `rel` is scanned under (scanRustItems' options): per unit it is compiled as, its // crate's features in that unit's feature context -- on for certain, and on maybe -- and the cfg @@ -2344,8 +2352,9 @@ function cachedScan(sources, path, content, build, templates = undefined) { // `#[cfg(feature = …)]`, and the build target's cfg set when one was given, which decides // `#[cfg(unix)]` and the like. `units`: file → the compile units it is compiled as (see // collectRustFilesFromDisk); `wantedUnits` gives each wanted root's, for the walk. -export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null, formats = null, units = null } = {}) { +export function buildRustTree(sources, { roots = [], baseDir = null, cargo = null, formats = null, units: given = null } = {}) { const ctx = cargo ?? (baseDir ? createCargoContext(baseDir, { entries: roots }) : null) + const units = withEntryUnits(given, roots, ctx) const rootSet = crateRoots(sources, roots, ctx) const resolutions = new Map() const missing = [] @@ -2912,7 +2921,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, for (const u of more) have.add(u) return true } - for (const e of entries) addUnits(e, unitsOf(units, e)) + withEntryUnits(units, entries, ctx) // Files already walked that are now compiled as more units: gone through again (their build // changed, and so may what they name, and as what). const revisit = new Set(entries.filter((e) => sources.has(e) && !formats?.has(e))) diff --git a/tests/bundle-cmd.test.js b/tests/bundle-cmd.test.js index a0cf65ad..41d7b606 100644 --- a/tests/bundle-cmd.test.js +++ b/tests/bundle-cmd.test.js @@ -3445,8 +3445,8 @@ test('CLI: bundle (JS) fails loudly when the oxc-parser dependency is missing', // exited 0 with no warning at all. The setup error must propagate with its // install hint instead. Exercised against a copy of stasis whose node_modules // carries only the zero-dep @exodus/stasis-core (so the moved-module shims - // resolve), @preventive/lockfile (whose TOML and .gitmodules readers the loaders - // import) and @exodus/bytes (its dependency, and the loaders' UTF-8 decoder), so + // resolve), @preventive/lockfile (whose TOML, .gitmodules and Cargo readers the + // loaders import) and @exodus/bytes (its dependency, and the loaders' UTF-8 decoder), so // the bundle command loads, but no oxc-parser, so the lazy lookup (createRequire // from src/scan.js) genuinely misses. const stasisCopy = join(tmp, 'stasis') diff --git a/tests/fixtures/rust-bundle/cargo-recorded/Cargo.lock b/tests/fixtures/rust-bundle/cargo-recorded/Cargo.lock new file mode 100644 index 00000000..b1c27f90 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/Cargo.lock @@ -0,0 +1,79 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "app" +version = "0.1.0" +dependencies = [ + "autocfg", + "cfg-if", + "itoa 0.4.8", + "itoa 1.0.0", + "itoa 1.0.18", + "lib", + "macros", + "memchr", + "paste", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "cfg-if" +version = "1.0.0" + +[[package]] +name = "extra" +version = "0.3.0" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "itoa" +version = "1.0.0" +source = "git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lib" +version = "0.2.0" +dependencies = [ + "extra", + "memchr", +] + +[[package]] +name = "macros" +version = "0.1.0" +dependencies = [ + "memchr", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" diff --git a/tests/fixtures/rust-bundle/cargo-recorded/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/Cargo.toml new file mode 100644 index 00000000..32f880a3 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[workspace] +members = ["crates/*"] +exclude = ["crates/extra", "patched/cfg-if"] + +[workspace.package] +version = "0.2.0" +edition = "2021" + +[workspace.dependencies] +memchr = { version = "2", default-features = false } +itoa = "1" + +[features] +default = ["fast"] +fast = ["memchr/std", "lib/std"] +fmt = ["dep:itoa04", "itoa04?/i128"] +git = ["dep:itoa-git"] + +[dependencies] +lib = { path = "crates/lib" } +macros = { path = "crates/macros" } +memchr = { workspace = true } +itoa = { workspace = true } +itoa04 = { package = "itoa", version = "0.4", optional = true, default-features = false } +itoa-git = { package = "itoa", git = "https://github.com/dtolnay/itoa", tag = "1.0.0", optional = true } + +[target.'cfg(windows)'.dependencies] +cfg-if = "1" + +[build-dependencies] +autocfg = "1" + +[dev-dependencies] +paste = "1" + +[patch.crates-io] +cfg-if = { path = "patched/cfg-if" } diff --git a/tests/fixtures/rust-bundle/cargo-recorded/crates/extra/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/crates/extra/Cargo.toml new file mode 100644 index 00000000..36c6a094 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/crates/extra/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "extra" +version = "0.3.0" + +[dependencies] +cfg-if = "1" + +[dev-dependencies] +paste = "1" diff --git a/tests/fixtures/rust-bundle/cargo-recorded/crates/lib/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/crates/lib/Cargo.toml new file mode 100644 index 00000000..7be8f0f9 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/crates/lib/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "lib" +version.workspace = true +edition.workspace = true + +[features] +std = ["memchr?/std"] +extra = ["dep:extra"] + +[dependencies] +memchr = { workspace = true, optional = true } +extra = { path = "../extra", optional = true } + +[target.'cfg(unix)'.dependencies] +memchr = { workspace = true, features = ["alloc"] } diff --git a/tests/fixtures/rust-bundle/cargo-recorded/crates/macros/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/crates/macros/Cargo.toml new file mode 100644 index 00000000..2bf953cd --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/crates/macros/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "macros" +version = "0.1.0" +edition = "2018" + +[lib] +proc-macro = true + +[dependencies] +memchr = { version = "2", default-features = false, features = ["alloc"] } diff --git a/tests/fixtures/rust-bundle/cargo-recorded/patched/cfg-if/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/patched/cfg-if/Cargo.toml new file mode 100644 index 00000000..6d0b3012 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/patched/cfg-if/Cargo.toml @@ -0,0 +1,4 @@ +[package] +name = "cfg-if" +version = "1.0.0" +edition = "2018" diff --git a/tests/fixtures/rust-bundle/cargo-recorded/recorded.json b/tests/fixtures/rust-bundle/cargo-recorded/recorded.json new file mode 100644 index 00000000..a20dd8f8 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/recorded.json @@ -0,0 +1,500 @@ +{ + "host": "x86_64-unknown-linux-gnu", + "cfg": { + "x86_64-unknown-linux-gnu": [ + "debug_assertions", + "panic=\"unwind\"", + "target_abi=\"\"", + "target_arch=\"x86_64\"", + "target_endian=\"little\"", + "target_env=\"gnu\"", + "target_family=\"unix\"", + "target_feature=\"fxsr\"", + "target_feature=\"sse\"", + "target_feature=\"sse2\"", + "target_has_atomic=\"16\"", + "target_has_atomic=\"32\"", + "target_has_atomic=\"64\"", + "target_has_atomic=\"8\"", + "target_has_atomic=\"ptr\"", + "target_os=\"linux\"", + "target_pointer_width=\"64\"", + "target_vendor=\"unknown\"", + "unix" + ], + "x86_64-pc-windows-msvc": [ + "debug_assertions", + "panic=\"unwind\"", + "target_abi=\"\"", + "target_arch=\"x86_64\"", + "target_endian=\"little\"", + "target_env=\"msvc\"", + "target_family=\"windows\"", + "target_feature=\"cmpxchg16b\"", + "target_feature=\"fxsr\"", + "target_feature=\"sse\"", + "target_feature=\"sse2\"", + "target_feature=\"sse3\"", + "target_has_atomic=\"128\"", + "target_has_atomic=\"16\"", + "target_has_atomic=\"32\"", + "target_has_atomic=\"64\"", + "target_has_atomic=\"8\"", + "target_has_atomic=\"ptr\"", + "target_os=\"windows\"", + "target_pointer_width=\"64\"", + "target_vendor=\"pc\"", + "windows" + ], + "aarch64-apple-darwin": [ + "debug_assertions", + "panic=\"unwind\"", + "target_abi=\"\"", + "target_arch=\"aarch64\"", + "target_endian=\"little\"", + "target_env=\"\"", + "target_family=\"unix\"", + "target_feature=\"aes\"", + "target_feature=\"crc\"", + "target_feature=\"dit\"", + "target_feature=\"dotprod\"", + "target_feature=\"dpb\"", + "target_feature=\"dpb2\"", + "target_feature=\"fcma\"", + "target_feature=\"fhm\"", + "target_feature=\"flagm\"", + "target_feature=\"fp16\"", + "target_feature=\"frintts\"", + "target_feature=\"jsconv\"", + "target_feature=\"lor\"", + "target_feature=\"lse\"", + "target_feature=\"neon\"", + "target_feature=\"paca\"", + "target_feature=\"pacg\"", + "target_feature=\"pan\"", + "target_feature=\"pmuv3\"", + "target_feature=\"ras\"", + "target_feature=\"rcpc\"", + "target_feature=\"rcpc2\"", + "target_feature=\"rdm\"", + "target_feature=\"sb\"", + "target_feature=\"sha2\"", + "target_feature=\"sha3\"", + "target_feature=\"ssbs\"", + "target_feature=\"vh\"", + "target_has_atomic=\"128\"", + "target_has_atomic=\"16\"", + "target_has_atomic=\"32\"", + "target_has_atomic=\"64\"", + "target_has_atomic=\"8\"", + "target_has_atomic=\"ptr\"", + "target_os=\"macos\"", + "target_pointer_width=\"64\"", + "target_vendor=\"apple\"", + "unix" + ] + }, + "members": [ + "lib 0.2.0", + "macros 0.1.0", + "app 0.1.0" + ], + "manifests": { + "app 0.1.0": "Cargo.toml", + "cfg-if 1.0.0": "patched/cfg-if/Cargo.toml", + "extra 0.3.0": "crates/extra/Cargo.toml", + "lib 0.2.0": "crates/lib/Cargo.toml", + "macros 0.1.0": "crates/macros/Cargo.toml" + }, + "vendored": { + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)": "autocfg", + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)": "itoa-0.4.8", + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff)": "itoa-1.0.0", + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)": "itoa", + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index)": "memchr", + "paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)": "paste" + }, + "graph": { + "app 0.1.0": { + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)": [ + "build *" + ], + "cfg-if 1.0.0": [ + "normal cfg(windows)" + ], + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)": [ + "normal *" + ], + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff)": [ + "normal *" + ], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)": [ + "normal *" + ], + "lib 0.2.0": [ + "normal *" + ], + "macros 0.1.0": [ + "normal *" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index)": [ + "normal *" + ], + "paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)": [ + "dev *" + ] + }, + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)": {}, + "cfg-if 1.0.0": {}, + "extra 0.3.0": { + "cfg-if 1.0.0": [ + "normal *" + ] + }, + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)": {}, + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff)": {}, + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)": {}, + "lib 0.2.0": { + "extra 0.3.0": [ + "normal *" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index)": [ + "normal *", + "normal cfg(unix)" + ] + }, + "macros 0.1.0": { + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index)": [ + "normal *" + ] + }, + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index)": {}, + "paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)": {} + }, + "features": { + "1": [ + { + "packages": "all", + "built": { + "app 0.1.0 normal": [ + "default", + "fast" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc", + "std" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + }, + { + "packages": "all", + "dev": true, + "built": { + "app 0.1.0 normal": [ + "default", + "fast" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc", + "std" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ], + "paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index) host": [] + } + }, + { + "packages": [ + "app" + ], + "features": [ + "fmt,git" + ], + "target": "x86_64-pc-windows-msvc", + "built": { + "app 0.1.0 normal": [ + "default", + "fast", + "fmt", + "git" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "i128" + ], + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff) normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc", + "std" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + }, + { + "packages": [ + "app" + ], + "noDefaultFeatures": true, + "features": [ + "lib/extra memchr/alloc" + ], + "built": { + "app 0.1.0 normal": [], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "extra 0.3.0 normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "extra" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc" + ] + } + }, + { + "packages": [ + "lib", + "macros" + ], + "features": [ + "std" + ], + "target": "aarch64-apple-darwin", + "built": null + }, + { + "packages": "all", + "allFeatures": true, + "target": "aarch64-apple-darwin", + "built": { + "app 0.1.0 normal": [ + "default", + "fast", + "fmt", + "git" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "extra 0.3.0 normal": [], + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "i128" + ], + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff) normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "extra", + "memchr", + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc", + "std" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + } + ], + "2": [ + { + "packages": "all", + "built": { + "app 0.1.0 normal": [ + "default", + "fast" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + }, + { + "packages": "all", + "dev": true, + "built": { + "app 0.1.0 normal": [ + "default", + "fast" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ], + "paste 1.0.15 (registry+https://github.com/rust-lang/crates.io-index) host": [] + } + }, + { + "packages": [ + "app" + ], + "features": [ + "fmt,git" + ], + "target": "x86_64-pc-windows-msvc", + "built": { + "app 0.1.0 normal": [ + "default", + "fast", + "fmt", + "git" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "i128" + ], + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff) normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + }, + { + "packages": [ + "app" + ], + "noDefaultFeatures": true, + "features": [ + "lib/extra memchr/alloc" + ], + "built": { + "app 0.1.0 normal": [], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "extra 0.3.0 normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "extra" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc" + ] + } + }, + { + "packages": [ + "lib", + "macros" + ], + "features": [ + "std" + ], + "target": "aarch64-apple-darwin", + "built": { + "lib 0.2.0 normal": [ + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + }, + { + "packages": "all", + "allFeatures": true, + "target": "aarch64-apple-darwin", + "built": { + "app 0.1.0 normal": [ + "default", + "fast", + "fmt", + "git" + ], + "autocfg 1.5.1 (registry+https://github.com/rust-lang/crates.io-index) host": [], + "cfg-if 1.0.0 normal": [], + "extra 0.3.0 normal": [], + "itoa 0.4.8 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "i128" + ], + "itoa 1.0.0 (git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff) normal": [], + "itoa 1.0.18 (registry+https://github.com/rust-lang/crates.io-index) normal": [], + "lib 0.2.0 normal": [ + "extra", + "memchr", + "std" + ], + "macros 0.1.0 host": [], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) host": [ + "alloc" + ], + "memchr 2.8.3 (registry+https://github.com/rust-lang/crates.io-index) normal": [ + "alloc", + "std" + ] + } + } + ] + } +} diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/.cargo-checksum.json new file mode 100644 index 00000000..444c5213 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"5713fb5e9d7124071a4fb0d0f8f80535560c7cef1995c42fe168af1c2627a050","Cargo.lock":"dc8d767ba9ab277c030476ca736afce4ecb4999a01bd3db8c8210a12ef79e81c","Cargo.toml":"ed7c832b4250c3be736e040d68688c2b83a5b164a93ef533e61532cbe0d44e99","Cargo.toml.orig":"57d2aec0d4157bd9446f97705aa2acff86cc3d1883f470526fed089e7cf9754e","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"27995d58ad5c1145c1a8cd86244ce844886958a35eb2b78c6b772748669999ac","README.md":"1c6c43f1d22ab1c6411b82862c356c550947f3cecbb4c7eede5edefc8890ff32","examples/integers.rs":"589ff4271566dfa322becddf3e2c7b592e6e0bc97b02892ce75619b7e452e930","examples/nightly.rs":"ac8b5a9aa1e04465e44f5053b3c899b635e07af058c73aa8b45176bf4b5912f9","examples/paths.rs":"1b30e466b824ce8df7ad0a55334424131d9d2573d6cf9f7d5d50c09c8901d526","examples/traits.rs":"cbee6a3e1f7db60b02ae25b714926517144a77cb492021f492774cf0e1865a9e","examples/versions.rs":"38535e6d9f5bfae0de474a3db79a40e8f5da8ba9334c5ff4c363de9bc99d4d12","src/error.rs":"fd8ff67c64f7cd1b9f81325a81de4baa34c39d6ae298bdb33f9829cc91acac39","src/lib.rs":"772630d0e09d06f343fd72284c8330eacde87d2a4ee22f70af62a0e2119fbf05","src/rustc.rs":"a8a213ddb64a05c1a1af933bcb331a98879e942b167c33d8f94f9f60ebb14e29","src/tests.rs":"b39f4d880ad343e65307a9e0c381954ea27adce4732f825516ce7952e2e5a91d","src/version.rs":"4f7d23b36f01c7be1871be86c038d6cb4689e145d67c82d3793690e9aa05b133","tests/no_std.rs":"18859dc4992fe1769887bde05f03d28f1ce524eafd17646d3fbcb4379422761a","tests/rustflags.rs":"e8ded4d57ba25379a38ab48456d67df14f82abbbb5f6bb66221c6decbcb517a3","tests/support/mod.rs":"32087d365b438ac3f62df9bb066d8d648b80cb130a5c777afcb2f21fbb68d88e","tests/tests.rs":"39149fe69a45f577eeb531587a5e600d8bc9f8d937480172dac9402939d856f0","tests/wrappers.rs":"96aee2217fbfc7a3aff16e70609c885620a4704dfd708de8b2d0bad4a8c6247d"},"package":"f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/Cargo.toml new file mode 100644 index 00000000..2327046d --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/Cargo.toml @@ -0,0 +1,75 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +rust-version = "1.0" +name = "autocfg" +version = "1.5.1" +authors = ["Josh Stone "] +build = false +exclude = ["/.github/**"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Automatic cfg for Rust compiler features" +documentation = "https://docs.rs/autocfg/" +readme = "README.md" +keywords = [ + "rustc", + "build", + "autoconf", +] +categories = ["development-tools::build-utils"] +license = "Apache-2.0 OR MIT" +repository = "https://github.com/cuviper/autocfg" + +[lib] +name = "autocfg" +path = "src/lib.rs" + +[[example]] +name = "integers" +path = "examples/integers.rs" + +[[example]] +name = "nightly" +path = "examples/nightly.rs" + +[[example]] +name = "paths" +path = "examples/paths.rs" + +[[example]] +name = "traits" +path = "examples/traits.rs" + +[[example]] +name = "versions" +path = "examples/versions.rs" + +[[test]] +name = "no_std" +path = "tests/no_std.rs" + +[[test]] +name = "rustflags" +path = "tests/rustflags.rs" + +[[test]] +name = "tests" +path = "tests/tests.rs" + +[[test]] +name = "wrappers" +path = "tests/wrappers.rs" +harness = false diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json new file mode 100644 index 00000000..b816353b --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"47986564aed398bb6a83979cbb1fe5eff0f3aa61fbba131a0954180ce8eaca26",".clippy.toml":"e0f282c75faa7e46c2630843826bb78f39dd2a6dea78dab7f960d34f730fb396",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"51a75e85b378ebc683ad6cdf12767e26ec495a2b372b980d278b68ef48046ca9","Cargo.toml":"46f2cf3006d59c73bed95c276f43ddab1a9a9fa177ea0142777b7471c753f558","Cargo.toml.orig":"a089595f30fd2a22f6152a96884e76d4aac63cac56aff6dd64a03ab2737a55dc","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"5eac563c0123003363f4706f36927cf4d3b118965e35b51bc515343ce6a711ae","benches/bench.rs":"fc009678431563bfb1b51698fb5b83f5309a3cdab90dc7365800f4e32bc287be","src/lib.rs":"a7b37eab3872193cf666c1d7d4db8d7e82c6dc6d8121fc762b5f69ddb4df3b01","src/udiv128.rs":"1c9c374680162e12c734d369d1494ee8c91e11eb4b05500e96e6303155bdc623","tests/test.rs":"c9b124462b75078e2407e0239222edd7859c3112d65a114742dbb8c47dafc07c"},"package":"b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml new file mode 100644 index 00000000..02b43829 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml @@ -0,0 +1,29 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +name = "itoa" +version = "0.4.8" +authors = ["David Tolnay "] +exclude = ["performance.png"] +description = "Fast functions for printing integer primitives to an io::Write" +documentation = "https://docs.rs/itoa" +readme = "README.md" +categories = ["value-formatting"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/itoa" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[features] +default = ["std"] +i128 = [] +std = [] diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json new file mode 100644 index 00000000..2ca74031 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".clippy.toml":"33ffb83bbddb772575b3aa565b7136a8158ee386c216ffc2588fed9e83fa3826",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"e217196efe9a3efa3d374deebfde6980cda2cbfba14fecd9fdc86cdc1d5dcbe3","Cargo.toml":"39bf8df2e494ac7c8446decd837c0b242df393a1093d788c295002ecef27ddff","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"8e9d4e6ac0c740793958bf57c3dbd62827340cf77a282633c388aff70a7c1a22","benches/bench.rs":"636f3093bd461210ad3063289d455f90669c4a1be3273bcd30898de39f02c641","src/lib.rs":"2c013a4a736d514d8629fd9cc162e05e76fb3c1557d9d737a0f79c04bde0c971","src/udiv128.rs":"16394f767854452756372d74f8025f7329fd8b61a676d81edf489681a6332ee9","tests/test.rs":"fc4a9fed6ad3cd9f548b6916444c0d269fbbb7033a41a4f15a1712f6fb32f9cb"},"package":null} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml new file mode 100644 index 00000000..5cac345d --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml @@ -0,0 +1,48 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +rust-version = "1.36" +name = "itoa" +version = "1.0.0" +authors = ["David Tolnay "] +build = false +exclude = [ + "performance.png", + "chart/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Fast integer primitive to string conversion" +documentation = "https://docs.rs/itoa" +readme = "README.md" +categories = ["value-formatting"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/itoa" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +name = "itoa" +path = "src/lib.rs" + +[[test]] +name = "test" +path = "tests/test.rs" + +[[bench]] +name = "bench" +path = "benches/bench.rs" diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json new file mode 100644 index 00000000..1c33e56e --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"e0165305f3b22a9ab4217a121f85379aa56c2675caa7afedc0a7468fc187600b",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"89c4f60223daf03217ea7c1e8879135b11d2554bd1e57dd9eff5c02f1adc7909","Cargo.lock":"1cd64090b8e883963bb08d91e6968cc1e83ffdba4f4b2f47651ce9d351a8a947","Cargo.toml":"894c215e4e7e973b556b0e9627bf9b6a719a043fac9b5af9838660816392a09f","Cargo.toml.orig":"44ca1ae5d9b48743305b57f3367051289512037e7836b8b3693b68d48dec0483","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"1305bd79675db0194915d9bc91bd0b3cf9487bc1126e05bf46b48ab43edce44b","benches/bench.rs":"29a0bdb973848007d681e571c57487e21e1bfb7aed5980c2190dba354498b154","src/lib.rs":"e962338e1886873aa7c3399ec89aa5378ccbfed712b9690f043628079390a34f","src/u128_ext.rs":"498cc94f13805e551248fd82f47cc318f51d953a5002a9f8698858cd12fbae3c","tests/test.rs":"b22f8a8ed89edcbb4140b74f5edd7f2adf3cb0e363990dd6010cd6ebe0dc36b7"},"package":"8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml new file mode 100644 index 00000000..d60aa961 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml @@ -0,0 +1,66 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.68" +name = "itoa" +version = "1.0.18" +authors = ["David Tolnay "] +build = false +exclude = ["*.png"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Fast integer primitive to string conversion" +documentation = "https://docs.rs/itoa" +readme = "README.md" +keywords = ["integer"] +categories = [ + "value-formatting", + "no-std", + "no-std::no-alloc", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/itoa" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = [ + "--generate-link-to-definition", + "--generate-macro-expansion", + "--extern-html-root-url=core=https://doc.rust-lang.org", + "--extern-html-root-url=alloc=https://doc.rust-lang.org", + "--extern-html-root-url=std=https://doc.rust-lang.org", +] + +[lib] +name = "itoa" +path = "src/lib.rs" + +[[test]] +name = "test" +path = "tests/test.rs" + +[[bench]] +name = "bench" +path = "benches/bench.rs" +harness = false + +[dependencies.no-panic] +version = "0.1" +optional = true + +[target."cfg(not(miri))".dev-dependencies.criterion] +version = "0.8" +default-features = false diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json new file mode 100644 index 00000000..2c5ead36 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"739b57f156ae78ce5ad0f2bc44ff727f1a9d48e743bc266e8e3ab8c35baa5ecb",".ignore":"ae8b19032d4fc418b99ccae9e7cc3996b1386665d0bd5edc5634a158e7d2f6a2",".vim/coc-settings.json":"cdc5e2b88bddbdbd1b85f21389c4d882720e4c4488ad566c43fccd9124f2e3bf","AI_POLICY.md":"eb058d5b8387f03159cf6ef868100459d7e01f49499733028fcc916bc97c1cc4","CONTRIBUTING.md":"f5b2b4a9a3dbcd9a6f2654eb97dd7135e6af3717b9a3f037a9cbf714e387af76","COPYING":"01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f","Cargo.lock":"54b66d3eabcf4993f06c5b556ab1df0082dd14985529e301124590280dbe0554","Cargo.toml":"a14f68b8fa55f15fd03b15dced37bcf66a0a628f568eedc82817cdb7f9281951","Cargo.toml.orig":"2d004168044e88b00f971e65d7e6c988a555aa00d7ef4d0ceb5b5a9c228fb0c9","LICENSE-MIT":"0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f","README.md":"92a74aaffe011bdaa06fbc34a01686a6eba58ca1322e976759417a547fddf734","UNLICENSE":"7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c","rustfmt.toml":"1ca600239a27401c4a43f363cf3f38183a212affc1f31bff3ae93234bbaec228","src/arch/aarch64/memchr.rs":"5bb70f915084e629d940dbc322f5b9096b2e658cf63fea8a2f6e7550412e73a0","src/arch/aarch64/mod.rs":"44cd1a614bd66f1e66fc86c541d3c3b8d3a14a644c13e8bf816df3f555eac2d4","src/arch/aarch64/neon/memchr.rs":"e8c00b8fb2c7e2711832ae3cedefe59f32ebedd7dfa4d0ec6de2a566c979daea","src/arch/aarch64/neon/mod.rs":"eab6d56c2b2354db4ee395f40282cd49f97e2ab853547be5de6e65fbe1b2f634","src/arch/aarch64/neon/packedpair.rs":"32d3e4cd0dd9b6c8382e5308cbd896d20242c90b12862c44a5de6a8b4d6126df","src/arch/all/memchr.rs":"b0b1214aa573ed5d02ae62a77c42c773065566b50274d4096e37817d65ab1594","src/arch/all/mod.rs":"05f3fc2b069682eb1545fc6366d167bb620a454365dac8b8dd6cde6cd64de18a","src/arch/all/packedpair/default_rank.rs":"abffd1b5b8b7a3be95c03dd1105b905c246a379854dc56f1e846ea7c4408f2c7","src/arch/all/packedpair/mod.rs":"292b66042c5b5c78bba33db6526aeae6904db803d601fcdd29032b87b3eb3754","src/arch/all/rabinkarp.rs":"06038cce1cbead7b088e541ba41bcc1c095c34bde483b7df6ad9f77a49c1063c","src/arch/all/shiftor.rs":"0d79117f52a1e4795843603a3bb0b45397df4ad5e4184bbc923658dab9dc3b5f","src/arch/all/twoway.rs":"0b27eb1cd7402c1375a50466f8f66a68cb2806c3f69a647340b2759793eed4f8","src/arch/generic/memchr.rs":"ee2a77afcbf506655e352eb3cd42451a70cc2c39e3da487b2985638bc09ced3e","src/arch/generic/mod.rs":"1dd75f61e0ea2563b8205a08aaa7b55500130aa331d18b9e9f995724b66c7a39","src/arch/generic/packedpair.rs":"892d6ee28f0f5007c973d11d697e43e6f6888341b15ea95ce4571e9584069dc5","src/arch/mod.rs":"ca3960b7e2ed28d1b3c121710a870430531aad792f64d4dcb4ca4709d6cbda30","src/arch/wasm32/memchr.rs":"d88ac79f891d8530f505f5035062d3da274a05d66c611480c75430d52709d052","src/arch/wasm32/mod.rs":"a20377aa8fe07d68594879101dc73061e4f51d9c8d812b593b1f376e3c8add79","src/arch/wasm32/simd128/memchr.rs":"bac2c4c43fe710c83a6f2b1118fede043be89dd821d4b532907f129f09fdb5cf","src/arch/wasm32/simd128/mod.rs":"c157b373faedbfd65323be432e25bc411d97aa1b7bc58e76048614c7b2bf3bf6","src/arch/wasm32/simd128/packedpair.rs":"288ba6e5eee6a7a8e5e45c64cff1aa5d72d996c2a6bc228be372c75789f08e45","src/arch/x86_64/avx2/memchr.rs":"576ec0c30f49874f7fd9f6caeb490d56132c0fbbaa4d877b1aa532cafce19323","src/arch/x86_64/avx2/mod.rs":"0033d1b712d0b10f0f273ef9aa8caa53e05e49f4c56a64f39af0b9df97eec584","src/arch/x86_64/avx2/packedpair.rs":"87b69cb4301815906127db4f6370f572c7c5d5dad35c0946c00ad888dbcaec8c","src/arch/x86_64/memchr.rs":"4cf0611c926c3016d1b640880ef14e5654a8d1c1b480fe9a5965956be7eeeeef","src/arch/x86_64/mod.rs":"61b2aa876942fd3e78714c2ae21e356c8634545c06995020f443fa50218df027","src/arch/x86_64/sse2/memchr.rs":"0de0444e26d885eaf866220578752aac871e03bebee7b4f5de7fe8a35f5fa97f","src/arch/x86_64/sse2/mod.rs":"38b70ae52a64ec974dbb91d04d6ca8013d9e06d1fe4af852206bbc2faf1c59aa","src/arch/x86_64/sse2/packedpair.rs":"ad893c5ab9525966693e742673fb666f54ff9102949fc91c5e61642c8ad2f110","src/cow.rs":"2b053e393d3841f780425b3b0da5bec4f187603fa7c271b045ecb7d885b23395","src/ext.rs":"210f89d1e32211bc64414cbd56e97b4f56ce8a8832d321d77a9fe519634e27ea","src/lib.rs":"614f778a41e88a29ea0ceb8e92c839dbb6b5a61c967f8bfd962975e18f932c71","src/macros.rs":"3e4b39252bfa471fad384160a43f113ebfec7bec46a85d16f006622881dd2081","src/memchr.rs":"6ae779ec5d00f443075316e0105edf30b489a38e2e96325bec14ccecd014145b","src/memmem/mod.rs":"d869159f0d0c1915d7acc42a6a4aa3c97c049ddc043e6566afeb78ad290995d1","src/memmem/searcher.rs":"84f6a23bef907696cb672e6898c15fb87008058abc100dde58519d8ec5ebca5d","src/tests/memchr/mod.rs":"269f8e4b4f7f5ea458f27a3c174eb1020ffb2484eeba9464170beb51747df69b","src/tests/memchr/naive.rs":"6a0bee033e5edfb5b1d5769a5fa1c78388f7e9ff7bb91cb67f0ad029289e00e7","src/tests/memchr/prop.rs":"1854eea2338c405fe4635aac430f51e10d2069cd37a7489ddaff47da95f8720b","src/tests/mod.rs":"7cec8f809e279310a465c6a7725087970f219a676cc76c83de30c695bb490740","src/tests/packedpair.rs":"b02ec4fbb61a8653cb5f2268c31bc9168b8043347f2abdcc74081acf83b98e15","src/tests/substring/mod.rs":"c7660d10749363ac4687e7da2b5fda60768230425df8ba416c0c28b8d56a5c74","src/tests/substring/naive.rs":"df6f55d165382b8a53762ba4c324926cac13ebc62cde1805f4ce08740b326483","src/tests/substring/prop.rs":"38c15992609b5681a95d838ae6f2933e00a1219f2c971bfba245f96e0729fcdc","src/vector.rs":"9b104d0bda9b8e18b10c41633513637af84346282bf113f3be3813725605fe31"},"package":"cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml new file mode 100644 index 00000000..a03a2ad6 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml @@ -0,0 +1,89 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.61" +name = "memchr" +version = "2.8.3" +authors = [ + "Andrew Gallant ", + "bluss", +] +build = false +exclude = [ + "/.github", + "/benchmarks", + "/fuzz", + "/scripts", + "/tmp", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Provides extremely fast (uses SIMD on x86_64, aarch64 and wasm32) routines for +1, 2 or 3 byte search and single substring search. +""" +homepage = "https://github.com/BurntSushi/memchr" +documentation = "https://docs.rs/memchr/" +readme = "README.md" +keywords = [ + "memchr", + "memmem", + "substring", + "find", + "search", +] +license = "Unlicense OR MIT" +repository = "https://github.com/BurntSushi/memchr" + +[package.metadata.docs.rs] +rustdoc-args = ["--generate-link-to-definition"] + +[features] +alloc = [] +default = ["std"] +libc = [] +logging = ["dep:log"] +rustc-dep-of-std = ["core"] +std = ["alloc"] +use_std = ["std"] + +[lib] +name = "memchr" +path = "src/lib.rs" +bench = false + +[dependencies.core] +version = "1.0.0" +optional = true +package = "rustc-std-workspace-core" + +[dependencies.log] +version = "0.4.20" +optional = true + +[dev-dependencies.quickcheck] +version = "1.0.3" +default-features = false + +[profile.bench] +debug = 2 + +[profile.release] +debug = 2 + +[profile.test] +opt-level = 3 +debug = 2 diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json new file mode 100644 index 00000000..9fbc6043 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{".cargo_vcs_info.json":"7e77363b21d4f614eb3292289ef0816d753bbed1f3a3253698f2d548defa1bfd",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"8cddd19b88f90df53271d384c62997ba13fe1e9a017e6ff082401c81d46f3be6","Cargo.toml":"b639e399e32159609ed573ffdb1a4edf06fba8aee208d9259e9dba031834aaaa","Cargo.toml.orig":"0a9a6dd2be15fb91d0ad7c545a99a509bdd3a32a4ff795f85fd70b60e91fad0a","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"cd8f304fa9cca9566dff0bb3492beae55f47f3972ddfb8f1fa38c3da1068b4a8","build.rs":"dba46ae4291317fb644ba2143f44eaf54a8ab946ba1367a33d055d694715f68a","src/attr.rs":"e8301f019e107ef9f5d8a3bbed355ffa9c0fd73396646d7fedbba333d341b7bc","src/error.rs":"be08d9d48b4cb9984b5141b12f21c31bd7293942c18b3ce25754723930cf54f5","src/lib.rs":"12a0578bb1d011ae759a94aa4850e970fc1c74102d1ede62b9f405675d7e182e","src/segment.rs":"462be04eea3ad48028d0841e8000c13205eb049522b8003bcc2926f21e93cb78","tests/compiletest.rs":"022a8e400ef813d7ea1875b944549cee5125f6a995dc33e93b48cba3e1b57bd1","tests/test_attr.rs":"39a55dcce5a127b2f85e9127a810b4aa3318e408cac93ff678301a2120aac5b5","tests/test_doc.rs":"7afb22427cb1c54fcf75a581fec5503404d8e93274027040b4d901f8caa597bb","tests/test_expr.rs":"d25ff118a78bd8e7d38ff532f0b9abb046bd0fbe61db8080b973bff67743af4e","tests/test_item.rs":"ee373a6658b384d1b2a0053653ac18e19d4053bbd04c8024980340894c04dcdc","tests/ui/case-warning.rs":"6b7cb803527911fe698e7fbe080160bc5d07c0da36fac0ec29d599753ce8bd10","tests/ui/case-warning.stderr":"7b7d470edc2a11b0864534099b360e0c42d423e068f49cea7858e0b340c2ef3d","tests/ui/env-empty.rs":"093fb0844475f4710580b5e4cad162918896d4fb48d1516bae04eb39dddd265a","tests/ui/env-empty.stderr":"3462f06f6f21130a81efff3be3bc838b395f6107bfbf753e479ec5a60e09b15d","tests/ui/env-non-string.rs":"516460860e7790b412b0fcd203289720a155dfba2498616967e4d7e6e6ab4990","tests/ui/env-non-string.stderr":"fea08d588a238e0f4c06de5c5ae97ac195bb2bcfc070040fb8087bdf4d1f4eee","tests/ui/env-suffix.rs":"0f1756608efc7a6c5810b8b21528092546ac95371e048c6060064e747c1b0a27","tests/ui/env-suffix.stderr":"287fecb2a91abc61501e0bd71d7f111670a526960ab18a554f4d16ed83d940b5","tests/ui/env-unexpected.rs":"18f06317e72ea16dae6ebe7c20bf5ef26b46e01a5c54fd31dc2a351ba68e8ca5","tests/ui/env-unexpected.stderr":"14d9a40160daf0191efc2b53be5fdde5b066674053e76af170e58af08f4022aa","tests/ui/invalid-ident.rs":"c3f375cac04d56684e427834076128b6e77eeeb9e99067a06307e27693b0ff51","tests/ui/invalid-ident.stderr":"b56d1c26253dec71ba1f97ffa01602b3a4429fc85a6b5c87f5290aab0568ea30","tests/ui/missing-paren-on-env.rs":"05cafef972e8c99f82834e164e7713f6056ac1a24b1f31d0320ac4fe361a53bd","tests/ui/missing-paren-on-env.stderr":"baf7b420d201f033e2113eaebd85376047397d991c35c441f07e5c6f1ec2b685","tests/ui/no-env-var.rs":"330bb71f2f3d98d23176acc5aeb1c0b66443af14c0857bb5bc92bfd130eb4a62","tests/ui/no-env-var.stderr":"68af072d3ccb2537bca9a5cd6d5796bb5ab5408f59564c8e8c66b1590ac7c5a5","tests/ui/no-ident-after-colon.rs":"a5af9548ab8d7f5b6130e1410c90f2cc589c595eb99f6939e6deb3f54e7262a2","tests/ui/no-ident-after-colon.stderr":"efac036a78f077c9c10a1ed048d69531293dcb97784ed784af0b24788ecbab54","tests/ui/unexpected-group.rs":"6b2e398a9ad3e077ea49c68b7c134ec3268e398723f4d9d5ef7b4d7b8513ac44","tests/ui/unexpected-group.stderr":"1260b5a4c295639958e3dd0da475c385184f0a916f5fdc4f1f2bfe232688f43c","tests/ui/unexpected-modifier.rs":"8b7e4c93f06fe7e7be469375e8bbaf19fec5fdffe688a0c29d2de2944a420337","tests/ui/unexpected-modifier.stderr":"581212b89358ac6bd38632bfd4322c25c171046875711fd639cfb498e01629c8","tests/ui/unexpected-punct.rs":"430e98bc03abdcc46f72cea01264c278a9c25f3d1d63064a6d78f4ae71b6a2d9","tests/ui/unexpected-punct.stderr":"24ab2e32d2da8dad93ef04dc05a2ecf6fdeb11bdebeae9454ad4759cc2b16551","tests/ui/unsupported-literal.rs":"1b2d3634fba5dc35d5ac9012d488aa93e329d1fa4489df1eadee65613a010e59","tests/ui/unsupported-literal.stderr":"6b137ccd6bc2a341aeb1e0d5321a896aa6c8db9a4e2571f4e41bd744511760bf","tests/ui/unsupported-modifier.rs":"03bb0239880070e8b470a6072c06094e1483ea91256e7cb4b9f2e8761696fa5a","tests/ui/unsupported-modifier.stderr":"890d7f484fdbd46da72923db4d7aba0ac3eaece253bb7195b7a36ae9a918027e"},"package":"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml new file mode 100644 index 00000000..c5872871 --- /dev/null +++ b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml @@ -0,0 +1,72 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +rust-version = "1.31" +name = "paste" +version = "1.0.15" +authors = ["David Tolnay "] +build = "build.rs" +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Macros for all your token pasting needs" +documentation = "https://docs.rs/paste" +readme = "README.md" +keywords = ["macros"] +categories = [ + "development-tools", + "no-std", + "no-std::no-alloc", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/dtolnay/paste" + +[package.metadata.docs.rs] +rustdoc-args = ["--generate-link-to-definition"] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +name = "paste" +path = "src/lib.rs" +proc-macro = true + +[[test]] +name = "test_item" +path = "tests/test_item.rs" + +[[test]] +name = "test_attr" +path = "tests/test_attr.rs" + +[[test]] +name = "compiletest" +path = "tests/compiletest.rs" + +[[test]] +name = "test_doc" +path = "tests/test_doc.rs" + +[[test]] +name = "test_expr" +path = "tests/test_expr.rs" + +[dev-dependencies.paste-test-suite] +version = "0" + +[dev-dependencies.rustversion] +version = "1.0" + +[dev-dependencies.trybuild] +version = "1.0.49" +features = ["diff"] diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index 2b31a0de..1f59e9f0 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -2,20 +2,19 @@ import { test } from 'node:test' import { spawnSync } from 'node:child_process' import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' -import { dirname, join, parse } from 'node:path' +import { dirname, join, parse, posix } from 'node:path' import { fileURLToPath } from 'node:url' import { + HOST_UNIT, createCargoContext, evalCfg, findCargoLock, - parseCargoLock, parseCargoManifest, parseFeatureList, parseRustcCfg, resolutionFromMetadata, rustcTargetCfgs, - satisfiesCargoReq, } from '../stasis/src/loaders/cargo.js' import { buildRustBundle } from '../stasis/src/cmd/bundle.js' import { buildRustTree, collectRustFilesFromDisk } from '../stasis/src/loaders/rust.js' @@ -28,6 +27,58 @@ const enabledOf = (cargo, context) => Object.fromEntries([...cargo.resolvedFeatu // The packages built only maybe, or with features on only maybe: dir -> those features. const maybeOf = (cargo, context) => Object.fromEntries([...cargo.featureResolution(context)].filter(([dir, r]) => r.maybe.size > 0 || !cargo.resolvedFeatures(context).has(dir)).map(([dir, r]) => [dir, sorted(r.maybe)]).toSorted()) +// A throwaway project: `files` maps project-relative paths to their text. +const writeProject = (files) => { + const tmp = mkdtempSync(join(tmpdir(), 'stasis-cargo-')) + for (const [rel, text] of Object.entries(files)) { + mkdirSync(dirname(join(tmp, rel)), { recursive: true }) + writeFileSync(join(tmp, rel), text) + } + return tmp +} +const withProject = (files, fn) => { + const tmp = writeProject(files) + try { + return fn(tmp) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} +const captureWarningsAsync = async (fn) => { + const original = console.warn + const warnings = [] + console.warn = (...args) => warnings.push(args.join(' ')) + try { + return { result: await fn(), warnings } + } finally { + console.warn = original + } +} +const withProjectAsync = async (files, fn) => { + const tmp = writeProject(files) + try { + return await fn(tmp) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} + +// Warnings `fn` prints, and its result. +const captureWarningsSync = (fn) => { + const original = console.warn + const warnings = [] + console.warn = (...args) => warnings.push(args.join(' ')) + try { + return { result: fn(), warnings } + } finally { + console.warn = original + } +} +const vendoredPackage = (name, version) => ({ + [`vendor/${name}-${version}/Cargo.toml`]: `[package]\nname = "${name}"\nversion = "${version}"\n`, + [`vendor/${name}-${version}/src/lib.rs`]: '', +}) + // --- Cargo.toml / Cargo.lock --- test('parseCargoManifest reads multi-line arrays, feature tables, dependency kinds and patches', (t) => { @@ -42,43 +93,44 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin '[patch.crates-io]', 'plain = { path = "patches/plain" }', 'dotted.path = "patches/dotted"', '[patch.crates-io.subbed]', 'path = "patches/subbed"', ].join('\n')) - t.assert.deepEqual(m.package, { name: 'app', version: '0.1.0', versionFromWorkspace: false, edition: '2021', editionFromWorkspace: false, build: null }) - t.assert.equal(m.resolver, '2') - t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']]]) + t.assert.deepEqual(m.package, { name: 'app', version: '0.1.0', edition: '2021', build: null }) + t.assert.equal(m.resolver, 2) + // cargo's feature map: as written, and a feature for the optional dependency no `dep:` names + t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']], ['pm-crate', ['dep:pm-crate']]]) const dep = (k) => Object.fromEntries([...m.deps.get(k).kinds].toSorted()) - const ask = (version, more) => ({ version, path: null, package: null, workspace: false, optional: false, defaultFeatures: null, features: [], ...more }) + const ask = (pkg, version, more) => ({ version, path: null, source: 'registry', package: pkg, renamed: false, inherited: false, optional: false, defaultFeatures: true, features: [], ...more }) // Each dependency table is its own request, for the crate it names: a dev-dependency's features - // stay out of the normal one. `defaultFeatures` stays null until a table says (an inherited - // entry may only turn defaults on). - t.assert.deepEqual(dep('plain'), { dev: ask('1', { features: ['dev-only'] }), normal: ask('1') }) - t.assert.deepEqual(dep('opt'), { normal: ask('1', { optional: true, defaultFeatures: false, features: ['a'] }) }) - t.assert.deepEqual(dep('sub'), { normal: ask('2', { features: ['one', 'two'] }) }) + // stay out of the normal one. + t.assert.deepEqual(dep('plain'), { dev: ask('plain', '1', { features: ['dev-only'] }), normal: ask('plain', '1') }) + t.assert.deepEqual(dep('opt'), { normal: ask('opt', '1', { optional: true, defaultFeatures: false, features: ['a'] }) }) + t.assert.deepEqual(dep('sub'), { normal: ask('sub', '2', { features: ['one', 'two'] }) }) t.assert.deepEqual(Object.keys(dep('cc')), ['build']) // A target-specific table is a request of its own, beside the plain one. t.assert.deepEqual(Object.keys(dep('nix')), ['normal@cfg(unix)']) // the key is the `use` spelling, the name the manifest's (an optional dep's implicit feature name) t.assert.deepEqual([m.deps.get('pm_crate').key, m.deps.get('pm_crate').name, m.deps.get('pm_crate').kinds.get('normal').optional], ['pm_crate', 'pm-crate', true]) // `[patch.]` entries in every spelling: inline table, dotted key, sub-table. - t.assert.deepEqual([...m.patches], [['plain', 'patches/plain'], ['dotted', 'patches/dotted'], ['subbed', 'patches/subbed']]) + t.assert.deepEqual(Object.entries(m.cargo.patch['crates-io']).map(([name, spec]) => [name, spec.source.path]), [['plain', 'patches/plain'], ['dotted', 'patches/dotted'], ['subbed', 'patches/subbed']]) }) -test('parseCargoManifest reads a workspace-inherited edition', (t) => { - const m = parseCargoManifest(['[workspace]', '[workspace.package]', 'edition = "2021"', '[package]', 'name = "app"', 'version = "0.1.0"', 'edition.workspace = true'].join('\n')) - t.assert.equal(m.package.edition, null) - t.assert.equal(m.package.editionFromWorkspace, true) - t.assert.equal(m.workspacePackage.edition, '2021') - t.assert.equal(parseCargoManifest('[package]\nname = "app"\nedition = { workspace = true }\n').package.editionFromWorkspace, true) +test('parseCargoManifest reads a workspace-inherited edition, from its own [workspace] or the root given', (t) => { + const root = parseCargoManifest(['[workspace]', '[workspace.package]', 'edition = "2021"', '[package]', 'name = "app"', 'version = "0.1.0"', 'edition.workspace = true'].join('\n')) + t.assert.equal(root.package.edition, '2021') + const member = '[package]\nname = "lib"\nedition = { workspace = true }\n' + t.assert.equal(parseCargoManifest(member, 'crates/lib/Cargo.toml', root).package.edition, '2021') + t.assert.throws(() => parseCargoManifest(member, 'crates/lib/Cargo.toml'), { name: 'LockfileError', message: 'crates/lib/Cargo.toml: package.edition: inherits from a workspace, and no workspace root is given' }) }) test('parseCargoManifest splits dotted dependency keys and survives multi-line strings', (t) => { + const root = parseCargoManifest('[workspace]\n[workspace.dependencies]\nutil = { path = "crates/util" }\n') const m = parseCargoManifest([ '[package]', 'name = "app"', 'description = """', 'Not a table: [x]', 'nor a key = value', '"""', 'version = "0.4.0"', '[dependencies]', 'util.workspace = true', 'util.features = ["extra"]', "serde.version = '1'", 'serde.features = [', ' "derive",', ']', - ].join('\n')) + ].join('\n'), 'crates/app/Cargo.toml', root) t.assert.equal(m.package.version, '0.4.0') t.assert.deepEqual([...m.deps.keys()], ['util', 'serde']) - t.assert.equal(m.deps.get('util').kinds.get('normal').workspace, true) - t.assert.deepEqual(m.deps.get('util').kinds.get('normal').features, ['extra']) + const util = m.deps.get('util').kinds.get('normal') + t.assert.deepEqual([util.inherited, util.path, util.features], [true, 'crates/util', ['extra']]) // the path is the workspace root's t.assert.equal(m.deps.get('serde').kinds.get('normal').version, '1') t.assert.deepEqual(m.deps.get('serde').kinds.get('normal').features, ['derive']) }) @@ -91,7 +143,7 @@ test('parseCargoManifest reads a pair by the table it lands in, whichever way th '[dependencies]\nfoo = { version = "1", features = ["x"] }\n', ] for (const text of spellings) { - const normal = parseCargoManifest(text).deps.get('foo').kinds.get('normal') + const normal = parseCargoManifest(`[package]\nname = "app"\n${text}`).deps.get('foo').kinds.get('normal') t.assert.deepEqual([normal.version, normal.features], ['1', ['x']], text) } const m = parseCargoManifest([ @@ -101,31 +153,50 @@ test('parseCargoManifest reads a pair by the table it lands in, whichever way th '[patch.crates-io.plain]', 'path = "patches/plain"', '[workspace.package]', 'version = "0.9.0"', // a [workspace.*] table alone makes this a workspace root ].join('\n')) - t.assert.deepEqual(m.package, { name: 'app', version: null, versionFromWorkspace: true, edition: null, editionFromWorkspace: false, build: null }) + t.assert.deepEqual(m.package, { name: 'app', version: '0.9.0', edition: '2015', build: null }) t.assert.deepEqual([...m.deps.get('winapi').kinds].map(([request, r]) => [request, r.version]), [['dev@cfg(windows)', '0.3']]) - t.assert.deepEqual([...m.patches], [['plain', 'patches/plain']]) - t.assert.deepEqual([m.isWorkspace, m.workspacePackage.version], [true, '0.9.0']) + t.assert.deepEqual(Object.keys(m.cargo.patch['crates-io']), ['plain']) + t.assert.equal(m.isWorkspace, true) }) -test('parseCargoManifest and parseCargoLock refuse text that is not TOML, naming the file and line', (t) => { +test('parseCargoManifest refuses text that is not TOML, or not a manifest cargo reads, naming the file and line', (t) => { t.assert.throws(() => parseCargoManifest('[package]\nname = "app"\nversion = 0.1.0\n', 'crates/app/Cargo.toml'), { name: 'TomlError', message: 'crates/app/Cargo.toml: expected a value, found "0.1.0" at line 3', }) t.assert.throws(() => parseCargoManifest('[dependencies]\nserde = { version = "1", version = "2" }\n'), { message: 'duplicate key "version" at line 2' }) - t.assert.throws(() => parseCargoLock('version = 3\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["a" "b"]\n', 'Cargo.lock'), { - name: 'TomlError', message: 'Cargo.lock: expected "," or "]", found "\\"b\\"]" at line 6', + // what cargo would refuse, or read in a way the reader can't tell: @preventive/lockfile's word + t.assert.throws(() => parseCargoManifest('[package]\nname = "app"\n[features]\nstd = ["nothing"]\n', 'Cargo.toml'), { + name: 'LockfileError', message: 'Cargo.toml: features.std: "nothing" is neither a feature nor a dependency', }) + t.assert.throws(() => parseCargoManifest('[package]\nname = "app"\n[replace]\n"foo:1.0.0" = { path = "foo" }\n', 'Cargo.toml'), { name: 'LockfileError', message: 'Cargo.toml: replace: [replace] is not supported' }) // through the context, with the manifest's project-relative path - const tmp = mkdtempSync(join(tmpdir(), 'stasis-toml-')) - try { - mkdirSync(join(tmp, 'crates', 'app', 'src'), { recursive: true }) - writeFileSync(join(tmp, 'Cargo.toml'), '[workspace]\nmembers = ["crates/app"]\n') - writeFileSync(join(tmp, 'crates', 'app', 'Cargo.toml'), '[package]\nname = "app"\nversion = "0.1.0\n') - writeFileSync(join(tmp, 'crates', 'app', 'src', 'main.rs'), 'fn main() {}\n') + withProject({ + 'Cargo.toml': '[workspace]\nmembers = ["crates/app"]\n', + 'crates/app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0\n', + 'crates/app/src/main.rs': 'fn main() {}\n', + }, (tmp) => { t.assert.throws(() => createCargoContext(tmp).packageInfo('crates/app/src/main.rs'), { name: 'TomlError', message: 'crates/app/Cargo.toml: unterminated string at line 3', line: 2 }) - } finally { - rmSync(tmp, { recursive: true, force: true }) - } + }) +}) + +test('createCargoContext refuses a Cargo.lock older than version 3, or not TOML, naming it', (t) => { + const files = (lock) => ({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nrand = "0.8"\n', + 'Cargo.lock': lock, + 'src/main.rs': '', + ...vendoredPackage('rand', '0.8.5'), + }) + const v2 = '[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["rand"]\n\n[[package]]\nname = "rand"\nversion = "0.8.5"\nsource = "registry+https://github.com/rust-lang/crates.io-index"\n' + withProject(files(v2), (tmp) => { + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/main.rs'] }).resolveCrate('rand', 'src/main.rs'), { + name: 'LockfileError', message: 'Cargo.lock: version: no `version`: this is lockfile version 1 or 2, which is not read here, where 3 and 4 are', + }) + }) + withProject(files('version = 3\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["a" "b"]\n'), (tmp) => { + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/main.rs'] }).resolveCrate('rand', 'src/main.rs'), { + name: 'TomlError', message: 'Cargo.lock: expected "," or "]", found "\\"b\\"]" at line 6', + }) + }) }) test('parseFeatureList splits cargo\'s repeatable, comma- or space-separated feature flags', (t) => { @@ -156,62 +227,6 @@ test('findCargoLock finds the lock beside the workspace root (a member dir\'s is } }) -test('parseCargoLock indexes packages and their (possibly versioned) dependency edges', (t) => { - const lock = parseCargoLock([ - 'version = 3', '', '[[package]]', 'name = "app"', 'version = "0.1.0"', 'dependencies = [', ' "lib-a",', ' "winnowish 0.6.1",', ']', '', - '[[package]]', 'name = "lib-a"', 'version = "0.2.0"', 'dependencies = ["winnowish 0.5.0"]', '', - '[[package]]', 'name = "winnowish"', 'version = "0.5.0"', 'source = "registry+https://github.com/rust-lang/crates.io-index"', '', - '[[package]]', 'name = "winnowish"', 'version = "0.6.1"', - ].join('\n')) - t.assert.deepEqual(lock.byId.get('app 0.1.0').deps, [{ name: 'lib_a', version: null }, { name: 'winnowish', version: '0.6.1' }]) - t.assert.deepEqual(lock.byId.get('lib_a 0.2.0').deps, [{ name: 'winnowish', version: '0.5.0' }]) - t.assert.deepEqual(lock.byName.get('winnowish').map((p) => p.version), ['0.5.0', '0.6.1']) - t.assert.equal(parseCargoLock(null), null) -}) - -test('satisfiesCargoReq implements Cargo requirement semantics', (t) => { - // caret is the default; the leftmost non-zero part may not change - t.assert.equal(satisfiesCargoReq('1.5.0', '1'), true) - t.assert.equal(satisfiesCargoReq('2.0.0', '1'), false) - t.assert.equal(satisfiesCargoReq('1.2.9', '1.2'), true) - t.assert.equal(satisfiesCargoReq('1.1.0', '1.2'), false) - t.assert.equal(satisfiesCargoReq('0.9.3', '0.9'), true) - t.assert.equal(satisfiesCargoReq('0.10.3', '0.9'), false) - t.assert.equal(satisfiesCargoReq('0.10.3', '0.10'), true) - t.assert.equal(satisfiesCargoReq('0.0.3', '0.0.3'), true) - t.assert.equal(satisfiesCargoReq('0.0.4', '0.0.3'), false) - t.assert.equal(satisfiesCargoReq('0.5.0', '^0'), true) - t.assert.equal(satisfiesCargoReq('1.0.0', '^0'), false) - // a wildcard with an operator keeps the operator: the `*` only omits the component - t.assert.equal(satisfiesCargoReq('3.0.0', '>=1.*'), true) - t.assert.equal(satisfiesCargoReq('1.5.0', '>1.*'), false) - t.assert.equal(satisfiesCargoReq('2.0.0', '>1.*'), true) - t.assert.equal(satisfiesCargoReq('1.2.9', '=1.2.*'), true) - t.assert.equal(satisfiesCargoReq('1.3.0', '=1.2.*'), false) - t.assert.equal(satisfiesCargoReq('2.0.0', '1.*'), false) - // tilde, wildcard, exact, comparisons, several comparators - t.assert.equal(satisfiesCargoReq('1.2.9', '~1.2.3'), true) - t.assert.equal(satisfiesCargoReq('1.3.0', '~1.2.3'), false) - t.assert.equal(satisfiesCargoReq('1.7.0', '1.*'), true) - t.assert.equal(satisfiesCargoReq('1.7.0', '1.2.*'), false) - t.assert.equal(satisfiesCargoReq('1.2.3', '=1.2.3'), true) - t.assert.equal(satisfiesCargoReq('1.2.4', '=1.2.3'), false) - t.assert.equal(satisfiesCargoReq('1.2.4', '=1.2'), true) - t.assert.equal(satisfiesCargoReq('1.9.0', '>=1.2, <2.0'), true) - t.assert.equal(satisfiesCargoReq('2.0.0', '>=1.2, <2.0'), false) - // a partial bound covers its whole minor: `>1.2` is `>=1.3.0`, `<=1.2` is `<1.3.0` - t.assert.equal(satisfiesCargoReq('1.2.5', '>1.2'), false) - t.assert.equal(satisfiesCargoReq('1.3.0', '>1.2'), true) - t.assert.equal(satisfiesCargoReq('1.2.9', '<=1.2'), true) - t.assert.equal(satisfiesCargoReq('1.3.0', '<=1.2'), false) - t.assert.equal(satisfiesCargoReq('1.2.1', '>1.2.0'), true) - t.assert.equal(satisfiesCargoReq('1.2.0', '<=1.2.0'), true) - t.assert.equal(satisfiesCargoReq('9.9.9', '*'), true) - // a prerelease sorts below its release - t.assert.equal(satisfiesCargoReq('1.0.0-beta.1', '>=1.0.0'), false) - t.assert.equal(satisfiesCargoReq('junk', '1'), false) -}) - // --- feature resolution from the manifests --- test('createCargoContext resolves features like `cargo build` of the entry package: defaults, implications, dep requests', (t) => { @@ -264,42 +279,6 @@ test('createCargoContext falls back to the requirement when there is no Cargo.lo } }) -// A throwaway project: `files` maps project-relative paths to their text. -const writeProject = (files) => { - const tmp = mkdtempSync(join(tmpdir(), 'stasis-cargo-')) - for (const [rel, text] of Object.entries(files)) { - mkdirSync(dirname(join(tmp, rel)), { recursive: true }) - writeFileSync(join(tmp, rel), text) - } - return tmp -} -const withProject = (files, fn) => { - const tmp = writeProject(files) - try { - return fn(tmp) - } finally { - rmSync(tmp, { recursive: true, force: true }) - } -} -const captureWarningsAsync = async (fn) => { - const original = console.warn - const warnings = [] - console.warn = (...args) => warnings.push(args.join(' ')) - try { - return { result: await fn(), warnings } - } finally { - console.warn = original - } -} -const withProjectAsync = async (files, fn) => { - const tmp = writeProject(files) - try { - return await fn(tmp) - } finally { - rmSync(tmp, { recursive: true, force: true }) - } -} - test('createCargoContext applies a bare --cargo-features name to every root package that has it', (t) => { withProject({ 'Cargo.toml': '[workspace]\nmembers = ["a", "b"]\n', @@ -361,17 +340,21 @@ test('createCargoContext applies a weak `dep?/feat` once the dependency is activ t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'] }) t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], features: ['with-dep'] })), { '.': ['default', 'std', 'with-dep'], 'vendor/dep': ['std'] }) }) - // required without default features: active from the start - withProject(files(['[dependencies]', 'dep = { version = "1", default-features = false }']), (tmp) => { - t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) + // optional in one table, required in another: active from the start + withProject(files(['[dependencies]', 'dep = { version = "1", optional = true }', '[build-dependencies]', 'dep = { version = "1", default-features = false }']), (tmp) => { + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'] }), 'host'), { 'vendor/dep': ['std'] }) }) - // required only on a target: without one, the target table makes the dependency's `std` a maybe - withProject(files(["[target.'cfg(unix)'.dependencies]", 'dep = { version = "1", default-features = false }']), (tmp) => { + // optional, and required only on a target: without one, the target table makes the dependency's `std` a maybe + withProject(files(['[dependencies]', 'dep = { version = "1", optional = true }', "[target.'cfg(unix)'.dependencies]", 'dep = { version = "1", default-features = false }']), (tmp) => { const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) t.assert.deepEqual(enabledOf(cargo), { '.': ['default', 'std'] }) t.assert.deepEqual(maybeOf(cargo), { 'vendor/dep': ['std'] }) t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX })), { '.': ['default', 'std'], 'vendor/dep': ['std'] }) }) + // `dep?/std` of a dependency no table makes optional is a manifest cargo refuses + withProject(files(['[dependencies]', 'dep = { version = "1", default-features = false }']), (tmp) => { + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/main.rs'] }).featuresFor('src/main.rs'), { name: 'LockfileError', message: 'Cargo.toml: features.std: "dep?/std" names "dep", which is not an optional dependency' }) + }) }) test('createCargoContext applies a weak `dep?/feat` from --cargo-features once `default` has activated the dependency', (t) => { @@ -1291,22 +1274,6 @@ test('buildRustBundle declares a template\'s mod in each module of its package i }) }) -// Warnings `fn` prints, and its result. -const captureWarningsSync = (fn) => { - const original = console.warn - const warnings = [] - console.warn = (...args) => warnings.push(args.join(' ')) - try { - return { result: fn(), warnings } - } finally { - console.warn = original - } -} -const vendoredPackage = (name, version) => ({ - [`vendor/${name}-${version}/Cargo.toml`]: `[package]\nname = "${name}"\nversion = "${version}"\n`, - [`vendor/${name}-${version}/src/lib.rs`]: '', -}) - test('createCargoContext resolves each dependency table on its own: one version per table, the asking file\'s table', (t) => { withProject({ 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nrand = "0.7"\n[build-dependencies]\nrand = "0.8"\n[dev-dependencies]\nfake = { package = "rand", version = "0.8" }\n', @@ -1328,7 +1295,7 @@ test('createCargoContext takes a Cargo.lock pin only when the requirement allows withProject({ 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nrand = "0.8"\n', // out of date: the manifest moved on to 0.8, the lock still says 0.7.3 - 'Cargo.lock': 'version = 3\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["rand"]\n\n[[package]]\nname = "rand"\nversion = "0.7.3"\n', + 'Cargo.lock': 'version = 3\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["rand"]\n\n[[package]]\nname = "rand"\nversion = "0.7.3"\nsource = "registry+https://github.com/rust-lang/crates.io-index"\n', 'src/main.rs': '', ...vendoredPackage('rand', '0.7.3'), ...vendoredPackage('rand', '0.8.5'), }, (tmp) => { @@ -1395,3 +1362,226 @@ test('buildRustBundle ends when a crate root it wants is refused, and reports ev ]) }) }) + +// --- ninth round: @preventive/lockfile's reading and cargo's resolver --- + +// A workspace cargo locked and vendored, and in recorded.json what cargo itself made of it: the +// features `cargo build --unit-graph` built each package with for a few command lines under +// resolver 1 and 2, null where cargo refused one, and the targets' `rustc --print cfg`. It is +// @preventive/lockfile's fixture (PreventiveMeasures/libraries#56, MIT), recorded there by +// lockfile/scripts/record-cargo.js. +const cargoRecorded = join(fixtures, 'cargo-recorded') +const recordedProject = (resolver) => { + const tmp = mkdtempSync(join(tmpdir(), 'stasis-recorded-')) + cpSync(cargoRecorded, tmp, { recursive: true }) + if (resolver === '1') writeFileSync(join(tmp, 'Cargo.toml'), readFileSync(join(tmp, 'Cargo.toml'), 'utf8').replace('[workspace]\n', '[workspace]\nresolver = "1"\n')) + return tmp +} + +test('createCargoContext turns on the features cargo builds each package with, given the lockfile and the target', (t) => { + const recorded = JSON.parse(readFileSync(join(cargoRecorded, 'recorded.json'), 'utf8')) + const dirOf = new Map([ + ...Object.entries(recorded.manifests).map(([key, path]) => [key, dirname(path)]), + ...Object.entries(recorded.vendored).map(([key, directory]) => [key, `vendor/${directory}`]), + ]) + const memberDir = { app: '.', lib: 'crates/lib', macros: 'crates/macros' } + const platform = (triple) => ({ triple, cfgs: parseRustcCfg(recorded.cfg[triple].join('\n')) }) + for (const [resolver, builds] of Object.entries(recorded.features)) { + const tmp = recordedProject(resolver) + try { + for (const build of builds) { + const label = `resolver ${resolver}: ${JSON.stringify({ ...build, built: undefined })}` + const names = build.packages === 'all' ? ['lib', 'macros', 'app'] : build.packages + const entries = names.map((name) => posix.join(memberDir[name], build.dev ? 'tests/t.rs' : 'src/lib.rs')) + const cargo = () => createCargoContext(tmp, { + entries, + features: build.features ?? [], + noDefaultFeatures: build.noDefaultFeatures === true, + allFeatures: build.allFeatures === true, + target: platform(build.target ?? recorded.host), + host: platform(recorded.host), + }) + if (build.built === null) { + t.assert.throws(() => cargo().featuresFor('src/lib.rs'), { name: 'LockfileError' }, label) + continue + } + const ctx = cargo() + const got = (unit) => { + const [key, fk] = [unit.slice(0, unit.lastIndexOf(' ')), unit.slice(unit.lastIndexOf(' ') + 1)] + return ctx.resolvedFeatures(fk === 'normal' ? 'target' : 'host').get(dirOf.get(key)) + } + for (const [unit, on] of Object.entries(build.built)) t.assert.deepEqual(sorted(got(unit) ?? []), on, `${label}: ${unit}`) + if (resolver === '1') continue // one set per package, both ways + // Nothing else is built, but the proc-macro member for the target, which cargo's resolver + // lists in case it has more targets than its library. + for (const [context, fk] of [['target', 'normal'], ['host', 'host']]) { + for (const dir of ctx.resolvedFeatures(context).keys()) { + const key = [...dirOf].find(([, d]) => d === dir)[0] + if (!(`${key} ${fk}` in build.built)) t.assert.deepEqual([key, fk], ['macros 0.1.0', 'normal'], label) + } + } + } + } finally { + rmSync(tmp, { recursive: true, force: true }) + } + } +}) + +test('createCargoContext resolves through cargo\'s graph: each table\'s source, the [patch], the proc-macro for the host', (t) => { + const tmp = recordedProject('2') + try { + const recorded = JSON.parse(readFileSync(join(tmp, 'recorded.json'), 'utf8')) + const ctx = createCargoContext(tmp, { entries: ['src/lib.rs'], features: ['fmt,git'], target: { triple: recorded.host, cfgs: parseRustcCfg(recorded.cfg[recorded.host].join('\n')) } }) + for (const dir of ['vendor/itoa', 'vendor/itoa-0.4.8', 'vendor/itoa-1.0.0', 'patched/cfg-if', 'crates/lib', 'crates/macros']) mkdirSync(join(tmp, dir, 'src'), { recursive: true }) + for (const dir of ['vendor/itoa', 'vendor/itoa-0.4.8', 'vendor/itoa-1.0.0', 'patched/cfg-if', 'crates/lib', 'crates/macros']) writeFileSync(join(tmp, dir, 'src', 'lib.rs'), '') + // `itoa = "1"` from crates.io, `itoa04 = { package = "itoa", version = "0.4" }`, `itoa-git` from + // git at tag 1.0.0: three copies, each the one its table names + t.assert.equal(ctx.resolveCrate('itoa', 'src/lib.rs'), 'vendor/itoa/src/lib.rs') + t.assert.equal(ctx.resolveCrate('itoa04', 'src/lib.rs'), 'vendor/itoa-0.4.8/src/lib.rs') + t.assert.equal(ctx.resolveCrate('itoa_git', 'src/lib.rs'), 'vendor/itoa-1.0.0/src/lib.rs') + t.assert.equal(ctx.resolveCrate('cfg_if', 'src/lib.rs'), 'patched/cfg-if/src/lib.rs') // [patch.crates-io], a path + t.assert.equal(ctx.resolveCrate('macros', 'src/lib.rs'), 'crates/macros/src/lib.rs') + // the proc-macro and what it depends on are built for the host, with the host's features + t.assert.deepEqual(sorted(ctx.featuresFor('crates/macros/src/lib.rs', HOST_UNIT)), []) + t.assert.deepEqual(sorted(ctx.featuresFor('vendor/memchr/src/lib.rs', HOST_UNIT)), ['alloc']) + t.assert.deepEqual(sorted(ctx.featuresFor('vendor/memchr/src/lib.rs')), ['alloc', 'std']) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +}) + +test('createCargoContext stops the build where the lockfile and the vendored copies disagree, and replays the manifests where it lacks a package', (t) => { + const recorded = JSON.parse(readFileSync(join(cargoRecorded, 'recorded.json'), 'utf8')) + const target = { triple: recorded.host, cfgs: parseRustcCfg(recorded.cfg[recorded.host].join('\n')) } + const tmp = recordedProject('2') + try { + // a vendored copy whose checksum isn't the lockfile's: cargo refuses to build from it + const sums = join(tmp, 'vendor', 'paste', '.cargo-checksum.json') + const original = readFileSync(sums, 'utf8') + writeFileSync(sums, original.replace(/"package":"[\da-f]{64}"/u, `"package":"${'0'.repeat(64)}"`)) + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/lib.rs'], target }).featuresFor('src/lib.rs'), { name: 'LockfileError', message: /^Cargo\.lock: paste 1\.0\.15 .*: "paste" holds it with checksum 0{64}, where the lockfile has / }) + writeFileSync(sums, original) + // a manifest the lockfile is out of date with: lib's entry in it has no cfg-if + const manifest = readFileSync(join(tmp, 'crates', 'lib', 'Cargo.toml'), 'utf8') + writeFileSync(join(tmp, 'crates', 'lib', 'Cargo.toml'), manifest.replace('[target.', 'cfg-if = "1"\n[target.')) + t.assert.throws(() => createCargoContext(tmp, { entries: ['src/lib.rs'], target }).featuresFor('src/lib.rs'), { name: 'LockfileError', message: /^Cargo\.lock: lib 0\.2\.0: the lockfile resolves no "cfg-if", which the members' features turn on: is it out of date\?$/u }) + writeFileSync(join(tmp, 'crates', 'lib', 'Cargo.toml'), manifest) + // a package the lockfile has and the vendor directory doesn't: the replay decides, as without a lockfile + rmSync(join(tmp, 'vendor', 'paste'), { recursive: true }) + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/lib.rs'], target }))['.'], ['default', 'fast']) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +}) + +// A registry package's `.cargo-checksum.json` and lockfile checksum; a git checkout's has none. +const REGISTRY = 'registry+https://github.com/rust-lang/crates.io-index' +const sha = (c) => c.repeat(64) +const vendoredCopy = (dir, manifest, checksum) => ({ + [`vendor/${dir}/Cargo.toml`]: manifest, + [`vendor/${dir}/.cargo-checksum.json`]: JSON.stringify({ files: {}, package: checksum }), + [`vendor/${dir}/src/lib.rs`]: '', +}) + +test('createCargoContext resolves a host it doesn\'t know both ways: its target-specific tables only maybe', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[target.\'cfg(unix)\'.build-dependencies]\ncc = { version = "1", features = ["parallel"] }\n', + 'Cargo.lock': `version = 4\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["cc"]\n\n[[package]]\nname = "cc"\nversion = "1.0.0"\nsource = "${REGISTRY}"\nchecksum = "${sha('c')}"\n`, + 'src/lib.rs': '', 'build.rs': '', + ...vendoredCopy('cc', '[package]\nname = "cc"\nversion = "1.0.0"\n[features]\nparallel = []\n', sha('c')), + }, (tmp) => { + // the target known, the host not: the build-dependency's table is about the host + const cross = createCargoContext(tmp, { entries: ['src/lib.rs'], target: LINUX }) + t.assert.equal(cross.featuresFor('vendor/cc/src/lib.rs', HOST_UNIT).size, 0) + t.assert.deepEqual(sorted(cross.maybeFeaturesFor('vendor/cc/src/lib.rs', HOST_UNIT)), ['parallel']) + // the host known: decided + const native = createCargoContext(tmp, { entries: ['src/lib.rs'], target: LINUX, host: LINUX }) + t.assert.deepEqual(sorted(native.featuresFor('vendor/cc/src/lib.rs', HOST_UNIT)), ['parallel']) + t.assert.equal(native.maybeFeaturesFor('vendor/cc/src/lib.rs', HOST_UNIT), null) + }) +}) + +test('createCargoContext takes a git dependency\'s copy from a git checkout, a registry one\'s from the registry, lockfile or not', (t) => { + const git = 'git+https://github.com/dtolnay/itoa?tag=1.0.0#e6a8f6f2f193aa852a3d2d84f2721e75d4517bff' + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nitoa = "1"\nitoa-git = { package = "itoa", git = "https://github.com/dtolnay/itoa", tag = "1.0.0" }\n', + 'src/main.rs': '', + ...vendoredCopy('itoa', '[package]\nname = "itoa"\nversion = "1.0.18"\n', sha('a')), + ...vendoredCopy('itoa-1.0.0', '[package]\nname = "itoa"\nversion = "1.0.0"\n', null), + } + const check = (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + t.assert.equal(cargo.resolveCrate('itoa', 'src/main.rs'), 'vendor/itoa/src/lib.rs') // not the git 1.0.0, which "1" allows too + t.assert.equal(cargo.resolveCrate('itoa_git', 'src/main.rs'), 'vendor/itoa-1.0.0/src/lib.rs') // no version: any, but a git one + } + withProject(files, check) + withProject({ + ...files, + 'Cargo.lock': `version = 4\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = [\n "itoa 1.0.0",\n "itoa 1.0.18",\n]\n\n[[package]]\nname = "itoa"\nversion = "1.0.0"\nsource = "${git}"\n\n[[package]]\nname = "itoa"\nversion = "1.0.18"\nsource = "${REGISTRY}"\nchecksum = "${sha('a')}"\n`, + }, check) +}) + +test('createCargoContext turns a package\'s feature of a dependency\'s name on with `dep/feature`, written or implicit', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[features]\ndefault = ["full"]\nfull = ["serde/derive"]\nserde = ["dep:serde", "chrono?/serde", "extra"]\nextra = []\n[dependencies]\nserde = { version = "1", optional = true }\nchrono = { version = "0.4", optional = true }\n', + 'src/lib.rs': '', + 'vendor/serde/Cargo.toml': '[package]\nname = "serde"\nversion = "1.0.0"\n[features]\nderive = []\n', 'vendor/serde/src/lib.rs': '', + 'vendor/chrono/Cargo.toml': '[package]\nname = "chrono"\nversion = "0.4.0"\n[features]\nserde = []\n', 'vendor/chrono/src/lib.rs': '', + }, (tmp) => { + t.assert.deepEqual(enabledOf(createCargoContext(tmp, { entries: ['src/lib.rs'] })), { '.': ['default', 'extra', 'full', 'serde'], 'vendor/serde': ['derive'] }) + }) +}) + +test('createCargoContext resolves a proc-macro entry for the host, where cargo builds it', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "derive"\nversion = "0.1.0"\nedition = "2021"\n[lib]\nproc-macro = true\n[features]\ndefault = ["printing"]\nprinting = ["quote/std"]\n[dependencies]\nquote = { version = "1", default-features = false }\n', + 'src/lib.rs': '', + 'vendor/quote/Cargo.toml': '[package]\nname = "quote"\nversion = "1.0.0"\n[features]\ndefault = ["std"]\nstd = []\n', 'vendor/quote/src/lib.rs': '', + }, (tmp) => { + const cargo = createCargoContext(tmp, { entries: ['src/lib.rs'] }) + t.assert.deepEqual(enabledOf(cargo, 'host'), { '.': ['default', 'printing'], 'vendor/quote': ['std'] }) + t.assert.deepEqual(enabledOf(cargo, 'target'), {}) + t.assert.deepEqual(sorted(cargo.featuresFor('src/lib.rs', HOST_UNIT)), ['default', 'printing']) + }) +}) + +test('createCargoContext matches versions by the semver crate\'s rules: a requirement takes no prerelease it doesn\'t name', (t) => { + withProject({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nfoo = "1"\nbar = "=2.0.0-rc.1"\n', + 'src/main.rs': '', + ...vendoredPackage('foo', '1.0.0'), ...vendoredPackage('foo', '1.1.0-beta.1'), + ...vendoredPackage('bar', '2.0.0-rc.1'), + }, (tmp) => { + const { result, warnings } = captureWarningsSync(() => { + const cargo = createCargoContext(tmp, { entries: ['src/main.rs'] }) + return [cargo.resolveCrate('foo', 'src/main.rs'), cargo.resolveCrate('bar', 'src/main.rs')] + }) + t.assert.deepEqual(result, ['vendor/foo-1.0.0/src/lib.rs', 'vendor/bar-2.0.0-rc.1/src/lib.rs']) + t.assert.deepEqual(warnings, []) + }) +}) + +test('createCargoContext reads the lockfile of the entries\' workspace, below the bundle root too', (t) => { + withProject({ + 'proj/Cargo.toml': '[workspace]\nmembers = ["app"]\n', + 'proj/app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\n[dependencies]\nserde = "1"\n', + 'proj/app/src/lib.rs': '', + 'proj/Cargo.lock': `version = 4\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["serde"]\n\n[[package]]\nname = "serde"\nversion = "1.0.100"\nsource = "${REGISTRY}"\n`, + ...vendoredPackage('serde', '1.0.100'), ...vendoredPackage('serde', '1.0.200'), + }, (tmp) => { + const { result, warnings } = captureWarningsSync(() => createCargoContext(tmp, { entries: ['proj/app/src/lib.rs'] }).resolveCrate('serde', 'proj/app/src/lib.rs')) + t.assert.equal(result, 'vendor/serde-1.0.100/src/lib.rs') + t.assert.deepEqual(warnings, []) + }) +}) + +test('buildRustBundle scans a proc-macro entry as the host build, with the features cargo builds it with', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "derive"\nversion = "0.1.0"\nedition = "2021"\n[lib]\nproc-macro = true\n[features]\ndefault = ["printing"]\nprinting = []\nextra = []\n', + 'src/lib.rs': '#[cfg(feature = "printing")]\nmod printing;\n#[cfg(feature = "extra")]\nmod extra;\n', + 'src/printing.rs': '', 'src/extra.rs': '', + }, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/lib.rs'] }) + t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/lib.rs', 'src/printing.rs']) + }) +}) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index a3b7730d..001921f1 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -347,6 +347,13 @@ test('evalCfg decides test/doc-only predicates and leaves target/feature ones un t.assert.equal(evalCfg('unix'), null) t.assert.equal(evalCfg('all()'), true) t.assert.equal(evalCfg('any()'), false) + // the boolean literals (rustc 1.88), and a raw identifier as the name it spells + t.assert.equal(evalCfg('true'), true) + t.assert.equal(evalCfg('not(false)'), true) + t.assert.equal(evalCfg('all(unix, false)'), false) + t.assert.equal(evalCfg('r#test'), false) + t.assert.equal(evalCfg('r#unix', { target: new Set(['unix']) }), true) + t.assert.equal(evalCfg('r#true'), null) // a cfg named `true`, not the literal }) test('evalCfg decides a predicate an unknown leaf repeats in when it holds, or fails, whatever that leaf is', (t) => { @@ -764,29 +771,27 @@ test('parseCargoManifest reads package, lib, dependencies in every shape, and wo '[workspace.package]', 'version = "0.9.0"', '[workspace.dependencies]', 'shared = { path = "crates/shared" }', ].join('\n')) - t.assert.deepEqual(m.package, { name: 'my-app', version: null, versionFromWorkspace: true, edition: '2021', editionFromWorkspace: false, build: null }) + t.assert.deepEqual(m.package, { name: 'my-app', version: '0.9.0', edition: '2021', build: null }) // the version its own [workspace] gives t.assert.deepEqual(m.lib, { name: 'myapp_lib', path: 'src/the_lib.rs', procMacro: false }) const dep = (k) => { const d = m.deps.get(k) const [r] = d.kinds.values() // one table each here: a dependency's identity is its table's - return { path: r.path, package: r.package, workspace: r.workspace, kinds: [...d.kinds.keys()].toSorted() } + return { path: r.path, package: r.package, renamed: r.renamed, inherited: r.inherited, kinds: [...d.kinds.keys()].toSorted() } } t.assert.deepEqual([...m.deps.keys()].toSorted(), ['inline_sub', 'nix', 'serde', 'shared', 'tempfile', 'tools', 'util']) - t.assert.deepEqual(dep('inline_sub'), { path: '../sub', package: null, workspace: false, kinds: ['normal'] }) - t.assert.deepEqual(dep('nix'), { path: '../nix', package: null, workspace: false, kinds: ['normal@cfg(unix)'] }) // a target table: its own request - t.assert.deepEqual(dep('serde'), { path: null, package: null, workspace: false, kinds: ['normal'] }) - t.assert.deepEqual(dep('shared'), { path: null, package: null, workspace: true, kinds: ['normal'] }) - t.assert.deepEqual(dep('tempfile'), { path: null, package: null, workspace: false, kinds: ['dev'] }) - t.assert.deepEqual(dep('tools'), { path: '../tools', package: 'dev-tools', workspace: false, kinds: ['normal'] }) + t.assert.deepEqual(dep('inline_sub'), { path: '../sub', package: 'inline-sub', renamed: false, inherited: false, kinds: ['normal'] }) + t.assert.deepEqual(dep('nix'), { path: '../nix', package: 'nix', renamed: false, inherited: false, kinds: ['normal@cfg(unix)'] }) // a target table: its own request + t.assert.deepEqual(dep('serde'), { path: null, package: 'serde', renamed: false, inherited: false, kinds: ['normal'] }) + t.assert.deepEqual(dep('shared'), { path: 'crates/shared', package: 'shared', renamed: false, inherited: true, kinds: ['normal'] }) // relative to the workspace root + t.assert.deepEqual(dep('tempfile'), { path: null, package: 'tempfile', renamed: false, inherited: false, kinds: ['dev'] }) + t.assert.deepEqual(dep('tools'), { path: '../tools', package: 'dev-tools', renamed: true, inherited: false, kinds: ['normal'] }) t.assert.equal(m.isWorkspace, true) - t.assert.equal(m.workspacePackage.version, '0.9.0') - t.assert.deepEqual([...m.workspaceDeps.keys()], ['shared']) - t.assert.equal(m.workspaceDeps.get('shared').path, 'crates/shared') + t.assert.deepEqual(m.cargo.workspace.members, ['crates/*']) }) -test('parseCargoManifest returns no package without a name', (t) => { +test('parseCargoManifest returns no package for a virtual manifest, and refuses a package without a name', (t) => { t.assert.equal(parseCargoManifest('[workspace]\nmembers = ["a"]\n').package, null) - t.assert.equal(parseCargoManifest('[package]\nversion = "1.0.0"\n').package, null) + t.assert.throws(() => parseCargoManifest('[package]\nversion = "1.0.0"\n', 'Cargo.toml'), { name: 'LockfileError', message: 'Cargo.toml: package.name: expected a string, found nothing' }) }) test('createCargoContext identifies the owning package, resolving version.workspace through the root', (t) => { From a954598ea69928fcd9ef27e5fb92e115121def5e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 20:25:21 +0000 Subject: [PATCH 08/10] refactor(bundle): Rust loader simplification pass - scanRustItems: one cfg verdict per predicate per scan, and the open scopes' conjunction kept as they open and close, rather than joined and evaluated again at every token (a 220 KB file inside one #[cfg] block: 588 ms -> 36 ms; with two compile units, 1.3 s -> 28 ms). - cfg leaves: leafFalse evaluates a leaf by key and value (evalCfgKey) instead of rebuilding its text for evalCfg; leavesOfCfgs uses cfgLeaves; joinCfgs where it was inlined; one cache helper for the per-asker verdicts. - One walk driver (collectRustBundle) for buildRustBundle and loadRust: build scripts, compile units and the wantedRoots loop live in rust.js; the link-out-of-package check (withinRealDir), boundaryOf and addUnits are shared rather than copied into bundle.js. - cargo.js: each path dependency's directory resolved once, against the workspace root its inheritance was read from (depSpec gone); requests carry kind and target (no `kind@target` parsing); resolver 1's single context folded into nodeKey; one [patch] source list; the git/registry flag stored with the checksum; tableOf's TOML reused for the manifest; build script and build files memoized per package; one error-naming helper shared with toml.js; internals no longer exported. - rust.js: the macro-template helpers shared by the walk and the tree pass (keyed by the package's own template set, not its identity); providedFrom's per-module file set computed once; resolveModDecl's unused cfg branch removed. - The Rust-only flag checks in stasis.js and bundle.js table-driven. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- stasis/bin/stasis.js | 7 +- stasis/src/cmd/bundle.js | 91 ++-------- stasis/src/loaders/cargo.js | 293 ++++++++++++++----------------- stasis/src/loaders/rust.js | 336 ++++++++++++++++++++---------------- stasis/src/loaders/toml.js | 14 +- tests/rust-features.test.js | 5 +- tests/rust-loader.test.js | 6 +- 7 files changed, 349 insertions(+), 403 deletions(-) diff --git a/stasis/bin/stasis.js b/stasis/bin/stasis.js index 5fdaf4b6..f509fd89 100755 --- a/stasis/bin/stasis.js +++ b/stasis/bin/stasis.js @@ -308,11 +308,8 @@ if (command === '-v' || command === '--version') { // Cargo.toml, Cargo.lock and .cargo/config.toml too (the Rust counterpart of --package-json). const cargoManifests = Boolean(values['cargo-manifests']) if (!allRust) { - if (cargoFeatures.length > 0) usage('Error: --cargo-features is only valid for Rust bundles') - if (cargoNoDefaultFeatures) usage('Error: --cargo-no-default-features is only valid for Rust bundles') - if (cargoAllFeatures) usage('Error: --cargo-all-features is only valid for Rust bundles') - if (cargoTarget !== null) usage('Error: --cargo-target is only valid for Rust bundles') - if (cargoManifests) usage('Error: --cargo-manifests is only valid for Rust bundles') + const given = { 'cargo-features': cargoFeatures.length > 0, 'cargo-no-default-features': cargoNoDefaultFeatures, 'cargo-all-features': cargoAllFeatures, 'cargo-target': cargoTarget !== null, 'cargo-manifests': cargoManifests } + for (const [flag, on] of Object.entries(given)) if (on) usage(`Error: --${flag} is only valid for Rust bundles`) } if (values.scope && !allJs) usage('Error: --scope is only valid for JS bundles') if (values.scope && !['node_modules', 'full'].includes(values.scope)) { diff --git a/stasis/src/cmd/bundle.js b/stasis/src/cmd/bundle.js index 57d51e3d..d2ec65e2 100644 --- a/stasis/src/cmd/bundle.js +++ b/stasis/src/cmd/bundle.js @@ -26,8 +26,8 @@ import { } from '../loaders/solidity.js' import { decodeUtf8 } from '../loaders/solidity-ownership.js' import { buildBashTree, collectBashFilesFromDisk } from '../loaders/bash.js' -import { buildRustTree, collectRustFilesFromDisk } from '../loaders/rust.js' -import { TARGET_UNIT, createCargoContext } from '../loaders/cargo.js' +import { boundaryOf, collectRustBundle, withinRealDir } from '../loaders/rust.js' +import { createCargoContext } from '../loaders/cargo.js' import { bucketizePhpSources, buildPhpTree, @@ -132,28 +132,21 @@ function makeRustClassifier(cargo) { return (path) => { const pkg = cargo.packageInfo(path) if (!pkg) return null - const vendored = pkg.dir.startsWith(`${cargo.vendorDir}/`) - return { bucketDir: pkg.dir, name: pkg.name, version: pkg.version, ecosystem: vendored ? 'cargo' : undefined } + return { bucketDir: pkg.dir, name: pkg.name, version: pkg.version, ecosystem: cargo.isVendored(path) ? 'cargo' : undefined } } } // A file's key inside its bucket: the inverse of moduleFileKey. const fileInBucket = (bucketDir, path) => (bucketDir === '.' ? path : path.slice(bucketDir.length + 1)) -// A file of the bundle root, its bytes, by project-relative path, or null when it isn't a regular -// file there (a manifest the context saw may have gone since; a FIFO would hang a read) or, with -// `within` a directory, doesn't really lie in it (a vendored crate's manifest linked to a project -// file). -function readFileWithinBase(baseDir, rel, within = '.') { +// A file of the bundle root (`realBase` its real path), its bytes, by project-relative path, or +// null when it isn't a regular file there (a manifest the context saw may have gone since; a FIFO +// would hang a read) or, with `within` a directory, doesn't really lie in it (a vendored crate's +// manifest linked to a project file). +function readFileWithinBase(baseDir, realBase, rel, within = '.') { try { - assertRealPathWithinBase(realpathSync(baseDir), baseDir, rel) - if (within !== '.') { - const real = relative(realpathSync(join(baseDir, within)), realpathSync(join(baseDir, rel))).split(/[\\/]/u).join('/') - if (real.startsWith('..') || isAbsolute(real)) { - console.warn(`[loader.cargo] Refusing file outside its package: ${rel} (a link out of ${within})`) - return null - } - } + assertRealPathWithinBase(realBase, baseDir, rel) + if (!withinRealDir(baseDir, rel, within, { who: 'loader.cargo' })) return null if (!statSync(join(baseDir, rel)).isFile()) return null return readFileSync(join(baseDir, rel)) } catch (err) { @@ -463,65 +456,19 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa allFeatures: cargoAllFeatures, target: cargoTarget, }) - // `include_str!` / `include_bytes!` assets ride along as resources: the walk records their format. - const formats = new Map() - const sources = new Map() - // The crate roots: the entries, then -- with `cargoManifests` -- the build script of every - // package the walk reaches, each a crate root of its own, until no package is new. Each file is - // compiled as the units `units` records (cargo.js unitKey): a build script for the host, with - // its package's features, and so what it reaches; a proc-macro crate and what it depends on for - // the host too. - const roots = [...normalized] - const units = new Map() - // Record `file` as compiled as each of `more` too; whether that is news. - const addUnits = (file, more) => { - const have = units.get(file) ?? units.set(file, new Set()).get(file) - if (more.isSubsetOf(have)) return false - for (const u of more) have.add(u) - return true - } - const walk = async (wave) => { - await collectRustFilesFromDisk(baseDir, wave, { cargo: cargoCtx, formats, sources, units }) - if (!cargoManifests) return - const scripts = new Set() - for (const path of sources.keys()) { - const script = formats.has(path) ? null : cargoCtx.buildScriptOf(path) - if (script === null) continue - const as = new Set([...(units.get(path) ?? [TARGET_UNIT])].map((u) => cargoCtx.buildScriptUnit(u))) - if (addUnits(script, as)) scripts.add(script) - } - if (scripts.size === 0) return - for (const s of scripts) if (!roots.includes(s)) roots.push(s) - await walk([...scripts]) - } - await walk(normalized) - // The tree pass may name in-tree crate roots the walk left out (a crate whose name a file also - // binds as a value, `use crate::util::log;` beside `log::info!`), or walked as fewer units than - // it is named as: buildRustTree's wantedRoots. Load them and build again, until none is new. A - // root the walk was asked for and didn't load -- refused (a link out of its package) or gone -- - // is not asked for again: the walk has said why. - const tried = new Set() - const complete = async () => { - const built = buildRustTree(sources, { roots, baseDir, cargo: cargoCtx, formats, units }) - const wanted = built.wantedRoots.filter((r) => addUnits(r, built.wantedUnits.get(r)) || (!sources.has(r) && !tried.has(r))) - if (wanted.length === 0) return built - for (const r of wanted) { - tried.add(r) - if (!roots.includes(r)) roots.push(r) - } - await walk(wanted) - return complete() - } - const tree = await complete() + // `include_str!` / `include_bytes!` assets ride along as resources: the walk records their + // format. With `cargoManifests`, each package's build script is walked too (collectRustBundle). + const { sources, formats, tree } = await collectRustBundle(baseDir, normalized, { cargo: cargoCtx, buildScripts: cargoManifests }) if (cargoManifests) { // Each bundled package's manifests, and its workspace's lockfile and cargo config, carried as // written: whatever they hold (a registry token, a `git` URL's credentials) is in the bundle // too, as with --package-json. A file that isn't UTF-8 text is refused, not altered. const files = new Set() for (const path of sources.keys()) for (const f of cargoCtx.buildFilesFor(path)) files.add(f.path) + const realBase = realpathSync(baseDir) for (const rel of files) { if (sources.has(rel)) continue - const buf = readFileWithinBase(baseDir, rel, cargoCtx.isVendored(rel) ? cargoCtx.packageInfo(rel).dir : '.') + const buf = readFileWithinBase(baseDir, realBase, rel, boundaryOf(rel, cargoCtx)) if (buf === null) continue if (!isUtf8(buf)) throw new Error(`Rust manifest is not valid UTF-8: ${rel}`) sources.set(rel, buf.toString('utf8')) @@ -1160,12 +1107,8 @@ function classifyEntries(name, { cwd = process.cwd(), entries, mappingFile, mani // --cargo runs `cargo metadata` for the Rust feature/dependency resolution and the --cargo-* // flags steer that resolution; nothing else reads Cargo. if (kind !== 'rust') { - if (cargo) throw new Error(`${name}: --cargo is only valid for Rust bundles`) - if (Array.isArray(cargoFeatures) && cargoFeatures.length > 0) throw new Error(`${name}: --cargo-features is only valid for Rust bundles`) - if (cargoNoDefaultFeatures) throw new Error(`${name}: --cargo-no-default-features is only valid for Rust bundles`) - if (cargoAllFeatures) throw new Error(`${name}: --cargo-all-features is only valid for Rust bundles`) - if (cargoTarget) throw new Error(`${name}: --cargo-target is only valid for Rust bundles`) - if (cargoManifests) throw new Error(`${name}: --cargo-manifests is only valid for Rust bundles`) + const given = { cargo, 'cargo-features': Array.isArray(cargoFeatures) && cargoFeatures.length > 0, 'cargo-no-default-features': cargoNoDefaultFeatures, 'cargo-all-features': cargoAllFeatures, 'cargo-target': cargoTarget, 'cargo-manifests': cargoManifests } + for (const [flag, on] of Object.entries(given)) if (on) throw new Error(`${name}: --${flag} is only valid for Rust bundles`) } if (scope !== undefined && kind !== 'js') { throw new Error(`${name}: --scope is only valid for JS bundles`) diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index f2540c2b..680ec27a 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -15,7 +15,7 @@ import { toPosix } from '@exodus/stasis-core/util' import { LockfileError, linkCargo, parseCargoConfig, parseCargoLock, parseCargoManifest as readCargoManifest, readCargoVendor, resolveCargoFeatures } from '@preventive/lockfile/cargo.js' import { matches, parseVersion, parseVersionReq } from '@preventive/lockfile/rust-semver.js' -import { TomlError, isTomlTable, readToml, splitTopLevel } from './toml.js' +import { TomlError, isTomlTable, nameErrors, readToml, splitTopLevel } from './toml.js' // `cargo vendor` copies registry crates in-tree under this dir. export const VENDOR_DIR = 'vendor' @@ -51,9 +51,6 @@ export function normalizeRel(dir, sub) { // --- Cargo.toml ----------------------------------------------------------------------- -// A request's kind (`normal`/`dev`/`build`), a target-specific table's `@` suffix dropped. -const kindOf = (request) => request.split('@')[0] - // Entries of a `[features]` list beyond a plain feature name: `dep:key` and `key/feat` / `key?/feat`. const DEP_IMPLICATION_RE = /^dep:(.+)$/u const DEP_FEATURE_RE = /^([^/?]+)(\?)?\/(.+)$/u @@ -73,24 +70,7 @@ export function isTestTargetPath(pkgDir, fileRel) { // `read()`, with a TomlError or LockfileError it throws naming `file`: what @preventive/lockfile // refuses stops the build, saying which file. -function readNamed(file, read) { - try { - return read() - } catch (err) { - if (file === null) throw err - if (err instanceof TomlError) { - const named = new TomlError(`${file}: ${err.message}`) - named.line = err.line - throw named - } - if (err instanceof LockfileError) { - const named = new LockfileError(`${file}: ${err.message}`) - named.where = err.where - throw named - } - throw err - } -} +const readNamed = (file, read) => nameErrors(file, read, [TomlError, LockfileError]) // A Cargo.toml as the loader reads it: @preventive/lockfile's reading (`cargo`) -- the package, its // features (with the one each optional dependency no `dep:` names turns on) and every dependency @@ -101,10 +81,12 @@ function readNamed(file, read) { // (`-` → `_`); each table is a request of its own, `` or `@` (`normal`, // `build`, `dev@cfg(windows)`): `[dependencies] rand = "0.7"` beside `[build-dependencies] rand = // "0.8"` are two crates, and sha2's dev-dependency on digest asks digest for nothing a build of -// sha2's dependents sees. Throws a TomlError or LockfileError naming `file`. -export function parseCargoManifest(text, file = null, workspace = null) { +// sha2's dependents sees; each entry has its `kind` and `target` (null for none) apart too. Throws +// a TomlError or LockfileError naming `file`. +export const parseCargoManifest = (text, file = null, workspace = null) => manifestOf(text, readToml(text, file), file, workspace) +// parseCargoManifest of `text` read as `doc` already. +function manifestOf(text, doc, file, workspace) { const root = workspace?.cargo.workspace === undefined ? undefined : workspace.cargo - const doc = readToml(text, file) const cargo = readNamed(file, () => readCargoManifest(text, isTomlTable(doc.workspace) ? undefined : root)) const pkg = cargo.package const table = (v) => (isTomlTable(v) ? v : null) @@ -115,6 +97,8 @@ export function parseCargoManifest(text, file = null, workspace = null) { const key = normName(d.name) if (!deps.has(key)) deps.set(key, { key, name: d.name, kinds: new Map() }) deps.get(key).kinds.set(d.target === undefined ? d.kind : `${d.kind}@${d.target}`, { + kind: d.kind, + target: d.target ?? null, version: d.version ?? null, path: d.source.type === 'path' ? d.source.path : null, source: d.source.type, @@ -175,8 +159,7 @@ export function findCargoLock(baseDir) { // `my = "a b"` are two values. export const normalizeCfg = (pred) => pred.replaceAll(/"(?:[^"\\]|\\.)*"|\s+/gu, (m) => (m.startsWith('"') ? m : ' ')).trim() -const FEATURE_CFG_RE = /^(?:r#)?feature\s*=\s*"([^"]*)"$/u -// Any other leaf: `unix`, `target_os = "linux"`, a raw identifier's `r#` dropped. +// A leaf: `unix`, `target_os = "linux"`, a raw identifier's `r#` dropped. const CFG_LEAF_RE = /^(?:r#)?([A-Za-z_]\w*)\s*(?:=\s*"([^"]*)")?$/u // The cfg keys rustc sets from the target alone (`rustc --print cfg --target `), decided by @@ -201,10 +184,7 @@ export const TARGET_CFG_KEYS = new Set(['unix', 'windows', 'target_abi', 'target // `#[cfg(any(test, kani))] mod tests { #[cfg(not(kani))] mod compatibility { … } }` -- never holds. const MAX_FREE_CFG_LEAVES = 6 export function evalCfg(pred, env = {}) { - if (env.units) { - const each = env.units.map((u) => evalCfg(pred, u)) - return each.every((r) => r === false) ? false : (each.every((r) => r === true) ? true : null) - } + if (env.units) return unanimous(env.units.map((u) => evalCfg(pred, u))) const free = { counts: null } // unknown leaf → how often it occurs, once one is met const r = evalCfgWith(pred, env, null, free) if (r !== null || free.counts === null || free.counts.size > MAX_FREE_CFG_LEAVES || [...free.counts.values()].every((n) => n === 1)) return r @@ -223,9 +203,11 @@ function evalCfgWith(pred, env, assume, free = null) { const p = pred.trim() const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) if (!m) { - const known = evalCfgLeaf(p, env) + if (p === 'true' || p === 'false') return p === 'true' // the literals (rustc 1.88); `r#true` is a name + const kv = CFG_LEAF_RE.exec(p) + const known = kv === null ? null : evalCfgKey(kv[1], kv[2] ?? null, env) if (known !== null) return known - const kv = CFG_LEAF_RE.exec(p) // the same leaf however it is spaced, its value as written + // The same leaf however it is spaced, its value as written. const leaf = kv === null ? p : (kv[2] === undefined ? kv[1] : `${kv[1]}="${kv[2]}"`) if (assume?.has(leaf)) return assume.get(leaf) if (free !== null) (free.counts ??= new Map()).set(leaf, (free.counts.get(leaf) ?? 0) + 1) @@ -236,17 +218,18 @@ function evalCfgWith(pred, env, assume, free = null) { if (m[1] === 'all') return args.includes(false) ? false : (args.every((a) => a === true) ? true : null) return args.includes(true) ? true : (args.every((a) => a === false) ? false : null) } -function evalCfgLeaf(p, env) { - if (p === 'true' || p === 'false') return p === 'true' // the literals (rustc 1.88); `r#true` is a name - const name = p.startsWith('r#') ? p.slice(2) : p - if (name === 'test') return env.test === true - if (name === 'doctest' || name === 'doc') return false - const feature = FEATURE_CFG_RE.exec(p) - if (feature) return env.features ? (env.features.has(feature[1]) ? true : (env.maybeFeatures?.has(feature[1]) ? null : false)) : null - const leaf = CFG_LEAF_RE.exec(p) - if (leaf && env.target && TARGET_CFG_KEYS.has(leaf[1])) return env.target.has(leaf[2] === undefined ? leaf[1] : `${leaf[1]}="${leaf[2]}"`) +// The verdict on one leaf other than a literal, by its key and value (`target_os = "linux"`; null +// for a bare `unix`), as evalCfg gives it -- `env.units` too. +export function evalCfgKey(key, value, env) { + if (env.units) return unanimous(env.units.map((u) => evalCfgKey(key, value, u))) + if (value === null && key === 'test') return env.test === true + if (value === null && (key === 'doctest' || key === 'doc')) return false + if (key === 'feature' && value !== null) return env.features ? (env.features.has(value) ? true : (env.maybeFeatures?.has(value) ? null : false)) : null + if (env.target && TARGET_CFG_KEYS.has(key)) return env.target.has(value === null ? key : `${key}="${value}"`) return null } +// Code compiled several ways: false only when false in each, true only when true in each. +const unanimous = (each) => (each.every((r) => r === false) ? false : (each.every((r) => r === true) ? true : null)) // `rustc --print cfg` output → the set of cfg leaves as printed, one per line (`unix`, // `target_os="linux"`). @@ -310,7 +293,7 @@ export function rustcTargetCfgs(target, baseDir = null) { // (doc/file-formats.md has the full caveat); never run on a bundle root unasked. With a lockfile // present, `--locked` keeps the resolution the one the build uses. The feature flags pass through; // `platform`, a target triple, restricts the graph to that target's dependencies. -export function runCargoMetadata(baseDir, { features = [], noDefaultFeatures = false, allFeatures = false, platform = null } = {}) { +function runCargoMetadata(baseDir, { features = [], noDefaultFeatures = false, allFeatures = false, platform = null } = {}) { const args = ['metadata', '--format-version', '1'] // The workspace's lock may sit above the bundle root: never let metadata rewrite it. if (findCargoLock(baseDir) !== null) args.push('--locked') @@ -493,18 +476,18 @@ function hostUndecided(graph, on) { // --- Context ---------------------------------------------------------------------------- // No custom cfg a build sets (see cfgsSetFor). -export const NO_CFGS_SET = { names: new Set(), any: false } +const NO_CFGS_SET = { names: new Set(), any: false } // A compile unit: the feature context a file's code sees (`target`: the build's; `host`: that of // what is built for the host -- build-dependencies, proc-macro crates and their dependencies -- // which resolver 2 keeps apart) and the platform it is compiled for, as `|`. // A package's code is compiled as `target|target`, its build script as `target|host`, a // build-dependency's or proc-macro's code as `host|host`; one file may be compiled as several. -export const unitKey = (featureCtx, platform) => `${featureCtx}|${platform}` +const unitKey = (featureCtx, platform) => `${featureCtx}|${platform}` export const TARGET_UNIT = unitKey('target', 'target') export const HOST_UNIT = unitKey('host', 'host') const unitFeatureCtx = (unit) => unit.slice(0, unit.indexOf('|')) -export const unitPlatform = (unit) => unit.slice(unit.indexOf('|') + 1) +const unitPlatform = (unit) => unit.slice(unit.indexOf('|') + 1) // Per-bundle Cargo state: manifest lookup (memoized per directory), crate-name resolution against // in-tree sources, and feature resolution for the packages owning `entries` (the crate roots being @@ -545,12 +528,18 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return null } + // A manifest as parseCargoManifest reads it, with each path dependency's directory, project- + // relative, on its entry (`dir`; null outside the bundle root): an inherited one's path is the + // workspace root's. const readManifest = (dir) => { if (!manifests.has(dir)) { const raw = tableOf(dir) const root = raw === null ? null : rootOf(dir, raw.doc) - const workspace = root === null ? null : readManifest(root) - manifests.set(dir, raw === null ? null : { dir, ...parseCargoManifest(raw.text, raw.file, workspace) }) + const m = raw === null ? null : { dir, ...manifestOf(raw.text, raw.doc, raw.file, root === null ? null : readManifest(root)) } + for (const d of m?.deps.values() ?? []) { + for (const r of d.kinds.values()) r.dir = r.path === null ? null : normalizeRel(r.inherited ? (root ?? dir) : dir, r.path) + } + manifests.set(dir, m) } return manifests.get(dir) } @@ -598,6 +587,16 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return m.libRoot } const libName = (m) => normName(m.lib.name ?? m.package?.name ?? '') + // The package's build script when it is on disk (see buildScriptOf); memoized on the manifest. + const buildScript = (m) => { + if (m.buildScript === undefined) { + const rel = m.package.build === false ? null : normalizeRel(m.dir, m.package.build ?? 'build.rs') + const inside = rel !== null && (!isVendoredDir(m.dir) || rel.startsWith(`${m.dir}/`)) + if (rel !== null && !inside && isFile(join(baseDir, rel))) console.warn(`[loader.cargo] Refusing build script outside its package: ${m.package.build} in ${m.dir}`) + m.buildScript = inside && isFile(join(baseDir, rel)) ? rel : null + } + return m.buildScript + } // Whether `fileRel` belongs to a test or bench target of its package (compiled with `cfg(test)`). const isTestTarget = (fileRel) => isTestTargetPath(packageFor(fileRel)?.dir ?? '.', fileRel) const version = (m) => m.package.version @@ -626,7 +625,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const m = readManifest(`${vendorDir}/${d}`) if (!m?.package) continue const sums = checksumOf(m.dir) - const entry = { version: version(m), dir: m.dir, git: sums === null ? null : sums.package === null } + const entry = { version: version(m), dir: m.dir, git: sums?.git ?? null } for (const [index, key] of [[vendorIndex.byName, normName(m.package.name)], [vendorIndex.byLib, libName(m)]]) { if (!index.has(key)) index.set(key, []) index.get(key).push(entry) @@ -649,7 +648,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef throw new Error(`${file}: not JSON: ${err.message}`, { cause: err }) } } - checksums.set(dir, value === null ? null : { text, package: value?.package ?? null }) + // A registry's copy has a package checksum; a git checkout's has none. + checksums.set(dir, value === null ? null : { text, git: (value?.package ?? null) === null }) } return checksums.get(dir) } @@ -691,20 +691,6 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef locate: (name, ver) => (typeof name === 'string' ? vendored().byName.get(normName(name))?.find((c) => c.version === ver)?.dir ?? null : null), }) : null - // A dependency as the package sees it through one of its tables (`request`, an entry of - // `dep.kinds`; what a member inherits from its workspace is already in it): its path relative to - // the manifest's directory, or the workspace root's where the entry is inherited. - const depSpec = (m, dep, request) => ({ - key: dep.key, - version: request.version, - path: request.path, - source: request.source, - package: request.package, - renamed: request.renamed, - defaultFeatures: request.defaultFeatures, - features: request.features, - relTo: request.inherited ? (workspaceFor(m.dir)?.dir ?? m.dir) : m.dir, - }) // The `[patch]` tables of the cargo config of every directory from `dir` up to the bundle root, // the nearest first, as @preventive/lockfile reads them, each with the directory its paths are // relative to (the one holding its `.cargo`); the config's text too, for the reading as cargo @@ -723,16 +709,19 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return configsMemo.get(dir) } + // Where a build rooted at `root` takes its `[patch]` tables from, in the order they apply: the + // cargo config's, then the root manifest's, as `{ patch, relTo, from }` (`relTo` the directory + // its paths are relative to, `from` the file). + const patchSources = (root) => [ + ...configsFrom(root?.dir ?? '.').map((c) => ({ patch: c.patch, relTo: c.dir, from: c.file })), + ...(root ? [{ patch: root.cargo.patch, relTo: root.dir, from: posix.join(root.dir, 'Cargo.toml') }] : []), + ] // The `[patch]` of crate `crate` in a build rooted at `root` (the workspace root's manifest, else - // the bundle root's), as `{ spec, dir, from }`: the cargo config's first, then the root - // manifest's -- `spec` as @preventive/lockfile reads it, `dir` the patch's directory for a path - // (null outside the bundle root), `from` the file saying so. Undefined when nothing patches it. + // the bundle root's), as `{ spec, dir, from }`, the first patchSources has -- `spec` as + // @preventive/lockfile reads it, `dir` the patch's directory for a path (null outside the bundle + // root), `from` the file saying so. Undefined when nothing patches it. const patchFor = (root, crate) => { - const patches = [ - ...configsFrom(root?.dir ?? '.').map((c) => ({ patch: c.patch, relTo: c.dir, from: c.file })), - ...(root ? [{ patch: root.cargo.patch, relTo: root.dir, from: posix.join(root.dir, 'Cargo.toml') }] : []), - ] - for (const { patch, relTo, from } of patches) { + for (const { patch, relTo, from } of patchSources(root)) { for (const specs of Object.values(patch)) { for (const [name, spec] of Object.entries(specs)) { if (normName(name) !== crate) continue @@ -750,8 +739,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const lockKeysOf = (lk, m) => { const keys = lk.byId.get(`${normName(m.package.name)} ${version(m)}`) ?? [] if (!isVendoredDir(m.dir)) return keys.filter((k) => lk.lock.packages[k].source === undefined) - const sums = checksumOf(m.dir) - const kind = sums === null ? null : (sums.package === null ? 'git' : 'registry') + const git = checksumOf(m.dir)?.git + const kind = git === undefined ? null : (git ? 'git' : 'registry') return keys.filter((k) => lk.lock.packages[k].source !== undefined && (kind === null || sourceKind(lk.lock.packages[k].source) === kind)) } // Dependency → package, memoized per (package, request): the fixed-point loop asks many times. @@ -782,19 +771,17 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // cargo metadata knows exactly which package each dependency edge points at. const known = metadata?.deps.get(m.dir)?.get(dep.key) if (known !== undefined) return asPackage(known) - const spec = depSpec(m, dep, request) const who = `${m.package.name} ${version(m)}` - if (spec.path) { - const dir = normalizeRel(spec.relTo, spec.path) - const t = asPackage(dir) - if (dir === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} is a path outside the bundle root: ${spec.path}`) - else if (t === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} names ${dir}, which holds no Cargo.toml with a [package]`) + if (request.path) { + const t = asPackage(request.dir) + if (request.dir === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} is a path outside the bundle root: ${request.path}`) + else if (t === null) console.warn(`[loader.cargo] ${who}'s dependency ${dep.name} names ${request.dir}, which holds no Cargo.toml with a [package]`) return t } - const crate = normName(spec.package) - const req = spec.version + const crate = normName(request.package) + const req = request.version const fits = (ver) => req === null || satisfies(ver, req) - let kind = spec.source + let kind = request.source const patch = patchFor(workspaceFor(m.dir) ?? readManifest('.'), crate) if (patch?.spec.source.type === 'path') { if (patch.dir === null) { @@ -864,27 +851,26 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // Whether a table applies in a build for `platform`: `yes`, `no`, or `maybe` when the platform's // cfgs aren't known or don't decide it. A build-dependency table is about the host its build // script runs on. Resolver 1 counts every platform's table, as cargo's v1 unification does. - const tableApplies = (request, platform) => { - const at = request.indexOf('@') - if (at === -1 || resolverVersion() === 1) return 'yes' - const info = platformInfo(kindOf(request) === 'build' ? 'host' : platform) + const tableApplies = (r, platform) => { + if (r.target === null || resolverVersion() === 1) return 'yes' + const info = platformInfo(r.kind === 'build' ? 'host' : platform) if (info === null) return 'maybe' - const spec = request.slice(at + 1) - const cfg = /^cfg\((.*)\)$/u.exec(spec) - const holds = cfg ? evalCfg(cfg[1], { target: info.cfgs }) : spec === info.triple + const cfg = /^cfg\((.*)\)$/u.exec(r.target) + const holds = cfg ? evalCfg(cfg[1], { target: info.cfgs }) : r.target === info.triple return holds === true ? 'yes' : (holds === false ? 'no' : 'maybe') } // The context a dependency of a package built in context `c` is built in: the host's for a // build-dependency (its build script's) and for a proc-macro crate, else `c`'s. - const depCtx = (c, request, t) => featureCtx(kindOf(request) === 'build' || isProcMacroPkg(t) ? 'host' : c) + const depCtx = (c, r, t) => (r.kind === 'build' || isProcMacroPkg(t) ? 'host' : c) const isOptional = (d) => [...d.kinds.values()].some((r) => r.optional) // A feature's list, from cargo's feature map (an optional dependency no `dep:` names has a // feature of its own name, as the manifest spells it: `proc-macro-crate`), or null for none. const featureImplications = (m, f) => m.features.get(f) ?? null // The feature context a root package's own code is resolved in: the host's for a proc-macro // crate, which cargo builds for the host; the target's for any other. - const rootCtx = (m) => featureCtx(isProcMacroPkg(m) ? 'host' : 'target') - const nodeKey = (c, dir) => `${c}\0${dir}` + const rootCtx = (m) => (isProcMacroPkg(m) ? 'host' : 'target') + // A package in a context, as the resolutions key it: resolver 1's one context is `target`'s. + const nodeKey = (c, dir) => `${featureCtx(c)}\0${dir}` // Enabled features per (context, package dir), for every package the root packages' builds pull // in: the roots start from `default` (or the flags), then features imply features, activate @@ -928,15 +914,15 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // Dev-dependencies count for a root package under resolver 1, and for one whose entries include a // test/bench target (that build is `cargo test`'s, which links them). const testRootDirs = new Set(entries.filter((e) => isTestTarget(e)).map((e) => packageFor(e)?.dir)) - const kindApplies = (m, request) => kindOf(request) !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) - const applies = (request, c) => { - const a = tableApplies(request, c) + const kindApplies = (m, r) => r.kind !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) + const applies = (r, c) => { + const a = tableApplies(r, c) return a === 'yes' || (includeMaybe && a === 'maybe') } - // The tables of `d` that take part in `m`'s build in context `c`, as [request, entry] pairs, - // an optional one only once activated. - const activeRequests = (c, m, d) => [...d.kinds] - .filter(([request, r]) => kindApplies(m, request) && applies(request, c) && (!r.optional || active.get(nodeKey(c, m.dir))?.has(d.key) === true)) + // The tables of `d` (entries of `d.kinds`) that take part in `m`'s build in context `c`, an + // optional one only once activated. + const activeRequests = (c, m, d) => [...d.kinds.values()] + .filter((r) => kindApplies(m, r) && applies(r, c) && (!r.optional || active.get(nodeKey(c, m.dir))?.has(d.key) === true)) // One entry of a feature's list: `other`, `dep:key`, `key/feat`, `key?/feat`. Returns whether it named anything. const applyImplication = (c, m, imp) => { const explicitDep = DEP_IMPLICATION_RE.exec(imp) @@ -955,9 +941,9 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef activate(c, m, d.key) if (featureImplications(m, d.name) !== null) enable(c, m, d.name) } - for (const [request, r] of activeRequests(c, m, d)) { + for (const r of activeRequests(c, m, d)) { const t = resolveDep(m, d, r) - if (t) enable(depCtx(c, request, t), t, depFeature[3]) + if (t) enable(depCtx(c, r, t), t, depFeature[3]) } return true } @@ -1005,14 +991,13 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const imp of featureImplications(m, f) ?? []) applyImplication(c, m, imp) } for (const d of m.deps.values()) { - for (const [request, r] of activeRequests(c, m, d)) { - const spec = depSpec(m, d, r) + for (const r of activeRequests(c, m, d)) { const t = resolveDep(m, d, r) if (!t) continue - const dc = depCtx(c, request, t) + const dc = depCtx(c, r, t) inGraph(dc, t) - if (spec.defaultFeatures) enable(dc, t, 'default') - for (const f of spec.features) enable(dc, t, f) + if (r.defaultFeatures) enable(dc, t, 'default') + for (const f of r.features) enable(dc, t, f) } } } @@ -1072,8 +1057,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const dir of out) { for (const d of readManifest(dir).deps.values()) { for (const r of d.kinds.values()) { - if (r.path === null) continue - const sub = normalizeRel(r.inherited ? root.dir : dir, r.path) + const sub = r.dir if (sub !== null && inRoot(sub) && !isExcluded(sub) && readManifest(sub)?.package) out.add(sub) } } @@ -1111,7 +1095,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const keyOf = new Map() // dir -> package key const dirOf = new Map() // package key -> dir const queue = [...memberDirs] - for (const { patch, relTo } of [...configsFrom(root.dir).map((c) => ({ patch: c.patch, relTo: c.dir })), { patch: root.cargo.patch, relTo: root.dir }]) { + for (const { patch, relTo } of patchSources(root)) { for (const specs of Object.values(patch)) { for (const spec of Object.values(specs)) { if (spec.source.type !== 'path') continue @@ -1131,9 +1115,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const d of m.deps.values()) { for (const r of d.kinds.values()) { if (r.path === null) continue - const sub = normalizeRel(r.inherited ? (workspaceFor(dir)?.dir ?? dir) : dir, r.path) - if (sub === null) return null - queue.push(sub) + if (r.dir === null) return null + queue.push(r.dir) } } } @@ -1181,17 +1164,15 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef sure = featuresOf(graph, { host: platform(hostInfo), targets: [platform(targetInfo)] }) all = sure } - // Per (context, dir), as resolveFeatures keys them: resolver 1's one set under `target`. + // Per (context, dir), as resolveFeatures keys them: resolver 1's one set under `target`, the + // normal one where there is one. const byNode = (result) => { const out = new Map() for (const [key, { normal, host: onHost }] of Object.entries(result)) { const dir = dirOf.get(key) if (dir === undefined) continue - if (resolverVersion() === 1) out.set(nodeKey('target', dir), new Set(normal ?? onHost)) - else { - if (normal !== undefined) out.set(nodeKey('target', dir), new Set(normal)) - if (onHost !== undefined) out.set(nodeKey('host', dir), new Set(onHost)) - } + if (onHost !== undefined) out.set(nodeKey('host', dir), new Set(onHost)) + if (normal !== undefined) out.set(nodeKey('target', dir), new Set(normal)) } return out } @@ -1273,9 +1254,9 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } const ROLE_KINDS = { build: ['build'], test: ['normal', 'dev'], normal: ['normal'] } const requestsFor = (d, role) => { - const all = [...d.kinds] - const own = all.filter(([request]) => ROLE_KINDS[role].includes(kindOf(request))) - return (own.length > 0 ? own : all).map(([, r]) => r) + const all = [...d.kinds.values()] + const own = all.filter((r) => ROLE_KINDS[role].includes(r.kind)) + return own.length > 0 ? own : all } // depCrate from the exact resolution: the package's dependency of that key, else the unrenamed // one whose lib is named that, from the asking file's tables when it has one there, as the @@ -1325,6 +1306,26 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return null } + // The files describing package `m`'s build, on disk (see buildFilesFor). + const buildFiles = (m) => { + const out = [{ path: posix.join(m.dir, 'Cargo.toml'), kind: 'manifest' }] + if (isVendoredDir(m.dir)) { + const checksum = posix.join(m.dir, '.cargo-checksum.json') + if (isFile(join(baseDir, checksum))) out.push({ path: checksum, kind: 'checksum' }) + return out + } + const ws = workspaceFor(m.dir) + if (ws && ws.dir !== m.dir) out.push({ path: posix.join(ws.dir, 'Cargo.toml'), kind: 'manifest' }) + const lockPath = posix.join(ws?.dir ?? m.dir, 'Cargo.lock') + if (isFile(join(baseDir, lockPath))) out.push({ path: lockPath, kind: 'lock' }) + for (let dir = m.dir; ; dir = posix.dirname(dir)) { + const config = cargoConfigIn(baseDir, dir) + if (config !== null) out.push({ path: config, kind: 'config' }) + if (dir === '.') break + } + return out + } + return { // Where `cargo vendor` put the registry crates (project-relative): `vendor`, or what // .cargo/config.toml names. A package under it is a vendored dependency. @@ -1352,8 +1353,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const lib = libPath(m) if (lib && lib !== fromFile) return lib } - const script = m === null || m.package.build === false ? null : normalizeRel(m.dir, m.package.build ?? 'build.rs') - return depCrate(m, norm, fromFile === script ? 'build' : (isTestTarget(fromFile) ? 'test' : 'normal')) + return depCrate(m, norm, m !== null && fromFile === buildScript(m) ? 'build' : (isTestTarget(fromFile) ? 'test' : 'normal')) }, isTestTarget, // Whether `fromFile`'s package declares a dependency of that name (in any table, by the key @@ -1377,23 +1377,8 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef buildFilesFor(fileRel) { const m = packageFor(fileRel) if (!m) return [] - const out = [{ path: posix.join(m.dir, 'Cargo.toml'), kind: 'manifest' }] - if (isVendoredDir(m.dir)) { - const checksum = posix.join(m.dir, '.cargo-checksum.json') - if (isFile(join(baseDir, checksum))) out.push({ path: checksum, kind: 'checksum' }) - return out - } - const ws = workspaceFor(m.dir) - if (ws && ws.dir !== m.dir) out.push({ path: posix.join(ws.dir, 'Cargo.toml'), kind: 'manifest' }) - const top = ws?.dir ?? m.dir - const lockPath = posix.join(top, 'Cargo.lock') - if (isFile(join(baseDir, lockPath))) out.push({ path: lockPath, kind: 'lock' }) - for (let dir = m.dir; ; dir = posix.dirname(dir)) { - const config = cargoConfigIn(baseDir, dir) - if (config !== null) out.push({ path: config, kind: 'config' }) - if (dir === '.') break - } - return out + m.buildFiles ??= buildFiles(m) + return m.buildFiles }, // Whether `fileRel` belongs to a vendored package (a registry crate `cargo vendor` copied in). isVendored(fileRel) { @@ -1405,14 +1390,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // vendored package's may not name a file outside the package (a published crate never does). buildScriptOf(fileRel) { const m = packageFor(fileRel) - if (!m || m.package.build === false) return null - const rel = normalizeRel(m.dir, m.package.build ?? 'build.rs') - if (rel === null || !isFile(join(baseDir, rel))) return null - if (this.isVendored(fileRel) && !rel.startsWith(`${m.dir}/`)) { - console.warn(`[loader.cargo] Refusing build script outside its package: ${m.package.build} in ${m.dir}`) - return null - } - return rel + return m ? buildScript(m) : null }, // The custom cfgs the build of `fileRel`'s package may set (a `--cfg` a default build lacks is // otherwise presumed off): `{ names, any }` -- the names its build script prints as @@ -1423,7 +1401,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const memo = m?.dir ?? '\0' if (!cfgsSetMemo.has(memo)) { rustflagCfgs ??= rustflagsCfgsOf(baseDir) - const script = m ? this.buildScriptOf(fileRel) : null + const script = m ? buildScript(m) : null const printed = script === null ? { names: new Set(), any: false } : cfgsPrinted(buildScriptTexts(script)) const names = new Set([...rustflagCfgs, ...printed.names]) cfgsSetMemo.set(memo, names.size === 0 && !printed.any ? NO_CFGS_SET : { names, any: printed.any }) @@ -1437,7 +1415,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef featuresFor(fileRel, unit = TARGET_UNIT) { const m = packageFor(fileRel) if (!m) return null - const node = nodeKey(featureCtx(unitFeatureCtx(unit)), m.dir) + const node = nodeKey(unitFeatureCtx(unit), m.dir) const { sure, all } = ensureResolved() return sure.get(node) ?? (all.has(node) ? NO_FEATURES : null) }, @@ -1447,27 +1425,24 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef maybeFeaturesFor(fileRel, unit = TARGET_UNIT) { const m = packageFor(fileRel) if (!m) return null - return maybeOf(nodeKey(featureCtx(unitFeatureCtx(unit)), m.dir)) + return maybeOf(nodeKey(unitFeatureCtx(unit), m.dir)) }, // Every package of a context the build may pull in: dir -> { on, maybe }, the features on for // certain and those on only maybe (a package only an undecided table pulls in has every // feature of its maybe). For diagnostics and tests. featureResolution(context = 'target') { const out = new Map() + const prefix = nodeKey(context, '') for (const node of ensureResolved().all.keys()) { - const cut = node.indexOf('\0') - if (node.slice(0, cut) !== featureCtx(context)) continue - out.set(node.slice(cut + 1), { on: ensureResolved().sure.get(node) ?? NO_FEATURES, maybe: maybeOf(node) ?? NO_FEATURES }) + if (node.startsWith(prefix)) out.set(node.slice(prefix.length), { on: ensureResolved().sure.get(node) ?? NO_FEATURES, maybe: maybeOf(node) ?? NO_FEATURES }) } return out }, // Every resolved package of a context: dir -> Set. For diagnostics and tests. resolvedFeatures(context = 'target') { const out = new Map() - for (const [node, set] of ensureResolved().sure) { - const cut = node.indexOf('\0') - if (node.slice(0, cut) === featureCtx(context)) out.set(node.slice(cut + 1), set) - } + const prefix = nodeKey(context, '') + for (const [node, set] of ensureResolved().sure) if (node.startsWith(prefix)) out.set(node.slice(prefix.length), set) return out }, // The compile unit of a crate root `libFile` named from code compiled as `fromUnit`: a proc-macro diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index c9725d83..03380518 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -13,8 +13,8 @@ import { realpathSync } from 'node:fs' import { readFile } from 'node:fs/promises' import { dirname, extname, isAbsolute, join, posix, relative, resolve } from 'node:path' -import { assertRealPathWithinBase } from '@exodus/stasis-core/util' -import { TARGET_CFG_KEYS, TARGET_UNIT, VENDOR_DIR, createCargoContext, evalCfg, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' +import { assertRealPathWithinBase, toPosix } from '@exodus/stasis-core/util' +import { TARGET_CFG_KEYS, TARGET_UNIT, VENDOR_DIR, createCargoContext, evalCfg, evalCfgKey, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' import { matchClose, splitTopLevel } from './toml.js' // Leads of the expression-position paths anchored on the module tree rather than on a name. @@ -453,8 +453,29 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, let visNext = null // the previous token was a `pub` visibility (parseVisibility): the coming item has it // Open blocks a live `#[cfg]` gates -- `if #[cfg(unix)] { mod imp; }` in a `cfg_if!` body, a // `#[cfg(x)] mod m { mod a; }` -- as `{ depth, cfg }`: everything declared inside is gated too. + // `scopeCfg` is their conjunction, kept as they open and close. const cfgScopes = [] - const scopeCfgs = () => cfgScopes.map((s) => s.cfg) + let scopeCfg = null + const openScope = (d, cfg) => { + cfgScopes.push({ depth: d, cfg }) + scopeCfg = joinCfgs(cfgScopes.map((s) => s.cfg)) + } + const closeScopes = (d) => { + if (cfgScopes.length === 0 || cfgScopes.at(-1).depth <= d) return + while (cfgScopes.length > 0 && cfgScopes.at(-1).depth > d) cfgScopes.pop() + scopeCfg = joinCfgs(cfgScopes.map((s) => s.cfg)) + } + // A predicate's verdict in this scan's build (evalCfg), once per predicate: the open scopes' + // conjunction is asked at every token inside them. + const verdicts = new Map() + const verdict = (pred) => { + let v = verdicts.get(pred) + if (v === undefined) { + v = evalCfg(pred, env) + verdicts.set(pred, v) + } + return v + } // The cfg of a `#[cfg(…)] name! { … }`: it gates the invocation, so everything the body // declares (mio's `#[cfg(unix)] cfg_os_poll! { mod unix; }`); taken up by the body's `{`. let macroBodyCfg = null @@ -503,6 +524,12 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, let templateName = null let templateMacro = null // the top-level `macro_rules!` whose body the scan is in: what its body calls and includes is its own, nested definitions' too + // Whether the token at `at` sits in a `macro_rules!` body: a template, expanded where the + // macro is invoked. + const inTemplate = (at) => at < macroUntil && macroName === 'macro_rules' + // Whether what is declared at `at` under `cfg` may not exist as an item: a cfg not known to + // hold, an inline module so gated around it, or a macro invocation's body. + const conditionalAt = (cfg, at) => (cfg !== null && verdict(cfg) !== true) || stack.some((s) => s.conditional) || at < macroUntil const inlinePath = () => stack.map((s) => s.name) const closeTo = (targetDepth, at) => { while (stack.length > 0 && stack.at(-1).depth > targetDepth) { @@ -603,7 +630,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, const attr = parseAttr(code.slice(open + 1, close), masked.slice(open + 1, close)) // `#[cfg_attr(, derive(serde::Serialize))]` with pred never holding applies nothing: // the paths in its text are dead code too, and so is whatever else it would apply. - const inert = attr.pred !== null && evalCfg(attr.pred, env) === false + const inert = attr.pred !== null && verdict(attr.pred) === false if (inert) deadSpans.push([i, close + 1]) if (pending.length === 0) attrStart = i pending.push(inert ? INERT_ATTR : attr) @@ -613,7 +640,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // the inline module it opens. const attr = parseAttr(code.slice(open + 1, close), masked.slice(open + 1, close)) // (An inner attribute inside a macro invocation's body is the macro's business.) - if (attr.cfg !== null && evalCfg(attr.cfg, env) === false && i >= macroUntil) { + if (attr.cfg !== null && verdict(attr.cfg) === false && i >= macroUntil) { if (depth === 0) return { mods: [], refs: [], externCrates: [], bindings: new Set(), imports: [], macros: [], invocations: new Set(), calls: new Map(), includes: [], unfollowed: 0, defined: [], inlineModules: [], inlineModuleVis: new Map(), skipped } const top = stack.at(-1) if (top !== undefined && top.depth === depth) { @@ -625,7 +652,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, } } // `#![doc = include_str!("../README.md")]`: the module's own include, applied here. - const conditional = stack.some((s) => s.conditional) || i < macroUntil + const conditional = conditionalAt(null, i) for (const inc of attr.includes) includes.push({ ...inc, conditional }) pending = [] } @@ -639,13 +666,13 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, const attrCfg = word === 'pub' ? null : joinCfgs(pending.map((a) => a.cfg).filter((c) => c !== null)) // A `{` opening a branch of a cfg chain is gated on the earlier branches' cfgs not holding too. const own = ch === '{' ? branchCfg(i, attrCfg) ?? attrCfg : attrCfg - const cfg = joinCfgs(own === null ? scopeCfgs() : [...scopeCfgs(), own]) + const cfg = own === null ? scopeCfg : joinCfgs([...cfgScopes.map((s) => s.cfg), own]) // An item gated on a cfg that never holds in the build is dead code for the bundle: skip it // whole -- from its first attribute (a `#[derive(serde::Serialize)]` on it names nothing // live) through a declaration's `;`, a field's or variant's `,`, or a body's or block's `}` // -- without recording anything in it, whatever token starts it: a keyword or a name, a // `{ … }` block statement, a `(a, b)` / `[a, ..]` pattern or tuple type, a `&x`, `*x` or literal. - if (cfg !== null && evalCfg(cfg, env) === false) { + if (cfg !== null && verdict(cfg) === false) { pending = [] visNext = null const end = skipItem(word === null ? i : i + word.length, word !== null && ITEM_KEYWORDS.has(word), { elseChain: ch !== '{', toSemicolon: word === 'let' }) @@ -657,7 +684,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // The item is live, so the includes its attributes name (`#[doc = include_str!("docs/x.md")]`) // are too; a `pub` keeps them pending for the keyword after it. if (word !== 'pub' && pending.length > 0) { - const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil + const conditional = conditionalAt(cfg, i) for (const a of pending) { for (const inc of a.includes) includes.push({ ...inc, conditional }) a.includes = [] @@ -667,7 +694,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, depth++ const gate = own ?? macroBodyCfg macroBodyCfg = null - if (gate !== null) cfgScopes.push({ depth, cfg: gate }) + if (gate !== null) openScope(depth, gate) pending = [] i++ continue @@ -675,7 +702,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, if (ch === '}') { depth-- closeTo(depth, i) - while (cfgScopes.length > 0 && cfgScopes.at(-1).depth > depth) cfgScopes.pop() + closeScopes(depth) endBranch(i + 1, depth + 1) pending = [] i++ @@ -698,7 +725,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, if (name !== null) { open = skipWs(open + name.length) const gate = GATE_MACRO_RE.test(name) && masked[open] === '{' ? gatePredicate(code, masked, open, matchClose(masked, open)) : undefined - macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], calls: new Set(), offset: i, at: i, template: i < macroUntil && macroName === 'macro_rules' ? templateName : null, gate }) + macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], calls: new Set(), offset: i, at: i, template: inTemplate(i) ? templateName : null, gate }) if (i >= macroUntil) { templateName = name templateMacro = macros.at(-1) @@ -726,7 +753,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // it where the macro is invoked, not where it is written -- and a recursive arm's // `m!(…)` is no invocation of this file's. if (!pathQualified) { - if (i < macroUntil && macroName === 'macro_rules' && templateMacro !== null) templateMacro.calls.add(word) + if (inTemplate(i) && templateMacro !== null) templateMacro.calls.add(word) else { invocations.add(word) ;(calls.get(word) ?? calls.set(word, []).get(word)).push(i) @@ -735,13 +762,13 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, if (INCLUDE_MACROS.has(word)) { const text = code.slice(open + 1, close) const arg = includeArg(text) - const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil + const conditional = conditionalAt(cfg, i) // Inside a `macro_rules!` body the path is relative to whichever file invokes the // macro: recorded on the definition, resolved per invoking file. Build output // (`concat!(env!("OUT_DIR"), …)`) is nobody's to follow; any other unreadable argument // is counted, for a warning. if (arg === null) unfollowed += /\bOUT_DIR\b/u.test(text) ? 0 : 1 - else if (i < macroUntil && macroName === 'macro_rules' && templateMacro !== null) templateMacro.includes.push({ kind: word, ...arg }) + else if (inTemplate(i) && templateMacro !== null) templateMacro.includes.push({ kind: word, ...arg }) else includes.push({ kind: word, ...arg, conditional }) } } @@ -763,7 +790,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // `impl`/`trait` body's associated one, nor a `macro_rules!` template's (that is some invoking // module's item); `const fn` / `static mut` / lazy_static's `static ref` are not items named // `fn` / `mut` / `ref`. - if (DEFINING_KEYWORDS.has(word) && atModuleLevel(i) && !isAssociated(i, depth) && !(i < macroUntil && macroName === 'macro_rules') && !(word === 'const' && masked[skipWsBack(i - 1)] === '*')) { + if (DEFINING_KEYWORDS.has(word) && atModuleLevel(i) && !isAssociated(i, depth) && !inTemplate(i) && !(word === 'const' && masked[skipWsBack(i - 1)] === '*')) { let at = skipWs(i + word.length) if (masked.startsWith('mut ', at) || masked.startsWith('ref ', at)) at = skipWs(at + 3) if (masked.startsWith('r#', at)) at += 2 @@ -798,19 +825,19 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, const attrs = pending pending = [] // Dead cfgs were skipped above; a decidable-true one (`not(test)`) is as firm as no cfg at all. - const conditional = (cfg !== null && evalCfg(cfg, env) !== true) || stack.some((s) => s.conditional) || i < macroUntil + const conditional = conditionalAt(cfg, i) if (masked[k] === ';') { // rustc applies the first `#[path]` / `#[cfg_attr(, path = …)]` whose predicate holds. // A variant whose predicate can't hold names nothing in this build; one whose predicate // holds ends the list -- outright the `#[path]` when nothing undecided precedes it -- and // either way the default `.rs` lookup never happens. - const live = attrs.flatMap((a) => a.paths).filter((p) => p.cfg === null || evalCfg(p.cfg, env) !== false) - const applies = live.findIndex((p) => p.cfg === null || evalCfg(p.cfg, env) === true) + const live = attrs.flatMap((a) => a.paths).filter((p) => p.cfg === null || verdict(p.cfg) !== false) + const applies = live.findIndex((p) => p.cfg === null || verdict(p.cfg) === true) const paths = applies === -1 ? live : (applies === 0 ? [{ path: live[0].path, cfg: null }] : live.slice(0, applies + 1)) mods.push({ name, inlinePath: inlinePath(), inlineDirs: stack.map((s) => s.dir), cfg, conditional, paths, noDefault: applies !== -1, vis, macroUse: attrs.some((a) => a.macroUse === true), macro: i < macroUntil ? macroName : null, offset: i, at: i, - template: i < macroUntil && macroName === 'macro_rules' ? templateName : null, + template: inTemplate(i) ? templateName : null, }) i = k + 1 continue @@ -820,7 +847,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // `#[path = ""] mod non_bpf_modules { mod account_keys; }` right beside the file. const dir = attrs.flatMap((a) => a.paths).find((p) => p.cfg === null)?.path ?? name depth++ - if (own !== null) cfgScopes.push({ depth, cfg: own }) + if (own !== null) openScope(depth, own) stack.push({ name, dir, depth, conditional, start: k + 1, vis }) i = k + 1 continue @@ -999,11 +1026,24 @@ export function resolveExplicitModPath(explicitPath, fromFile, { knownSources, b // Whether project-relative `rel` lies in directory `dir` (`.`: anywhere in the bundle root). const withinDir = (rel, dir) => dir === '.' || rel === dir || rel.startsWith(`${dir}/`) +// Whether the file at project-relative `rel` really lies in directory `within` (`.`: anywhere in +// the bundle root, which assertRealPathWithinBase holds it to): a vendored crate's file linked out +// of its package -- to the project's `.env`, say -- doesn't, and is refused like an include naming +// the target outright, warned as `who`'s. `realDirs` keeps the directories' real paths. +export function withinRealDir(baseDir, rel, within, { who = 'loader.rust', realDirs = new Map() } = {}) { + if (within === '.') return true + if (!realDirs.has(within)) realDirs.set(within, realpathSync(join(baseDir, within))) + const back = toPosix(relative(realDirs.get(within), realpathSync(join(baseDir, rel)))) + if (!back.startsWith('..') && !isAbsolute(back)) return true + console.warn(`[${who}] Refusing file outside its package: ${rel} (a link out of ${within})`) + return false +} + // The directory a file's `#[path]`, include and build-script paths may reach into: a published // crate never legitimately names a file outside its own package, so a file of a vendored one is // held to that package's directory (`vendor/`); the project's own code -- the root package, // workspace members, path dependencies -- may reach anywhere in the bundle root (`.`). -function boundaryOf(file, ctx) { +export function boundaryOf(file, ctx) { return ctx?.isVendored(file) === true ? ctx.packageInfo(file).dir : '.' } @@ -1025,17 +1065,12 @@ function includeTarget(inc, file, ctx) { // Every file a `mod` declaration can denote, as `{ cfg, file, explicit }`: an unconditional // `#[path]` names one outright; otherwise each `#[cfg_attr(, path = …)]` names one under // its predicate (`cfg`), and the default `.rs`/`/mod.rs` lookup is the fallback (`cfg` -// null). A variant whose predicate can't hold in the build is dropped: the scanner already did so -// for the build it scanned under, and `opts.features` / `opts.test` / `opts.target`, when given, -// describe one here. `explicit` marks a file a `#[path]` named (its own submodules then sit -// beside it). The inline modules the declaration sits in contribute their directories -// (`inlineDirs`: an inline module's own `#[path]` replaces its name, an empty one adds nothing), -// else their names. +// null); the scanner has dropped those whose predicate can't hold in the build it scanned under. +// `explicit` marks a file a `#[path]` named (its own submodules then sit beside it). The inline +// modules the declaration sits in contribute their directories (`inlineDirs`: an inline module's +// own `#[path]` replaces its name, an empty one adds nothing), else their names. export function resolveModDecl(decl, fromFile, opts = {}) { const o = { ...opts, inlinePath: decl.inlineDirs ?? decl.inlinePath } - const env = opts.features === undefined && opts.test === undefined && opts.target === undefined - ? null - : { features: opts.features ?? null, test: opts.test === true, target: opts.target ?? null } const unconditional = decl.paths.find((p) => p.cfg === null) if (unconditional) { const file = resolveExplicitModPath(unconditional.path, fromFile, o) @@ -1043,7 +1078,6 @@ export function resolveModDecl(decl, fromFile, opts = {}) { } const out = [] for (const { path, cfg } of decl.paths) { - if (env !== null && evalCfg(cfg, env) === false) continue const file = resolveExplicitModPath(path, fromFile, o) if (file && !out.some((x) => x.file === file)) out.push({ cfg, file, explicit: true }) } @@ -1286,8 +1320,7 @@ function gatePredicate(code, masked, open, close) { preds.push(normalizeCfg(code.slice(paren + 1, matchClose(masked, paren)))) } if (preds.length === 0) return null - const uniq = [...new Set(preds)].toSorted() - const pred = uniq.length === 1 ? uniq[0] : `all(${uniq.join(', ')})` + const pred = joinCfgs([...new Set(preds)].toSorted()) if (seen !== null && seen !== pred) return null seen = pred i = end @@ -1398,14 +1431,15 @@ const eitherLeaves = (a, b, sets) => { // target's cfgs (`unix`, `target_os = "linux"`, …: what every file of the build is compiled // under) -- against a candidate's: exclusive candidates are skipped; no asker (an internal query) // skips nothing. The verdict is kept per pair of sets in `compat`, shared by the askers of one set. -const compatible = (asker, set) => { - if (asker === undefined || set.key === '') return true - let verdict = asker.compat.get(set.key) - if (verdict === undefined) { - verdict = !cfgExclusive(asker.set.leaves, set.leaves) - asker.compat.set(set.key, verdict) +const compatible = (asker, set) => asker === undefined || set.key === '' || cached(asker.compat, set.key, () => !cfgExclusive(asker.set.leaves, set.leaves)) +// `map`'s value for `key`, computed on the first ask (the verdicts per pair of leaf sets here). +function cached(map, key, compute) { + let value = map.get(key) + if (value === undefined) { + value = compute() + map.set(key, value) } - return verdict + return value } // Whether a leaf list can't hold in the asker's build (scanRustItems' build, see buildOf): one of // its leaves the build decides is false -- a feature that is off, a platform the build isn't -- @@ -1414,31 +1448,14 @@ const compatible = (asker, set) => { // build may compile, and only when there is none -- a path written behind the same gate, whose // code the build doesn't compile either, means what it would mean where it compiles. For an // asker whose own leaves can't hold (`deadHere`) the build says nothing. -const EVALUABLE_CFG_KEYS = new Set([...TARGET_CFG_KEYS, 'feature', 'test', 'doc', 'doctest', 'false']) -const leafFalseMemo = new WeakMap() // build → leaf → verdict (builds are interned per settled context, see buildOf) const leafFalse = (l, build) => { if (l.alts !== undefined) return l.alts.every((alt) => alt.some((m) => leafFalse(m, build))) - if (!EVALUABLE_CFG_KEYS.has(l.key)) return false - const memo = leafFalseMemo.get(build) ?? leafFalseMemo.set(build, new Map()).get(build) - const pred = l.value === null ? l.key : `${l.key} = "${l.value}"` - const text = l.neg ? `not(${pred})` : pred - let verdict = memo.get(text) - if (verdict === undefined) { - verdict = evalCfg(text, build) === false - memo.set(text, verdict) - } - return verdict + if (l.key === 'false' && l.value === null) return !l.neg // the literal (cfgLeaves) + const verdict = evalCfgKey(l.key, l.value, build) + return l.neg ? verdict === true : verdict === false } const deadUnder = (leaves, build) => build !== undefined && leaves.some((l) => leafFalse(l, build)) -const deadFor = (asker, set) => { - if (asker === undefined || asker.deadHere || set.key === '') return false - let verdict = asker.dead.get(set.key) - if (verdict === undefined) { - verdict = deadUnder(set.leaves, asker.build) - asker.dead.set(set.key, verdict) - } - return verdict -} +const deadFor = (asker, set) => asker !== undefined && !asker.deadHere && set.key !== '' && cached(asker.dead, set.key, () => deadUnder(set.leaves, asker.build)) // Whether every leaf of a candidate's set is among the asker's own (`keys`, by leafKey): the two // are certainly compiled together, not merely possibly (compatible). libc's aix/mod.rs, mounted // under `target_os = "aix"`, asks `crate::fsid_t`: the aix glob's, under that same leaf, is the @@ -1448,15 +1465,7 @@ const deadFor = (asker, set) => { // else: for it, candidates under cfgs of their own are each only maybe the answer, and all of // them are (see withAlternatives), not the first in written order. const holds = (l, keys, custom) => keys.has(leafKey(l)) || (l.alts === undefined ? l.neg && custom(l.key) : l.alts.some((alt) => alt.every((m) => holds(m, keys, custom)))) -const entailed = (asker, set) => { - if (asker === undefined || set.key === '') return true - let verdict = asker.sure.get(set.key) - if (verdict === undefined) { - verdict = set.leaves.every((l) => holds(l, asker.keys, asker.custom)) - asker.sure.set(set.key, verdict) - } - return verdict -} +const entailed = (asker, set) => asker === undefined || set.key === '' || cached(asker.sure, set.key, () => set.leaves.every((l) => holds(l, asker.keys, asker.custom))) // The cfgs rustc and cargo set: anything else in a positive leaf is a custom `--cfg` (`loom`, // `docsrs`, `tokio_unstable`, mio's `mio_unsupported_force_poll_poll`), off in a default build // unless a build script or the rustflags set it -- so a candidate under one is `doubtful`: taken @@ -1486,13 +1495,7 @@ const hasCustom = (leaves, custom) => leaves.some((l) => (l.alts === undefined ? const doubtful = (asker, set) => { if (asker === undefined || set.key === '') return false if (set.custom === undefined) set.custom = hasCustom(set.leaves, customKey) - if (!set.custom) return false - let verdict = asker.doubt.get(set.key) - if (verdict === undefined) { - verdict = set.leaves.some((l) => !holds(l, asker.keys, asker.custom) && doubtfulLeaf(l, asker.set.leaves, asker.custom)) - asker.doubt.set(set.key, verdict) - } - return verdict + return set.custom && cached(asker.doubt, set.key, () => set.leaves.some((l) => !holds(l, asker.keys, asker.custom) && doubtfulLeaf(l, asker.set.leaves, asker.custom))) } // --- memoization ----------------------------------------------------------------------------- @@ -1709,9 +1712,7 @@ function cfgTextOf(leaves) { const positive = own.filter((l) => l.alts === undefined && !l.neg) const anyPositive = own.filter((l) => l.alts !== undefined && l.alts.every((alt) => alt.every((m) => m.alts === undefined && !m.neg))) const shown = positive.length + anyPositive.length > 0 ? [...positive, ...anyPositive] : (own.length > 0 ? own : leaves) - if (shown.length === 0) return null - const texts = [...new Set(shown.map((l) => (l.key === 'variant' ? l.value : leafText(l))))].toSorted() - return texts.length === 1 ? texts[0] : `all(${texts.join(', ')})` + return joinCfgs([...new Set(shown.map((l) => (l.key === 'variant' ? l.value : leafText(l))))].toSorted()) } // The answer of several candidates none of which the asker's cfgs entail (pick, definedIn): each @@ -1730,12 +1731,8 @@ function withAlternatives(found, asker) { if (answer.file === undefined || seen.has(answer.file)) continue seen.add(answer.file) // Kept per asker set (the text leaves out what the asker holds itself). - let text = asker?.texts.get(leaves.key) - if (text === undefined) { - text = cfgTextOf(leaves.leaves.filter((l) => asker === undefined || !asker.keys.has(leafKey(l)))) - asker?.texts.set(leaves.key, text) - } - const base = cfgKey(text) + const textOf = () => cfgTextOf(leaves.leaves.filter((l) => asker === undefined || !asker.keys.has(leafKey(l)))) + const base = cfgKey(asker === undefined ? textOf() : cached(asker.texts, leaves.key, textOf)) let key = base for (let k = 2; alternatives.has(key); k++) key = `${base}#${k}` alternatives.set(key, answer.file) @@ -1905,8 +1902,9 @@ function definedIn(root, at, name, seeing, ctx, asker, ns = null) { // providedAll: a `use log;` in such a file asks for `log` in its own module, which is this import. function providedFrom(root, at, name, ctx, file, asker, ns = null) { const of = ctx.imports.get(root)?.get(at) - const several = of !== undefined && [...of.globs, ...[...of.named.values()].flat()].some((im) => im.file !== file) - if (!several) return provided(root, at, name, at, ctx, asker, ns) + // The files holding the module's imports, gathered once (they are all in place by now). + const files = of === undefined ? null : (of.files ??= new Set([...of.globs, ...[...of.named.values()].flat()].map((im) => im.file))) + if (files === null || files.size === (files.has(file) ? 1 : 0)) return provided(root, at, name, at, ctx, asker, ns) const memo = ctx.provided.get(root) ?? ctx.provided.set(root, new Map()).get(root) const seeing = at.split('::').length const key = `${at}\0${name}\0@${file}` @@ -2187,14 +2185,14 @@ function globMayProvide(root, at, lead, ctx) { const seeing = at.split('::').length const closure = globClosure(root, at, seeing, ctx) const positions = closureIndex(closure) - reach = { unseen: new Map(), crates: [] } + reach = { unseen: new Set(), crates: [] } for (const [module, globs] of opaqueGlobsOf(root, ctx)) { let sees = module === at ? seeing : -1 for (const i of positions.get(module) ?? NONE) sees = Math.max(sees, closure[i].seeing) if (sees === -1) continue for (const { im, source } of globs) { if (im.scopeDepth > sees) continue - if (source === null) reach.unseen.set(im.segments[0], (reach.unseen.get(im.segments[0]) ?? 0) + 1) + if (source === null) reach.unseen.add(im.segments[0]) else reach.crates.push({ im, source }) } } @@ -2204,10 +2202,7 @@ function globMayProvide(root, at, lead, ctx) { // bundle sees: anything, for a crate not in-tree; else what the module has, what its `pub` // globs bring in from its own crate (providedAll), or may from a third. const inCrate = (file, module) => !ctx.trees.has(file) || hasAll(file, module, lead, 1, ctx).length > 0 || providedAll(file, module, lead, 1, ctx).get(0) !== undefined || globMayProvide(file, module, lead, ctx) - let answer = false - let unseen = 0 - for (const [name, count] of reach.unseen) if (name !== lead) unseen += count - if (unseen > 0) answer = true + let answer = reach.unseen.size > (reach.unseen.has(lead) ? 1 : 0) for (const { im, source } of reach.crates) { if (answer || im.segments[0] === lead) break // The glob's path on from the crate it names (`sync` of `use tokio::sync::*;`), when the @@ -2285,11 +2280,38 @@ function effectiveInvokers(name, invokers, templateCalls, seen = new Set()) { for (const [caller, called] of templateCalls) if (called.has(name)) for (const f of effectiveInvokers(caller, invokers, templateCalls, seen)) out.add(f) return out } +// Note what file `path`'s scan (`items`) invokes by bare name, and what its `macro_rules!` +// templates call, for effectiveInvokers. +function noteMacroCalls(invokers, templateCalls, path, items) { + for (const name of items.invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(path) + for (const m of items.macros) for (const name of m.calls) (templateCalls.get(m.name) ?? templateCalls.set(m.name, new Set()).get(m.name)).add(name) +} +// The files of `from`'s package in effect invoking macro `name` (`packageOf`: file → package): +// where a `mod` its template declares is declared -- a macro of the same name elsewhere is likely +// another (see buildRustTree). +const hostsOf = (name, from, invokers, templateCalls, packageOf) => [...effectiveInvokers(name, invokers, templateCalls)].filter((f) => packageOf(f) === packageOf(from)) +// TEMPLATE_MACROS less the ones a package defines itself (`own`), one object per set of them. +const templatesLessMemo = new Map() +const templatesLess = (own) => { + const key = [...own].toSorted().join() + return templatesLessMemo.get(key) ?? templatesLessMemo.set(key, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(key) +} // The compile units a file is compiled as (see cargo.js unitKey): what `units` recorded, else // the target's alone. const DEFAULT_UNITS = new Set([TARGET_UNIT]) const unitsOf = (units, rel) => units?.get(rel) ?? DEFAULT_UNITS +// Record `file` as compiled as each of `more` too, in `units`; whether that is news. +function addUnits(units, file, more) { + const have = units.get(file) + if (have === undefined) { + units.set(file, new Set(more)) + return true + } + if (more.isSubsetOf(have)) return false + for (const u of more) have.add(u) + return true +} // `units` (a new map for none) with each entry it doesn't name compiled as its package is // (unitOfCrate): a proc-macro crate's for the host, as cargo builds it, any other's for the target. function withEntryUnits(units, entries, ctx) { @@ -2381,21 +2403,16 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul for (const [path, items] of scanned) { for (const m of items.macros) if (TEMPLATE_MACROS.has(m.name)) (ownTemplates.get(packageOf(path)) ?? ownTemplates.set(packageOf(path), new Set()).get(packageOf(path))).add(m.name) } - const reducedTemplates = new Map() // per package's own set: TEMPLATE_MACROS less those, one object for (const [path, items] of scanned) { const own = items.skipped.size > 0 ? ownTemplates.get(packageOf(path)) : undefined if (own === undefined || own.isDisjointFrom(items.skipped)) continue - const templates = reducedTemplates.get(own) ?? reducedTemplates.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) - scanned.set(path, cachedScan(sources, path, sources.get(path), buildOf(path, ctx, units), templates)) + scanned.set(path, cachedScan(sources, path, sources.get(path), buildOf(path, ctx, units), templatesLess(own))) } // Bare macro invocations per file, for what a `macro_rules!` body holds: rustc expands it where // the macro is invoked -- through the templates that call it, the outermost (effectiveInvokers). const invokers = new Map() const templateCalls = new Map() - for (const [path, items] of scanned) { - for (const name of items.invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(path) - for (const m of items.macros) for (const name of m.calls) (templateCalls.get(m.name) ?? templateCalls.set(m.name, new Set()).get(m.name)).add(name) - } + for (const [path, items] of scanned) noteMacroCalls(invokers, templateCalls, path, items) // A `mod` a `macro_rules!` template declares is declared in each module invoking the macro, and // its file found from there (serde_core's src/crate_root.rs holds `crate_root! { … pub mod de; // … }`, which lib.rs invokes: src/de/mod.rs). Only its own package's invocations count -- a @@ -2409,7 +2426,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul for (const [path, items] of scanned) { for (const decl of items.mods) { if (decl.template === null) continue - const hosts = [...effectiveInvokers(decl.template, invokers, templateCalls)].filter((f) => packageOf(f) === packageOf(path)) + const hosts = hostsOf(decl.template, path, invokers, templateCalls, packageOf) if (hosts.length === 0) continue hostedAway.add(decl) for (const f of hosts) (hosted.get(f) ?? hosted.set(f, []).get(f)).push({ ...decl, offset: scanned.get(f).calls.get(decl.template)?.[0] ?? Infinity, definedIn: path }) @@ -2537,10 +2554,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // build script (known only when the target is the host); a file compiled for both platforms // asks under the cfgs they share, which the loader knows only when they are one. const platformLeaves = new Map() // cfg set -> its leaf set - const leavesOfCfgs = (cfgs) => platformLeaves.get(cfgs) ?? platformLeaves.set(cfgs, leafSet([...cfgs].flatMap((cfg) => { - const m = /^([\w-]+)(?:="([^"]*)")?$/u.exec(cfg) - return m && TARGET_CFG_KEYS.has(m[1]) ? [{ key: m[1], value: m[2] ?? null, neg: false }] : [] - }), leafSets)).get(cfgs) + const leavesOfCfgs = (cfgs) => platformLeaves.get(cfgs) ?? platformLeaves.set(cfgs, leafSet([...cfgs].flatMap(cfgLeaves).filter((l) => TARGET_CFG_KEYS.has(l.key)), leafSets)).get(cfgs) const platformCfgsOf = (path) => { const each = new Set([...unitsOf(units, path)].map((u) => ctx?.platformOf(u)?.cfgs ?? null)) return each.size === 1 ? [...each][0] : null @@ -2585,6 +2599,9 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const fileMacros = new Map() const externPrelude = new Map() const perRoot = (map, root) => map.get(root) ?? map.set(root, new Map()).get(root) + // An import's or item's leaves: those its file (`here`) was mounted under, its own cfg's, and + // its gate macro's. + const itemLeaves = (here, x, path) => unionLeaves(here.leaves, leafSet([...cfgLeaves(x.cfg), ...gateLeaves(x.macro, gatesOf(path))], leafSets), leafSets) const scopeDepthOf = (vis, modulePath) => visibilityScope(vis, modulePath).split('::').length // Bare `name!` invocations resolve in textual scope, as rustc has it: a `macro_rules!` is in // scope from its definition to the end of its file, in the files the `mod`s after it mount (and @@ -2610,7 +2627,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const modulePath = [here.modulePath, ...im.inlinePath].join('::') const byModule = perRoot(imports, here.root) const of = byModule.get(modulePath) ?? byModule.set(modulePath, { named: new Map(), globs: [] }).get(modulePath) - const entry = { ...im, file: path, module: modulePath, scopeDepth: scopeDepthOf(im.vis, modulePath), leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(im.cfg), ...gateLeaves(im.macro, gatesOf(path))], leafSets), leafSets) } + const entry = { ...im, file: path, module: modulePath, scopeDepth: scopeDepthOf(im.vis, modulePath), leaves: itemLeaves(here, im, path) } if (im.glob) of.globs.push(entry) else if (im.binding !== null) (of.named.get(im.binding) ?? of.named.set(im.binding, []).get(im.binding)).push(entry) } @@ -2618,7 +2635,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const modulePath = [here.modulePath, ...d.inlinePath].join('::') const byModule = perRoot(defined, here.root) const names = byModule.get(modulePath) ?? byModule.set(modulePath, new Map()).get(modulePath) - ;(names.get(d.name) ?? names.set(d.name, []).get(d.name)).push({ file: path, scopeDepth: scopeDepthOf(d.vis, modulePath), ns: d.ns, leaves: unionLeaves(here.leaves, leafSet([...cfgLeaves(d.cfg), ...gateLeaves(d.macro, gatesOf(path))], leafSets), leafSets) }) + ;(names.get(d.name) ?? names.set(d.name, []).get(d.name)).push({ file: path, scopeDepth: scopeDepthOf(d.vis, modulePath), ns: d.ns, leaves: itemLeaves(here, d, path) }) } const scopes = perRoot(modScope, here.root) const specMap = resolutions.get(path) @@ -2888,6 +2905,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, // file → the directory it must really lie in (boundaryOf, from the file that named it): a // vendored crate's files stay in its package, symlinks included. const boundaries = new Map(entries.map((e) => [e, boundaryOf(e, ctx)])) + const realDirs = new Map() // those directories' real paths // An include macro in a `macro_rules!` body names a file relative to each file invoking the // macro -- through the templates that call it, the outermost (effectiveInvokers): definitions // and invocations arrive in any order, so each side is kept and paired. Every definition of a @@ -2902,24 +2920,9 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, const packageOf = (rel) => ctx.packageInfo(rel)?.dir ?? '.' const ownTemplates = new Map() // package → the template names it defines const skippedIn = new Map() // package → its files that skipped a template body - const reduced = new Map() // per package's own set: TEMPLATE_MACROS less those, one object const scanFor = (relPath, content) => { const own = ownTemplates.get(packageOf(relPath)) - const build = buildOf(relPath, ctx, units) - if (own === undefined) return cachedScan(sources, relPath, content, build) - const set = reduced.get(own) ?? reduced.set(own, new Set([...TEMPLATE_MACROS].filter((name) => !own.has(name)))).get(own) - return cachedScan(sources, relPath, content, build, set) - } - // Record `file` as compiled as each of `more` too; whether that is news. - const addUnits = (file, more) => { - const have = units.get(file) - if (have === undefined) { - units.set(file, new Set(more)) - return true - } - if (more.isSubsetOf(have)) return false - for (const u of more) have.add(u) - return true + return cachedScan(sources, relPath, content, buildOf(relPath, ctx, units), own === undefined ? undefined : templatesLess(own)) } withEntryUnits(units, entries, ctx) // Files already walked that are now compiled as more units: gone through again (their build @@ -2940,7 +2943,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, } } if (!boundaries.has(file)) boundaries.set(file, within) - const grew = addUnits(file, as) + const grew = addUnits(units, file, as) if (!sources.has(file)) next.push(file) else if (grew && rust && !formats?.has(file)) revisit.add(file) } @@ -2967,16 +2970,13 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, queue(from, file, { rust: true }) } } - // The files of `from`'s package in effect invoking macro `name`: where its template's `mod`s - // are declared (see buildRustTree). - const hostsOf = (name, from) => [...effectiveInvokers(name, invokers, templateCalls)].filter((f) => packageOf(f) === packageOf(from)) // The include macros of every template, relative to every file in effect invoking it; its // `mod`s relative to those of its own package. const expandTemplates = () => { for (const [name, incs] of templateIncludes) { for (const f of effectiveInvokers(name, invokers, templateCalls)) for (const inc of incs) includeFrom(inc, f) } - for (const [name, decls] of templateMods) for (const { decl, from } of decls.values()) for (const f of hostsOf(name, from)) modFrom(decl, f) + for (const [name, decls] of templateMods) for (const { decl, from } of decls.values()) for (const f of hostsOf(name, from, invokers, templateCalls, packageOf)) modFrom(decl, f) } // A template's `mod`s that nothing in its package invokes by bare name stand beside its @@ -2985,7 +2985,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, const placeUninvoked = () => { for (const [name, decls] of templateMods) { for (const { decl, from } of decls.values()) { - if (placed.has(decl) || hostsOf(name, from).length > 0) continue + if (placed.has(decl) || hostsOf(name, from, invokers, templateCalls, packageOf).length > 0) continue placed.add(decl) modFrom(decl, from) } @@ -2999,16 +2999,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, toLoad.map(async (relPath) => { try { assertRealPathWithinBase(realBase, baseDir, relPath) - // A vendored crate's file must really lie in its package: a symlink out of it (to the - // project's `.env`, say) is refused like an include naming the target outright. - const within = boundaries.get(relPath) - if (within !== undefined && within !== '.') { - const rel = relative(realpathSync(join(baseDir, within)), realpathSync(join(baseDir, relPath))).split(/[\\/]/u).join('/') - if (rel.startsWith('..') || isAbsolute(rel)) { - console.warn(`[loader.rust] Refusing file outside its package: ${relPath} (a link out of ${within})`) - return null - } - } + if (!withinRealDir(baseDir, relPath, boundaries.get(relPath) ?? '.', { realDirs })) return null // Rust source (a module, a build script, an `include!`d file) and an `include_str!` asset // are UTF-8 text, or rustc rejects them: one that isn't is refused, never carried with // its bytes replaced. An `include_bytes!` asset is any bytes, carried as base64 if need be. @@ -3039,7 +3030,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, const handle = (relPath, content) => { const here = unitsOf(units, relPath) const items = scanFor(relPath, content) - const { mods, refs, externCrates, bindings, includes, macros, invocations } = items + const { mods, refs, externCrates, bindings, includes, macros } = items const pkg = packageOf(relPath) if (items.skipped.size > 0) (skippedIn.get(pkg) ?? skippedIn.set(pkg, new Set()).get(pkg)).add(relPath) for (const m of macros) { @@ -3062,13 +3053,11 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, // (expandTemplates), against the files invoking it. for (const inc of includes) includeFrom(inc, relPath) for (const m of macros) { - if (m.includes.length > 0) { - const list = templateIncludes.get(m.name) ?? templateIncludes.set(m.name, []).get(m.name) - for (const inc of m.includes) if (!list.some((x) => x.kind === inc.kind && x.path === inc.path && x.base === inc.base)) list.push(inc) - } - for (const name of m.calls) (templateCalls.get(m.name) ?? templateCalls.set(m.name, new Set()).get(m.name)).add(name) + if (m.includes.length === 0) continue + const list = templateIncludes.get(m.name) ?? templateIncludes.set(m.name, []).get(m.name) + for (const inc of m.includes) if (!list.some((x) => x.kind === inc.kind && x.path === inc.path && x.base === inc.base)) list.push(inc) } - for (const name of invocations) (invokers.get(name) ?? invokers.set(name, new Set()).get(name)).add(relPath) + noteMacroCalls(invokers, templateCalls, relPath, items) // A reference to an in-tree crate pulls its root in (a crate root by role, whatever its // name -- `[lib] path` may point anywhere); the root's own `mod` edges follow next wave. const leads = new Set() @@ -3106,6 +3095,51 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, return sources } +// The files of a Rust bundle of `entries`, from disk, and its tree: the walk +// (collectRustFilesFromDisk) and -- with `buildScripts` -- the build script of every package it +// reaches, each a crate root of its own, until no package is new; then the tree pass +// (buildRustTree), again with the in-tree crate roots it names that the walk left out (a crate +// whose name a file also binds as a value, `use crate::util::log;` beside `log::info!`) or walked +// as fewer units than it is named as (`wantedRoots`), until none is new. Each file is compiled as the units `units` records (cargo.js +// unitKey): a build script for the host, with its package's features, and so what it reaches; a +// proc-macro crate and what it depends on for the host too. A root the walk was asked for and +// didn't load -- refused (a link out of its package) or gone -- is not asked for again: the walk +// has said why. Returns `{ sources, formats, tree }`. +export async function collectRustBundle(baseDir, entries, { cargo, buildScripts = false }) { + const formats = new Map() + const sources = new Map() + const units = new Map() + const roots = [...entries] + const walk = async (wave) => { + await collectRustFilesFromDisk(baseDir, wave, { cargo, formats, sources, units }) + if (!buildScripts) return + const scripts = new Set() + for (const path of sources.keys()) { + const script = formats.has(path) ? null : cargo.buildScriptOf(path) + if (script === null) continue + const as = new Set([...unitsOf(units, path)].map((u) => cargo.buildScriptUnit(u))) + if (addUnits(units, script, as)) scripts.add(script) + } + if (scripts.size === 0) return + for (const s of scripts) if (!roots.includes(s)) roots.push(s) + await walk([...scripts]) + } + await walk(entries) + const tried = new Set() + const complete = async () => { + const tree = buildRustTree(sources, { roots, baseDir, cargo, formats, units }) + const wanted = tree.wantedRoots.filter((r) => addUnits(units, r, tree.wantedUnits.get(r)) || (!sources.has(r) && !tried.has(r))) + if (wanted.length === 0) return tree + for (const r of wanted) { + tried.add(r) + if (!roots.includes(r)) roots.push(r) + } + await walk(wanted) + return complete() + } + return { sources, formats, tree: await complete() } +} + // Reject absolute paths and `..`-escaping paths in a `.rs.txt` listing so a // malicious or sloppy listing can't read files outside the listing's own dir. function assertWithinBase(baseDir, candidate, label) { @@ -3130,8 +3164,6 @@ export async function loadRust(rsTxtFile) { } for (const line of lines) assertWithinBase(baseDir, line, 'Entry path') - const cargo = createCargoContext(baseDir, { entries: lines }) - const formats = new Map() - const sources = await collectRustFilesFromDisk(baseDir, lines, { cargo, formats }) - return { ...buildRustTree(sources, { roots: lines, baseDir, cargo, formats }), formats } + const { tree, formats } = await collectRustBundle(baseDir, lines, { cargo: createCargoContext(baseDir, { entries: lines }) }) + return { ...tree, formats } } diff --git a/stasis/src/loaders/toml.js b/stasis/src/loaders/toml.js index bb76b5ab..6283eea5 100644 --- a/stasis/src/loaders/toml.js +++ b/stasis/src/loaders/toml.js @@ -51,14 +51,16 @@ export function splitTopLevel(text) { // The table tree of TOML `text` (parseToml): tables are null-prototype objects, arrays arrays, and // values strings, booleans, numbers (an integer past 2^53 a BigInt), TomlFloat and TomlDateTime. // Throws a TomlError on anything else, naming `file` when given. -export function readToml(text, file = null) { +export const readToml = (text, file = null) => nameErrors(file, () => parseToml(text)) + +// `read()`, with an error of one of `classes` it throws (@preventive/lockfile's, whose message +// leads with where it is) leading with `file` too, its fields kept; null `file` names none. +export function nameErrors(file, read, classes = [TomlError]) { try { - return parseToml(text) + return read() } catch (err) { - if (file === null || !(err instanceof TomlError)) throw err - const named = new TomlError(`${file}: ${err.message}`) - named.line = err.line - throw named + if (file === null || !classes.some((c) => err instanceof c)) throw err + throw Object.assign(new err.constructor(`${file}: ${err.message}`), err) } } diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index 1f59e9f0..3d9df636 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -98,15 +98,16 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin // cargo's feature map: as written, and a feature for the optional dependency no `dep:` names t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']], ['pm-crate', ['dep:pm-crate']]]) const dep = (k) => Object.fromEntries([...m.deps.get(k).kinds].toSorted()) - const ask = (pkg, version, more) => ({ version, path: null, source: 'registry', package: pkg, renamed: false, inherited: false, optional: false, defaultFeatures: true, features: [], ...more }) + const ask = (pkg, version, more) => ({ kind: 'normal', target: null, version, path: null, source: 'registry', package: pkg, renamed: false, inherited: false, optional: false, defaultFeatures: true, features: [], ...more }) // Each dependency table is its own request, for the crate it names: a dev-dependency's features // stay out of the normal one. - t.assert.deepEqual(dep('plain'), { dev: ask('plain', '1', { features: ['dev-only'] }), normal: ask('plain', '1') }) + t.assert.deepEqual(dep('plain'), { dev: ask('plain', '1', { kind: 'dev', features: ['dev-only'] }), normal: ask('plain', '1') }) t.assert.deepEqual(dep('opt'), { normal: ask('opt', '1', { optional: true, defaultFeatures: false, features: ['a'] }) }) t.assert.deepEqual(dep('sub'), { normal: ask('sub', '2', { features: ['one', 'two'] }) }) t.assert.deepEqual(Object.keys(dep('cc')), ['build']) // A target-specific table is a request of its own, beside the plain one. t.assert.deepEqual(Object.keys(dep('nix')), ['normal@cfg(unix)']) + t.assert.deepEqual([dep('nix')['normal@cfg(unix)'].kind, dep('nix')['normal@cfg(unix)'].target], ['normal', 'cfg(unix)']) // the key is the `use` spelling, the name the manifest's (an optional dep's implicit feature name) t.assert.deepEqual([m.deps.get('pm_crate').key, m.deps.get('pm_crate').name, m.deps.get('pm_crate').kinds.get('normal').optional], ['pm_crate', 'pm-crate', true]) // `[patch.]` entries in every spelling: inline table, dotted key, sub-table. diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 001921f1..86879454 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -729,11 +729,7 @@ test('resolveModDecl lists cfg_attr variants under their predicate plus the defa { cfg: 'windows', file: 'src/sys/windows.rs', explicit: true }, { cfg: null, file: 'src/sys.rs', explicit: false }, ]) - // Given the build (`test`, `features`), a variant whose predicate can't hold is dropped here too. - const withMock = { ...decl, paths: [...decl.paths, { path: 'sys/mock.rs', cfg: 'test' }] } - t.assert.deepEqual(resolveModDecl(withMock, 'src/lib.rs', { knownSources: known, test: false }).map((x) => x.file), ['src/sys/unix.rs', 'src/sys/windows.rs', 'src/sys.rs']) - t.assert.deepEqual(resolveModDecl(withMock, 'src/lib.rs', { knownSources: known, test: true }).map((x) => x.file), ['src/sys/unix.rs', 'src/sys/windows.rs', 'src/sys/mock.rs', 'src/sys.rs']) - // Without one, the scanner is trusted to have dropped them for the build it scanned under. + // A variant whose predicate can't hold is the scanner's to drop, for the build it scans under. const src = '#[cfg_attr(unix, path = "sys/unix.rs")]\n#[cfg_attr(test, path = "sys/mock.rs")]\nmod sys;\n' t.assert.deepEqual(scanRustItems(src).mods[0].paths, [{ path: 'sys/unix.rs', cfg: 'unix' }]) // rustc applies the first variant whose predicate holds. One that holds in the scanned build ends From f4b66821fb4e317094c48515dc137f59247239cb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 1 Oct 2026 21:47:42 +0000 Subject: [PATCH 09/10] fix(bundle): Rust loader review 10: platform tables, build-script crates, proc-macro roots, out-of-root workspaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Crate resolution: - A dependency name is resolved from the tables the asking code links: per crate root the file is compiled in (a build script's modules and a file it shares with the lib through #[path] take the [build-dependencies]; tests, benches and examples the dev ones too), and only tables for the platforms the code is compiled for. Tables that may each apply (no target to decide them, or two roles) give cfg-keyed alternatives, each carried. Both on the replay and on cargo's resolver. - A crate's lib-name fallback resolves only dependencies that may have that lib name: `u8::MAX` no longer resolves every declared dependency and warns about versions. - A declared dependency the bundle lacks is reported when only an expression names it (`serde_json::to_string(…)`). - Macros of other crates get edges: `use dep::mac;` (an alias, a prelude glob too), `dep::mac!(…)` and `#[macro_use] extern crate`. Cargo: - The replay seeds a proc-macro root in the target context too, as cargo activates a member it is asked to build (fixes a regression from f0287e0). - A member bundled from its own directory reads its workspace root above the bundle root (never bundled) for what it inherits and the resolver, when that root's members take it. - [patch] tables apply to their own source only: a git URL's not to the crates.io dependency. - The vendor directory comes from the cargo config nearest the entries (a nested workspace's own). - `stasis bundle` says when the features come from the replay and why; a vendored copy the lockfile doesn't list no longer forces the replay. EXODUS_STASIS_DEBUG labels the mode and prints both contexts. - A vendored file whose bytes aren't those .cargo-checksum.json lists stops the bundle, as cargo refuses to build it. cfgs and resolution: - `r#true` / `r#false` are custom cfgs, not the literals, in every reader. - A child module under a custom cfg gives way to a glob of its name, as any doubtful candidate does. - An include in a nested macro_rules! is that macro's, resolved where it is invoked. - A refused or missing crate root is warned about once. - Dense cycles of cfg-gated globs no longer blow up: leaf keys are cached, absorbed alternatives dropped, and alternatives capped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- doc/file-formats.md | 77 +++++-- stasis/bin/stasis.js | 2 +- stasis/src/cmd/bundle.js | 26 ++- stasis/src/loaders/cargo.js | 446 +++++++++++++++++++++++++----------- stasis/src/loaders/rust.js | 234 ++++++++++++------- tests/bundle-cmd.test.js | 7 +- tests/rust-features.test.js | 267 ++++++++++++++++++++- tests/rust-loader.test.js | 57 ++++- 8 files changed, 858 insertions(+), 258 deletions(-) diff --git a/doc/file-formats.md b/doc/file-formats.md index ec0a1e43..60f824b1 100644 --- a/doc/file-formats.md +++ b/doc/file-formats.md @@ -515,12 +515,23 @@ stay as cfg-keyed candidates. Each root gets its own module tree, so a lib and its bin bundled together don't collide on `crate::`. A `tests/*.rs` or `benches/*.rs` entry is compiled the way `cargo test` does: `cfg(test)` holds, its `#[test]` fns and `#[cfg(test)]` modules are live, and -the package's dev-dependencies take part in the feature resolution. A `use`/`extern crate` naming a crate found in-tree pulls +the package's dev-dependencies take part in the feature resolution; an +`examples/*.rs` entry links the dev-dependencies too, without `cfg(test)`. A `use`/`extern crate` naming a crate found in-tree pulls that crate's root in: the package's own lib target (`use my_app::…` from `main.rs`), a Cargo `path` dependency (incl. `workspace = true` ones and `package = …` renames, honouring `[lib] path`), or a `cargo vendor`ed crate under -`vendor/` (or the `directory` of the source the project's cargo config replaces -crates.io with, through any chain of `replace-with`s, whatever its name). A +`vendor/` (or the `directory` of the source the cargo config replaces crates.io +with, through any chain of `replace-with`s, whatever its name -- the config +nearest the entries', so a nested workspace's own, relative to the directory +holding its `.cargo`). Which of a package's dependency tables a name means is the +asking code's, as cargo links them: a build script's, and its modules' -- a file +it shares with the lib through `#[path]` too -- the `[build-dependencies]`, a +test, bench or example's the `[dependencies]` and `[dev-dependencies]`, other +code's the `[dependencies]`; and of those only the tables of the platforms the +code is compiled for (`[target.'cfg(windows)'.dependencies] foo = "2"` is no +crate of a Linux build). Tables that may each apply -- a `cfg(windows)` and a +`cfg(unix)` one without a target, or a shared file's two -- are each followed, +and the edge is a cfg-keyed map of their crates. A `path` or `[patch]` dependency outside the bundle root is not in-tree, and never bound to a vendored crate of its name instead: the loader says so, and so it does of a `path` naming a directory with no `Cargo.toml` holding a `[package]`. @@ -528,8 +539,9 @@ Registry dependencies live in `~/.cargo/registry`, outside the bundle root, so they're never read: vendor them first (`cargo vendor`). Whenever a bundle references crates it lacks -- none in-tree, or one whose root the walk refused (a symlink leading out of the bundle root) -- `stasis bundle` lists -them, `vendor/` dir or not; with no `vendor/` dir it also suggests `cargo -vendor`. +them, `vendor/` dir or not, a dependency the package declares whether a `use` +or only an expression (`serde_json::to_string(…)`) names it; with no `vendor/` +dir it also suggests `cargo vendor`. A path edge (`crate::a::b::Item`, `super::x`, a relative `child::y`) resolves module by module along the crate's tree, inline `mod x { … }` blocks included @@ -579,11 +591,11 @@ set it: a name its build script prints as `cargo:rustc-cfg=…`, or that a rustflags `--cfg` sets (the cargo config's `build`/`target.<…>` `rustflags`, `RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, `CARGO_BUILD_RUSTFLAGS`), is neither presumed off nor on, and a build script that formats a name or uses `autocfg` -makes every custom cfg of its package so. A child module the build rules out -gives way to whatever else its module has of the name; one under a custom cfg -only to an import or item of its name in the same module (serde's docsrs-only -`mod de` beside the `pub use serde_core::de` of every other build) -- what a -glob brings in never shadows a module that is there. With `--cargo-target`, every file compiled +makes every custom cfg of its package so. A child module the build rules out, +or one under a custom cfg it presumably lacks, gives way to whatever else its +module has of the name, as such a candidate does to any other: serde's +docsrs-only `mod de` to the `pub use serde_core::de` of every other build, a +`#[cfg(loom)] mod imp` to the `imp` a `use other::*` brings in. With `--cargo-target`, every file compiled for the target is under the target's cfgs too, so a `windows` candidate is out for every file and a `target_os = "linux"` file takes the `any(android, linux)` branch over the `any(aix, solaris)` one (mio's `sys::Waker` is the @@ -710,10 +722,15 @@ test code reaches for. Two kinds of cfg are decided: manifests, a vendored copy whose checksum isn't the lockfile's, a feature asked of a package that hasn't it: each stops the build. A host the loader doesn't know (a target that isn't `host`) is resolved both ways: what the - host's target-specific tables turn on is on *maybe*. + host's target-specific tables turn on is on *maybe*. A vendored copy the + lockfile doesn't list needs no `.cargo-checksum.json`: cargo never reads it. - **The replay**, otherwise -- no lockfile, no target, a locked package not - vendored: the roots start from their `default` feature (a proc-macro crate - is resolved for the host, where cargo builds it), features imply features + vendored or without its checksums, an entry outside the workspace -- and + `stasis bundle` says so and why (`[stasis] Rust features from a replay of + the manifests, not cargo's resolver: no --cargo-target`): the roots start + from their `default` feature (a proc-macro crate is resolved for the host, + where cargo builds it, and for the target too, where cargo activates a + member it is asked to build), features imply features (`std = ["alloc", "dep:serde", "serde?/std"]`; `serde/derive` turns on the package's feature `serde` too, written or the one an optional dependency gets), enable optional dependencies and request dependency features, and @@ -737,8 +754,7 @@ test code reaches for. Two kinds of cfg are decided: `rand = "0.7"` under `[dependencies]` beside `rand = "0.8"` under `[build-dependencies]` (or a `[dev-dependencies] fake = { package = "rand", version = "0.8" }`) is two crates, and a file is followed into the one its - target uses -- the build script into the build-dependency, a test target - into the normal or dev one, anything else into the normal one. The vendored + code uses (see above). The vendored copy a dependency resolves to is from where the dependency says: a git dependency's (or one a git `[patch]` replaces) a git checkout's -- a copy whose `.cargo-checksum.json` has no package checksum -- any other's a @@ -752,15 +768,22 @@ test code reaches for. Two kinds of cfg are decided: takes that one. A `[patch]` -- in the root manifest, or in the cargo config of a directory from the package's up to the bundle root (nearest first, ahead of the manifest's, its `path` relative to the directory holding - `.cargo`) -- replaces a crate only where its version fits the dependent's - requirement, as cargo applies it; one that doesn't is reported and not used. + `.cargo`) -- replaces a crate of its table's source only (`[patch.crates-io]` + crates.io's, `[patch."https://github.com/…"]` that repository's), and only + where its version fits the dependent's requirement, as cargo applies it; one + that doesn't is reported and not used. When the locked version isn't vendored, or no vendored version fits, or several do and no lock says which, the loader warns and doesn't follow the dependency rather than guess: cargo would build none of them from what the bundle holds. A package the resolved build doesn't pull in has unknown features, and its - gated code is kept. + gated code is kept. A member bundled from its own directory, its workspace's + root above the bundle root, takes what it inherits (`edition.workspace = true`, + `dep = { workspace = true }`) and the resolver from that root, as cargo does: + the loader reads that manifest for those -- never bundling it, nor reading the + lockfile beside it -- when its `members` take the package and its `exclude` + doesn't. - With `--cargo-target=`, **target cfgs** are decided too: the loader asks `rustc --print cfg --target ` (`host`: the running @@ -799,7 +822,8 @@ test code reaches for. Two kinds of cfg are decided: --filter-platform`. `all(…)`/`any(…)`/`not(…)` compose, and `true` and `false` are what they say (a -raw `r#name` is the name); a predicate that reduces to true (`not(test)`, +raw `r#name` is the name; `r#true` and `r#false` are names, custom cfgs like any +other); a predicate that reduces to true (`not(test)`, an enabled feature) is as firm as no cfg, so a missing module behind it is fatal. A leaf's value is taken as written, spacing outside its quotes aside: `my = "a b"` and `my = "ab"` (or `"a b"`) are different values. @@ -827,7 +851,7 @@ everything cargo would ever compile for the workspace, tests included, and can enable features (and so bundle modules) that a plain `cargo build` of the entries' packages leaves off; the loader's own resolution describes that build. Set `EXODUS_STASIS_DEBUG=1` to have `stasis bundle` print the resolved features per -package, in either mode. +package and context (target, host), and which resolution they come from. `stasis bundle --cargo-manifests` also carries what describes each bundled package's build, the way `--package-json` carries npm manifests. For a package @@ -839,7 +863,9 @@ reads when run there (`.cargo/config` before `.cargo/config.toml`), those that lie inside the bundle root (a workspace's may sit above). For a vendored crate: its `Cargo.toml` and the `.cargo-checksum.json` cargo checks its files against -- the lock and config it was published with play no part in a build that -depends on it. All are `resource` files in their package's bucket, or the +depends on it. A vendored file the bundle reads is checked against that list +in any case: one changed since `cargo vendor` stops the bundle, as cargo refuses +to build it. All are `resource` files in their package's bucket, or the workspace bucket for the root-level ones. They are carried as written, as `--package-json` carries `package.json`: stasis doesn't edit them, so whatever they hold -- a `[registries]` token or `[http]` proxy in the cargo config, the credentials in a `git = …` URL or a @@ -895,7 +921,8 @@ resolves relative to each file that invokes the macro, as rustc expands it -- through the macros whose bodies invoke it, the outermost call (the edge is that file's, at the invocation); so are the bare macro calls a body makes made and resolved where it is invoked -- the body's own, even past a -`macro_rules!` it defines inside. `!` marks a `macro_rules!` +`macro_rules!` it defines inside, whose own includes are its, resolved where it +is invoked in turn. `!` marks a `macro_rules!` invoked by bare name and defined in another file the invocation can see -- in textual scope, as rustc has it: the files above it in the module tree, each up to the `mod` that mounts the next (a macro defined after `mod early;` is not @@ -912,7 +939,11 @@ declares at the invocation in each file hosting it (see above: serde's `#[macro_use] mod macros;`, so `de` sees `forward_to_deserialize_any!`), and, in the crate root's file, whose items they are, the `#[macro_export]`ed ones -- or by path, through the module's imports (`use crate::combinator::dispatch;`, a `use -super::*` from the root); a `macro_rules!` in a sibling file is out of scope +super::*` from the root), into another crate too (`use dep::mac;` or an alias +of it, a prelude's `pub use dep::mac;` behind a glob, the macro in the file +defining it); a crate root's `#[macro_use] extern crate dep;` brings dep's +exported macros to every module; `dep::mac!(…)` has an edge to that file +beside the crate's. A `macro_rules!` in a sibling file is out of scope otherwise. A file whose inner `#![cfg(…)]` can never hold is carried but compiled empty, so nothing in it is followed (a `#![cfg]` inside a macro invocation's body gates nothing). A `mod` diff --git a/stasis/bin/stasis.js b/stasis/bin/stasis.js index f509fd89..3c2c7f25 100755 --- a/stasis/bin/stasis.js +++ b/stasis/bin/stasis.js @@ -45,7 +45,7 @@ function usage(prefix = '') { (Rust: each crate's Cargo features are resolved from Cargo.toml/Cargo.lock like "cargo build" of the entries' packages, so #[cfg(feature = ...)] code that is off stays out -- by cargo's own resolver when there is a Cargo.lock (version 3 or 4; an older one stops the build), --cargo-target and every - locked package in-tree, by replaying the manifests otherwise; --cargo takes the resolution + locked package in-tree, by replaying the manifests otherwise (said, with why); --cargo takes the resolution from "cargo metadata" instead -- it runs cargo, so only on a project you trust; the --cargo-* feature flags are cargo's --features / --no-default-features / --all-features for those packages; --cargo-target asks rustc for the target's cfgs so #[cfg(unix)]-style code for other targets stays diff --git a/stasis/src/cmd/bundle.js b/stasis/src/cmd/bundle.js index d2ec65e2..5193111e 100644 --- a/stasis/src/cmd/bundle.js +++ b/stasis/src/cmd/bundle.js @@ -470,6 +470,7 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa if (sources.has(rel)) continue const buf = readFileWithinBase(baseDir, realBase, rel, boundaryOf(rel, cargoCtx)) if (buf === null) continue + cargoCtx.checkVendoredFile(rel, buf) if (!isUtf8(buf)) throw new Error(`Rust manifest is not valid UTF-8: ${rel}`) sources.set(rel, buf.toString('utf8')) formats.set(rel, 'resource') @@ -488,15 +489,24 @@ export async function buildRustBundle({ cwd = process.cwd(), entries, cargo = fa throw new Error(`Rust bundle has unresolved modules:\n${issues.map((s) => ` ${s}`).join('\n')}`) } - // EXODUS_STASIS_DEBUG=1: show the feature resolution the cfg decisions came from, per package, so - // the manifest replay and `cargo metadata` (which unifies dev/build deps like resolver 1) can be compared. + // Where the features the cfg decisions rest on come from: cargo's resolver, or -- something it + // takes missing -- a replay of the manifests, said so (a project without Cargo has neither). + const resolution = cargoCtx.resolution() + if (resolution.mode === 'replay' && normalized.some((e) => cargoCtx.packageInfo(e) !== null)) { + console.warn(`[stasis] Rust features from a replay of the manifests, not cargo's resolver: ${resolution.why}`) + } + // EXODUS_STASIS_DEBUG=1: show the feature resolution the cfg decisions came from, per package and + // context, so the replay, cargo's resolver and `cargo metadata` (which unifies dev/build deps + // like resolver 1) can be compared. if (process.env.EXODUS_STASIS_DEBUG === '1' || process.env.EXODUS_STASIS_DEBUG === 'true') { - const resolved = [...cargoCtx.resolvedFeatures()].toSorted(([a], [b]) => (a < b ? -1 : 1)) - const mode = cargo ? 'cargo metadata' : 'Cargo.toml + Cargo.lock' - console.warn(`[stasis] Rust features (${mode}), ${resolved.length} package${resolved.length === 1 ? '' : 's'}:`) - for (const [dir, set] of resolved) { - const pkg = cargoCtx.packageInfo(moduleFileKey(dir, 'Cargo.toml')) - console.warn(`[stasis] ${pkg?.name ?? '?'}@${pkg?.version ?? '?'} (${dir}): ${[...set].toSorted().join(', ') || '(none)'}`) + const mode = { cargo: "cargo's resolver", metadata: 'cargo metadata', replay: 'manifest replay' }[resolution.mode] + for (const context of ['target', 'host']) { + const resolved = [...cargoCtx.resolvedFeatures(context)].toSorted(([a], [b]) => (a < b ? -1 : 1)) + console.warn(`[stasis] Rust features (${mode}, ${context}), ${resolved.length} package${resolved.length === 1 ? '' : 's'}:`) + for (const [dir, set] of resolved) { + const pkg = cargoCtx.packageInfo(moduleFileKey(dir, 'Cargo.toml')) + console.warn(`[stasis] ${pkg?.name ?? '?'}@${pkg?.version ?? '?'} (${dir}): ${[...set].toSorted().join(', ') || '(none)'}`) + } } } diff --git a/stasis/src/loaders/cargo.js b/stasis/src/loaders/cargo.js index 680ec27a..799f22ef 100644 --- a/stasis/src/loaders/cargo.js +++ b/stasis/src/loaders/cargo.js @@ -6,6 +6,7 @@ // known; a replay of the manifests otherwise. import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { existsSync, readdirSync, readFileSync, realpathSync, statSync } from 'node:fs' import { homedir } from 'node:os' import { dirname, isAbsolute, join, parse, posix, relative, resolve, sep } from 'node:path' @@ -63,10 +64,19 @@ export function parseFeatureList(values) { // Whether a source file belongs to a test or bench target of the package at `pkgDir` (`tests/*.rs`, // `benches/*.rs` and what they declare): rustc compiles those with `cfg(test)`. export function isTestTargetPath(pkgDir, fileRel) { - const inside = pkgDir === '.' || pkgDir === '' ? fileRel : (fileRel.startsWith(`${pkgDir}/`) ? fileRel.slice(pkgDir.length + 1) : fileRel) - const first = inside.split('/')[0] + const first = targetDirOf(pkgDir, fileRel) return first === 'tests' || first === 'benches' } +// Whether a file belongs to a target cargo builds with the dev-dependencies: a test, bench or +// example (`examples/*.rs`), which unlike the other two isn't compiled with `cfg(test)`. +export function linksDevDepsPath(pkgDir, fileRel) { + return isTestTargetPath(pkgDir, fileRel) || targetDirOf(pkgDir, fileRel) === 'examples' +} +// The first directory of a file's path inside its package. +function targetDirOf(pkgDir, fileRel) { + const inside = pkgDir === '.' || pkgDir === '' ? fileRel : (fileRel.startsWith(`${pkgDir}/`) ? fileRel.slice(pkgDir.length + 1) : fileRel) + return inside.split('/')[0] +} // `read()`, with a TomlError or LockfileError it throws naming `file`: what @preventive/lockfile // refuses stops the build, saying which file. @@ -102,6 +112,9 @@ function manifestOf(text, doc, file, workspace) { version: d.version ?? null, path: d.source.type === 'path' ? d.source.path : null, source: d.source.type, + // where from: a git dependency's repository URL, another registry's name or index than + // crates.io's + origin: d.source.type === 'git' ? d.source.url : (d.source.type === 'registry' ? d.source.registry ?? d.source.index ?? null : null), package: d.package, renamed: d.package !== d.name, inherited: d.inherited, @@ -159,6 +172,10 @@ export function findCargoLock(baseDir) { // `my = "a b"` are two values. export const normalizeCfg = (pred) => pred.replaceAll(/"(?:[^"\\]|\\.)*"|\s+/gu, (m) => (m.startsWith('"') ? m : ' ')).trim() +// A cfg name as written, a raw identifier's `r#` dropped -- but `r#true` and `r#false`, names apart +// from the literals `true` and `false`. +export const cfgName = (written) => (/^r#(?:true|false)$/u.test(written) ? written : written.replace(/^r#/u, '')) + // A leaf: `unix`, `target_os = "linux"`, a raw identifier's `r#` dropped. const CFG_LEAF_RE = /^(?:r#)?([A-Za-z_]\w*)\s*(?:=\s*"([^"]*)")?$/u @@ -358,19 +375,35 @@ function cargoConfigIn(baseDir, dir) { } const cargoConfigOf = (baseDir) => cargoConfigIn(baseDir, '.') -// The directory `cargo vendor` filled, as the project's cargo config says: the `directory` of the -// source `[source.crates-io] replace-with` names, through any chain of replacements (`cargo -// vendor` suggests `vendored-sources`, but any name does), else of the one source some -// `replace-with` names; the default `vendor` when the config replaces nothing with a directory. -// A configured directory that doesn't exist is warned about (a stale `vendor/` beside it is not -// what cargo builds from). Throws a TomlError when the config isn't TOML. -function vendorDirOf(baseDir) { - const rel = cargoConfigOf(baseDir) - const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) - if (text === null) return VENDOR_DIR +// The directory `cargo vendor` filled, as the cargo config cargo reads when run in `from` (a +// project-relative dir: the build's, so a nested workspace's own config) says, the nearest first: +// the `directory` of the source `[source.crates-io] replace-with` names, through any chain of +// replacements (`cargo vendor` suggests `vendored-sources`, but any name does), else of the one +// source some `replace-with` names -- relative to the directory holding the config's `.cargo`; +// the default `vendor` (at the bundle root) when no config replaces anything with a directory. A +// configured directory that doesn't exist is warned about (a stale `vendor/` beside it is not what +// cargo builds from). Throws a TomlError when a config isn't TOML. +function vendorDirOf(baseDir, from = '.') { + for (let at = from; ; at = posix.dirname(at)) { + const rel = cargoConfigIn(baseDir, at) + const text = rel === null ? null : readFileOrNull(join(baseDir, rel)) + const dir = text === null ? null : configuredVendorDir(readToml(text, rel)) + if (dir !== null) { + const norm = normalizeRel(at, dir)?.replace(/\/+$/u, '') || null + if (norm === null) { + console.warn(`[loader.cargo] ${rel} names a vendored source directory outside the bundle root: ${dir}`) + return VENDOR_DIR + } + if (!existsSync(join(baseDir, norm))) console.warn(`[loader.cargo] ${rel} names a vendored source directory that doesn't exist: ${norm}`) + return norm + } + if (at === '.' || at === '') return VENDOR_DIR + } +} +// The directory a cargo config's sources replace crates.io's with (see vendorDirOf), or null. +function configuredVendorDir(config) { // source name -> { replaceWith, directory } const sources = new Map() - const config = readToml(text, rel) for (const [name, fields] of Object.entries(isTomlTable(config.source) ? config.source : {})) { if (!isTomlTable(fields)) continue const str = (v) => (typeof v === 'string' ? v : null) @@ -385,19 +418,10 @@ function vendorDirOf(baseDir) { } return null } - let dir = follow('crates-io') - if (dir === null) { - const replaced = new Set([...sources.values()].map((s) => (s.replaceWith === null ? null : follow(s.replaceWith))).filter((d) => d !== null)) - if (replaced.size === 1) [dir] = replaced - } - if (dir === null) return VENDOR_DIR - const norm = normalizeRel('.', dir)?.replace(/\/+$/u, '') || null - if (norm === null) { - console.warn(`[loader.cargo] ${rel} names a vendored source directory outside the bundle root: ${dir}`) - return VENDOR_DIR - } - if (!existsSync(join(baseDir, norm))) console.warn(`[loader.cargo] ${rel} names a vendored source directory that doesn't exist: ${norm}`) - return norm + const dir = follow('crates-io') + if (dir !== null) return dir + const replaced = new Set([...sources.values()].map((s) => (s.replaceWith === null ? null : follow(s.replaceWith))).filter((d) => d !== null)) + return replaced.size === 1 ? [...replaced][0] : null } // The `--cfg` names in rustflags: `--cfg name`, `--cfg=name`, `--cfg 'name="value"'`. @@ -406,8 +430,8 @@ function cfgFlags(flags) { for (let i = 0; i < flags.length; i++) { const flag = flags[i] const arg = flag === '--cfg' ? flags[++i] : (flag.startsWith('--cfg=') ? flag.slice('--cfg='.length) : null) - const name = typeof arg === 'string' ? /^[A-Za-z_]\w*/u.exec(arg)?.[0] : undefined - if (name !== undefined) out.push(name) + const name = typeof arg === 'string' ? /^(?:r#)?[A-Za-z_]\w*/u.exec(arg)?.[0] : undefined + if (name !== undefined) out.push(cfgName(name)) } return out } @@ -435,20 +459,42 @@ function rustflagsCfgsOf(baseDir, env = process.env) { // What a build script prints as `cargo:rustc-cfg=` (or `cargo::`), among the texts of its // files: the names it may set, and whether it may set one the loader can't read (a name it // formats, `autocfg`'s probes). -const RUSTC_CFG_RE = /cargo::?rustc-cfg=([A-Za-z_]\w*)?/gu +const RUSTC_CFG_RE = /cargo::?rustc-cfg=((?:r#)?[A-Za-z_]\w*)?/gu function cfgsPrinted(texts) { const names = new Set() let any = false for (const text of texts) { for (const m of text.matchAll(RUSTC_CFG_RE)) { if (m[1] === undefined) any = true - else names.add(m[1]) + else names.add(cfgName(m[1])) } if (/\bautocfg\b/u.test(text)) any = true } return { names, any } } +// Whether a JSON value is a plain object (not an array, not null). +const isPlainObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v) + +// A source URL as cargo compares them: no trailing `/` or `.git`, the scheme and host lowercased. +const canonicalUrl = (url) => url.trim().replace(/\/+$/u, '').replace(/\.git$/u, '').replace(/^([a-z][\w+.-]*:\/\/)([^/]*)/iu, (m) => m.toLowerCase()) +// crates.io's index, as a `[patch.]` may name it. +const CRATES_IO_INDEXES = new Set(['https://github.com/rust-lang/crates.io-index', 'sparse+https://index.crates.io'].map(canonicalUrl)) + +// A `members` pattern's segment as a regex (glob's syntax, as cargo takes it: `*`, `?`, `[…]`). +const globSegment = (seg) => new RegExp(`^${seg.replaceAll(/[.+^${}()|\\]/gu, '\\$&').replaceAll('[!', '[^').replaceAll('*', '[^/]*').replaceAll('?', '[^/]')}$`, 'u') +// Whether `members` pattern `pattern` takes the relative path `rel` (`**` for any depth). +function globMatches(pattern, rel) { + const segs = pattern.split('/').filter((x) => x !== '' && x !== '.') + const parts = rel.split('/').filter((x) => x !== '' && x !== '.') + const at = (i, j) => { + if (i === segs.length) return j === parts.length + if (segs[i] === '**') return at(i + 1, j) || (j < parts.length && at(i, j + 1)) + return j < parts.length && globSegment(segs[i]).test(parts[j]) && at(i + 1, j + 1) + } + return at(0, 0) +} + // --- An undecided host ---------------------------------------------------------------- // The stand-in platforms an undecided host is resolved against (hostUndecided): the target's cfgs @@ -513,31 +559,65 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return tables.get(dir) } // The workspace root a manifest inherits from: its `[package] workspace` path, else the nearest - // manifest above it with a [workspace] table, inside the bundle root -- null for a root, a - // vendored crate (a published manifest inherits nothing) or a package outside any workspace. + // manifest above it with a [workspace] table -- inside the bundle root, a project-relative dir; + // else above it (outerWorkspace) -- null for a root, a vendored crate (a published manifest + // inherits nothing) or a package outside any workspace. const rootOf = (dir, doc) => { if (isTomlTable(doc.workspace) || isVendoredDir(dir)) return null const named = (isTomlTable(doc.package) ? doc.package : null)?.workspace - if (typeof named === 'string') { - const rel = normalizeRel(dir, named) - return rel === null ? null : rel - } + if (typeof named === 'string') return normalizeRel(dir, named) ?? outerWorkspace(join(baseAbs, dir, named), null) for (let d = dir; d !== '.' && d !== '';) { d = posix.dirname(d) if (isTomlTable(tableOf(d)?.doc.workspace)) return d } - return null + for (let abs = dirname(baseAbs); ; abs = dirname(abs)) { + const found = outerWorkspace(abs, dir) + if (found !== undefined) return found + if (dirname(abs) === abs) return null + } + } + // The workspace root at absolute `abs`, above the bundle root, as cargo finds one for a package + // of the bundle (`member`, project-relative; null for a root its manifest names): the manifest + // there with a [workspace] whose `members` take the package and whose `exclude` doesn't. Read + // for what members inherit and the resolver it sets -- never bundled, nor its lockfile read -- + // as `{ dir: null, outside: abs, … }` (see readManifest); null for a [workspace] that doesn't + // take the package (cargo then builds it outside any), undefined for no [workspace] there. + const baseAbs = resolve(baseDir) + const outer = new Map() + const outerWorkspace = (abs, member) => { + if (!outer.has(abs)) { + const file = join(abs, 'Cargo.toml') + const text = readFileOrNull(file) + const label = toPosix(relative(baseAbs, file)) + const doc = text === null ? null : readToml(text, label) + outer.set(abs, isTomlTable(doc?.workspace) ? { dir: null, outside: abs, ...manifestOf(text, doc, label, null) } : undefined) + } + const ws = outer.get(abs) + if (ws === undefined || member === null) return ws + const rel = toPosix(relative(abs, join(baseAbs, member))) + const ws1 = ws.cargo.workspace + const under = (d) => rel === posix.normalize(d).replace(/\/+$/u, '') || rel.startsWith(`${posix.normalize(d).replace(/\/+$/u, '')}/`) + return ws1.members.some((pattern) => globMatches(pattern, rel)) && !ws1.exclude.some(under) ? ws : null } // A manifest as parseCargoManifest reads it, with each path dependency's directory, project- // relative, on its entry (`dir`; null outside the bundle root): an inherited one's path is the // workspace root's. + // `workspaceRoot` is the manifest of the workspace root it inherits from (rootOf), or null. const readManifest = (dir) => { if (!manifests.has(dir)) { const raw = tableOf(dir) const root = raw === null ? null : rootOf(dir, raw.doc) - const m = raw === null ? null : { dir, ...manifestOf(raw.text, raw.doc, raw.file, root === null ? null : readManifest(root)) } + const ws = root === null ? null : (typeof root === 'string' ? readManifest(root) : root) + const m = raw === null ? null : { dir, ...manifestOf(raw.text, raw.doc, raw.file, ws), workspaceRoot: ws } + // A path relative to the workspace root above the bundle root: project-relative, or null + // outside it. + const fromOutside = (path) => normalizeRel('.', toPosix(relative(baseAbs, join(ws.outside, path)))) for (const d of m?.deps.values() ?? []) { - for (const r of d.kinds.values()) r.dir = r.path === null ? null : normalizeRel(r.inherited ? (root ?? dir) : dir, r.path) + for (const r of d.kinds.values()) { + if (r.path === null) r.dir = null + else if (!r.inherited || ws === null) r.dir = normalizeRel(dir, r.path) + else r.dir = ws.outside === undefined ? normalizeRel(ws.dir, r.path) : fromOutside(r.path) + } } manifests.set(dir, m) } @@ -599,6 +679,9 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } // Whether `fileRel` belongs to a test or bench target of its package (compiled with `cfg(test)`). const isTestTarget = (fileRel) => isTestTargetPath(packageFor(fileRel)?.dir ?? '.', fileRel) + // Whether `fileRel` belongs to a target built with the dev-dependencies (tests, benches, + // examples). + const linksDevDeps = (fileRel) => linksDevDepsPath(packageFor(fileRel)?.dir ?? '.', fileRel) const version = (m) => m.package.version // --- dependency resolution @@ -609,7 +692,12 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // differs: `md-5` → `md5`). `git`: whether the copy is of a git repository -- its // `.cargo-checksum.json` has no package checksum, which a registry's always does -- or null // without that file. - const vendorDir = vendorDirOf(baseDir) + // The vendor directory of the build the entries are in: the cargo config nearest the first + // entry's says (a nested workspace's own), else the bundle root's -- an entry in the bundle + // root's vendor directory is a vendored crate, whose own config cargo never reads. + const rootVendorDir = vendorDirOf(baseDir) + const ownEntry = entries.find((e) => !e.startsWith(`${rootVendorDir}/`)) + const vendorDir = ownEntry === undefined ? rootVendorDir : vendorDirOf(baseDir, posix.dirname(ownEntry)) // Whether a package directory is a vendored crate's (a registry crate `cargo vendor` copied in). const isVendoredDir = (dir) => dir !== vendorDir && dir.startsWith(`${vendorDir}/`) let vendorIndex = null @@ -648,8 +736,9 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef throw new Error(`${file}: not JSON: ${err.message}`, { cause: err }) } } - // A registry's copy has a package checksum; a git checkout's has none. - checksums.set(dir, value === null ? null : { text, git: (value?.package ?? null) === null }) + // A registry's copy has a package checksum; a git checkout's has none. `files`: each file's + // sha256, as `cargo vendor` listed them, path relative to the package. + checksums.set(dir, value === null ? null : { text, git: (value?.package ?? null) === null, files: isPlainObject(value?.files) ? value.files : {} }) } return checksums.get(dir) } @@ -720,9 +809,10 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // the bundle root's), as `{ spec, dir, from }`, the first patchSources has -- `spec` as // @preventive/lockfile reads it, `dir` the patch's directory for a path (null outside the bundle // root), `from` the file saying so. Undefined when nothing patches it. - const patchFor = (root, crate) => { + const patchFor = (root, crate, r) => { for (const { patch, relTo, from } of patchSources(root)) { - for (const specs of Object.values(patch)) { + for (const [source, specs] of Object.entries(patch)) { + if (!patchesSource(source, r)) continue for (const [name, spec] of Object.entries(specs)) { if (normName(name) !== crate) continue return { spec, dir: spec.source.type === 'path' ? normalizeRel(relTo, spec.source.path) : null, from } @@ -731,6 +821,16 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } return undefined } + // Whether a `[patch.]` table patches what request `r` comes from: crates.io's + // (`crates-io`, or its index URL) for a crates.io dependency, the registry's name or index for + // another, the repository for a git dependency -- URLs compared as cargo canonicalizes them (no + // trailing `/` or `.git`, the scheme's and host's case aside). + const patchesSource = (source, r) => { + if (r.source === 'git') return r.origin !== null && canonicalUrl(source) === canonicalUrl(r.origin) + if (r.source !== 'registry') return false + if (r.origin === null) return source === 'crates-io' || CRATES_IO_INDEXES.has(canonicalUrl(source)) + return source === r.origin || canonicalUrl(source) === canonicalUrl(r.origin) + } // Where a lockfile's package comes from: `path`, `git` or `registry` (sparse or not). const sourceKind = (source) => (source === undefined ? 'path' : (source.startsWith('git+') ? 'git' : 'registry')) // The keys of the build's lockfile that are package `m`: its path package, or for a vendored copy @@ -782,7 +882,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const req = request.version const fits = (ver) => req === null || satisfies(ver, req) let kind = request.source - const patch = patchFor(workspaceFor(m.dir) ?? readManifest('.'), crate) + const patch = patchFor(workspaceFor(m.dir) ?? readManifest('.'), crate, request) if (patch?.spec.source.type === 'path') { if (patch.dir === null) { console.warn(`[loader.cargo] ${patch.from} patches ${crate} with a path outside the bundle root`) @@ -837,7 +937,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const resolverVersion = () => { if (resolverMemo !== null) return resolverMemo const pkg = entries.map(packageFor).find(Boolean) - const root = (pkg ? workspaceFor(pkg.dir) ?? pkg : null) ?? readManifest('.') + const root = (pkg ? workspaceFor(pkg.dir) ?? pkg.workspaceRoot ?? pkg : null) ?? readManifest('.') resolverMemo = root?.resolver ?? (Number(root?.package?.edition ?? 0) >= 2021 ? 2 : 1) return resolverMemo } @@ -851,12 +951,15 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // Whether a table applies in a build for `platform`: `yes`, `no`, or `maybe` when the platform's // cfgs aren't known or don't decide it. A build-dependency table is about the host its build // script runs on. Resolver 1 counts every platform's table, as cargo's v1 unification does. - const tableApplies = (r, platform) => { - if (r.target === null || resolverVersion() === 1) return 'yes' - const info = platformInfo(r.kind === 'build' ? 'host' : platform) + const tableApplies = (r, platform) => (resolverVersion() === 1 ? 'yes' : tableOn(r.target, r.kind === 'build' ? 'host' : platform)) + // Whether a table for platform `spec` (a `cfg(…)` or a triple; null or undefined for none) is + // one for `platform`: `yes`, `no`, or `maybe` when its cfgs aren't known or don't decide it. + const tableOn = (spec, platform) => { + if (spec === null || spec === undefined) return 'yes' + const info = platformInfo(platform) if (info === null) return 'maybe' - const cfg = /^cfg\((.*)\)$/u.exec(r.target) - const holds = cfg ? evalCfg(cfg[1], { target: info.cfgs }) : r.target === info.triple + const cfg = /^cfg\((.*)\)$/u.exec(spec) + const holds = cfg ? evalCfg(cfg[1], { target: info.cfgs }) : spec === info.triple return holds === true ? 'yes' : (holds === false ? 'no' : 'maybe') } // The context a dependency of a package built in context `c` is built in: the host's for a @@ -866,9 +969,10 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // A feature's list, from cargo's feature map (an optional dependency no `dep:` names has a // feature of its own name, as the manifest spells it: `proc-macro-crate`), or null for none. const featureImplications = (m, f) => m.features.get(f) ?? null - // The feature context a root package's own code is resolved in: the host's for a proc-macro - // crate, which cargo builds for the host; the target's for any other. - const rootCtx = (m) => (isProcMacroPkg(m) ? 'host' : 'target') + // The feature contexts a root package is resolved in: the target's, and for a proc-macro crate + // the host's too -- cargo builds it for the host, but a member selected for the build is + // activated in both, so what it asks of a package the target build shares reaches that build. + const rootCtxs = (m) => (isProcMacroPkg(m) ? ['target', 'host'] : ['target']) // A package in a context, as the resolutions key it: resolver 1's one context is `target`'s. const nodeKey = (c, dir) => `${featureCtx(c)}\0${dir}` @@ -913,7 +1017,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const rootDirs = new Set(roots.map((r) => r.dir)) // Dev-dependencies count for a root package under resolver 1, and for one whose entries include a // test/bench target (that build is `cargo test`'s, which links them). - const testRootDirs = new Set(entries.filter((e) => isTestTarget(e)).map((e) => packageFor(e)?.dir)) + const testRootDirs = new Set(entries.filter((e) => linksDevDeps(e)).map((e) => packageFor(e)?.dir)) const kindApplies = (m, r) => r.kind !== 'dev' || (rootDirs.has(m.dir) && (resolver === 1 || testRootDirs.has(m.dir))) const applies = (r, c) => { const a = tableApplies(r, c) @@ -953,11 +1057,13 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } for (const m of roots) { - inGraph(rootCtx(m), m) - if (allFeatures) { - for (const f of m.features.keys()) enable(rootCtx(m), m, f) - } else if (!noDefaultFeatures) { - enable(rootCtx(m), m, 'default') + for (const c of rootCtxs(m)) { + inGraph(c, m) + if (allFeatures) { + for (const f of m.features.keys()) enable(c, m, f) + } else if (!noDefaultFeatures) { + enable(c, m, 'default') + } } } // `--features a,b,pkg/c`: a bare name is a feature of every root package; `x/c` is a feature of @@ -971,7 +1077,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const targets = scoped ? roots.filter((m) => normName(m.package.name) === pkg) : roots const imp = scoped ? flag.slice(slash + 1) : flag // Whether a name is known doesn't depend on what is active yet, so the check happens once. - if (targets.filter((m) => applyImplication(rootCtx(m), m, imp)).length === 0 && includeMaybe) { + if (targets.filter((m) => rootCtxs(m).filter((c) => applyImplication(c, m, imp)).length > 0).length === 0 && includeMaybe) { console.warn(`[stasis] --cargo-features: '${flag}' names no feature of the entries' packages, nor a dependency of theirs`) } requested.push({ targets, imp }) @@ -981,7 +1087,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef changed = false // The requested features re-apply on every pass like the manifest's entries do: a weak // `dep?/x` asked for on the command line takes effect once `default` has activated `dep`. - for (const { targets, imp } of requested) for (const m of targets) applyImplication(rootCtx(m), m, imp) + for (const { targets, imp } of requested) for (const m of targets) for (const c of rootCtxs(m)) applyImplication(c, m, imp) // Map/Set iteration is live: packages and features added mid-pass are visited in this pass. for (const node of enabled.keys()) { const cut = node.indexOf('\0') @@ -1026,7 +1132,7 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (let i = 0; i < all.length; i++) all.push(...subdirs(all[i]).filter((d) => !posix.basename(d).startsWith('.'))) next.push(...all) } else if (/[*?[]/u.test(seg)) { - const re = new RegExp(`^${seg.replaceAll(/[.+^${}()|\\]/gu, '\\$&').replaceAll('[!', '[^').replaceAll('*', '[^/]*').replaceAll('?', '[^/]')}$`, 'u') + const re = globSegment(seg) next.push(...subdirs(dir).filter((d) => re.test(posix.basename(d)))) } else { const rel = normalizeRel(dir, seg) @@ -1080,27 +1186,34 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef // gives them -- or null when something it takes is missing, for the replay to decide: no // lockfile, no target, a package not vendored, a path outside the bundle root. let exactMemo + let exactWhy = null // why there is none, for the replay to say const exact = () => { if (exactMemo === undefined) exactMemo = exactUncached() return exactMemo } + const notExact = (why) => { + exactWhy = why + return null + } const exactUncached = () => { - if (metadata || targetInfo === null) return null + if (metadata) return null + if (targetInfo === null) return notExact('no --cargo-target') const lk = lockfile() const root = buildRoot() - if (lk === null || root === null) return null + if (lk === null) return notExact(`no ${posix.join(root?.dir ?? '.', 'Cargo.lock')}`) + if (root === null) return notExact('no package owns the entries') const memberDirs = membersOf(root) - if (memberDirs === null) return null + if (memberDirs === null) return notExact(`a members pattern of ${posix.join(root.dir, 'Cargo.toml')} leaves the bundle root`) // The path packages: the members, the path dependencies of each, the path [patch]es. const keyOf = new Map() // dir -> package key const dirOf = new Map() // package key -> dir const queue = [...memberDirs] - for (const { patch, relTo } of patchSources(root)) { + for (const { patch, relTo, from } of patchSources(root)) { for (const specs of Object.values(patch)) { for (const spec of Object.values(specs)) { if (spec.source.type !== 'path') continue const dir = normalizeRel(relTo, spec.source.path) - if (dir === null) return null + if (dir === null) return notExact(`${from} patches with a path outside the bundle root`) queue.push(dir) } } @@ -1108,32 +1221,35 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef for (const dir of queue) { if (keyOf.has(dir)) continue const m = readManifest(dir) - if (!m?.package) return null + if (!m?.package) return notExact(`${posix.join(dir, 'Cargo.toml')} isn't a package's`) const key = `${m.package.name} ${m.package.version}` keyOf.set(dir, key) dirOf.set(key, dir) for (const d of m.deps.values()) { for (const r of d.kinds.values()) { if (r.path === null) continue - if (r.dir === null) return null + if (r.dir === null) return notExact(`${m.package.name}'s path dependency ${r.path} lies outside the bundle root`) queue.push(r.dir) } } } - // The vendor directory as a directory source reads it, and every package of the lockfile there. + // The vendor directory as a directory source reads it, and every package of the lockfile there + // (a copy the lockfile doesn't list needs no checksums: cargo never reads it). const vendor = Object.create(null) - const held = new Set() + const held = new Map() // `name version` -> whether its copy has its checksums for (const sub of subdirs(vendorDir)) { const name = posix.basename(sub) const m = name.startsWith('.') ? null : readManifest(sub) if (!m?.package) continue const sums = checksumOf(sub) - if (sums === null) return null - vendor[name] = { manifest: tableOf(sub).text, checksum: sums.text } - held.add(`${m.package.name} ${m.package.version}`) + const id = `${m.package.name} ${m.package.version}` + held.set(id, held.get(id) === true || sums !== null) + if (sums !== null) vendor[name] = { manifest: tableOf(sub).text, checksum: sums.text } } for (const [key, p] of Object.entries(lk.lock.packages)) { - if (p.source === undefined ? !dirOf.has(key) : !held.has(`${p.name} ${p.version}`)) return null + if (p.source === undefined && !dirOf.has(key)) return notExact(`${lk.file} locks the path package ${p.name} ${p.version}, which isn't in-tree`) + if (p.source !== undefined && !held.has(`${p.name} ${p.version}`)) return notExact(`${p.name} ${p.version} is locked but not vendored`) + if (p.source !== undefined && !held.get(`${p.name} ${p.version}`)) return notExact(`the vendored copy of ${p.name} ${p.version} has no .cargo-checksum.json`) } const copies = readNamed(lk.file, () => readCargoVendor(lk.lock, vendor)) const manifestsByKey = Object.create(null) @@ -1146,12 +1262,13 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef } const memberKeys = memberDirs.map((dir) => keyOf.get(dir)) const entryKeys = [...new Set(entries.map(packageFor).filter(Boolean).map((m) => keyOf.get(m.dir)))] - if (entryKeys.length === 0 || entryKeys.some((k) => !memberKeys.includes(k))) return null + if (entryKeys.length === 0) return notExact('no package owns the entries') + if (entryKeys.some((k) => !memberKeys.includes(k))) return notExact('an entry\'s package isn\'t a member of the workspace') const configs = configsFrom(root.dir) const config = readNamed(configs[0]?.file ?? null, () => parseCargoConfig(configs.map((c) => c.text))) const graph = readNamed(lk.file, () => linkCargo(lk.lock, manifestsByKey, { workspace: root.cargo, members: memberKeys, config })) const rootFile = posix.join(root.dir, 'Cargo.toml') - const build = { packages: entryKeys, features, allFeatures, noDefaultFeatures, dev: entries.some((e) => isTestTarget(e)) } + const build = { packages: entryKeys, features, allFeatures, noDefaultFeatures, dev: entries.some((e) => linksDevDeps(e)) } const platform = (info, marks = []) => ({ name: info.triple, cfg: [...info.cfgs, ...marks] }) const featuresOf = (g, platforms) => readNamed(rootFile, () => resolveCargoFeatures(g, { ...build, ...platforms })) let sure @@ -1234,76 +1351,118 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef return texts } - // The in-tree crate root a name resolves to from package `m` (null: no owning package), leaving - // out `m`'s own lib (which depends on the asking file): the package its dependency of that name - // resolves to (path dep, `[patch]`, or the vendored version Cargo.lock says); a dependency whose - // lib is named that (`md-5` is used as `md5`); or -- for a file no manifest claims -- the one - // vendored crate of that lib or package name. A package's own name for a crate is its manifest's - // word: a declared dependency that doesn't resolve in-tree (a path outside the bundle root, a - // version no vendored copy has) never falls back to some vendored copy of that name, and a - // name it doesn't declare is no crate of its. Which of the dependency's tables names the crate - // is the asking file's `role`: a build script's are the `[build-dependencies]`, a test or bench - // target's the `[dependencies]` and `[dev-dependencies]`, other code's the `[dependencies]` (any - // table when none of those has it). Memoized per (package, name, role): every file of a package - // asks for the same few crates. + // The in-tree crate roots a name resolves to from package `m` (null: no owning package), leaving + // out `m`'s own lib (which depends on the asking file), as `[{ file, key }]`: the package its + // dependency of that name resolves to (path dep, `[patch]`, or the vendored version Cargo.lock + // says); a dependency whose lib is named that (`md-5` is used as `md5`); or -- for a file no + // manifest claims -- the one vendored crate of that lib or package name. A package's own name for + // a crate is its manifest's word: a declared dependency that doesn't resolve in-tree (a path + // outside the bundle root, a version no vendored copy has) never falls back to some vendored copy + // of that name, and a name it doesn't declare is no crate of its. Which of the dependency's + // tables count is the asking code's (`roles`, one per crate root the file is compiled in): a build + // script's are the `[build-dependencies]`, a test, bench or example target's the + // `[dependencies]` and `[dev-dependencies]`, other code's the `[dependencies]` (any table when + // none of those has it) -- each only where its platform may be one the code is compiled for + // (`platforms`, the file's units'; a build script's the host): a table the target's cfgs rule out + // is no dependency of a build for it, under either resolver. Tables that may each apply -- one + // per platform when no target decides them, or one per role -- give one candidate each, `key` + // naming which (`*` for the untargeted table, else its platform; `build-script ` before a build + // script's beside another role's). Memoized per (package, name, roles, platforms): every file of + // a package asks for the same few crates. const crateTargets = new Map() - const depCrate = (m, norm, role) => { - const memo = `${m?.dir ?? ''}\0${norm}\0${role}` - if (!crateTargets.has(memo)) crateTargets.set(memo, depCrateUncached(m, norm, role)) + const depCrate = (m, norm, roles, platforms) => { + const memo = `${m?.dir ?? ''}\0${norm}\0${roles.join()}\0${platforms.join()}` + if (!crateTargets.has(memo)) crateTargets.set(memo, depCrateUncached(m, norm, roles, platforms)) return crateTargets.get(memo) } const ROLE_KINDS = { build: ['build'], test: ['normal', 'dev'], normal: ['normal'] } - const requestsFor = (d, role) => { - const all = [...d.kinds.values()] - const own = all.filter((r) => ROLE_KINDS[role].includes(r.kind)) - return own.length > 0 ? own : all + // The role of the code of crate root `root` in package `m` (see depCrate). + const roleOf = (m, root) => (root === buildScript(m) ? 'build' : (linksDevDeps(root) ? 'test' : 'normal')) + // Of a dependency's tables (`{ kind, target, … }`, the manifest's or the linked graph's, in + // order), the packages the asking code links: for each role, the first table of each platform + // that may apply and that `find` finds a package for, as `{ t, key }` (see depCrate). `any`: a + // role with no table of its kinds takes any table instead (a declared name). + const linkedTables = (all, roles, platforms, find, any) => { + const out = [] + const seen = new Set() + for (const role of roles) { + const own = all.filter((r) => ROLE_KINDS[role].includes(r.kind)) + const plats = role === 'build' ? ['host'] : platforms + for (const r of own.length === 0 && any ? all : own) { + const on = r.target ?? null + const key = `${role === 'build' && roles.length > 1 ? 'build-script ' : ''}${on === null ? '*' : (/^cfg\((.*)\)$/su.exec(on)?.[1] ?? on)}` + if (seen.has(key) || !plats.some((p) => tableOn(on, r.kind === 'build' ? 'host' : p) !== 'no')) continue + const t = find(r) + if (t === null) continue + seen.add(key) + out.push({ t, key }) + } + } + return out + } + // The candidates as crate roots, one per file. + const asRoots = (found) => { + const out = [] + for (const { t, key } of found) { + const file = libPath(t) + if (file !== null && !out.some((c) => c.file === file)) out.push({ file, key }) + } + return out } // depCrate from the exact resolution: the package's dependency of that key, else the unrenamed - // one whose lib is named that, from the asking file's tables when it has one there, as the - // lockfile resolves it. - const exactCrate = (x, key, norm, role) => { + // one whose lib is named that, as the lockfile resolves it. + const exactCrate = (x, key, norm, roles, platforms) => { const deps = x.graph.packages[key].dependencies - const libOf = (d) => readManifest(x.dirOf.get(d.resolved)) - const pick = (list) => { - const own = list.filter((d) => ROLE_KINDS[role].includes(d.kind)) - const d = (own.length > 0 ? own : list).find((dep) => dep.resolved !== undefined) - return d === undefined ? null : libPath(libOf(d)) - } + const packageOf = (d) => (d.resolved === undefined ? null : readManifest(x.dirOf.get(d.resolved))) const declared = deps.filter((d) => normName(d.name) === norm) - if (declared.length > 0) return pick(declared) - return pick(deps.filter((d) => d.name === d.package && d.resolved !== undefined && libName(libOf(d)) === norm)) + if (declared.length > 0) return asRoots(linkedTables(declared, roles, platforms, packageOf, true)) + return asRoots(linkedTables(deps.filter((d) => d.name === d.package && d.resolved !== undefined && libName(packageOf(d)) === norm), roles, platforms, packageOf, false)) } - const depCrateUncached = (m, norm, role) => { + const depCrateUncached = (m, norm, roles, platforms) => { const x = m ? exact() : null const key = x?.keyOf.get(m.dir) - if (key !== undefined) return exactCrate(x, key, norm, role) + if (key !== undefined) return exactCrate(x, key, norm, roles, platforms) if (m) { const d = m.deps.get(norm) - if (d) { - for (const r of requestsFor(d, role)) { - const t = resolveDep(m, d, r) - if (t) return libPath(t) - } - return null + if (d) return asRoots(linkedTables([...d.kinds.values()], roles, platforms, (r) => resolveDep(m, d, r), true)) + // A dependency whose lib is named `norm`: only one that may be is resolved (a path + // dependency's manifest says, a vendored crate's must have that lib name), so a name that is + // no crate (`u8::MAX`) resolves -- and warns about -- nothing. + const mayBe = (r) => { + if (!r.path) return vendored().byLib.has(norm) + const t = r.dir === null ? null : readManifest(r.dir) + return Boolean(t?.package) && libName(t) === norm } for (const other of m.deps.values()) { - for (const r of requestsFor(other, role)) { - if (r.renamed) continue // a rename is used by its key, not its lib name + const named = [...other.kinds.values()].filter((r) => !r.renamed && mayBe(r)) // a rename is used by its key, not its lib name + const found = linkedTables(named, roles, platforms, (r) => { const t = resolveDep(m, other, r) - if (t && libName(t) === norm) return libPath(t) - } + return t && libName(t) === norm ? t : null + }, false) + if (found.length > 0) return asRoots(found) } - return null + return [] } const { byLib, byName } = vendored() const found = new Set() for (const index of [byLib, byName]) for (const c of index.get(norm) ?? []) found.add(c.dir) if (found.size > 1) { console.warn(`[loader.cargo] Several vendored crates are named ${norm} (${[...found].join(', ')}) and no manifest says which`) - return null + return [] } - for (const dir of found) return libPath(readManifest(dir)) - return null + return asRoots([...found].map((dir) => ({ t: readManifest(dir), key: '*' }))) + } + + const crateCandidates = (name, fromFile, { roots = null, units = null } = {}) => { + const norm = normName(name) + const m = packageFor(fromFile) + if (m && libName(m) === norm) { + const lib = libPath(m) + if (lib && lib !== fromFile) return [{ file: lib, key: '*' }] + } + const roles = m === null ? ['normal'] : [...new Set((roots === null || roots.size === 0 ? [fromFile] : [...roots]).map((r) => roleOf(m, r)))] + const platforms = units === null || units.size === 0 ? ['target'] : [...new Set([...units].map(unitPlatform))] + return depCrate(m, norm, roles, platforms) } // The files describing package `m`'s build, on disk (see buildFilesFor). @@ -1342,18 +1501,22 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef const m = packageFor(fileRel) return m !== null && libPath(m) === fileRel }, - // Resolve a crate name as used in source (`use name::…`, `extern crate name`) from `fromFile` - // to a crate root in-tree: the owning package's own lib (from another of its files), else what - // depCrate finds. Null for anything else (a registry dep that isn't vendored, std, a name that - // isn't a crate). - resolveCrate(name, fromFile) { - const norm = normName(name) - const m = packageFor(fromFile) - if (m && libName(m) === norm) { - const lib = libPath(m) - if (lib && lib !== fromFile) return lib - } - return depCrate(m, norm, m !== null && fromFile === buildScript(m) ? 'build' : (isTestTarget(fromFile) ? 'test' : 'normal')) + // The in-tree crate roots a crate name as used in source (`use name::…`, `extern crate name`) + // names from `fromFile`, as `[{ file, key }]`: the owning package's own lib (from another of + // its files), else what depCrate finds -- for the code of each crate root the file is compiled + // in (`roots`; the file's own role without them), on the platforms of its compile units + // (`units`; the target's without them). Several when tables that may each apply name different + // packages (see depCrate); none for anything else (a registry dep that isn't vendored, std, a + // name that isn't a crate). + crateCandidates, + // The first of crateCandidates, or null. + resolveCrate: (name, fromFile, opts) => crateCandidates(name, fromFile, opts)[0]?.file ?? null, + // Where the features come from: `{ mode, why }` -- `cargo` (cargo's resolver over the + // lockfile), `metadata` (`cargo metadata`), or `replay` (the manifests replayed) with `why` + // cargo's resolver couldn't run. + resolution() { + ensureResolved() + return metadata ? { mode: 'metadata', why: null } : (exact() === null ? { mode: 'replay', why: exactWhy } : { mode: 'cargo', why: null }) }, isTestTarget, // Whether `fromFile`'s package declares a dependency of that name (in any table, by the key @@ -1380,6 +1543,17 @@ export function createCargoContext(baseDir, { entries = [], features = [], noDef m.buildFiles ??= buildFiles(m) return m.buildFiles }, + // Throws when `fileRel` is a vendored package's file whose bytes (`buf`) aren't the ones its + // `.cargo-checksum.json` lists -- edited after `cargo vendor`: cargo refuses to build it, and + // so the bundle does. A file the list doesn't name, or a package without one, passes. + checkVendoredFile(fileRel, buf) { + const m = packageFor(fileRel) + if (m === null || !isVendoredDir(m.dir)) return + const listed = checksumOf(m.dir)?.files[fileRel.slice(m.dir.length + 1)] + if (typeof listed !== 'string') return + const actual = createHash('sha256').update(buf).digest('hex') + if (actual !== listed) throw new Error(`${fileRel} isn't the file ${m.dir}/.cargo-checksum.json lists (sha256 ${actual}, listed ${listed}): changed since \`cargo vendor\`, which cargo refuses to build`) + }, // Whether `fileRel` belongs to a vendored package (a registry crate `cargo vendor` copied in). isVendored(fileRel) { const m = packageFor(fileRel) diff --git a/stasis/src/loaders/rust.js b/stasis/src/loaders/rust.js index 03380518..5495270b 100644 --- a/stasis/src/loaders/rust.js +++ b/stasis/src/loaders/rust.js @@ -14,7 +14,7 @@ import { readFile } from 'node:fs/promises' import { dirname, extname, isAbsolute, join, posix, relative, resolve } from 'node:path' import { assertRealPathWithinBase, toPosix } from '@exodus/stasis-core/util' -import { TARGET_CFG_KEYS, TARGET_UNIT, VENDOR_DIR, createCargoContext, evalCfg, evalCfgKey, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' +import { TARGET_CFG_KEYS, TARGET_UNIT, VENDOR_DIR, cfgName, createCargoContext, evalCfg, evalCfgKey, isFile, isTestTargetPath, normName, normalizeCfg, normalizeRel } from './cargo.js' import { matchClose, splitTopLevel } from './toml.js' // Leads of the expression-position paths anchored on the module tree rather than on a name. @@ -522,7 +522,15 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // `crate_root! { … mod de; … }`) appears where the macro is invoked, not where it is defined, so // such a `mod` takes the offset of the macro's first bare invocation in this file (`calls`). let templateName = null - let templateMacro = null // the top-level `macro_rules!` whose body the scan is in: what its body calls and includes is its own, nested definitions' too + let templateMacro = null // the top-level `macro_rules!` whose body the scan is in: what its body calls is its own, nested definitions' too + // The `macro_rules!` bodies the scan is in, outermost first, `{ macro, end }`: an include in a + // nested one's is that one's, resolved where it is invoked (which may be another directory than + // where the outer one is). + const defs = [] + const innermostDef = (at) => { + while (defs.length > 0 && defs.at(-1).end < at) defs.pop() + return defs.at(-1)?.macro ?? templateMacro + } // Whether the token at `at` sits in a `macro_rules!` body: a template, expanded where the // macro is invoked. @@ -720,12 +728,14 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, const pathQualified = masked[i - 1] === ':' && masked[i - 2] === ':' if (masked[bang] === '!') { let open = skipWs(bang + 1) + let definition = null // the `macro_rules!` this word defines if (word === 'macro_rules') { const name = readWord(open) if (name !== null) { open = skipWs(open + name.length) const gate = GATE_MACRO_RE.test(name) && masked[open] === '{' ? gatePredicate(code, masked, open, matchClose(masked, open)) : undefined macros.push({ name, exported: pending.some((a) => a.macroExport === true), includes: [], calls: new Set(), offset: i, at: i, template: inTemplate(i) ? templateName : null, gate }) + definition = macros.at(-1) if (i >= macroUntil) { templateName = name templateMacro = macros.at(-1) @@ -734,6 +744,10 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, } if (masked[open] === '{' || masked[open] === '(' || masked[open] === '[') { const close = matchClose(masked, open) + if (definition !== null) { + innermostDef(i) + defs.push({ macro: definition, end: close }) + } if (templates.has(word) && !localMacros.has(word)) { // serde_derive's `quote! { use #path as _serde; … _serde::__private::Ok(…) }`, bare or as // `quote::quote! { … }`: skipped whole. (A crate's own `macro_rules! quote` is no template.) @@ -768,7 +782,7 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, // (`concat!(env!("OUT_DIR"), …)`) is nobody's to follow; any other unreadable argument // is counted, for a warning. if (arg === null) unfollowed += /\bOUT_DIR\b/u.test(text) ? 0 : 1 - else if (inTemplate(i) && templateMacro !== null) templateMacro.includes.push({ kind: word, ...arg }) + else if (inTemplate(i) && templateMacro !== null) innermostDef(i).includes.push({ kind: word, ...arg }) else includes.push({ kind: word, ...arg, conditional }) } } @@ -887,10 +901,11 @@ export function scanRustItems(content, { features = null, maybeFeatures = null, if (word === 'extern') { EXTERN_CRATE_RE.lastIndex = i const m = EXTERN_CRATE_RE.exec(masked) + const macroUse = pending.some((a) => a.macroUse === true) pending = [] if (m) { const ip = inlinePath() - externCrates.push({ name: m[1], alias: m[2] ?? null, inlinePath: ip }) + externCrates.push({ name: m[1], alias: m[2] ?? null, inlinePath: ip, macroUse }) if (m[2] !== undefined) bindings.add(m[2]) // `extern crate self as x;` names this crate's root; any other, a crate by its own name. const selfCrate = m[1] === 'self' @@ -1128,7 +1143,8 @@ export function crateRoots(sources, explicit = [], cargo = null) { // file → its declarations' `{ cfg, macro }`; a file declared under several is compiled under any // one of them: an `any` leaf, see cfgLeaves and gateLeaves; `gatesOf`: file → the gate macros // of its package, see gateLeaves) and, for a `mod` with several files, a `variant` leaf naming -// which (two variants of one `mod` never both apply) -- and `parent`, the file that mounted it. +// which (two variants of one `mod` never both apply) -- `parent`, the file that mounted it, and +// `roots`, every crate root whose tree reaches it (a file a build script and its lib share). export function buildModuleTrees(sources, resolutions, roots, inlineModules = null, mountCfgs = null, gatesOf = null) { const trees = new Map() const files = new Map() @@ -1147,7 +1163,8 @@ export function buildModuleTrees(sources, resolutions, roots, inlineModules = nu const queue = [[root, 'crate', 0, [], null]] for (let qi = 0; qi < queue.length; qi++) { const [path, modulePath, depth, leaves, parent] = queue[qi] - if (!files.has(path)) files.set(path, { root, modulePath, leaves, parent }) + if (!files.has(path)) files.set(path, { root, modulePath, leaves, parent, roots: new Set() }) + files.get(path).roots.add(root) if (seen.has(path) || depth >= MAX_MODULE_DEPTH) continue seen.add(path) // Module files first, then the inline modules that hold nested declarations, then the other @@ -1269,8 +1286,8 @@ function cfgLeaves(pred) { const m = /^(all|any|not)\s*\(([\s\S]*)\)$/u.exec(p) if (!m) { if (p === 'true') return [] // holds: decides nothing (`false` is a leaf the build rules out, see leafFalse) - const kv = /^(?:r#)?([\w-]+)\s*(?:=\s*"([^"]*)")?$/u.exec(p) - return kv ? [{ key: kv[1], value: kv[2] ?? null, neg: false }] : [] + const kv = /^((?:r#)?[\w-]+)\s*(?:=\s*"([^"]*)")?$/u.exec(p) + return kv ? [{ key: cfgName(kv[1]), value: kv[2] ?? null, neg: false }] : [] } const parts = splitTopLevel(m[2]).filter((part) => part.trim() !== '').map(cfgLeaves) if (m[1] === 'all') return parts.flat() @@ -1278,14 +1295,23 @@ function cfgLeaves(pred) { return parts.length === 1 ? negated(parts[0]) : [] } // The leaves holding when any of `alts` (conjunctions) does: the one alternative when there is -// one (the same twice counts once), an `any` leaf of them, nothing when one decides nothing. +// one (the same twice counts once, and one holding whenever another does adds nothing: `a ∨ (a ∧ +// b)` is `a`), an `any` leaf of them, nothing when one decides nothing -- or when there are more +// than MAX_ALTERNATIVES, which a module reached along every path of a dense glob cycle may be: +// that many decide too little to be worth telling apart. +const MAX_ALTERNATIVES = 16 const anyOf = (alts) => { const uniq = new Map() for (const alt of alts) { if (alt.length === 0) return [] uniq.set(alt.map(leafKey).toSorted().join('&'), alt) } - const list = [...uniq.values()] + let list = [...uniq.values()] + if (list.length > 1) { + const keys = list.map((alt) => new Set(alt.map(leafKey))) + list = list.filter((_, i) => !keys.some((other, j) => j !== i && other.size < keys[i].size && other.isSubsetOf(keys[i]))) + } + if (list.length > MAX_ALTERNATIVES) return [] if (list.length <= 1) return list[0] ?? [] return [{ key: 'any', value: null, neg: false, alts: list }] } @@ -1395,8 +1421,17 @@ function prepared(leaves) { return prep } -// A stable text for one leaf; an `any` leaf's names its alternatives. -const leafKey = (l) => (l.alts === undefined ? `${l.neg ? '!' : ''}${l.key}${l.site === undefined ? '' : `@${l.site}`}${l.value === null ? '' : `=${l.value}`}` : `any(${l.alts.map((alt) => alt.map(leafKey).toSorted().join('&')).toSorted().join(',')})`) +// A stable text for one leaf; an `any` leaf's names its alternatives. Kept per leaf: an `any` +// leaf's is asked for again and again as sets of them are interned and compared. +const leafKeys = new WeakMap() +const leafKey = (l) => { + let key = leafKeys.get(l) + if (key === undefined) { + key = l.alts === undefined ? `${l.neg ? '!' : ''}${l.key}${l.site === undefined ? '' : `@${l.site}`}${l.value === null ? '' : `=${l.value}`}` : `any(${l.alts.map((alt) => alt.map(leafKey).toSorted().join('&')).toSorted().join(',')})` + leafKeys.set(l, key) + } + return key +} // Leaf sets are interned per tree (`sets`: key → `{ leaves, key }`), one object per distinct set: // the union of the cfgs along a chain of globs stays small however long the chain, and the @@ -1610,14 +1645,6 @@ function provided(root, at, name, from, ctx, asker, ns = null) { return pick(providedAll(root, at, name, commonDepth(from, at), ctx), root, ctx, asker, ns) // scopes are ancestors of `at`: `from` is inside those this deep or shallower } -// What module `at` binds `name` to itself, as seen from `from` -- a `use` of the name or an item -// it defines (see providedAll), not what its globs bring in -- or null. -function bound(root, at, name, from, ctx, asker, ns = null) { - const list = [...importsOf(root, at, name, commonDepth(from, at), ctx)] - for (const d of ctx.defined.get(root)?.get(at)?.get(name) ?? NONE) list.push({ answer: { kind: 'item', file: d.file }, leaves: d.leaves, ns: d.ns }) - return pick({ get: (i) => list[i] }, root, ctx, asker, ns) -} - // The answer of the first of `candidates` (see providedAll: in written order) whose cfgs the // asker's entail (entailed) -- else the first compatible with them under no custom cfg, else the // first compatible at all (doubtful) -- in the namespace `ns` asks for: for a path's lead or @@ -1667,6 +1694,8 @@ function pick(candidates, root, ctx, asker, ns = null) { const depth = ctx.walking++ im.following = depth r = followImport(im, root, ctx, asker, ns) + // The name a macro an import leads to is defined by: the import's (`use dep::mac as m;`). + if (ns === 'macro' && r?.kind === 'item' && r.macroName === undefined && im.segments.length > 0) r = { ...r, macroName: im.segments.at(-1) } im.following = undefined ctx.walking-- ctx.hits.delete(depth) @@ -2033,7 +2062,11 @@ function followImport(im, root, ctx, asker, ns = null) { const head = im.segments[0] if (NON_CRATE_LEADS.has(head)) return { kind: 'item', file: im.file } const target = ctx.resolveCrate(head, im.file) - return target ? { kind: 'crate', name: head, file: target } : null + if (!target) return null + // A macro of another crate (`use dep::mac;`), followed as one: the file defining it, where + // `#[macro_export]` puts it at that crate's root. + const macro = ns === 'macro' && im.segments.length === 2 ? ctx.macros.get(target)?.get(im.segments[1]) : undefined + return macro === undefined ? { kind: 'crate', name: head, file: target } : { kind: 'item', file: macro, macroName: im.segments[1] } } // An item answer for what definedIn found. @@ -2112,12 +2145,13 @@ function walkPath(segments, root, from, ctx, { ns = null, file, asker } = {}) { let p if (isModule(child, last)) { // A child module the asker's build doesn't compile (under a cfg off in it) gives way to what - // else the module has of the name, if anything; one it presumably doesn't (under a custom - // cfg, doubtful: serde's docsrs-only `mod de` beside the `pub use serde_core::de` of every - // other build) only to what the module binds the name to itself, which can stand beside the - // module only under other cfgs. What a glob brings in never shadows a module that is there. + // else the module has of the name, if anything; so does one it presumably doesn't (under a + // custom cfg, doubtful: serde's docsrs-only `mod de` beside the `pub use serde_core::de` of + // every other build; a `#[cfg(loom)] mod imp` beside the `use other::*` that brings `imp` + // in), as a doubtful candidate gives way to any other: where the module isn't there, a + // glob's name is the module's name. const set = ctx.files.get(tree.get(child))?.leaves - p = set === undefined ? null : (deadFor(asker, set) ? lookup(at, name, want) : (doubtful(asker, set) ? bound(root, at, name, from, ctx, asker, want) : null)) + p = set === undefined || !(deadFor(asker, set) || doubtful(asker, set)) ? null : lookup(at, name, want) if (p === null || p === VALUE_ONLY) { cur.push(name) i++ @@ -2246,7 +2280,8 @@ function resolvePathRef(ref, file, bindings, ctx) { return r === VALUE_ONLY ? null : r } const target = ctx.resolveCrate(name, file) - if (target) return { kind: 'crate', name, file: target } + // `dep::mac!(…)`: the crate, and the file defining its exported macro. + if (target) return { kind: 'crate', name, file: target, alternatives: ctx.crateAlternatives?.(name, file), macro: ref.macroCall && segments.length === 2 ? ctx.macros.get(target)?.get(segments[1]) : undefined } const macro = segments.length === 1 && from === 'crate' ? ctx.macros.get(here.root)?.get(lead) : undefined if (macro !== undefined) return { kind: 'item', file: macro } // A glob into a crate that isn't in-tree (`use serde::*;`) may bring the lead in -- unless the @@ -2301,11 +2336,12 @@ const templatesLess = (own) => { // the target's alone. const DEFAULT_UNITS = new Set([TARGET_UNIT]) const unitsOf = (units, rel) => units?.get(rel) ?? DEFAULT_UNITS -// Record `file` as compiled as each of `more` too, in `units`; whether that is news. -function addUnits(units, file, more) { - const have = units.get(file) +// Add each of `more` to `map`'s set for `file` (the units it is compiled as, the crates it is +// compiled in); whether that is news. +function addTo(map, file, more) { + const have = map.get(file) if (have === undefined) { - units.set(file, new Set(more)) + map.set(file, new Set(more)) return true } if (more.isSubsetOf(have)) return false @@ -2738,14 +2774,30 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul inheritedMemo.set(file, out) return out } - // Crate names resolve per package (a path dep's, a rename), so per file; the same (name, file) - // comes up once per reference to the crate, and every glob import's source resolution asks. + // Crate names resolve per package (a path dep's, a rename) and per the code asking (a build + // script's tables, the platforms it is compiled for), so per file; the same (name, file) comes up + // once per reference to the crate, and every glob import's source resolution asks. `{ file, + // alternatives }`: the crate root, and -- when tables that may each apply name different + // packages (cargo.js crateCandidates) -- each of them by the table's platform. const crates = new Map() - const resolveCrate = (name, from) => { + const crateOf = (name, from) => { const key = `${name}\0${from}` - if (!crates.has(key)) crates.set(key, ctx?.resolveCrate(name, from) ?? resolveVendoredCrate(name, { knownSources: sources })) + if (!crates.has(key)) { + const found = ctx ? ctx.crateCandidates(name, from, { roots: files.get(from)?.roots ?? null, units: unitsOf(units, from) }) : [] + const file = found[0]?.file ?? resolveVendoredCrate(name, { knownSources: sources }) + crates.set(key, { file, alternatives: found.length > 1 ? new Map(found.map((c) => [cfgKey(c.key), c.file])) : undefined }) + } return crates.get(key) } + const resolveCrate = (name, from) => crateOf(name, from).file + const crateAlternatives = (name, from) => crateOf(name, from).alternatives + // The crates a crate root's `#[macro_use] extern crate`s bring the exported macros of into every + // module's scope, as their roots. + const macroUse = new Map() + const macroUseCrates = (root) => { + if (!macroUse.has(root)) macroUse.set(root, (scanned.get(root)?.externCrates ?? []).filter((e) => e.macroUse && !NON_CRATE_LEADS.has(e.name)).map((e) => resolveCrate(e.name, root)).filter(Boolean)) + return macroUse.get(root) + } // One context for every path of every file: `provided` memoizes across them (see walkPath). // Per crate root, name → the modules with anything of the name (a child module, an item, an // import binding it; the root for an exported macro): what hasAll may answer for. @@ -2755,7 +2807,7 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul for (const [root, byModule] of imports) for (const [module, of] of byModule) for (const name of of.named.keys()) note(having, root, name, module) for (const [root, byName] of macros) for (const name of byName.keys()) note(having, root, name, 'crate') const declaresCrate = (name, from) => ctx?.declaresCrate?.(name, from) === true - const pathCtx = { trees, children, files, resolveCrate, declaresCrate, imports, defined, modScope, macros, fileMacros, externPrelude, leafSets, having, provided: new Map(), closures: new Map(), opaque: new Map(), opaqueGlobs: new Map(), walking: 0, hits: new Set() } + const pathCtx = { trees, children, files, resolveCrate, crateAlternatives, declaresCrate, imports, defined, modScope, macros, fileMacros, externPrelude, leafSets, having, provided: new Map(), closures: new Map(), opaque: new Map(), opaqueGlobs: new Map(), walking: 0, hits: new Set() } // Every path of every file, on top of the `mod` / include edges (`resolutions`): once, and once // more when the first pass left glob closures short of an import cycle (stabilizeClosures). const resolveAll = () => { @@ -2789,13 +2841,15 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // A crate resolved to a root the walk didn't load (its name was bound in the walking file as // a value, see resolvePathRef), or loaded as fewer units than this file names it as: the // caller may load it and build again. - const addCrate = (name, target) => { - if (target && ctx) { - const want = new Set([...unitsOf(units, path)].map((u) => ctx.unitOfCrate(target, u))) - const have = sources.has(target) ? unitsOf(units, target) : new Set() - if (!want.isSubsetOf(have)) for (const u of want) (wanted.get(target) ?? wanted.set(target, new Set()).get(target)).add(u) - } else if (target && !sources.has(target)) wanted.set(target, DEFAULT_UNITS) - add(`use ${name}`, target) + const addCrate = (name, target, alternatives) => { + for (const file of alternatives === undefined ? [target] : alternatives.values()) { + if (file && ctx) { + const want = new Set([...unitsOf(units, path)].map((u) => ctx.unitOfCrate(file, u))) + const have = sources.has(file) ? unitsOf(units, file) : new Set() + if (!want.isSubsetOf(have)) for (const u of want) (wanted.get(file) ?? wanted.set(file, new Set()).get(file)).add(u) + } else if (file && !sources.has(file)) wanted.set(file, DEFAULT_UNITS) + } + add(`use ${name}`, target, alternatives) } // A crate name nothing in-tree satisfied -- sysroot crates aside, and an uppercase lead is an // `Enum::Variant` import, not a crate. @@ -2807,10 +2861,15 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul for (const ref of items.refs) { const r = resolvePathRef(ref, path, items.bindings, pathCtx) if (r === null) continue - if (r.kind === 'crate') addCrate(r.name, r.file) + if (r.kind === 'crate') { + addCrate(r.name, r.file, r.alternatives) + if (r.macro !== undefined) add(`${ref.spec}!`, r.macro) + } else if (r.kind === 'unresolved') { - // Only a `use` reliably says its lead names a crate (see scanRustItems' `fromUse`). - if (ref.fromUse) noteUnresolved(r.name) + // Only a `use` reliably says its lead names a crate (see scanRustItems' `fromUse`) -- or + // the package declaring a dependency of that name (`serde_json::to_string(…)`), which it + // then is. + if (ref.fromUse || declaresCrate(r.name, path)) noteUnresolved(r.name) continue } else add(ref.macroCall ? `${ref.spec}!` : ref.spec, r.file, r.alternatives) // `crate::helper!()` beside `crate::helper()`: two edges // The module an import was followed from, when the path's own segments name it (`super`, @@ -2823,8 +2882,8 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul } for (const { name } of items.externCrates) { if (NON_CRATE_LEADS.has(name)) continue // `extern crate self as x;` / `extern crate alloc;` - const target = resolveCrate(name, path) - if (target) addCrate(name, target) + const { file: target, alternatives } = crateOf(name, path) + if (target) addCrate(name, target, alternatives) else noteUnresolved(name) } // A macro invoked by bare name and defined by `macro_rules!` in textual scope (see mounts): @@ -2837,12 +2896,21 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul if (here !== undefined) { const above = inherited(path) const exported = here.modulePath === 'crate' ? macros.get(here.root) : undefined - // The file defining the macro `name` a bare call at `offset` names. + // The macro a bare call of `name` at `offset` names, `{ file, name }` (the file defining it, + // by the name it defines it under): one in textual scope; else one an import brings in -- + // a file that defines a macro of that name, not a `fn write` beside `write!` -- or a + // `#[macro_use] extern crate` of the crate root does. const macroFileAt = (name, offset) => { const inScope = macroAt(path, name, offset, offset, above) ?? exported?.get(name) - if (inScope !== undefined) return inScope - const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { ns: 'macro', file: path, asker: here.asker })?.file - return byPath !== undefined && fileMacros.get(byPath)?.has(name) === true ? byPath : undefined // a file that defines a macro of that name, not a `fn write` beside `write!` + if (inScope !== undefined) return { file: inScope, name } + const byPath = walkPath([name], here.root, here.modulePath, pathCtx, { ns: 'macro', file: path, asker: here.asker }) + const as = byPath?.macroName ?? name + if (byPath?.file !== undefined && fileMacros.get(byPath.file)?.has(as) === true) return { file: byPath.file, name: as } + for (const crate of macroUseCrates(here.root)) { + const file = macros.get(crate)?.get(name) + if (file !== undefined) return { file, name } + } + return undefined } for (const [name, offsets] of items.calls) { // Scope is decided at each call: a `macro_rules!` later in the file doesn't shadow what an @@ -2855,10 +2923,10 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul const expand = (called) => { if (seen.has(called)) return seen.add(called) - const macroFile = macroFileAt(called, offset) - if (macroFile === undefined) return - found.add(`${called}!\0${macroFile}`) - for (const m of scanned.get(macroFile)?.macros ?? []) if (m.name === called) for (const inner of m.calls) expand(inner) + const macro = macroFileAt(called, offset) + if (macro === undefined) return + found.add(`${called}!\0${macro.file}`) + for (const m of scanned.get(macro.file)?.macros ?? []) if (m.name === macro.name) for (const inner of m.calls) expand(inner) } expand(name) } @@ -2893,8 +2961,11 @@ export function buildRustTree(sources, { roots = [], baseDir = null, cargo = nul // units it is compiled as (cargo.js unitKey), filled in as the walk goes -- an entry the map // doesn't name is the target's; what a file declares or includes is compiled as that file is, // and a crate it names as the Cargo context says (unitOfCrate: a proc-macro crate, or a crate -// named from code built for the host, is built for the host). -export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, formats = null, sources = new Map(), units = new Map() } = {}) { +// named from code built for the host, is built for the host). `crateRoots`: file → the crate roots +// it is compiled in, filled in the same way (an entry is its own; a build script's modules are the +// script's, and name the crates of its `[build-dependencies]`). `unloaded` gets the files the walk +// refused (a link out of their package) or found gone, each warned about. +export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, formats = null, sources = new Map(), units = new Map(), crateRoots: rootsOf = new Map(), unloaded = new Set() } = {}) { const realBase = realpathSync(baseDir) const roots = new Set(entries) const pathLoaded = new Set() // files a `#[path]` named: their submodules sit beside them @@ -2925,15 +2996,16 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, return cachedScan(sources, relPath, content, buildOf(relPath, ctx, units), own === undefined ? undefined : templatesLess(own)) } withEntryUnits(units, entries, ctx) - // Files already walked that are now compiled as more units: gone through again (their build - // changed, and so may what they name, and as what). + for (const e of entries) addTo(rootsOf, e, new Set([e])) + // Files already walked that are now compiled as more units, or in more crates: gone through + // again (their build changed, and so may what they name, and as what). const revisit = new Set(entries.filter((e) => sources.has(e) && !formats?.has(e))) // What a Rust file names, queued for the next wave: compiled as the naming file is -- as its - // units, a crate as unitOfCrate says (`as`) -- and, for a vendored crate's, within its package - // (`within`). + // units, a crate as unitOfCrate says (`as`); in its crates, or the crate it is the root of + // (`crate`) -- and, for a vendored crate's, within its package (`within`). let next = [] - const queue = (from, file, { rust, within = boundaryOf(from, ctx), as = unitsOf(units, from) }) => { + const queue = (from, file, { rust, within = boundaryOf(from, ctx), as = unitsOf(units, from), crate = null }) => { if (rust) { rustFiles.add(file) // Named as Rust source after being carried as an asset: read again, as source. @@ -2943,7 +3015,8 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, } } if (!boundaries.has(file)) boundaries.set(file, within) - const grew = addUnits(units, file, as) + const inCrates = addTo(rootsOf, file, crate === null ? rootsOf.get(from) ?? new Set([from]) : new Set([crate])) + const grew = addTo(units, file, as) || inCrates if (!sources.has(file)) next.push(file) else if (grew && rust && !formats?.has(file)) revisit.add(file) } @@ -2999,12 +3072,16 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, toLoad.map(async (relPath) => { try { assertRealPathWithinBase(realBase, baseDir, relPath) - if (!withinRealDir(baseDir, relPath, boundaries.get(relPath) ?? '.', { realDirs })) return null + if (!withinRealDir(baseDir, relPath, boundaries.get(relPath) ?? '.', { realDirs })) { + unloaded.add(relPath) + return null + } // Rust source (a module, a build script, an `include!`d file) and an `include_str!` asset // are UTF-8 text, or rustc rejects them: one that isn't is refused, never carried with // its bytes replaced. An `include_bytes!` asset is any bytes, carried as base64 if need be. const kind = assets.get(relPath) const buf = await readFile(join(baseDir, relPath)) + ctx.checkVendoredFile?.(relPath, buf) const utf8 = isUtf8(buf) if (!utf8 && kind !== 'include_bytes') throw new Error(`${kind === undefined ? 'Rust source' : 'include_str! file'} is not valid UTF-8: ${relPath}`) if (kind === undefined) return [relPath, buf.toString('utf8'), false] @@ -3013,6 +3090,7 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, } catch (err) { if (err.code === 'ENOENT') { console.warn(`[loader.rust] Missing file: ${relPath}`) + unloaded.add(relPath) return null } // resolveModPath gates on isFile so a dir is normally never queued; guard EISDIR anyway. @@ -3067,10 +3145,10 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, if (name !== null) leads.add(name) } for (const name of leads) { - const lib = ctx.resolveCrate(name, relPath) - if (!lib) continue - roots.add(lib) - queue(relPath, lib, { rust: true, within: boundaryOf(lib, ctx), as: new Set([...here].map((u) => ctx.unitOfCrate(lib, u))) }) + for (const { file: lib } of ctx.crateCandidates(name, relPath, { roots: rootsOf.get(relPath) ?? null, units: here })) { + roots.add(lib) + queue(relPath, lib, { rust: true, within: boundaryOf(lib, ctx), as: new Set([...here].map((u) => ctx.unitOfCrate(lib, u))), crate: lib }) + } } } for (const entry of reads) { @@ -3100,25 +3178,27 @@ export async function collectRustFilesFromDisk(baseDir, entries, { cargo = null, // reaches, each a crate root of its own, until no package is new; then the tree pass // (buildRustTree), again with the in-tree crate roots it names that the walk left out (a crate // whose name a file also binds as a value, `use crate::util::log;` beside `log::info!`) or walked -// as fewer units than it is named as (`wantedRoots`), until none is new. Each file is compiled as the units `units` records (cargo.js -// unitKey): a build script for the host, with its package's features, and so what it reaches; a -// proc-macro crate and what it depends on for the host too. A root the walk was asked for and -// didn't load -- refused (a link out of its package) or gone -- is not asked for again: the walk -// has said why. Returns `{ sources, formats, tree }`. +// as fewer units than it is named as (`wantedRoots`), until none is new. Each file is compiled as +// the units `units` records (cargo.js unitKey): a build script for the host, with its package's +// features, and so what it reaches; a proc-macro crate and what it depends on for the host too. +// A root the walk was asked for and didn't load -- refused (a link out of its package) or gone -- +// is not asked for again: the walk has said why. Returns `{ sources, formats, tree }`. export async function collectRustBundle(baseDir, entries, { cargo, buildScripts = false }) { const formats = new Map() const sources = new Map() const units = new Map() + const rootsOf = new Map() + const unloaded = new Set() // what the walk refused or found gone: said once const roots = [...entries] const walk = async (wave) => { - await collectRustFilesFromDisk(baseDir, wave, { cargo, formats, sources, units }) + await collectRustFilesFromDisk(baseDir, wave, { cargo, formats, sources, units, crateRoots: rootsOf, unloaded }) if (!buildScripts) return const scripts = new Set() for (const path of sources.keys()) { const script = formats.has(path) ? null : cargo.buildScriptOf(path) if (script === null) continue const as = new Set([...unitsOf(units, path)].map((u) => cargo.buildScriptUnit(u))) - if (addUnits(units, script, as)) scripts.add(script) + if (addTo(units, script, as)) scripts.add(script) } if (scripts.size === 0) return for (const s of scripts) if (!roots.includes(s)) roots.push(s) @@ -3128,7 +3208,7 @@ export async function collectRustBundle(baseDir, entries, { cargo, buildScripts const tried = new Set() const complete = async () => { const tree = buildRustTree(sources, { roots, baseDir, cargo, formats, units }) - const wanted = tree.wantedRoots.filter((r) => addUnits(units, r, tree.wantedUnits.get(r)) || (!sources.has(r) && !tried.has(r))) + const wanted = tree.wantedRoots.filter((r) => !unloaded.has(r) && (addTo(units, r, tree.wantedUnits.get(r)) || (!sources.has(r) && !tried.has(r)))) if (wanted.length === 0) return tree for (const r of wanted) { tried.add(r) diff --git a/tests/bundle-cmd.test.js b/tests/bundle-cmd.test.js index 41d7b606..47cb2781 100644 --- a/tests/bundle-cmd.test.js +++ b/tests/bundle-cmd.test.js @@ -3917,7 +3917,7 @@ test('buildRustBundle resolves a mod declared inside inline modules under their test('buildRustBundle leaves test/doc-only modules and the dev-deps they reach out of the bundle', async (t) => { const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: join(rustFixtures, 'cfg-test'), entries: ['src/lib.rs'] })) - t.assert.deepEqual(warnings, []) + t.assert.deepEqual(warnings, ["[stasis] Rust features from a replay of the manifests, not cargo's resolver: no --cargo-target"]) // Not bundled: src/tests/mod.rs, src/prop/strategies.rs, src/doc_only.rs, src/sys/mock.rs, src/maybe.rs (its feature // is off), vendor/proptest, vendor/quickcheck, serde's test helpers. t.assert.deepEqual([...bundle.sources.keys()].toSorted(), [ @@ -4005,14 +4005,15 @@ test('CLI: bundle --cargo-target keeps only the named target\'s #[cfg_attr(…, test('CLI: EXODUS_STASIS_DEBUG=1 prints the resolved Rust features per package', (t) => { const r = runCli(['bundle', '-o', '/dev/null', 'src/main.rs'], { cwd: join(rustFixtures, 'features'), env: { ...cleanEnv, EXODUS_STASIS_DEBUG: '1' } }) t.assert.equal(r.status, 0, r.stderr) - t.assert.match(r.stderr, /^\[stasis\] Rust features \(Cargo\.toml \+ Cargo\.lock\), 6 packages:$/mu) + t.assert.match(r.stderr, /^\[stasis\] Rust features \(manifest replay, target\), 6 packages:$/mu) + t.assert.match(r.stderr, /^\[stasis\] Rust features \(manifest replay, host\), 0 packages:$/mu) t.assert.match(r.stderr, /^\[stasis\] {3}app@0\.1\.0 \(\.\): default, fast$/mu) t.assert.match(r.stderr, /^\[stasis\] {3}lib-a@0\.2\.0 \(crates\/lib-a\): default, extra, extra-dep, std$/mu) t.assert.match(r.stderr, /^\[stasis\] {3}extra-dep@1\.0\.0 \(vendor\/extra-dep\): \(none\)$/mu) t.assert.match(r.stderr, /^\[stasis\] {3}winnowish@0\.6\.1 \(vendor\/winnowish\): default, std$/mu) t.assert.match(r.stderr, /^\[stasis\] {3}winnowish@0\.5\.0 \(vendor\/winnowish-0\.5\.0\): default, std$/mu) const quiet = runCli(['bundle', '-o', '/dev/null', 'src/main.rs'], { cwd: join(rustFixtures, 'features') }) - t.assert.doesNotMatch(quiet.stderr, /Rust features/u) + t.assert.doesNotMatch(quiet.stderr, /Rust features \(/u) }) test('CLI: bundle --cargo-features enables a root feature (repeatable, comma-separated)', withTmp((t, tmp) => { diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index 3d9df636..92f1f006 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -1,5 +1,6 @@ import { test } from 'node:test' import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { homedir, tmpdir } from 'node:os' import { dirname, join, parse, posix } from 'node:path' @@ -17,7 +18,7 @@ import { rustcTargetCfgs, } from '../stasis/src/loaders/cargo.js' import { buildRustBundle } from '../stasis/src/cmd/bundle.js' -import { buildRustTree, collectRustFilesFromDisk } from '../stasis/src/loaders/rust.js' +import { buildRustTree, collectRustBundle, collectRustFilesFromDisk } from '../stasis/src/loaders/rust.js' const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'rust-bundle') const featuresFixture = join(fixtures, 'features') @@ -74,6 +75,8 @@ const captureWarningsSync = (fn) => { console.warn = original } } +// Warnings, less the notice that the features come from a replay of the manifests (see the test of it). +const besidesReplay = (warnings) => warnings.filter((w) => !w.startsWith('[stasis] Rust features from a replay of the manifests')) const vendoredPackage = (name, version) => ({ [`vendor/${name}-${version}/Cargo.toml`]: `[package]\nname = "${name}"\nversion = "${version}"\n`, [`vendor/${name}-${version}/src/lib.rs`]: '', @@ -98,7 +101,7 @@ test('parseCargoManifest reads multi-line arrays, feature tables, dependency kin // cargo's feature map: as written, and a feature for the optional dependency no `dep:` names t.assert.deepEqual([...m.features], [['default', ['std']], ['std', []], ['full', ['std', 'dep:opt', 'sub/two', 'opt?/extra']], ['pm-crate', ['dep:pm-crate']]]) const dep = (k) => Object.fromEntries([...m.deps.get(k).kinds].toSorted()) - const ask = (pkg, version, more) => ({ kind: 'normal', target: null, version, path: null, source: 'registry', package: pkg, renamed: false, inherited: false, optional: false, defaultFeatures: true, features: [], ...more }) + const ask = (pkg, version, more) => ({ kind: 'normal', target: null, version, path: null, source: 'registry', origin: null, package: pkg, renamed: false, inherited: false, optional: false, defaultFeatures: true, features: [], ...more }) // Each dependency table is its own request, for the crate it names: a dev-dependency's features // stay out of the normal one. t.assert.deepEqual(dep('plain'), { dev: ask('plain', '1', { kind: 'dev', features: ['dev-only'] }), normal: ask('plain', '1') }) @@ -1205,7 +1208,7 @@ test('buildRustBundle reads a package-defined quote! body as macro input while w const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/lib.rs', 'vendor/other/src/lib.rs'] })) t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/lib.rs', 'src/m.rs', 'src/m/inner.rs', 'src/user.rs', 'src/user/hidden.rs', 'vendor/other/src/lib.rs']) t.assert.deepEqual(Object.fromEntries(bundle.imports.get('rust').get('src/user.rs')), { 'mod hidden': 'src/user/hidden.rs', 'quote!': 'src/m/inner.rs' }) - t.assert.deepEqual(warnings, []) + t.assert.deepEqual(besidesReplay(warnings), []) }) }) @@ -1268,7 +1271,7 @@ test('buildRustBundle declares a template\'s mod in each module of its package i t.assert.equal(imports.get('src/lib.rs').get('gm1::X'), 'src/a/gm1.rs') t.assert.equal(imports.get('src/lib.rs').get('mod solo'), 'src/solo.rs') t.assert.equal(imports.get('src/b.rs').get('app::a::gm1::X'), 'src/a/gm1.rs') - t.assert.deepEqual(warnings, []) + t.assert.deepEqual(besidesReplay(warnings), []) const cargo = createCargoContext(tmp, { entries }) const sources = await collectRustFilesFromDisk(tmp, entries, { cargo }) t.assert.deepEqual(sorted(buildRustTree(sources, { roots: entries, baseDir: tmp, cargo }).unresolvedCrates), []) @@ -1356,7 +1359,7 @@ test('buildRustBundle ends when a crate root it wants is refused, and reports ev symlinkSync(join('..', '..', '..', 'secret.rs'), join(tmp, 'vendor', 'linked', 'src', 'lib.rs')) const { result: bundle, warnings } = await captureWarningsAsync(() => buildRustBundle({ cwd: tmp, entries: ['src/main.rs'] })) t.assert.deepEqual([...bundle.sources.keys()], ['src/main.rs']) - t.assert.deepEqual([...new Set(warnings)], [ + t.assert.deepEqual([...new Set(besidesReplay(warnings))], [ '[loader.cargo] app 0.1.0\'s dependency far is a path outside the bundle root: ../far', '[loader.rust] Refusing file outside its package: vendor/linked/src/lib.rs (a link out of vendor/linked)', '[stasis] 3 crates referenced but not in the bundle: far, linked (vendor/linked/src/lib.rs), missing', @@ -1440,7 +1443,11 @@ test('createCargoContext resolves through cargo\'s graph: each table\'s source, t.assert.equal(ctx.resolveCrate('itoa', 'src/lib.rs'), 'vendor/itoa/src/lib.rs') t.assert.equal(ctx.resolveCrate('itoa04', 'src/lib.rs'), 'vendor/itoa-0.4.8/src/lib.rs') t.assert.equal(ctx.resolveCrate('itoa_git', 'src/lib.rs'), 'vendor/itoa-1.0.0/src/lib.rs') - t.assert.equal(ctx.resolveCrate('cfg_if', 'src/lib.rs'), 'patched/cfg-if/src/lib.rs') // [patch.crates-io], a path + // `[target.'cfg(windows)'.dependencies] cfg-if`: no crate of a Linux build; a Windows one's is + // the [patch.crates-io] path + t.assert.equal(ctx.resolveCrate('cfg_if', 'src/lib.rs'), null) + const windows = createCargoContext(tmp, { entries: ['src/lib.rs'], features: ['fmt,git'], target: { triple: 'x86_64-pc-windows-msvc', cfgs: parseRustcCfg(recorded.cfg['x86_64-pc-windows-msvc'].join('\n')) } }) + t.assert.equal(windows.resolveCrate('cfg_if', 'src/lib.rs'), 'patched/cfg-if/src/lib.rs') t.assert.equal(ctx.resolveCrate('macros', 'src/lib.rs'), 'crates/macros/src/lib.rs') // the proc-macro and what it depends on are built for the host, with the host's features t.assert.deepEqual(sorted(ctx.featuresFor('crates/macros/src/lib.rs', HOST_UNIT)), []) @@ -1541,7 +1548,8 @@ test('createCargoContext resolves a proc-macro entry for the host, where cargo b }, (tmp) => { const cargo = createCargoContext(tmp, { entries: ['src/lib.rs'] }) t.assert.deepEqual(enabledOf(cargo, 'host'), { '.': ['default', 'printing'], 'vendor/quote': ['std'] }) - t.assert.deepEqual(enabledOf(cargo, 'target'), {}) + // cargo activates a proc-macro member it is asked to build in the target's context too + t.assert.deepEqual(enabledOf(cargo, 'target'), { '.': ['default', 'printing'], 'vendor/quote': ['std'] }) t.assert.deepEqual(sorted(cargo.featuresFor('src/lib.rs', HOST_UNIT)), ['default', 'printing']) }) }) @@ -1586,3 +1594,248 @@ test('buildRustBundle scans a proc-macro entry as the host build, with the featu t.assert.deepEqual([...bundle.sources.keys()].toSorted(), ['src/lib.rs', 'src/printing.rs']) }) }) + +// --- tenth review: platform tables, build-script crates, proc-macro roots, out-of-root workspaces --- + +// The crate-root edges of a bundle collected from `tmp`: file -> `use` spec -> target (a cfg-keyed +// map as [key, file] pairs), and the files carried. +const crateEdges = async (tmp, entries, opts = {}) => { + const cargo = createCargoContext(tmp, { entries, ...opts }) + const { sources, tree } = await collectRustBundle(tmp, entries, { cargo, buildScripts: true }) + const edges = {} + for (const [file, specs] of tree.resolutions) { + for (const [spec, target] of specs) if (spec.startsWith('use ')) (edges[file] ??= {})[spec] = target instanceof Map ? [...target] : target + } + return { edges, files: [...sources.keys()].toSorted(), tree } +} + +const platformTables = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[target.\'cfg(windows)\'.dependencies]\nfoo = "2"\n[target.\'cfg(unix)\'.dependencies]\nfoo = "1"\n[target.\'cfg(windows)\'.build-dependencies]\nbar = "2"\n[target.\'cfg(unix)\'.build-dependencies]\nbar = "1"\n', + 'src/main.rs': 'use foo::f;\nfn main() { f() }\n', + 'build.rs': 'use bar::b;\nfn main() { b() }\n', +} +const platformLock = `version = 4\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = [\n "bar 1.0.0",\n "bar 2.0.0",\n "foo 1.0.0",\n "foo 2.0.0",\n]\n${[['bar', '1.0.0', 'c'], ['bar', '2.0.0', 'd'], ['foo', '1.0.0', 'a'], ['foo', '2.0.0', 'b']].map(([n, v, c]) => `\n[[package]]\nname = "${n}"\nversion = "${v}"\nsource = "${REGISTRY}"\nchecksum = "${sha(c)}"\n`).join('')}` + +test('createCargoContext links only the dependency tables of the platforms the code is built for, and keeps each one that may be', async (t) => { + const vendored = { + ...vendoredCopy('foo', '[package]\nname = "foo"\nversion = "1.0.0"\n', sha('a')), ...vendoredCopy('foo-2.0.0', '[package]\nname = "foo"\nversion = "2.0.0"\n', sha('b')), + ...vendoredCopy('bar', '[package]\nname = "bar"\nversion = "1.0.0"\n', sha('c')), ...vendoredCopy('bar-2.0.0', '[package]\nname = "bar"\nversion = "2.0.0"\n', sha('d')), + } + // cargo's resolver (the lockfile, the target and the host known): the unix tables on Linux, + // whichever table comes first + await Promise.all([platformTables['Cargo.toml'], platformTables['Cargo.toml'].replace(/(\[target.'cfg\(windows\)'.dependencies\]\nfoo = "2"\n)(\[target.'cfg\(unix\)'.dependencies\]\nfoo = "1"\n)/u, '$2$1')].map((toml) => + withProjectAsync({ ...platformTables, 'Cargo.toml': toml, 'Cargo.lock': platformLock, ...vendored }, async (tmp) => { + const { edges, files } = await crateEdges(tmp, ['src/main.rs'], { target: LINUX, host: LINUX }) + t.assert.deepEqual(edges, { 'src/main.rs': { 'use foo': 'vendor/foo/src/lib.rs' }, 'build.rs': { 'use bar': 'vendor/bar/src/lib.rs' } }) + t.assert.deepEqual(files.filter((f) => f.startsWith('vendor/')), ['vendor/bar/src/lib.rs', 'vendor/foo/src/lib.rs']) + }))) + await withProjectAsync({ ...platformTables, ...vendored }, async (tmp) => { + // the replay, the same + const known = await crateEdges(tmp, ['src/main.rs'], { target: LINUX, host: LINUX }) + t.assert.deepEqual(known.edges, { 'src/main.rs': { 'use foo': 'vendor/foo/src/lib.rs' }, 'build.rs': { 'use bar': 'vendor/bar/src/lib.rs' } }) + // no target: either table may apply, so either version is carried, keyed by its platform + const any = await crateEdges(tmp, ['src/main.rs']) + t.assert.deepEqual(any.edges, { + 'src/main.rs': { 'use foo': [['windows', 'vendor/foo-2.0.0/src/lib.rs'], ['unix', 'vendor/foo/src/lib.rs']] }, + 'build.rs': { 'use bar': [['windows', 'vendor/bar-2.0.0/src/lib.rs'], ['unix', 'vendor/bar/src/lib.rs']] }, + }) + // the target known, the host not: the build script's tables still both maybe + const cross = await crateEdges(tmp, ['src/main.rs'], { target: LINUX }) + t.assert.deepEqual(cross.edges['src/main.rs'], { 'use foo': 'vendor/foo/src/lib.rs' }) + t.assert.deepEqual(cross.edges['build.rs'], { 'use bar': [['windows', 'vendor/bar-2.0.0/src/lib.rs'], ['unix', 'vendor/bar/src/lib.rs']] }) + }) +}) + +test('createCargoContext resolves a build script\'s modules, and a file it shares with the lib, from the [build-dependencies]', async (t) => { + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n[build-dependencies]\nfoo = "2"\n', + 'build.rs': 'mod probe;\n#[path = "shared.rs"]\nmod shared;\nfn main() { probe::run(); }\n', + 'probe.rs': 'use foo::v2;\npub fn run() { v2() }\n', + 'shared.rs': 'use foo as f;\n', + 'src/lib.rs': '#[path = "../shared.rs"]\npub mod shared;\npub fn lib_uses() { foo::v1() }\n', + ...vendoredPackage('foo', '1.0.0'), ...vendoredPackage('foo', '2.0.0'), + } + await withProjectAsync(files, async (tmp) => { + const { edges, files: carried } = await crateEdges(tmp, ['src/lib.rs'], { target: LINUX, host: LINUX }) + t.assert.equal(edges['probe.rs']['use foo'], 'vendor/foo-2.0.0/src/lib.rs') + t.assert.equal(edges['src/lib.rs']['use foo'], 'vendor/foo-1.0.0/src/lib.rs') + // compiled into both crates: both versions, the build script's keyed as its + t.assert.deepEqual(edges['shared.rs']['use foo'], [['*', 'vendor/foo-1.0.0/src/lib.rs'], ['build-script *', 'vendor/foo-2.0.0/src/lib.rs']]) + t.assert.ok(carried.includes('vendor/foo-1.0.0/src/lib.rs') && carried.includes('vendor/foo-2.0.0/src/lib.rs')) + }) +}) + +test('createCargoContext resolves a proc-macro entry beside another in both contexts, as cargo activates it', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[workspace]\nmembers = ["app", "pm", "shared"]\nresolver = "2"\n', + 'app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nshared = { path = "../shared" }\n', + 'app/src/lib.rs': 'pub use shared::f;\n', + 'pm/Cargo.toml': '[package]\nname = "pm"\nversion = "0.1.0"\nedition = "2021"\n[lib]\nproc-macro = true\n[dependencies]\nshared = { path = "../shared", features = ["h"] }\n', + 'pm/src/lib.rs': 'pub use shared::f;\n', + 'shared/Cargo.toml': '[package]\nname = "shared"\nversion = "0.1.0"\nedition = "2021"\n[features]\nh = []\n', + 'shared/src/lib.rs': '#[cfg(not(feature = "h"))]\nmod noth;\npub fn f() {}\n', + 'shared/src/noth.rs': 'compile_error!("cargo build -p app -p pm never compiles this");\n', + }, async (tmp) => { + const { files } = await crateEdges(tmp, ['app/src/lib.rs', 'pm/src/lib.rs']) + t.assert.ok(!files.includes('shared/src/noth.rs')) + const cargo = createCargoContext(tmp, { entries: ['app/src/lib.rs', 'pm/src/lib.rs'] }) + t.assert.deepEqual([sorted(cargo.featuresFor('shared/src/lib.rs')), sorted(cargo.featuresFor('shared/src/lib.rs', HOST_UNIT))], [['h'], ['h']]) + }) +}) + +test('buildRustTree reports a declared dependency that isn\'t in the bundle, used in expressions only', async (t) => { + await Promise.all(['fn main() { let _ = serde_json::to_string(&1); }\n', 'fn main() { let _ = serde_json::json!({}); }\n', 'use serde_json::to_string;\nfn main() {}\n'].map((main) => + withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nserde_json = "1"\n', + 'src/main.rs': main, + ...vendoredPackage('other', '1.0.0'), + }, async (tmp) => { + const { tree } = await crateEdges(tmp, ['src/main.rs']) + t.assert.deepEqual([...tree.unresolvedCrates], ['serde_json'], main) + }))) +}) + +test('createCargoContext reads the workspace a member bundled from its own directory inherits from, above the bundle root', async (t) => { + await withProjectAsync({ + 'ws/Cargo.toml': '[workspace]\nmembers = ["crates/*"]\nresolver = "2"\n[workspace.package]\nversion = "0.3.0"\nedition = "2015"\n[workspace.dependencies]\ndep = { path = "crates/app/dep" }\n', + 'ws/crates/app/Cargo.toml': '[package]\nname = "app"\nversion.workspace = true\nedition.workspace = true\n[features]\ndefault = ["fast"]\nfast = []\n[dependencies]\ndep = { workspace = true }\n[dev-dependencies]\ndep = { workspace = true, features = ["extra"] }\n', + 'ws/crates/app/src/lib.rs': '#[cfg(feature = "fast")]\nmod fast;\npub use dep::x;\n', + 'ws/crates/app/src/fast.rs': '', + 'ws/crates/app/dep/Cargo.toml': '[package]\nname = "dep"\nversion = "0.1.0"\n[features]\nextra = []\n', + 'ws/crates/app/dep/src/lib.rs': '#[cfg(feature = "extra")]\nmod extra;\n#[cfg(not(feature = "extra"))]\nmod plain;\npub fn x() {}\n', + 'ws/crates/app/dep/src/extra.rs': '', 'ws/crates/app/dep/src/plain.rs': '', + }, async (tmp) => { + const root = join(tmp, 'ws', 'crates', 'app') + const { files } = await crateEdges(root, ['src/lib.rs']) + // the inherited edition and the workspace's dependency read; its resolver 2 keeps the + // dev-dependency's feature out of the build, which resolver 1 of the member's edition wouldn't + t.assert.deepEqual(files, ['dep/src/lib.rs', 'dep/src/plain.rs', 'src/fast.rs', 'src/lib.rs']) + t.assert.equal(createCargoContext(root, { entries: ['src/lib.rs'] }).packageInfo('src/lib.rs').version, '0.3.0') + // a [workspace] above that doesn't take the package is none of its + writeFileSync(join(tmp, 'ws', 'Cargo.toml'), '[workspace]\nmembers = ["other"]\n[workspace.package]\nversion = "0.3.0"\nedition = "2015"\n') + t.assert.throws(() => createCargoContext(root, { entries: ['src/lib.rs'] }).packageInfo('src/lib.rs'), { name: 'LockfileError', message: /no workspace root is given/u }) + }) +}) + +test('createCargoContext says why the features come from a replay, and a vendored copy the lockfile doesn\'t list needs no checksums', (t) => { + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n', + 'Cargo.lock': `version = 4\n\n[[package]]\nname = "app"\nversion = "0.1.0"\ndependencies = ["foo"]\n\n[[package]]\nname = "foo"\nversion = "1.0.0"\nsource = "${REGISTRY}"\nchecksum = "${sha('a')}"\n`, + 'src/main.rs': '', + ...vendoredCopy('foo', '[package]\nname = "foo"\nversion = "1.0.0"\n', sha('a')), + ...vendoredPackage('junk', '0.1.0'), // no .cargo-checksum.json, and nothing locks it + } + withProject(files, (tmp) => { + t.assert.deepEqual(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }).resolution(), { mode: 'cargo', why: null }) + t.assert.deepEqual(createCargoContext(tmp, { entries: ['src/main.rs'] }).resolution(), { mode: 'replay', why: 'no --cargo-target' }) + rmSync(join(tmp, 'vendor/foo/.cargo-checksum.json')) + t.assert.deepEqual(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }).resolution(), { mode: 'replay', why: 'the vendored copy of foo 1.0.0 has no .cargo-checksum.json' }) + rmSync(join(tmp, 'vendor/foo'), { recursive: true }) + t.assert.deepEqual(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }).resolution(), { mode: 'replay', why: 'foo 1.0.0 is locked but not vendored' }) + rmSync(join(tmp, 'Cargo.lock')) + t.assert.deepEqual(createCargoContext(tmp, { entries: ['src/main.rs'], target: LINUX }).resolution(), { mode: 'replay', why: 'no Cargo.lock' }) + }) +}) + +test('createCargoContext applies a [patch] to the dependencies of its source only: a git URL\'s not to crates.io\'s', (t) => { + const base = { + 'src/lib.rs': '', + ...vendoredPackage('foo', '1.0.0'), + 'patched/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "1.0.0"\n', 'patched/foo/src/lib.rs': '', + } + const manifest = '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n' + for (const [where, files] of [ + ['manifest', { 'Cargo.toml': `${manifest}[patch."https://github.com/example/foo"]\nfoo = { path = "patched/foo" }\n` }], + ['config', { 'Cargo.toml': manifest, '.cargo/config.toml': '[patch."https://github.com/example/foo"]\nfoo = { path = "patched/foo" }\n' }], + ['crates-io', { 'Cargo.toml': `${manifest}[patch.crates-io]\nfoo = { path = "patched/foo" }\n` }], + ]) { + withProject({ ...base, ...files }, (tmp) => { + t.assert.equal(createCargoContext(tmp, { entries: ['src/lib.rs'] }).resolveCrate('foo', 'src/lib.rs'), where === 'crates-io' ? 'patched/foo/src/lib.rs' : 'vendor/foo-1.0.0/src/lib.rs', where) + }) + } +}) + +test('buildRustBundle refuses a vendored file that isn\'t the one its .cargo-checksum.json lists, as cargo does', async (t) => { + const lib = 'pub fn f() {}\n' + const listed = createHash('sha256').update(lib).digest('hex') + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n', + 'src/main.rs': 'fn main() { foo::f() }\n', + 'vendor/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "1.0.0"\n', + 'vendor/foo/.cargo-checksum.json': JSON.stringify({ files: { 'src/lib.rs': listed }, package: sha('a') }), + 'vendor/foo/src/lib.rs': lib, + } + await withProjectAsync(files, async (tmp) => { + const bundle = await buildRustBundle({ cwd: tmp, entries: ['src/main.rs'] }) + t.assert.ok(bundle.sources.has('vendor/foo/src/lib.rs')) + writeFileSync(join(tmp, 'vendor/foo/src/lib.rs'), 'mod injected;\npub fn f() {}\n') + await t.assert.rejects(buildRustBundle({ cwd: tmp, entries: ['src/main.rs'] }), { message: new RegExp(`^vendor/foo/src/lib\\.rs isn't the file vendor/foo/\\.cargo-checksum\\.json lists \\(sha256 [0-9a-f]{64}, listed ${listed}\\)`, 'u') }) + }) +}) + +test('buildRustTree takes `r#false` for a custom cfg, not the literal `false`', async (t) => { + const files = { + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n', + 'src/lib.rs': '#[cfg(r#false)]\nmod imp;\nmod other;\nuse crate::other::*;\npub fn f() { imp::g() }\n', + 'src/imp.rs': 'pub fn g() {}\n', 'src/other.rs': 'pub mod imp;\n', 'src/other/imp.rs': 'pub fn g() {}\n', + } + t.assert.equal(evalCfg('r#false', {}), null) + t.assert.equal(evalCfg('false', {}), false) + const imp = async (more) => withProjectAsync({ ...files, ...more }, async (tmp) => (await crateEdges(tmp, ['src/lib.rs'])).tree.resolutions.get('src/lib.rs').get('imp::g')) + // presumed off, as any custom cfg: the glob's `imp`; set by the rustflags: the module + t.assert.equal(await imp({}), 'src/other/imp.rs') + t.assert.equal(await imp({ '.cargo/config.toml': '[build]\nrustflags = ["--cfg", "r#false"]\n' }), 'src/imp.rs') +}) + +test('createCargoContext takes a nested workspace\'s vendored crates from the directory its own cargo config names', async (t) => { + await withProjectAsync({ + 'sub/Cargo.toml': '[workspace]\nmembers = ["app"]\n', + 'sub/app/Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n', + 'sub/app/src/lib.rs': 'pub use foo::f;\n', + 'sub/.cargo/config.toml': '[source.crates-io]\nreplace-with = "vendored-sources"\n[source.vendored-sources]\ndirectory = "vendor"\n', + 'sub/vendor/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "1.0.1"\n', 'sub/vendor/foo/src/lib.rs': '', + 'vendor/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "1.0.0"\n', 'vendor/foo/src/lib.rs': '', // the outer project's + }, async (tmp) => { + const { files } = await crateEdges(tmp, ['sub/app/src/lib.rs']) + t.assert.deepEqual(files, ['sub/app/src/lib.rs', 'sub/vendor/foo/src/lib.rs']) + }) +}) + +test('createCargoContext resolves an example with the dev-dependencies', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\ndep = { path = "dep" }\n[dev-dependencies]\ndep = { path = "dep", features = ["extra"] }\n', + 'src/lib.rs': '', + 'examples/demo.rs': 'fn main() { dep::f() }\n', + 'dep/Cargo.toml': '[package]\nname = "dep"\nversion = "0.1.0"\n[features]\nextra = []\n', + 'dep/src/lib.rs': '#[cfg(feature = "extra")]\nmod extra;\npub fn f() {}\n', 'dep/src/extra.rs': '', + }, async (tmp) => { + t.assert.ok((await crateEdges(tmp, ['examples/demo.rs'])).files.includes('dep/src/extra.rs')) + }) +}) + +test('createCargoContext resolves no dependency for a name no crate has, and so warns of none', async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dev-dependencies]\nbar = "3"\n[target.\'cfg(windows)\'.dependencies]\nqux = "9"\n', + 'src/lib.rs': 'pub fn f() -> u8 { let _ = str::from_utf8(b""); undeclared::x(); u8::MAX }\n', + ...vendoredPackage('bar', '1.0.0'), ...vendoredPackage('qux', '1.0.0'), + }, async (tmp) => { + // a dev-dependency and another platform's are no dependencies of this build; `u8::MAX`, + // `str::from_utf8` and `undeclared::x` name no crate + const { warnings } = await captureWarningsAsync(() => crateEdges(tmp, ['src/lib.rs'], { target: LINUX })) + t.assert.deepEqual(besidesReplay(warnings), []) + }) +}) + +test('collectRustBundle warns once of a crate root it refuses, however often the tree asks for it', { skip: process.platform === 'win32' ? 'symlinks' : false }, async (t) => { + await withProjectAsync({ + 'Cargo.toml': '[package]\nname = "app"\nversion = "0.1.0"\nedition = "2021"\n[dependencies]\nfoo = "1"\n', + 'src/lib.rs': 'pub use foo::f;\nfn g() { foo::f() }\n', + 'secret.rs': 'pub fn f() {}\n', + 'vendor/foo/Cargo.toml': '[package]\nname = "foo"\nversion = "1.0.0"\n', + }, async (tmp) => { + mkdirSync(join(tmp, 'vendor/foo/src')) + symlinkSync('../../../secret.rs', join(tmp, 'vendor/foo/src/lib.rs')) + const { warnings } = await captureWarningsAsync(() => crateEdges(tmp, ['src/lib.rs'])) + t.assert.deepEqual(warnings.filter((w) => w.includes('Refusing file outside its package')), ['[loader.rust] Refusing file outside its package: vendor/foo/src/lib.rs (a link out of vendor/foo)']) + }) +}) diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 86879454..7fc4de06 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -920,7 +920,7 @@ test('buildModuleTrees maps module paths to files per crate root, and files back t.assert.equal(tree.get('crate'), 'src/main.rs') t.assert.equal(tree.get('crate::foo'), 'src/foo.rs') t.assert.equal(tree.get('crate::foo::bar'), 'src/foo/bar.rs') - t.assert.deepEqual(files.get('src/foo/bar.rs'), { root: 'src/main.rs', modulePath: 'crate::foo::bar', leaves: [], parent: 'src/foo.rs' }) + t.assert.deepEqual(files.get('src/foo/bar.rs'), { root: 'src/main.rs', modulePath: 'crate::foo::bar', leaves: [], parent: 'src/foo.rs', roots: new Set(['src/main.rs']) }) }) test('buildModuleTrees keeps a lib and a bin apart (no shared `crate` key) whatever the entry order', (t) => { @@ -2281,7 +2281,7 @@ test('buildRustTree keeps a template\'s calls and includes with the template whe t.assert.deepEqual(edges(resolutions.get('src/user.rs')), { 'outer!': 'src/lib.rs', 'include_str data.txt': 'src/data.txt', 'helper!': 'src/lib.rs' }) }) -test('buildRustTree lets a local module under a custom cfg stand against a glob, giving way only to the module\'s own binding', (t) => { +test('buildRustTree lets a local module under a custom cfg give way to a glob or a binding of the module, as a doubtful candidate does', (t) => { const base = [ ['src/lib.rs', 'mod other;\n#[cfg(docsrs)]\nmod util;\nuse other::*;\nfn f() { util::g(); }\n'], ['src/other.rs', 'pub mod util;\n'], @@ -2289,7 +2289,9 @@ test('buildRustTree lets a local module under a custom cfg stand against a glob, ['src/util.rs', 'pub fn g() {}\n'], ] const r = (sources) => edges(buildRustTree(new Map(sources), { roots: ['src/lib.rs'] }).resolutions.get('src/lib.rs'))['util::g'] - t.assert.equal(r(base), 'src/util.rs') // a glob never shadows a module that is there + // a default build has no `mod util`: the glob's `util` is the name (rustc builds it with the glob's) + t.assert.equal(r(base), 'src/other/util.rs') + t.assert.equal(r(base.filter(([f]) => f !== 'src/other.rs' && f !== 'src/other/util.rs').concat([['src/other.rs', '']])), 'src/util.rs') // nothing else of the name: the module // serde: the docsrs-only module beside the `pub use` of every other build. t.assert.equal(r([...base, ['src/lib.rs', 'mod other;\n#[cfg(docsrs)]\nmod util;\n#[cfg(not(docsrs))]\nuse other::util;\nfn f() { util::g(); }\n']]), 'src/other/util.rs') }) @@ -2316,3 +2318,52 @@ test('evalCfg and buildRustTree keep cfg values that differ only in spaces apart t.assert.deepEqual(edges(resolutions.get('src/lib.rs')), { 'mod x': 'src/x.rs' }) } }) + +// --- tenth review: macros of other crates, nested templates --- + +test('buildRustTree gives a macro of another crate an edge, through an import, an alias, a path and #[macro_use] extern crate', (t) => { + const sources = new Map([ + ['src/lib.rs', 'mod a;\nmod b;\nmod c;\nmod prelude;\nuse prelude::*;\nfn f() { pm!(); }\n'], + ['src/a.rs', 'use dep::mac;\nfn f() { mac!(); }\n'], + ['src/b.rs', 'use dep::other as o;\nfn f() { o!(); }\n'], + ['src/c.rs', 'fn f() { dep::other!(); }\n'], + ['src/prelude.rs', 'pub use dep::mac as pm;\n'], + ['tool/main.rs', '#[macro_use]\nextern crate dep;\nfn main() { other!(); }\n'], + ['vendor/dep/src/lib.rs', '#[macro_use]\nmod macros;\n#[macro_export]\nmacro_rules! mac { () => {} }\n'], + ['vendor/dep/src/macros.rs', '#[macro_export]\nmacro_rules! other { () => {} }\n'], + ]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs', 'tool/main.rs'] }) + t.assert.equal(resolutions.get('src/a.rs').get('mac!'), 'vendor/dep/src/lib.rs') + t.assert.equal(resolutions.get('src/b.rs').get('o!'), 'vendor/dep/src/macros.rs') // the macro the alias names, in the file defining it + t.assert.equal(resolutions.get('src/c.rs').get('dep::other!'), 'vendor/dep/src/macros.rs') + t.assert.equal(resolutions.get('src/lib.rs').get('pm!'), 'vendor/dep/src/lib.rs') // through the prelude glob + t.assert.equal(resolutions.get('tool/main.rs').get('other!'), 'vendor/dep/src/macros.rs') +}) + +test('buildRustTree resolves an include in a nested macro_rules! where that macro is invoked, not where the outer one is', (t) => { + const nested = 'macro_rules! outer { () => { macro_rules! inner { () => { include_str!("data.txt") } } } }\n' + const sources = new Map([ + ['src/lib.rs', '#[macro_use]\nmod macros;\nouter!();\npub mod sub;\n'], + ['src/macros.rs', nested], + ['src/sub/mod.rs', 'pub mod user;\n'], + ['src/sub/user.rs', 'pub fn f() -> &\'static str { inner!() }\n'], + ['src/sub/data.txt', 'sub'], + ['src/data.txt', 'a decoy rustc never reads'], + ]) + const items = scanRustItems(nested) + t.assert.deepEqual(items.macros.map((m) => [m.name, m.includes.map((i) => i.path)]), [['outer', []], ['inner', ['data.txt']]]) + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.equal(resolutions.get('src/sub/user.rs').get('include_str data.txt'), 'src/sub/data.txt') + t.assert.equal(resolutions.get('src/lib.rs').get('include_str data.txt'), undefined) +}) + +test('buildRustTree resolves through a dense cycle of cfg-gated globs in time: each module globbing every other', (t) => { + // every path through the cycle a distinct `any(…)` of cfgs: kept to the ones that tell anything apart + const N = 6 + const sources = new Map([['src/lib.rs', `${Array.from({ length: N }, (_, k) => `pub mod m${k};\n`).join('')}${Array.from({ length: N * N }, (_, i) => `pub use crate::m${Math.floor(i / N)}::item${i % N} as u${i};\n`).join('')}`]]) + for (let k = 0; k < N; k++) sources.set(`src/m${k}.rs`, `${Array.from({ length: N }, (_, j) => (j === k ? '' : `#[cfg(feature = "f${k}_${j}")]\npub use crate::m${j}::*;\n`)).join('')}pub fn item${k}() {}\n`) + const started = performance.now() + const { resolutions } = buildRustTree(sources, { roots: ['src/lib.rs'] }) + t.assert.ok(performance.now() - started < 10_000, `${Math.round(performance.now() - started)} ms`) + t.assert.equal(resolutions.get('src/lib.rs').get('crate::m0::item0'), 'src/m0.rs') +}) From 8e6e4e0e39d0f16c31f45cdbf53427835dd05868 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 08:54:09 +0000 Subject: [PATCH 10/10] test(bundle): hold the Rust loader's new fixture projects in one tests/fixtures/rust-bundle.json.br `includes` and `cargo-recorded` (30 files, 44 KB) become one brotli-compressed JSON file (10 KB): project name -> project-relative path -> the file's text, or `{ base64 }` for one that isn't UTF-8. tests/rust-fixtures.helper.js writes a project out once per test process, to a temporary directory removed when the process exits, and `rustFixture(name)` gives its path; `node tests/rust-fixtures.helper.js unpack|pack ` edits them. The fixtures main already holds stay as they are. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015nbp2vC4jLPbGN8cmorxAL --- tests/bundle-cmd.test.js | 3 +- tests/fixtures/rust-bundle.json.br | Bin 0 -> 10126 bytes .../rust-bundle/cargo-recorded/Cargo.lock | 79 --- .../rust-bundle/cargo-recorded/Cargo.toml | 42 -- .../cargo-recorded/crates/extra/Cargo.toml | 9 - .../cargo-recorded/crates/lib/Cargo.toml | 15 - .../cargo-recorded/crates/macros/Cargo.toml | 10 - .../cargo-recorded/patched/cfg-if/Cargo.toml | 4 - .../rust-bundle/cargo-recorded/recorded.json | 500 ------------------ .../vendor/autocfg/.cargo-checksum.json | 1 - .../cargo-recorded/vendor/autocfg/Cargo.toml | 75 --- .../vendor/itoa-0.4.8/.cargo-checksum.json | 1 - .../vendor/itoa-0.4.8/Cargo.toml | 29 - .../vendor/itoa-1.0.0/.cargo-checksum.json | 1 - .../vendor/itoa-1.0.0/Cargo.toml | 48 -- .../vendor/itoa/.cargo-checksum.json | 1 - .../cargo-recorded/vendor/itoa/Cargo.toml | 66 --- .../vendor/memchr/.cargo-checksum.json | 1 - .../cargo-recorded/vendor/memchr/Cargo.toml | 89 ---- .../vendor/paste/.cargo-checksum.json | 1 - .../cargo-recorded/vendor/paste/Cargo.toml | 72 --- .../fixtures/rust-bundle/includes/Cargo.lock | 7 - .../fixtures/rust-bundle/includes/Cargo.toml | 4 - tests/fixtures/rust-bundle/includes/README.md | 3 - tests/fixtures/rust-bundle/includes/build.rs | 8 - .../rust-bundle/includes/build/helper.rs | 1 - .../rust-bundle/includes/data/blob.bin | Bin 4 -> 0 bytes .../rust-bundle/includes/data/table.txt | 2 - .../rust-bundle/includes/src/gated.rs | 4 - .../includes/src/generated/consts.rs | 1 - .../fixtures/rust-bundle/includes/src/lib.rs | 18 - .../rust-bundle/includes/src/macros.rs | 5 - tests/rust-features.test.js | 5 +- tests/rust-fixtures.helper.js | 77 +++ tests/rust-loader.test.js | 3 +- 35 files changed, 84 insertions(+), 1101 deletions(-) create mode 100644 tests/fixtures/rust-bundle.json.br delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/Cargo.lock delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/extra/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/lib/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/crates/macros/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/patched/cfg-if/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/recorded.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/autocfg/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json delete mode 100644 tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/includes/Cargo.lock delete mode 100644 tests/fixtures/rust-bundle/includes/Cargo.toml delete mode 100644 tests/fixtures/rust-bundle/includes/README.md delete mode 100644 tests/fixtures/rust-bundle/includes/build.rs delete mode 100644 tests/fixtures/rust-bundle/includes/build/helper.rs delete mode 100644 tests/fixtures/rust-bundle/includes/data/blob.bin delete mode 100644 tests/fixtures/rust-bundle/includes/data/table.txt delete mode 100644 tests/fixtures/rust-bundle/includes/src/gated.rs delete mode 100644 tests/fixtures/rust-bundle/includes/src/generated/consts.rs delete mode 100644 tests/fixtures/rust-bundle/includes/src/lib.rs delete mode 100644 tests/fixtures/rust-bundle/includes/src/macros.rs create mode 100644 tests/rust-fixtures.helper.js diff --git a/tests/bundle-cmd.test.js b/tests/bundle-cmd.test.js index 47cb2781..589d277c 100644 --- a/tests/bundle-cmd.test.js +++ b/tests/bundle-cmd.test.js @@ -19,6 +19,7 @@ import { outermostDir, } from '../stasis/src/cmd/bundle.js' import { diffCommand } from '../stasis/src/cmd/diff.js' +import { rustFixture } from './rust-fixtures.helper.js' const here = dirname(fileURLToPath(import.meta.url)) const cli = join(here, '..', 'stasis', 'bin', 'stasis.js') @@ -3970,7 +3971,7 @@ test('buildBundle rejects --cargo-target and --cargo-manifests for a non-Rust bu test('CLI: bundle --cargo-manifests adds the package manifest, lockfile and build script to a Rust bundle', withTmp((t, tmp) => { const outPath = join(tmp, 'out.stasis.code.br') - const cwd = join(rustFixtures, 'includes') + const cwd = rustFixture('includes') const plain = runCli(['bundle', '-o', outPath, 'src/lib.rs'], { cwd }) t.assert.equal(plain.status, 0, plain.stderr) t.assert.match(plain.stderr, /Bundled 7 files in 1 package/u) diff --git a/tests/fixtures/rust-bundle.json.br b/tests/fixtures/rust-bundle.json.br new file mode 100644 index 0000000000000000000000000000000000000000..8c446593f6c4a3752dcaf32c6fe60970b44d969b GIT binary patch literal 10126 zcmV;9Cvn&tCB0GPb^xiM{Hq4rhCS#G(#L6$qY&6QAb|4619Tq7#;nGl6A`fpu65~R z{Iw@3n?Jg@24`T~$e-loy-d~q{=8s1$w{`BR7rJd3k(*ZJEUtW8~iw$ zHeWX7_fMx%j&q$_pum=xW>>gA5m;n9H-)>1w4PJqkH`-QzIA&mLL7 zFnEM<=p2&-QYkIv*tV82Ewy&y_@QA0k`O?9o`!5ySQGAED-=tSOkw~181D6`SgaF7 zKS?dXsMiW|@cN2kdULxzJ z%to8cavDti{znrXXJf*<^}-X1Y8!C6Gvd~;KSaabwreZf0Me=J@d+15X&8fuZJuzj zq#L)Ze0u41?TT45*FtQQpuClkC|;54P$9nwvtt~P3EUwmbtu_qV2|(P9$>me1hH9% zgTjo31Y{2*f6S-31@!=yjYk3zqsO3>9Ze(TV>t1_C~eUy@<<|SXOYOa)+3nGFS=lo zzk9uT%PizZe z*Zld#sRMtD`TxJz|M&b4S}V?u4wI`~sd#8Yj+}5^M-;v9hVC20h&HERp)z)VsA$Hg9u zw@a&WOAg1>0u(&Jq4W41n!)C44ug&A&3i`B+sio4|BmzCaYsnjSH&uOg+@hLCod@B!bFnEWcAIiy> zVFgK=_VPP32#~5BEhvN7a~>DiXNh?(psgQ@YM!StJVBJ=OJo<;(A_eg+D`b*TDSga(uTCjJAO zmY9^!l#8n1^A=qXyjH|mRUkchX46MLTd{Z7X~s7rW0g)lYk|G zUZPPMj!&uG*ui@L0!k^1)E{sF6C@cW1DeyWUnqPFB0W6Ow4ZM|RdrGt!|!6U=2Xo3 zRQ$}d_&t1)kKsAw2#QpeG;0$?y{aK96G~RS)`L+B4ni|ojq>&6^A;nrzvd-myZ;Pe zsh1OQ>NJdk?)5)|wEji5&140ec-Y4>W2Ylg|7JX&+WGSXJfYDu!+4iCy`MMKCEo(Y zL-cwV1ZosBN-p!sT+_I*4D!>JiJ;HAc>Q1f;IG4r{?VrrV`FLg#&v5;gTw;!r~Ao{ zUmpy{m@Lo!qbnKfet&xB0_a(jJ~V2F!EnCY>~p~)&9)frm~ zHqf4?k9NV>T(}%Q2b#~vyjx(Yuuvo|I;zzw#>fb23bM}TRL^jI%zS+1Pn{G|PQFgY z$HAPfX{W9d$5ZM&t5sywIx#dezJ3iV(Ju{MZO+xuYO89V>u_ryW>+N8IU2CBOF{tK zwtv#bZ`*Cwz@V`v&vCleW;Ufv;XaNBw;2MfKP5fLX4o?l)m6_a6rqhY8V|}0F?90j6F&HWvhnY+ju6NX8e=eX-WtI! z z-}OK{0;`o~qpIk~^22lC;}BQbE6^_P@oS41^0jP^BMCT`T*JV{1t_Z;Jb!yJQ=iZ+TGZhqBQH>B!ce9Y z{jGi4$IZK<`nW$}5Vu#xR@ck>iZinSduZBU5JDmCS#IEiTd52y^>*w9Qwu3P?lrar zfw8cNM;%<+Xi~|0!n}sc^m<`LHWSx=*d8iMgo=l18%9vQCq(Sh9klAyyVvbK8jRo* z2PL~5h+KacuCkzyUF}5JyWKlDJ@UKHr^0SJx}qqfS&T1$n>wRN-l7a>4wK|Uzu;z@ zWj)tC!iKA}$6MvcGIca+weXjkE->E;7^EcRe zflOL~v)q*=k8`Z{Wb_pAWl+8sSTCCro4aRSYBwWe7+A;wI~Kk%qL3HNu+_8fLnXV1 z&Y@1JU{z)>3I{wGs!^%J@_{UmxOTJHY3?7>>Pe z4>-nNvqdh7j`<53_GC-gtO9Lh2FCXE7kvFMz0X(U&_i<`XVNXYWrX)E=m8JK;aefu zx`S?F!`EHRo1*q=!=v7Hls${1nnTpad=BM;s;f%Qy48sA*=$#r zpjX660eZ`eeT{wN=|K3fX$wa^I-|Mh2D6DVi4J&;45iQA>$=VnUYEv1EM#kJJOxVL zYL@7!`)&!NhL^4ZO!SM3Np=Z9r>PDBO3n$j1HVZ|+u#56(AqtOetp}D>n_?l803pk zBPel@xiN@YZyM+tu6Rb`lhzP?j`A}FmoAfh!%xN#lR=xm@}WM#KEp2NvY6B2`nccNR7oZ@H_eHf$ zH?niGBubHQ5Snc)l2k$u8V@Q=5U~sTz~)LsY=6&yqq5x-h~O_6Dw)%MzKO9Ie9BO{ z;Jc(D$uf6HVx_PhUb;yMoMiQ#+}@6JM{PmX{$Nw4-gq5$8;N@$%nP=~6W#jJB-a;H zhh16Z^m=8nDjf^q!@PNN;ePWK!XTY#ychcu>@8!?=WBi4@GBIe$0qX@09VR7A$Qy- ziXGe@)9D~C2n{cV?xoUbfx=+G_h8@xIuS}$&$Zyq+Wv)jtqB}=h3m^>vW35gey?$; zGad-#yQB=Y^cTRcwpzF@LCSlSVgaQt*#p7`KhBqIHO%haaC6USCB&JF-~L1{yo#t& zJLV=WQhi6;1ySizNBPuRW=qtnvPN-kIc3}|&1!By?;9r*SvLB(9~!d$_@OQ^_YyCvJ*w zV_31GQXJ;zmH4Y7Lu4-WP4U^i35GfJJTF|l^|1t< z_F`izl;3od!%v)X9E< zJ<$9vB)lIY*@E`5wL~@ZPA^M?#-9sbe$FH}yC({IDx5(OBo)bMRLKI?zG1{1ixC#iRrZ)O_jy(cd%Rb7$B;TL;Xv3Zz>^b-<#1U-ieLZq zepinp-U~V809{oii5sUAe~&OuaZL%{~_szNo}=}?8+7_COHdcv$}BCv@6f~9nsA=!sM0j(4HwV_&=Y_cpA;Or&O+Y~k*tj3Li4K6~7@m`lM z=&g3RB%h%J*7Dl)1Jk**=hIMBmm4nfRn#7mb?aQbl~Z`aUPd19cV#WPK>Cpo^WTx9 z14-bx6_@$G#LOLtOIHXy|5aQdcH2amAwncRf^Y@$XJjn81UA&I*$QtI^O-=&ojIFJHt`XbG=Gi zu}8)b!a>$;ahx-!Y&I>X$}SCt<(%P~(lR316)e{cZT8r14X)8Q7s<_+Y#*lpS0+tElOM(qk7O9H+?oJ6!U7MhO{X-o@ITxh!diBH`I ze<<0)A6DtX;V0UhLwm+5%g75z@*i4z_1S9lCYId z+@th7NzC6prd|G}bz^mG>T3HFDNcbotQ%g6glmYmU!_P)e`}(<^lR<|w%klE)(+r2^2ty~k(nvl~KRef@tz2H0_NYBDchjo-w&dozblw6IS#^nM? z&zbVZ+px@S0(UM)T3{J7?FQ7n&6T&KHS{J>%TURN0~mte`E7CO?|HPPj3 zfOf~t{61XxM9UQ-Qy6ClW6nvxEjZCs7|Ax|lr$_g04#I>?+@8z$PYN9DgXK*oGK=V zm84i!=A@-wH_BE&RI9akD>ZP`kLAwD4=YDvUAW&d+gPsl$J&*$uRt5uh@SR(t~Vwy zjof35&U`O7*59K{YveF99q5sr7Go*YSRXno!e#yQ%Zq&NN4XAyNLSv|veXehQ&WQ- zdW0PLWFp83R-Q;ijhH>*dp&_9N6UTK^pCUZb9g4+@E2}@#%KqJA24dZ(>=F90MQ&@ zz>}0rX9kLU9ccX`jed_|kEQLuKihwL!(S=Z{w(L0L| z*ehROLs<^5GXw03vkhn0C!S^(@R3=+9WG&+0#HPkvED^1c|q@#Wyw6-P;~^x6_7aL z=DlZExXp&QsxN2qfjwRB+m|jp7m@iSzqT@>RP;Y=6b>0qn zzpGYIMsaGfJM9$cZ^|wyFcp#gGXR1XAhHn+aOt?F<7+P?N>!@>#`k#OZAok8BZ_Ny zNky{IT*~IIFYj%ucI;9B;w*9jd{V;4#_#AjFt`Y*?lvA{-=*4MV(HH9L%~8} zt+quk>p;>V$}gRt`)Y$rR0vt4oD`UDB0BAaw5BzUcgnZvMQ7`(AV}+65RO=u*Bs1m zuneU~?QqTRxjhvee%vu!C}koKd85zK=IB&)tItRcHq~NHW6t#l&~bg9FF9f`)zaJ9 zyOPJWy)!nKJg`q^PpR2U&#BG8L?c;yvvkxRHL=Px?^5asP|zFBOJAyXWfkw0SzXiX z#ila`j4_Ywe3#G0QjgB?${U!8wig_<$`G%|(}sYAUP2><(Gw}1)ryueB%8ziK7UTO z`Q==5lqzxFkpZezD)%S!+!;*OtQE**1k#p0(8shIH6;v@TIz!w-#ZPK7PO5jdQaJG z(7am-N>tc%s=Z@I->kk-nZ>9&Oe9|x@%jHPF9`=~?yP$UTRYl|dqW@a3?gVJ=p(UW z&rq2scDNz7Wlsd+=s85gW#AFsrZH79k6OE~DdLZ>KjzQsK#_0b8xx?1>{<$9 zc5!B9@%jYV&M@*;6U8BRe@ffbB~r;ba3j@dIr33#r; zFiH+NpcIUFWIqqEdzIgUkAYr;xYu*W2}fF)68w?R`rada_cPM(v>A8C3 zL9D=K*IxPYAAaf@%qM31;4z$JxSZd_*{dy17&T;ia9x=^;oev!Os{S)CKDg!h{c+~ zJlKvj&CSC`h^1S0xepBSCD~&@8*CGZfIXB|Q>hqy*b;CXrD>XKy|%l% z7b3~Buh>-OZYgv_>)|MmY7Z`}y)YIPePEdi1Si5*0)#OKggyUX$E11Tz{AfC@G$$$bB|BaXg`hzRA0ltxZR>5e3LUfqFe8eX&vw>^M@qAiLl5}kVEmZ*6!@Gw!w>orqZNfo{H2a+=l z!fR?)w`Eat5;ApgArBd9`W{&5M89ePX9#pg8JlEawKA6A${6M7ez zS9fknkfRaeozCH6uAKF8z9BvmkH=9isWIAC!n`%tX0hnr??11 zVEKl;Tq0-kUiPWb3nyq2L8OlLwqHOlc``zY>9cYth46GZ!Yp%p&+G-y5)aWOx(4IGkXIDeGcy>jj#74`E`CgsWUm@|yOrK7 z2I$FK2p4E3li*DrwUa;8Oftw;3#`-$j#@oR<^?Du%%hi%eU(S_(iWWZ-ZVYfZ~wfb zZWMwH^kj4_Mk72yV^(BiuXABB3hG1T<6a76CpFMpLhe}Upo4CN^I2;ANtm`nc{G&^Y$ov2KmKl)DuS>0HQIdxSDA5n}bL*eU>Pysj1f))H#$ zWI7^(9ljvrbj>0LB-jeL8o&vm!2*@IXIjW*C6TVAXkDCBs$r41@zIc2ksrg=kMZr} ztB1&mzJhn!fJ&$Veu`L>9S=2MTH)}q{T#=Z#QEtVpEc&*C!R(DG*c&?kKw_n926;{ z9-gXD5PfOyJ(QSYDedC$U{<+6zT9j`kZH{r+`$~I9FMeaeMjVpKW z;Bhf2TgK6-#OzJ%n}1cY%dYp#V0OeM%n6g@aL6;t^e*IE+$=y}yMrJBe1Ri&+^b~1 zQqJYfWkOAZ+!5!yDJ(~Ei?I8b9xMJuxu zvS+B6O71N6LT?P(U=yzfs!?&}J$?*w74K!J+1!P0lBC{T(Gx#TrIK~Pm&$sCcn0*UP%3*h+2l~!JfIUYZ-@cN0TNKn<%YW`;R+Ar!_UL6wh*Z5TKUDMLQ+ZF{Dkcmwu7a#|LmPx>iS1IY;B&Udp zg}QU}4l&B3$JdABPM8ils|&<>;CdCO9RfO?`>whR*YOgJ>3V46cgeZTG@3&On+ibL zlxm<1KE^v3jBGxFGQi48UYv{9!|p+aZ?h z9QWI~xBDmR4lPO)N_#>Ojqgir^0vINjz>H;S2^=ai^xtWy)Gjjpub*n>%)u@9w z0ir4zVcwjdpYt$tW-xElvFob_88r)8ArBUCLyER_-`$GM1oHx8@Fln8hC4xPgm2U- zvB5`1%Cl|Qg_nMpfCkiDB9!-B%UWOfDJ4`DsKpim5$4u~kIAYy^WnZe^w%HKSur}w zyEmLRUgiY_%c9A;BcFjl48PUwwAC@3e)|a(uPpfk&UGKWhzp1k~8`#R@!2RVz zDBJ;wu~bG%o>$=>vST52&?FwQud+lac$6-9Ae0Txm7NVOXH?eEg8LTdvuJ#HYM98{v6-ZUQ`_3dG@h!cLq zL$nLnPn}-*%1b`5TU+mCOLpDp&Wm~5a&zx$M8TsRnms8}Lcr{n37?ry90{o-S)-oz zP#9C+FcUq}hmIf?u8w%4obQS=7SJeP(0t!kkoS&*2{xl2aP@SvozvR{B$G!Tm5=mz zaE!sEY&=eC4cm$)Z4$e>hpW#r4Ndvf1n+^&11;Kj^=1pPXKQmqRBG5Y-#d+_isX7g zg~iYC4ZnAMSTt-b*D3QzN9zS3jYL!hFH;U)u|&?{*6gYpask3>S|$#2Xd(N1qy0{? zU~S>TC=*tsRq2cAC|Od+lNIbS5k+)^K1wD)^8s}OaYg_*Y_vF(fztznltl^N3M94l z*W7uu;f9uWCAFj%bvc=+Ni_;@?`;qWaBhd z!9t<$!e3{Cu}A&jWD{FfNpa2K9G5HnuWl1hfaDWcY~U#`o-C#2z(l6Ny{oQyfgeZXZe+H}g43dlhyUr@X|WhY%=1MgQA1F)zbW{lox8Y%ey2nU{EhM_a?F(fhygVw{jhfh*|-z zoR?bA_h+93tC4AI;$Vos$WyNBJ}wG24|VzvQ_E3cNj9CuG`LMQHc~a6&MfJHT?TWa zGXP{F5C^S!vJJY6ac_1t*x%MRyfj$j$dTYiW~C~hUE=ckhJ=TR}5--l!0qp2|tmzb6x5ZXCmI1 zK`&K_bTrA?kwVRY{;=y9=u}y7At&~_=l9"] -build = false -exclude = ["/.github/**"] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Automatic cfg for Rust compiler features" -documentation = "https://docs.rs/autocfg/" -readme = "README.md" -keywords = [ - "rustc", - "build", - "autoconf", -] -categories = ["development-tools::build-utils"] -license = "Apache-2.0 OR MIT" -repository = "https://github.com/cuviper/autocfg" - -[lib] -name = "autocfg" -path = "src/lib.rs" - -[[example]] -name = "integers" -path = "examples/integers.rs" - -[[example]] -name = "nightly" -path = "examples/nightly.rs" - -[[example]] -name = "paths" -path = "examples/paths.rs" - -[[example]] -name = "traits" -path = "examples/traits.rs" - -[[example]] -name = "versions" -path = "examples/versions.rs" - -[[test]] -name = "no_std" -path = "tests/no_std.rs" - -[[test]] -name = "rustflags" -path = "tests/rustflags.rs" - -[[test]] -name = "tests" -path = "tests/tests.rs" - -[[test]] -name = "wrappers" -path = "tests/wrappers.rs" -harness = false diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json deleted file mode 100644 index b816353b..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{".cargo_vcs_info.json":"47986564aed398bb6a83979cbb1fe5eff0f3aa61fbba131a0954180ce8eaca26",".clippy.toml":"e0f282c75faa7e46c2630843826bb78f39dd2a6dea78dab7f960d34f730fb396",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"51a75e85b378ebc683ad6cdf12767e26ec495a2b372b980d278b68ef48046ca9","Cargo.toml":"46f2cf3006d59c73bed95c276f43ddab1a9a9fa177ea0142777b7471c753f558","Cargo.toml.orig":"a089595f30fd2a22f6152a96884e76d4aac63cac56aff6dd64a03ab2737a55dc","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"5eac563c0123003363f4706f36927cf4d3b118965e35b51bc515343ce6a711ae","benches/bench.rs":"fc009678431563bfb1b51698fb5b83f5309a3cdab90dc7365800f4e32bc287be","src/lib.rs":"a7b37eab3872193cf666c1d7d4db8d7e82c6dc6d8121fc762b5f69ddb4df3b01","src/udiv128.rs":"1c9c374680162e12c734d369d1494ee8c91e11eb4b05500e96e6303155bdc623","tests/test.rs":"c9b124462b75078e2407e0239222edd7859c3112d65a114742dbb8c47dafc07c"},"package":"b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml deleted file mode 100644 index 02b43829..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-0.4.8/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -name = "itoa" -version = "0.4.8" -authors = ["David Tolnay "] -exclude = ["performance.png"] -description = "Fast functions for printing integer primitives to an io::Write" -documentation = "https://docs.rs/itoa" -readme = "README.md" -categories = ["value-formatting"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/dtolnay/itoa" -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[features] -default = ["std"] -i128 = [] -std = [] diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json deleted file mode 100644 index 2ca74031..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{".clippy.toml":"33ffb83bbddb772575b3aa565b7136a8158ee386c216ffc2588fed9e83fa3826",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"e217196efe9a3efa3d374deebfde6980cda2cbfba14fecd9fdc86cdc1d5dcbe3","Cargo.toml":"39bf8df2e494ac7c8446decd837c0b242df393a1093d788c295002ecef27ddff","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"8e9d4e6ac0c740793958bf57c3dbd62827340cf77a282633c388aff70a7c1a22","benches/bench.rs":"636f3093bd461210ad3063289d455f90669c4a1be3273bcd30898de39f02c641","src/lib.rs":"2c013a4a736d514d8629fd9cc162e05e76fb3c1557d9d737a0f79c04bde0c971","src/udiv128.rs":"16394f767854452756372d74f8025f7329fd8b61a676d81edf489681a6332ee9","tests/test.rs":"fc4a9fed6ad3cd9f548b6916444c0d269fbbb7033a41a4f15a1712f6fb32f9cb"},"package":null} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml deleted file mode 100644 index 5cac345d..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa-1.0.0/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2018" -rust-version = "1.36" -name = "itoa" -version = "1.0.0" -authors = ["David Tolnay "] -build = false -exclude = [ - "performance.png", - "chart/**", -] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Fast integer primitive to string conversion" -documentation = "https://docs.rs/itoa" -readme = "README.md" -categories = ["value-formatting"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/dtolnay/itoa" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[lib] -name = "itoa" -path = "src/lib.rs" - -[[test]] -name = "test" -path = "tests/test.rs" - -[[bench]] -name = "bench" -path = "benches/bench.rs" diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json deleted file mode 100644 index 1c33e56e..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{".cargo_vcs_info.json":"e0165305f3b22a9ab4217a121f85379aa56c2675caa7afedc0a7468fc187600b",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"89c4f60223daf03217ea7c1e8879135b11d2554bd1e57dd9eff5c02f1adc7909","Cargo.lock":"1cd64090b8e883963bb08d91e6968cc1e83ffdba4f4b2f47651ce9d351a8a947","Cargo.toml":"894c215e4e7e973b556b0e9627bf9b6a719a043fac9b5af9838660816392a09f","Cargo.toml.orig":"44ca1ae5d9b48743305b57f3367051289512037e7836b8b3693b68d48dec0483","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"1305bd79675db0194915d9bc91bd0b3cf9487bc1126e05bf46b48ab43edce44b","benches/bench.rs":"29a0bdb973848007d681e571c57487e21e1bfb7aed5980c2190dba354498b154","src/lib.rs":"e962338e1886873aa7c3399ec89aa5378ccbfed712b9690f043628079390a34f","src/u128_ext.rs":"498cc94f13805e551248fd82f47cc318f51d953a5002a9f8698858cd12fbae3c","tests/test.rs":"b22f8a8ed89edcbb4140b74f5edd7f2adf3cb0e363990dd6010cd6ebe0dc36b7"},"package":"8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml deleted file mode 100644 index d60aa961..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/itoa/Cargo.toml +++ /dev/null @@ -1,66 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -rust-version = "1.68" -name = "itoa" -version = "1.0.18" -authors = ["David Tolnay "] -build = false -exclude = ["*.png"] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Fast integer primitive to string conversion" -documentation = "https://docs.rs/itoa" -readme = "README.md" -keywords = ["integer"] -categories = [ - "value-formatting", - "no-std", - "no-std::no-alloc", -] -license = "MIT OR Apache-2.0" -repository = "https://github.com/dtolnay/itoa" - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] -rustdoc-args = [ - "--generate-link-to-definition", - "--generate-macro-expansion", - "--extern-html-root-url=core=https://doc.rust-lang.org", - "--extern-html-root-url=alloc=https://doc.rust-lang.org", - "--extern-html-root-url=std=https://doc.rust-lang.org", -] - -[lib] -name = "itoa" -path = "src/lib.rs" - -[[test]] -name = "test" -path = "tests/test.rs" - -[[bench]] -name = "bench" -path = "benches/bench.rs" -harness = false - -[dependencies.no-panic] -version = "0.1" -optional = true - -[target."cfg(not(miri))".dev-dependencies.criterion] -version = "0.8" -default-features = false diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json deleted file mode 100644 index 2c5ead36..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{".cargo_vcs_info.json":"739b57f156ae78ce5ad0f2bc44ff727f1a9d48e743bc266e8e3ab8c35baa5ecb",".ignore":"ae8b19032d4fc418b99ccae9e7cc3996b1386665d0bd5edc5634a158e7d2f6a2",".vim/coc-settings.json":"cdc5e2b88bddbdbd1b85f21389c4d882720e4c4488ad566c43fccd9124f2e3bf","AI_POLICY.md":"eb058d5b8387f03159cf6ef868100459d7e01f49499733028fcc916bc97c1cc4","CONTRIBUTING.md":"f5b2b4a9a3dbcd9a6f2654eb97dd7135e6af3717b9a3f037a9cbf714e387af76","COPYING":"01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f","Cargo.lock":"54b66d3eabcf4993f06c5b556ab1df0082dd14985529e301124590280dbe0554","Cargo.toml":"a14f68b8fa55f15fd03b15dced37bcf66a0a628f568eedc82817cdb7f9281951","Cargo.toml.orig":"2d004168044e88b00f971e65d7e6c988a555aa00d7ef4d0ceb5b5a9c228fb0c9","LICENSE-MIT":"0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f","README.md":"92a74aaffe011bdaa06fbc34a01686a6eba58ca1322e976759417a547fddf734","UNLICENSE":"7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c","rustfmt.toml":"1ca600239a27401c4a43f363cf3f38183a212affc1f31bff3ae93234bbaec228","src/arch/aarch64/memchr.rs":"5bb70f915084e629d940dbc322f5b9096b2e658cf63fea8a2f6e7550412e73a0","src/arch/aarch64/mod.rs":"44cd1a614bd66f1e66fc86c541d3c3b8d3a14a644c13e8bf816df3f555eac2d4","src/arch/aarch64/neon/memchr.rs":"e8c00b8fb2c7e2711832ae3cedefe59f32ebedd7dfa4d0ec6de2a566c979daea","src/arch/aarch64/neon/mod.rs":"eab6d56c2b2354db4ee395f40282cd49f97e2ab853547be5de6e65fbe1b2f634","src/arch/aarch64/neon/packedpair.rs":"32d3e4cd0dd9b6c8382e5308cbd896d20242c90b12862c44a5de6a8b4d6126df","src/arch/all/memchr.rs":"b0b1214aa573ed5d02ae62a77c42c773065566b50274d4096e37817d65ab1594","src/arch/all/mod.rs":"05f3fc2b069682eb1545fc6366d167bb620a454365dac8b8dd6cde6cd64de18a","src/arch/all/packedpair/default_rank.rs":"abffd1b5b8b7a3be95c03dd1105b905c246a379854dc56f1e846ea7c4408f2c7","src/arch/all/packedpair/mod.rs":"292b66042c5b5c78bba33db6526aeae6904db803d601fcdd29032b87b3eb3754","src/arch/all/rabinkarp.rs":"06038cce1cbead7b088e541ba41bcc1c095c34bde483b7df6ad9f77a49c1063c","src/arch/all/shiftor.rs":"0d79117f52a1e4795843603a3bb0b45397df4ad5e4184bbc923658dab9dc3b5f","src/arch/all/twoway.rs":"0b27eb1cd7402c1375a50466f8f66a68cb2806c3f69a647340b2759793eed4f8","src/arch/generic/memchr.rs":"ee2a77afcbf506655e352eb3cd42451a70cc2c39e3da487b2985638bc09ced3e","src/arch/generic/mod.rs":"1dd75f61e0ea2563b8205a08aaa7b55500130aa331d18b9e9f995724b66c7a39","src/arch/generic/packedpair.rs":"892d6ee28f0f5007c973d11d697e43e6f6888341b15ea95ce4571e9584069dc5","src/arch/mod.rs":"ca3960b7e2ed28d1b3c121710a870430531aad792f64d4dcb4ca4709d6cbda30","src/arch/wasm32/memchr.rs":"d88ac79f891d8530f505f5035062d3da274a05d66c611480c75430d52709d052","src/arch/wasm32/mod.rs":"a20377aa8fe07d68594879101dc73061e4f51d9c8d812b593b1f376e3c8add79","src/arch/wasm32/simd128/memchr.rs":"bac2c4c43fe710c83a6f2b1118fede043be89dd821d4b532907f129f09fdb5cf","src/arch/wasm32/simd128/mod.rs":"c157b373faedbfd65323be432e25bc411d97aa1b7bc58e76048614c7b2bf3bf6","src/arch/wasm32/simd128/packedpair.rs":"288ba6e5eee6a7a8e5e45c64cff1aa5d72d996c2a6bc228be372c75789f08e45","src/arch/x86_64/avx2/memchr.rs":"576ec0c30f49874f7fd9f6caeb490d56132c0fbbaa4d877b1aa532cafce19323","src/arch/x86_64/avx2/mod.rs":"0033d1b712d0b10f0f273ef9aa8caa53e05e49f4c56a64f39af0b9df97eec584","src/arch/x86_64/avx2/packedpair.rs":"87b69cb4301815906127db4f6370f572c7c5d5dad35c0946c00ad888dbcaec8c","src/arch/x86_64/memchr.rs":"4cf0611c926c3016d1b640880ef14e5654a8d1c1b480fe9a5965956be7eeeeef","src/arch/x86_64/mod.rs":"61b2aa876942fd3e78714c2ae21e356c8634545c06995020f443fa50218df027","src/arch/x86_64/sse2/memchr.rs":"0de0444e26d885eaf866220578752aac871e03bebee7b4f5de7fe8a35f5fa97f","src/arch/x86_64/sse2/mod.rs":"38b70ae52a64ec974dbb91d04d6ca8013d9e06d1fe4af852206bbc2faf1c59aa","src/arch/x86_64/sse2/packedpair.rs":"ad893c5ab9525966693e742673fb666f54ff9102949fc91c5e61642c8ad2f110","src/cow.rs":"2b053e393d3841f780425b3b0da5bec4f187603fa7c271b045ecb7d885b23395","src/ext.rs":"210f89d1e32211bc64414cbd56e97b4f56ce8a8832d321d77a9fe519634e27ea","src/lib.rs":"614f778a41e88a29ea0ceb8e92c839dbb6b5a61c967f8bfd962975e18f932c71","src/macros.rs":"3e4b39252bfa471fad384160a43f113ebfec7bec46a85d16f006622881dd2081","src/memchr.rs":"6ae779ec5d00f443075316e0105edf30b489a38e2e96325bec14ccecd014145b","src/memmem/mod.rs":"d869159f0d0c1915d7acc42a6a4aa3c97c049ddc043e6566afeb78ad290995d1","src/memmem/searcher.rs":"84f6a23bef907696cb672e6898c15fb87008058abc100dde58519d8ec5ebca5d","src/tests/memchr/mod.rs":"269f8e4b4f7f5ea458f27a3c174eb1020ffb2484eeba9464170beb51747df69b","src/tests/memchr/naive.rs":"6a0bee033e5edfb5b1d5769a5fa1c78388f7e9ff7bb91cb67f0ad029289e00e7","src/tests/memchr/prop.rs":"1854eea2338c405fe4635aac430f51e10d2069cd37a7489ddaff47da95f8720b","src/tests/mod.rs":"7cec8f809e279310a465c6a7725087970f219a676cc76c83de30c695bb490740","src/tests/packedpair.rs":"b02ec4fbb61a8653cb5f2268c31bc9168b8043347f2abdcc74081acf83b98e15","src/tests/substring/mod.rs":"c7660d10749363ac4687e7da2b5fda60768230425df8ba416c0c28b8d56a5c74","src/tests/substring/naive.rs":"df6f55d165382b8a53762ba4c324926cac13ebc62cde1805f4ce08740b326483","src/tests/substring/prop.rs":"38c15992609b5681a95d838ae6f2933e00a1219f2c971bfba245f96e0729fcdc","src/vector.rs":"9b104d0bda9b8e18b10c41633513637af84346282bf113f3be3813725605fe31"},"package":"cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml deleted file mode 100644 index a03a2ad6..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/memchr/Cargo.toml +++ /dev/null @@ -1,89 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -rust-version = "1.61" -name = "memchr" -version = "2.8.3" -authors = [ - "Andrew Gallant ", - "bluss", -] -build = false -exclude = [ - "/.github", - "/benchmarks", - "/fuzz", - "/scripts", - "/tmp", -] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = """ -Provides extremely fast (uses SIMD on x86_64, aarch64 and wasm32) routines for -1, 2 or 3 byte search and single substring search. -""" -homepage = "https://github.com/BurntSushi/memchr" -documentation = "https://docs.rs/memchr/" -readme = "README.md" -keywords = [ - "memchr", - "memmem", - "substring", - "find", - "search", -] -license = "Unlicense OR MIT" -repository = "https://github.com/BurntSushi/memchr" - -[package.metadata.docs.rs] -rustdoc-args = ["--generate-link-to-definition"] - -[features] -alloc = [] -default = ["std"] -libc = [] -logging = ["dep:log"] -rustc-dep-of-std = ["core"] -std = ["alloc"] -use_std = ["std"] - -[lib] -name = "memchr" -path = "src/lib.rs" -bench = false - -[dependencies.core] -version = "1.0.0" -optional = true -package = "rustc-std-workspace-core" - -[dependencies.log] -version = "0.4.20" -optional = true - -[dev-dependencies.quickcheck] -version = "1.0.3" -default-features = false - -[profile.bench] -debug = 2 - -[profile.release] -debug = 2 - -[profile.test] -opt-level = 3 -debug = 2 diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json deleted file mode 100644 index 9fbc6043..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/.cargo-checksum.json +++ /dev/null @@ -1 +0,0 @@ -{"files":{".cargo_vcs_info.json":"7e77363b21d4f614eb3292289ef0816d753bbed1f3a3253698f2d548defa1bfd",".github/FUNDING.yml":"b017158736b3c9751a2d21edfce7fe61c8954e2fced8da8dd3013c2f3e295bd9",".github/workflows/ci.yml":"8cddd19b88f90df53271d384c62997ba13fe1e9a017e6ff082401c81d46f3be6","Cargo.toml":"b639e399e32159609ed573ffdb1a4edf06fba8aee208d9259e9dba031834aaaa","Cargo.toml.orig":"0a9a6dd2be15fb91d0ad7c545a99a509bdd3a32a4ff795f85fd70b60e91fad0a","LICENSE-APACHE":"62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"cd8f304fa9cca9566dff0bb3492beae55f47f3972ddfb8f1fa38c3da1068b4a8","build.rs":"dba46ae4291317fb644ba2143f44eaf54a8ab946ba1367a33d055d694715f68a","src/attr.rs":"e8301f019e107ef9f5d8a3bbed355ffa9c0fd73396646d7fedbba333d341b7bc","src/error.rs":"be08d9d48b4cb9984b5141b12f21c31bd7293942c18b3ce25754723930cf54f5","src/lib.rs":"12a0578bb1d011ae759a94aa4850e970fc1c74102d1ede62b9f405675d7e182e","src/segment.rs":"462be04eea3ad48028d0841e8000c13205eb049522b8003bcc2926f21e93cb78","tests/compiletest.rs":"022a8e400ef813d7ea1875b944549cee5125f6a995dc33e93b48cba3e1b57bd1","tests/test_attr.rs":"39a55dcce5a127b2f85e9127a810b4aa3318e408cac93ff678301a2120aac5b5","tests/test_doc.rs":"7afb22427cb1c54fcf75a581fec5503404d8e93274027040b4d901f8caa597bb","tests/test_expr.rs":"d25ff118a78bd8e7d38ff532f0b9abb046bd0fbe61db8080b973bff67743af4e","tests/test_item.rs":"ee373a6658b384d1b2a0053653ac18e19d4053bbd04c8024980340894c04dcdc","tests/ui/case-warning.rs":"6b7cb803527911fe698e7fbe080160bc5d07c0da36fac0ec29d599753ce8bd10","tests/ui/case-warning.stderr":"7b7d470edc2a11b0864534099b360e0c42d423e068f49cea7858e0b340c2ef3d","tests/ui/env-empty.rs":"093fb0844475f4710580b5e4cad162918896d4fb48d1516bae04eb39dddd265a","tests/ui/env-empty.stderr":"3462f06f6f21130a81efff3be3bc838b395f6107bfbf753e479ec5a60e09b15d","tests/ui/env-non-string.rs":"516460860e7790b412b0fcd203289720a155dfba2498616967e4d7e6e6ab4990","tests/ui/env-non-string.stderr":"fea08d588a238e0f4c06de5c5ae97ac195bb2bcfc070040fb8087bdf4d1f4eee","tests/ui/env-suffix.rs":"0f1756608efc7a6c5810b8b21528092546ac95371e048c6060064e747c1b0a27","tests/ui/env-suffix.stderr":"287fecb2a91abc61501e0bd71d7f111670a526960ab18a554f4d16ed83d940b5","tests/ui/env-unexpected.rs":"18f06317e72ea16dae6ebe7c20bf5ef26b46e01a5c54fd31dc2a351ba68e8ca5","tests/ui/env-unexpected.stderr":"14d9a40160daf0191efc2b53be5fdde5b066674053e76af170e58af08f4022aa","tests/ui/invalid-ident.rs":"c3f375cac04d56684e427834076128b6e77eeeb9e99067a06307e27693b0ff51","tests/ui/invalid-ident.stderr":"b56d1c26253dec71ba1f97ffa01602b3a4429fc85a6b5c87f5290aab0568ea30","tests/ui/missing-paren-on-env.rs":"05cafef972e8c99f82834e164e7713f6056ac1a24b1f31d0320ac4fe361a53bd","tests/ui/missing-paren-on-env.stderr":"baf7b420d201f033e2113eaebd85376047397d991c35c441f07e5c6f1ec2b685","tests/ui/no-env-var.rs":"330bb71f2f3d98d23176acc5aeb1c0b66443af14c0857bb5bc92bfd130eb4a62","tests/ui/no-env-var.stderr":"68af072d3ccb2537bca9a5cd6d5796bb5ab5408f59564c8e8c66b1590ac7c5a5","tests/ui/no-ident-after-colon.rs":"a5af9548ab8d7f5b6130e1410c90f2cc589c595eb99f6939e6deb3f54e7262a2","tests/ui/no-ident-after-colon.stderr":"efac036a78f077c9c10a1ed048d69531293dcb97784ed784af0b24788ecbab54","tests/ui/unexpected-group.rs":"6b2e398a9ad3e077ea49c68b7c134ec3268e398723f4d9d5ef7b4d7b8513ac44","tests/ui/unexpected-group.stderr":"1260b5a4c295639958e3dd0da475c385184f0a916f5fdc4f1f2bfe232688f43c","tests/ui/unexpected-modifier.rs":"8b7e4c93f06fe7e7be469375e8bbaf19fec5fdffe688a0c29d2de2944a420337","tests/ui/unexpected-modifier.stderr":"581212b89358ac6bd38632bfd4322c25c171046875711fd639cfb498e01629c8","tests/ui/unexpected-punct.rs":"430e98bc03abdcc46f72cea01264c278a9c25f3d1d63064a6d78f4ae71b6a2d9","tests/ui/unexpected-punct.stderr":"24ab2e32d2da8dad93ef04dc05a2ecf6fdeb11bdebeae9454ad4759cc2b16551","tests/ui/unsupported-literal.rs":"1b2d3634fba5dc35d5ac9012d488aa93e329d1fa4489df1eadee65613a010e59","tests/ui/unsupported-literal.stderr":"6b137ccd6bc2a341aeb1e0d5321a896aa6c8db9a4e2571f4e41bd744511760bf","tests/ui/unsupported-modifier.rs":"03bb0239880070e8b470a6072c06094e1483ea91256e7cb4b9f2e8761696fa5a","tests/ui/unsupported-modifier.stderr":"890d7f484fdbd46da72923db4d7aba0ac3eaece253bb7195b7a36ae9a918027e"},"package":"57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"} \ No newline at end of file diff --git a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml b/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml deleted file mode 100644 index c5872871..00000000 --- a/tests/fixtures/rust-bundle/cargo-recorded/vendor/paste/Cargo.toml +++ /dev/null @@ -1,72 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2018" -rust-version = "1.31" -name = "paste" -version = "1.0.15" -authors = ["David Tolnay "] -build = "build.rs" -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Macros for all your token pasting needs" -documentation = "https://docs.rs/paste" -readme = "README.md" -keywords = ["macros"] -categories = [ - "development-tools", - "no-std", - "no-std::no-alloc", -] -license = "MIT OR Apache-2.0" -repository = "https://github.com/dtolnay/paste" - -[package.metadata.docs.rs] -rustdoc-args = ["--generate-link-to-definition"] -targets = ["x86_64-unknown-linux-gnu"] - -[lib] -name = "paste" -path = "src/lib.rs" -proc-macro = true - -[[test]] -name = "test_item" -path = "tests/test_item.rs" - -[[test]] -name = "test_attr" -path = "tests/test_attr.rs" - -[[test]] -name = "compiletest" -path = "tests/compiletest.rs" - -[[test]] -name = "test_doc" -path = "tests/test_doc.rs" - -[[test]] -name = "test_expr" -path = "tests/test_expr.rs" - -[dev-dependencies.paste-test-suite] -version = "0" - -[dev-dependencies.rustversion] -version = "1.0" - -[dev-dependencies.trybuild] -version = "1.0.49" -features = ["diff"] diff --git a/tests/fixtures/rust-bundle/includes/Cargo.lock b/tests/fixtures/rust-bundle/includes/Cargo.lock deleted file mode 100644 index fc3da2fe..00000000 --- a/tests/fixtures/rust-bundle/includes/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "inc" -version = "0.1.0" diff --git a/tests/fixtures/rust-bundle/includes/Cargo.toml b/tests/fixtures/rust-bundle/includes/Cargo.toml deleted file mode 100644 index 28a41d09..00000000 --- a/tests/fixtures/rust-bundle/includes/Cargo.toml +++ /dev/null @@ -1,4 +0,0 @@ -[package] -name = "inc" -version = "0.1.0" -edition = "2021" diff --git a/tests/fixtures/rust-bundle/includes/README.md b/tests/fixtures/rust-bundle/includes/README.md deleted file mode 100644 index 898d009c..00000000 --- a/tests/fixtures/rust-bundle/includes/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# inc - -A fixture crate. diff --git a/tests/fixtures/rust-bundle/includes/build.rs b/tests/fixtures/rust-bundle/includes/build.rs deleted file mode 100644 index 9561dc40..00000000 --- a/tests/fixtures/rust-bundle/includes/build.rs +++ /dev/null @@ -1,8 +0,0 @@ -// The package's build script: a crate root of its own, with a module beside it. -#[path = "build/helper.rs"] -mod helper; - -fn main() { - helper::emit(); - println!("cargo:rerun-if-changed=build.rs"); -} diff --git a/tests/fixtures/rust-bundle/includes/build/helper.rs b/tests/fixtures/rust-bundle/includes/build/helper.rs deleted file mode 100644 index 6b1e61a9..00000000 --- a/tests/fixtures/rust-bundle/includes/build/helper.rs +++ /dev/null @@ -1 +0,0 @@ -pub fn emit() {} diff --git a/tests/fixtures/rust-bundle/includes/data/blob.bin b/tests/fixtures/rust-bundle/includes/data/blob.bin deleted file mode 100644 index 717c6b3086b9625de0ba7338b4f3d5fdc3ca8a0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZSh|Bn#>1pWd4 diff --git a/tests/fixtures/rust-bundle/includes/data/table.txt b/tests/fixtures/rust-bundle/includes/data/table.txt deleted file mode 100644 index cfa20f81..00000000 --- a/tests/fixtures/rust-bundle/includes/data/table.txt +++ /dev/null @@ -1,2 +0,0 @@ -a,b -1,2 diff --git a/tests/fixtures/rust-bundle/includes/src/gated.rs b/tests/fixtures/rust-bundle/includes/src/gated.rs deleted file mode 100644 index e3b036e2..00000000 --- a/tests/fixtures/rust-bundle/includes/src/gated.rs +++ /dev/null @@ -1,4 +0,0 @@ -#![cfg(feature = "never")] - -mod ghost; -use phantom_crate::Thing; diff --git a/tests/fixtures/rust-bundle/includes/src/generated/consts.rs b/tests/fixtures/rust-bundle/includes/src/generated/consts.rs deleted file mode 100644 index 8dc64d84..00000000 --- a/tests/fixtures/rust-bundle/includes/src/generated/consts.rs +++ /dev/null @@ -1 +0,0 @@ -pub const LIMIT: u32 = 3; diff --git a/tests/fixtures/rust-bundle/includes/src/lib.rs b/tests/fixtures/rust-bundle/includes/src/lib.rs deleted file mode 100644 index 2d51fdbc..00000000 --- a/tests/fixtures/rust-bundle/includes/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -#![doc = include_str!("../README.md")] - -#[macro_use] -mod macros; - -include!("generated/consts.rs"); - -pub const TABLE: &str = include_str!("../data/table.txt"); -pub const BLOB: &[u8] = include_bytes!("../data/blob.bin"); - -pub fn go() -> u32 { - ready!(1) + LIMIT -} - -#[cfg(test)] -const TEST_ONLY: &str = include_str!("../data/test-only.txt"); - -mod gated; diff --git a/tests/fixtures/rust-bundle/includes/src/macros.rs b/tests/fixtures/rust-bundle/includes/src/macros.rs deleted file mode 100644 index f7692b32..00000000 --- a/tests/fixtures/rust-bundle/includes/src/macros.rs +++ /dev/null @@ -1,5 +0,0 @@ -macro_rules! ready { - ($e:expr) => { - $e - }; -} diff --git a/tests/rust-features.test.js b/tests/rust-features.test.js index 92f1f006..54de356c 100644 --- a/tests/rust-features.test.js +++ b/tests/rust-features.test.js @@ -19,6 +19,7 @@ import { } from '../stasis/src/loaders/cargo.js' import { buildRustBundle } from '../stasis/src/cmd/bundle.js' import { buildRustTree, collectRustBundle, collectRustFilesFromDisk } from '../stasis/src/loaders/rust.js' +import { rustFixture } from './rust-fixtures.helper.js' const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'rust-bundle') const featuresFixture = join(fixtures, 'features') @@ -1106,7 +1107,7 @@ test('createCargoContext finds vendored crates in the directory .cargo/config.to // --- bundling --- test('buildRustBundle carries include!d source and include_str!/include_bytes! assets as resources', async (t) => { - const bundle = await buildRustBundle({ cwd: join(fixtures, 'includes'), entries: ['src/lib.rs'] }) + const bundle = await buildRustBundle({ cwd: rustFixture('includes'), entries: ['src/lib.rs'] }) t.assert.deepEqual(sorted(bundle.sources.keys()), ['README.md', 'data/blob.bin', 'data/table.txt', 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs']) t.assert.deepEqual([bundle.formats.get('src/generated/consts.rs'), bundle.formats.get('data/table.txt'), bundle.formats.get('data/blob.bin')], ['rust', 'resource', 'resource:base64']) t.assert.equal(bundle.sources.get('data/blob.bin'), Buffer.from([0, 0xff, 0xfe, 1]).toString('base64')) @@ -1374,7 +1375,7 @@ test('buildRustBundle ends when a crate root it wants is refused, and reports ev // resolver 1 and 2, null where cargo refused one, and the targets' `rustc --print cfg`. It is // @preventive/lockfile's fixture (PreventiveMeasures/libraries#56, MIT), recorded there by // lockfile/scripts/record-cargo.js. -const cargoRecorded = join(fixtures, 'cargo-recorded') +const cargoRecorded = rustFixture('cargo-recorded') const recordedProject = (resolver) => { const tmp = mkdtempSync(join(tmpdir(), 'stasis-recorded-')) cpSync(cargoRecorded, tmp, { recursive: true }) diff --git a/tests/rust-fixtures.helper.js b/tests/rust-fixtures.helper.js new file mode 100644 index 00000000..802466df --- /dev/null +++ b/tests/rust-fixtures.helper.js @@ -0,0 +1,77 @@ +import { isUtf8 } from 'node:buffer' +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join, relative } from 'node:path' +import { fileURLToPath } from 'node:url' +import { brotliCompressSync, brotliDecompressSync, constants } from 'node:zlib' + +// The Rust fixture projects the loader tests read from disk, held in one file, +// tests/fixtures/rust-bundle.json.br: brotli-compressed JSON of project name -> project-relative +// path -> the file's text, or `{ base64 }` for one that isn't UTF-8. +// +// - `includes`: a crate with every include form, its assets and a build script. +// - `cargo-recorded`: @preventive/lockfile's Cargo workspace fixture with the builds cargo +// recorded for it (from PreventiveMeasures/libraries#56, MIT). +// +// To edit them: `node tests/rust-fixtures.helper.js unpack `, change the files under +// `//`, then `node tests/rust-fixtures.helper.js pack `. + +const archive = fileURLToPath(new URL('fixtures/rust-bundle.json.br', import.meta.url)) + +const read = () => JSON.parse(brotliDecompressSync(readFileSync(archive)).toString('utf8')) +const write = (dir, files) => { + for (const [rel, content] of Object.entries(files)) { + mkdirSync(dirname(join(dir, rel)), { recursive: true }) + writeFileSync(join(dir, rel), typeof content === 'string' ? content : Buffer.from(content.base64, 'base64')) + } +} + +// The directory holding fixture project `name`, written out once per process (and removed when it +// exits) for the tests to read; never to be written to. +let projects = null +let root = null +const written = new Map() +export function rustFixture(name) { + if (!written.has(name)) { + projects ??= read() + if (!Object.hasOwn(projects, name)) throw new Error(`No Rust fixture project ${name} in ${archive}`) + if (root === null) { + root = mkdtempSync(join(tmpdir(), 'stasis-rust-fixtures-')) + process.on('exit', () => rmSync(root, { recursive: true, force: true })) + } + write(join(root, name), projects[name]) + written.set(name, join(root, name)) + } + return written.get(name) +} + +// `//…` -> the archive, every file of every project, in sorted order. +function pack(dir) { + const projectsOut = {} + for (const name of readdirSync(dir).toSorted()) { + const files = {} + const walk = (at) => { + for (const entry of readdirSync(at, { withFileTypes: true }).toSorted((a, b) => (a.name < b.name ? -1 : 1))) { + const path = join(at, entry.name) + if (entry.isDirectory()) walk(path) + else { + const buf = readFileSync(path) + files[relative(join(dir, name), path).split('\\').join('/')] = isUtf8(buf) ? buf.toString('utf8') : { base64: buf.toString('base64') } + } + } + } + walk(join(dir, name)) + projectsOut[name] = files + } + writeFileSync(archive, brotliCompressSync(`${JSON.stringify(projectsOut, null, 1)}\n`, { params: { [constants.BROTLI_PARAM_QUALITY]: 11, [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT } })) +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const [command, dir] = process.argv.slice(2) + if (command === 'pack' && dir) pack(dir) + else if (command === 'unpack' && dir) for (const [name, files] of Object.entries(read())) write(join(dir, name), files) + else { + console.error('usage: node tests/rust-fixtures.helper.js (pack|unpack) ') + process.exitCode = 1 + } +} diff --git a/tests/rust-loader.test.js b/tests/rust-loader.test.js index 7fc4de06..c3ea33cc 100644 --- a/tests/rust-loader.test.js +++ b/tests/rust-loader.test.js @@ -18,6 +18,7 @@ import { resolveVendoredCrate, scanRustItems, } from '../stasis/src/loaders/rust.js' +import { rustFixture } from './rust-fixtures.helper.js' const fixtures = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'rust-bundle') @@ -1102,7 +1103,7 @@ test('buildRustTree records include edges and edges to the file defining a macro }) test('collectRustFilesFromDisk follows include!, carries include_str!/include_bytes! assets with their format, and leaves a dead include alone', async (t) => { - const baseDir = join(fixtures, 'includes') + const baseDir = rustFixture('includes') const formats = new Map() const sources = await collectRustFilesFromDisk(baseDir, ['src/lib.rs'], { formats }) t.assert.deepEqual([...sources.keys()].toSorted(), ['README.md', 'data/blob.bin', 'data/table.txt', 'src/gated.rs', 'src/generated/consts.rs', 'src/lib.rs', 'src/macros.rs'])