diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 420e6045c0..91859de949 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -322,6 +322,11 @@ openshell sandbox create -- codex ### Rust build cache +The `openshell-core` build script embeds a version derived from Git. It +reruns when HEAD, local branches or tags change, in clones and linked +worktrees, even if no Rust source changed. Source archives without Git +metadata use the package version. + Mise preserves an existing `SCCACHE_DIR` so each environment can choose where to store compiler cache entries. When `SCCACHE_DIR` is unset, OpenShell uses the worktree-local `.cache/sccache` directory. To make cache entries available diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index 2779e78fb9..c9f172ecff 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -4,6 +4,7 @@ use std::env; use std::path::{Path, PathBuf}; +mod build_git; mod build_version; const PROTO_REL: &str = "../../proto"; @@ -35,11 +36,10 @@ fn main() -> Result<(), Box> { // Docker/CI builds where .git is absent, this silently does nothing and // the binary falls back to CARGO_PKG_VERSION (which is already sed-patched // by the build pipeline). - for path in git_rerun_paths() { - println!("cargo:rerun-if-changed={}", path.display()); - } - - if let Some(version) = git_version() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); + let source_root = manifest_dir.ancestors().nth(2).unwrap_or(&manifest_dir); + build_git::emit_rerun_if_changed(source_root); + if let Some(version) = build_git::version(source_root) { println!("cargo:rustc-env=OPENSHELL_GIT_VERSION={version}"); } @@ -59,7 +59,6 @@ fn main() -> Result<(), Box> { None }; - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); let proto_root = manifest_dir.join(PROTO_REL); let mut proto_files = Vec::new(); collect_proto_files(&proto_root, &mut proto_files)?; @@ -104,74 +103,3 @@ fn collect_proto_files(dir: &Path, out: &mut Vec) -> std::io::Result<() } Ok(()) } - -/// Git metadata files whose changes affect [`git_version`]. -/// -/// In a linked worktree `.git` is a file, so `HEAD` and `logs/HEAD` live in the -/// per-worktree git dir while branch and tag refs live in the common git dir. Cargo treats a -/// missing `rerun-if-changed` path as always stale, so only existing paths are -/// returned. Without git metadata nothing is returned. -fn git_rerun_paths() -> Vec { - let resolve = |args: &[&str]| { - let dir = PathBuf::from(git_output(args)?); - // Relative output is relative to the build script's working directory. - Some(env::current_dir().ok()?.join(dir)) - }; - let mut paths = Vec::new(); - if let Some(git_dir) = resolve(&["rev-parse", "--git-dir"]) { - paths.push(git_dir.join("HEAD")); - paths.push(git_dir.join("logs/HEAD")); - } - if let Some(common_dir) = resolve(&["rev-parse", "--git-common-dir"]) { - paths.push(common_dir.join("refs/tags")); - paths.push(common_dir.join("packed-refs")); - // A new worktree has no `logs/HEAD` until its first ref update, so also - // watch the checked-out branch ref, which moves on every commit. - if let Some(branch) = git_output(&["symbolic-ref", "-q", "HEAD"]) { - paths.push(common_dir.join(branch)); - } - } - paths.retain(|path| path.exists()); - paths -} - -/// Derive the release or development version from git metadata. -/// -/// Implements the "guess-next-dev" convention used by the release pipeline -/// (`tasks/scripts/release.py`): exact stable and prerelease tags retain their -/// version. Otherwise, the latest merged stable release gets a patch bump and -/// `-dev.+g` is appended. -/// -/// Examples: -/// on tag v0.1.0-pre.1 → "0.1.0-pre.1" -/// 3 commits past v0.0.3 → "0.0.4-dev.3+g2bf9969ab" -/// -/// Returns `None` when git metadata cannot be read. -fn git_version() -> Option { - let exact_tags = git_output(&["tag", "--points-at", "HEAD"])?; - if let Some(version) = build_version::exact_release_version(exact_tags.lines()) { - return Some(version); - } - - let merged_tags = git_output(&["tag", "--merged", "HEAD", "--list", "v*.*.*"])?; - let latest_tag = build_version::latest_stable_tag(merged_tags.lines()); - let revision_range = latest_tag - .as_deref() - .map_or_else(|| "HEAD".to_string(), |tag| format!("{tag}..HEAD")); - let distance = git_output(&["rev-list", "--count", &revision_range])? - .parse() - .ok()?; - let sha = git_output(&["rev-parse", "--short=9", "HEAD"])?; - - build_version::next_dev_version(latest_tag.as_deref(), distance, &sha) -} - -fn git_output(args: &[&str]) -> Option { - let output = std::process::Command::new("git").args(args).output().ok()?; - if !output.status.success() { - return None; - } - String::from_utf8(output.stdout) - .ok() - .map(|output| output.trim().to_string()) -} diff --git a/crates/openshell-core/build_git.rs b/crates/openshell-core/build_git.rs new file mode 100644 index 0000000000..4cb4cfe139 --- /dev/null +++ b/crates/openshell-core/build_git.rs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeSet; +use std::path::Path; +use std::process::Command; + +use crate::build_version; + +/// Track Git's resolved metadata, including creation of previously packed refs. +/// +/// `source_root` is the source checkout. Git metadata is used only when that +/// directory has its own `.git`, so an archive nested in another repository +/// neither adopts nor watches that repository. +pub fn emit_rerun_if_changed(source_root: &Path) { + // Ref directories must be watched whole: a packed branch has no loose file + // until its next update. Watching only existing individual refs misses it. + // Only heads and tags feed the version; remotes and stash would rebuild + // on every fetch or stash. reftable/ covers that ref backend. + let mut paths = BTreeSet::new(); + for name in [ + "HEAD", + "refs/heads", + "refs/tags", + "packed-refs", + "reftable", + "shallow", + ] { + let Some(path) = git_output(source_root, &["rev-parse", "--git-path", name]) else { + continue; + }; + paths.insert(source_root.join(path)); + } + // A linked worktree's reftable holds its private refs, including HEAD. + // Shared branches and tags live in the common directory's separate table. + if let Some(common_dir) = git_output(source_root, &["rev-parse", "--git-common-dir"]) { + paths.insert(source_root.join(common_dir).join("reftable")); + } + for path in paths { + // Missing Git metadata is normal in archives and container builds. + // Cargo reruns indefinitely for a watched file that never exists. + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + } + } +} + +/// Derive the release or development version from git metadata. +/// +/// Implements the "guess-next-dev" convention used by the release pipeline +/// (`tasks/scripts/release.py`): exact stable and prerelease tags retain their +/// version. Otherwise, the latest merged stable release gets a patch bump and +/// `-dev.+g` is appended. +/// +/// Examples: +/// on tag v0.1.0-pre.1 → "0.1.0-pre.1" +/// 3 commits past v0.0.3 → "0.0.4-dev.3+g2bf9969ab" +/// +/// Returns `None` when `source_root` has no `.git` of its own or its metadata +/// cannot be read. +pub fn version(source_root: &Path) -> Option { + let exact_tags = git_output(source_root, &["tag", "--points-at", "HEAD"])?; + if let Some(version) = build_version::exact_release_version(exact_tags.lines()) { + return Some(version); + } + + let merged_tags = git_output( + source_root, + &["tag", "--merged", "HEAD", "--list", "v*.*.*"], + )?; + let latest_tag = build_version::latest_stable_tag(merged_tags.lines()); + let revision_range = latest_tag + .as_deref() + .map_or_else(|| "HEAD".to_string(), |tag| format!("{tag}..HEAD")); + let distance = git_output(source_root, &["rev-list", "--count", &revision_range])? + .parse() + .ok()?; + let sha = git_output(source_root, &["rev-parse", "--short=9", "HEAD"])?; + + build_version::next_dev_version(latest_tag.as_deref(), distance, &sha) +} + +fn git_output(source_root: &Path, args: &[&str]) -> Option { + // A linked worktree has a `.git` file and a regular checkout a directory. + // Without either, Git would search upward into an enclosing repository. + if !source_root.join(".git").exists() { + return None; + } + let output = Command::new("git") + .args(args) + .current_dir(source_root) + .output() + .ok()?; + if !output.status.success() { + return None; + } + String::from_utf8(output.stdout) + .ok() + .map(|output| output.trim().to_string()) +} diff --git a/crates/openshell-core/tests/git_build_invalidation.rs b/crates/openshell-core/tests/git_build_invalidation.rs new file mode 100644 index 0000000000..a5d8319223 --- /dev/null +++ b/crates/openshell-core/tests/git_build_invalidation.rs @@ -0,0 +1,358 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exercise Cargo's actual invalidation with the production Git/version helper. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +struct Fixture { + temp: tempfile::TempDir, + repo: PathBuf, + checkout: PathBuf, + target: PathBuf, +} + +struct Build { + version: String, + runs: usize, +} + +fn output(command: &mut Command) -> String { + let output = command.output().expect("command should start"); + assert!( + output.status.success(), + "{command:?} failed: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + String::from_utf8(output.stdout) + .expect("command output should be UTF-8") + .trim() + .to_owned() +} + +fn git(repo: &Path, args: &[&str]) -> String { + output(git_command(repo).args(args)) +} + +fn isolate_git(command: &mut Command) -> &mut Command { + // Hooks and CI can export Git context. Fixture ref writes must never use + // the parent checkout's Git directory, object store, config, or hooks. + for name in [ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CONFIG", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_PARAMETERS", + "GIT_TEMPLATE_DIR", + ] { + command.env_remove(name); + } + command.env("GIT_CONFIG_NOSYSTEM", "1").env( + "GIT_CONFIG_GLOBAL", + if cfg!(windows) { "NUL" } else { "/dev/null" }, + ) +} + +fn git_command(repo: &Path) -> Command { + let mut command = Command::new("git"); + isolate_git(&mut command).arg("-C").arg(repo); + command +} + +fn object(repo: &Path, kind: &str, body: &str) -> String { + let mut child = git_command(repo) + .args(["hash-object", "-t", kind, "-w", "--stdin"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("object writer should start"); + child + .stdin + .take() + .unwrap() + .write_all(body.as_bytes()) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + String::from_utf8(output.stdout).unwrap().trim().to_owned() +} + +impl Fixture { + fn new(linked: bool) -> Self { + Self::with_ref_format(linked, None) + } + + fn with_ref_format(linked: bool, ref_format: Option<&str>) -> Self { + let temp = tempfile::tempdir().unwrap(); + let repo = temp.path().join("repository with spaces"); + fs::create_dir(&repo).unwrap(); + let mut init = git_command(&repo); + init.args(["init", "--quiet", "--initial-branch=main"]); + if let Some(ref_format) = ref_format { + init.arg(format!("--ref-format={ref_format}")); + } + output(&mut init); + // Inert objects exercise ref movement without user identity, signing, + // hooks, or a commit command that can prompt for a hardware key. + let tree = object(&repo, "tree", ""); + let initial = object( + &repo, + "commit", + &format!( + "tree {tree}\nauthor Fixture 1700000000 +0000\ncommitter Fixture 1700000000 +0000\n\ninitial\n" + ), + ); + git(&repo, &["update-ref", "refs/heads/main", &initial]); + git(&repo, &["update-ref", "refs/tags/v0.1.0", &initial]); + let checkout = if linked { + let checkout = temp.path().join("linked worktree"); + git( + &repo, + &[ + "worktree", + "add", + "--quiet", + "-b", + "linked", + checkout.to_str().unwrap(), + ], + ); + checkout + } else { + repo.clone() + }; + let fixture = Self { + target: temp.path().join("target"), + temp, + repo, + checkout, + }; + fixture.install_crate(); + fixture + } + + fn install_crate(&self) { + fs::create_dir_all(self.checkout.join("src")).unwrap(); + fs::write(self.checkout.join("Cargo.toml"), "[package]\nname = \"git-version-probe\"\nversion = \"9.8.7\"\nedition = \"2021\"\n[workspace]\n").unwrap(); + fs::write( + self.checkout.join("src/main.rs"), + "fn main() { println!(\"{}\", env!(\"PROBE_VERSION\")); }\n", + ) + .unwrap(); + fs::write( + self.checkout.join("build_git.rs"), + include_bytes!("../build_git.rs"), + ) + .unwrap(); + fs::write( + self.checkout.join("build_version.rs"), + include_bytes!("../build_version.rs"), + ) + .unwrap(); + fs::write(self.checkout.join("build.rs"), r#" +mod build_git; +mod build_version; +use std::io::Write; +fn main() { + let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let manifest = std::path::Path::new(&manifest); + // Stand in for the core build script's explicit protobuf input tracking. + println!("cargo:rerun-if-changed=build.rs"); + build_git::emit_rerun_if_changed(manifest); + let version = build_git::version(manifest).unwrap_or_else(|| "9.8.7".into()); + println!("cargo:rustc-env=PROBE_VERSION={version}"); + let mut runs = std::fs::OpenOptions::new().create(true).append(true).open(manifest.join("probe-runs")).unwrap(); + writeln!(runs, "run").unwrap(); +} +"#).unwrap(); + } + + fn advance(&self) -> String { + let parent = git(&self.checkout, &["rev-parse", "HEAD"]); + let tree = git(&self.checkout, &["rev-parse", "HEAD:"]); + let next = object( + &self.repo, + "commit", + &format!( + "tree {tree}\nparent {parent}\nauthor Fixture 1700000001 +0000\ncommitter Fixture 1700000001 +0000\n\nnext\n" + ), + ); + git(&self.checkout, &["update-ref", "HEAD", &next]); + next + } + + fn build(&self) -> Build { + // Each fixture has its own target; nested Cargo never contends for the + // parent test binary's target lock or a sibling worktree's artifacts. + output( + isolate_git(&mut Command::new(env!("CARGO"))) + .current_dir(&self.checkout) + .args(["build", "--offline", "--quiet"]) + .env("CARGO_TARGET_DIR", &self.target) + .env_remove("RUSTC_WRAPPER") + .env_remove("RUSTC_WORKSPACE_WRAPPER"), + ); + let executable = self + .target + .join("debug") + .join(format!("git-version-probe{}", std::env::consts::EXE_SUFFIX)); + Build { + version: output(&mut Command::new(executable)), + runs: fs::read_to_string(self.checkout.join("probe-runs")) + .unwrap() + .lines() + .count(), + } + } + + fn assert_fresh(&self, previous: &Build) { + let next = self.build(); + assert_eq!(next.version, previous.version); + assert_eq!( + next.runs, previous.runs, + "unchanged inputs must not rerun the build script" + ); + } +} + +#[test] +fn normal_repository_reuses_build_and_refreshes_revision() { + let fixture = Fixture::new(false); + let first = fixture.build(); + assert_eq!(first.version, "0.1.0"); + fixture.assert_fresh(&first); + fixture.advance(); + let next = fixture.build(); + let sha = git(&fixture.checkout, &["rev-parse", "--short=9", "HEAD"]); + assert_eq!(next.version, format!("0.1.1-dev.1+g{sha}")); + assert_eq!(next.runs, first.runs + 1); + fixture.assert_fresh(&next); + // Remote-tracking refs do not feed the version; a fetch must not rebuild. + git( + &fixture.repo, + &["update-ref", "refs/remotes/origin/main", "HEAD"], + ); + fixture.assert_fresh(&next); +} + +#[test] +fn linked_worktree_detects_packed_to_loose_branch_and_tag_changes() { + assert_branch_and_tag_changes(Fixture::new(true)); +} + +#[test] +fn linked_reftable_worktree_refreshes_shared_tags() { + // Older Git installations cannot create reftable repositories. Keep the + // files-backend regressions active there and report this optional skip. + let help = isolate_git(&mut Command::new("git")) + .args(["init", "-h"]) + .output() + .expect("Git should be installed"); + if !String::from_utf8_lossy(&help.stdout).contains("--ref-format") + && !String::from_utf8_lossy(&help.stderr).contains("--ref-format") + { + eprintln!("skipping reftable regression: Git lacks init --ref-format"); + return; + } + assert_branch_and_tag_changes(Fixture::with_ref_format(true, Some("reftable"))); +} + +fn assert_branch_and_tag_changes(fixture: Fixture) { + git(&fixture.repo, &["config", "core.logAllRefUpdates", "false"]); + let head_log = fixture.checkout.join(git( + &fixture.checkout, + &["rev-parse", "--git-path", "logs/HEAD"], + )); + if head_log.exists() { + fs::remove_file(&head_log).unwrap(); + } + git(&fixture.repo, &["pack-refs", "--all"]); + // The existing HEAD reflog must not mask creation of a previously packed + // branch. Neither file is present when Cargo first records its inputs. + assert!(!head_log.exists()); + let branch_ref = fixture.checkout.join(git( + &fixture.checkout, + &["rev-parse", "--git-path", "refs/heads/linked"], + )); + assert!(!branch_ref.exists()); + let first = fixture.build(); + fixture.assert_fresh(&first); + fixture.advance(); + let next = fixture.build(); + assert_ne!(next.version, first.version); + assert_eq!(next.runs, first.runs + 1); + fixture.assert_fresh(&next); + + git( + &fixture.checkout, + &["update-ref", "refs/tags/v0.2.0", "HEAD"], + ); + let tagged = fixture.build(); + assert_eq!(tagged.version, "0.2.0"); + assert_eq!(tagged.runs, next.runs + 1); + fixture.assert_fresh(&tagged); + git(&fixture.repo, &["pack-refs", "--all"]); + let packed = fixture.build(); + assert_eq!(packed.version, tagged.version); + fixture.assert_fresh(&packed); + git(&fixture.repo, &["update-ref", "-d", "refs/tags/v0.2.0"]); + let removed = fixture.build(); + assert_eq!(removed.version, next.version); + assert_eq!(removed.runs, packed.runs + 1); + fixture.assert_fresh(&removed); +} + +#[test] +fn branch_switch_and_detached_head_refresh_version() { + let fixture = Fixture::new(true); + fixture.advance(); + let first = fixture.build(); + git( + &fixture.checkout, + &["switch", "--quiet", "--detach", "v0.1.0"], + ); + let detached = fixture.build(); + assert_eq!(detached.version, "0.1.0"); + assert_eq!(detached.runs, first.runs + 1); + fixture.assert_fresh(&detached); + fixture.advance(); + let advanced = fixture.build(); + assert_ne!(advanced.version, detached.version); + assert_eq!(advanced.runs, detached.runs + 1); + fixture.assert_fresh(&advanced); + git(&fixture.checkout, &["switch", "--quiet", "linked"]); + let attached = fixture.build(); + assert_eq!(attached.version, first.version); + assert_eq!(attached.runs, advanced.runs + 1); + fixture.assert_fresh(&attached); +} + +#[test] +fn source_archive_reuses_build_without_git_metadata() { + let mut fixture = Fixture::new(false); + fixture.checkout = fixture.temp.path().join("archive"); + fixture.install_crate(); + let first = fixture.build(); + assert_eq!(first.version, "9.8.7"); + fixture.assert_fresh(&first); +} + +#[test] +fn archive_nested_in_a_repository_keeps_the_package_version() { + // The enclosing repository has a tagged commit. An archive inside it must + // not adopt that repository's version. + let mut fixture = Fixture::new(false); + fixture.checkout = fixture.repo.join("archive"); + fixture.install_crate(); + let first = fixture.build(); + assert_eq!(first.version, "9.8.7"); + fixture.assert_fresh(&first); +}