From f544a0d2c4292bebdd05fa76938ef3faad170254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Ci=C4=99=C5=BCarkiewicz?= Date: Fri, 24 Jul 2026 11:05:48 -0700 Subject: [PATCH] feat(source): replace source filters with filesets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Replace Flakebox’s bespoke source filtering API with a thin `flakeboxLib.source` namespace built on nixpkgs filesets. The new API validates literal paths, exposes upstream set composition, retains an explicit directory-pruning predicate escape hatch, and offers glob traversal as an explicit alternative. Migrate Flakebox, its template, checks, and documentation to raw path roots and remove the legacy API without compatibility aliases. ### Details Add `filesetFromPaths`, `fromFileset`, `fromPaths`, and feature-detected `fromGlobs`, plus canonical `all`, `any`, `not`, and `excludeDirectoriesNamed` filters. Focused fixtures verify source identity under excluded and selected content changes, literal validation, out-of-root rejection, missing and optional paths, overlap/duplicate set semantics, composition, symlinks, nested pruning, basename boundaries, combinator short-circuiting, and glob behavior. The migration also removes stale nonexistent `.cargo` selections that fileset validation exposed in two check fixtures. ### Reviews An independent holistic review passed with no actionable correctness, completeness, Nix semantics, API, test, documentation, or migration findings. GitHub lint subsequently caught invalid placeholder TOML in the focused fixtures; replace it with valid empty workspace manifests so the repository-wide Cargo check can parse them. ### Validation `nix develop -c selfci check` passed all main, treefmt/lint, and flake-check jobs after the rebase and fixture correction. The repository pre-commit hook, focused source test build, and `nix flake check --no-build` also passed. --- checks/default.nix | 1 + checks/nextest/default.nix | 7 +- checks/source-fixtures/baseline/Cargo.toml | 2 + .../baseline/nested/keep/value.txt | 1 + .../baseline/nested/manifest-link | 1 + .../baseline/nested/specs/ignored.txt | 1 + .../baseline/other/specs/ignored.txt | 1 + checks/source-fixtures/baseline/specs | 1 + .../source-fixtures/baseline/specs2/value.txt | 1 + .../excluded-change/Cargo.toml | 2 + .../excluded-change/nested/keep/value.txt | 1 + .../excluded-change/nested/manifest-link | 1 + .../excluded-change/nested/specs/ignored.txt | 1 + .../excluded-change/other/specs/ignored.txt | 1 + checks/source-fixtures/excluded-change/specs | 1 + .../excluded-change/specs2/value.txt | 1 + .../selected-change/Cargo.toml | 2 + .../selected-change/nested/keep/value.txt | 1 + .../selected-change/nested/manifest-link | 1 + .../selected-change/nested/specs/ignored.txt | 1 + .../selected-change/other/specs/ignored.txt | 1 + checks/source-fixtures/selected-change/specs | 1 + .../selected-change/specs2/value.txt | 1 + checks/source-tests.nix | 143 ++++++++++++++++++ checks/workspace-cross-compile/default.nix | 10 +- checks/workspace-sanity/default.nix | 10 +- docs/best-practices.md | 4 +- docs/building-new-project.md | 18 +-- docs/flakeboxLib.md | 142 +++++++++-------- docs/getting-started.md | 7 +- flake.nix | 7 +- lib/default.nix | 4 +- lib/filter/default.nix | 74 --------- lib/source/default.nix | 90 +++++++++++ templates/default/flake.nix | 7 +- 35 files changed, 364 insertions(+), 184 deletions(-) create mode 100644 checks/source-fixtures/baseline/Cargo.toml create mode 100644 checks/source-fixtures/baseline/nested/keep/value.txt create mode 120000 checks/source-fixtures/baseline/nested/manifest-link create mode 100644 checks/source-fixtures/baseline/nested/specs/ignored.txt create mode 100644 checks/source-fixtures/baseline/other/specs/ignored.txt create mode 100644 checks/source-fixtures/baseline/specs create mode 100644 checks/source-fixtures/baseline/specs2/value.txt create mode 100644 checks/source-fixtures/excluded-change/Cargo.toml create mode 100644 checks/source-fixtures/excluded-change/nested/keep/value.txt create mode 120000 checks/source-fixtures/excluded-change/nested/manifest-link create mode 100644 checks/source-fixtures/excluded-change/nested/specs/ignored.txt create mode 100644 checks/source-fixtures/excluded-change/other/specs/ignored.txt create mode 100644 checks/source-fixtures/excluded-change/specs create mode 100644 checks/source-fixtures/excluded-change/specs2/value.txt create mode 100644 checks/source-fixtures/selected-change/Cargo.toml create mode 100644 checks/source-fixtures/selected-change/nested/keep/value.txt create mode 120000 checks/source-fixtures/selected-change/nested/manifest-link create mode 100644 checks/source-fixtures/selected-change/nested/specs/ignored.txt create mode 100644 checks/source-fixtures/selected-change/other/specs/ignored.txt create mode 100644 checks/source-fixtures/selected-change/specs create mode 100644 checks/source-fixtures/selected-change/specs2/value.txt create mode 100644 checks/source-tests.nix delete mode 100644 lib/filter/default.nix create mode 100644 lib/source/default.nix diff --git a/checks/default.nix b/checks/default.nix index 096c979..76941da 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -20,6 +20,7 @@ onlyDrvs ( workspaceCross = callPackage ./workspace-cross-compile { inherit full; }; customStdenv = callPackage ./custom-stdenv { }; nextest = callPackage ./nextest { }; + sourceTests = callPackage ./source-tests.nix { }; mergeArgsTests = callPackage ./mergeArgs-tests.nix { }; craneMultiBuildTests = callPackage ./crane-multi-build-tests.nix { }; cargoCrapModule = callPackage ./cargo-crap-module.nix { inherit mkLib; }; diff --git a/checks/nextest/default.nix b/checks/nextest/default.nix index 5830d51..ae50818 100644 --- a/checks/nextest/default.nix +++ b/checks/nextest/default.nix @@ -1,9 +1,6 @@ { pkgs, flakeboxLib }: let - rootDir = builtins.path { - name = "nextest"; - path = ./.; - }; + rootDir = ./.; buildPaths = [ "Cargo.toml" @@ -14,7 +11,7 @@ let multiOutput = (flakeboxLib.craneMultiBuild { }) ( craneLib': let - src = flakeboxLib.filterSubPaths { + src = flakeboxLib.source.fromPaths { root = rootDir; paths = buildPaths; }; diff --git a/checks/source-fixtures/baseline/Cargo.toml b/checks/source-fixtures/baseline/Cargo.toml new file mode 100644 index 0000000..c4b464f --- /dev/null +++ b/checks/source-fixtures/baseline/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = [] diff --git a/checks/source-fixtures/baseline/nested/keep/value.txt b/checks/source-fixtures/baseline/nested/keep/value.txt new file mode 100644 index 0000000..2fa992c --- /dev/null +++ b/checks/source-fixtures/baseline/nested/keep/value.txt @@ -0,0 +1 @@ +keep diff --git a/checks/source-fixtures/baseline/nested/manifest-link b/checks/source-fixtures/baseline/nested/manifest-link new file mode 120000 index 0000000..38f27c5 --- /dev/null +++ b/checks/source-fixtures/baseline/nested/manifest-link @@ -0,0 +1 @@ +../Cargo.toml \ No newline at end of file diff --git a/checks/source-fixtures/baseline/nested/specs/ignored.txt b/checks/source-fixtures/baseline/nested/specs/ignored.txt new file mode 100644 index 0000000..2df1c76 --- /dev/null +++ b/checks/source-fixtures/baseline/nested/specs/ignored.txt @@ -0,0 +1 @@ +excluded one diff --git a/checks/source-fixtures/baseline/other/specs/ignored.txt b/checks/source-fixtures/baseline/other/specs/ignored.txt new file mode 100644 index 0000000..c03e796 --- /dev/null +++ b/checks/source-fixtures/baseline/other/specs/ignored.txt @@ -0,0 +1 @@ +excluded two diff --git a/checks/source-fixtures/baseline/specs b/checks/source-fixtures/baseline/specs new file mode 100644 index 0000000..5da2b6b --- /dev/null +++ b/checks/source-fixtures/baseline/specs @@ -0,0 +1 @@ +regular specs diff --git a/checks/source-fixtures/baseline/specs2/value.txt b/checks/source-fixtures/baseline/specs2/value.txt new file mode 100644 index 0000000..921019d --- /dev/null +++ b/checks/source-fixtures/baseline/specs2/value.txt @@ -0,0 +1 @@ +specs2 keep diff --git a/checks/source-fixtures/excluded-change/Cargo.toml b/checks/source-fixtures/excluded-change/Cargo.toml new file mode 100644 index 0000000..c4b464f --- /dev/null +++ b/checks/source-fixtures/excluded-change/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = [] diff --git a/checks/source-fixtures/excluded-change/nested/keep/value.txt b/checks/source-fixtures/excluded-change/nested/keep/value.txt new file mode 100644 index 0000000..2fa992c --- /dev/null +++ b/checks/source-fixtures/excluded-change/nested/keep/value.txt @@ -0,0 +1 @@ +keep diff --git a/checks/source-fixtures/excluded-change/nested/manifest-link b/checks/source-fixtures/excluded-change/nested/manifest-link new file mode 120000 index 0000000..38f27c5 --- /dev/null +++ b/checks/source-fixtures/excluded-change/nested/manifest-link @@ -0,0 +1 @@ +../Cargo.toml \ No newline at end of file diff --git a/checks/source-fixtures/excluded-change/nested/specs/ignored.txt b/checks/source-fixtures/excluded-change/nested/specs/ignored.txt new file mode 100644 index 0000000..4036708 --- /dev/null +++ b/checks/source-fixtures/excluded-change/nested/specs/ignored.txt @@ -0,0 +1 @@ +changed excluded diff --git a/checks/source-fixtures/excluded-change/other/specs/ignored.txt b/checks/source-fixtures/excluded-change/other/specs/ignored.txt new file mode 100644 index 0000000..c03e796 --- /dev/null +++ b/checks/source-fixtures/excluded-change/other/specs/ignored.txt @@ -0,0 +1 @@ +excluded two diff --git a/checks/source-fixtures/excluded-change/specs b/checks/source-fixtures/excluded-change/specs new file mode 100644 index 0000000..5da2b6b --- /dev/null +++ b/checks/source-fixtures/excluded-change/specs @@ -0,0 +1 @@ +regular specs diff --git a/checks/source-fixtures/excluded-change/specs2/value.txt b/checks/source-fixtures/excluded-change/specs2/value.txt new file mode 100644 index 0000000..921019d --- /dev/null +++ b/checks/source-fixtures/excluded-change/specs2/value.txt @@ -0,0 +1 @@ +specs2 keep diff --git a/checks/source-fixtures/selected-change/Cargo.toml b/checks/source-fixtures/selected-change/Cargo.toml new file mode 100644 index 0000000..c4b464f --- /dev/null +++ b/checks/source-fixtures/selected-change/Cargo.toml @@ -0,0 +1,2 @@ +[workspace] +members = [] diff --git a/checks/source-fixtures/selected-change/nested/keep/value.txt b/checks/source-fixtures/selected-change/nested/keep/value.txt new file mode 100644 index 0000000..6ebf573 --- /dev/null +++ b/checks/source-fixtures/selected-change/nested/keep/value.txt @@ -0,0 +1 @@ +changed selected diff --git a/checks/source-fixtures/selected-change/nested/manifest-link b/checks/source-fixtures/selected-change/nested/manifest-link new file mode 120000 index 0000000..38f27c5 --- /dev/null +++ b/checks/source-fixtures/selected-change/nested/manifest-link @@ -0,0 +1 @@ +../Cargo.toml \ No newline at end of file diff --git a/checks/source-fixtures/selected-change/nested/specs/ignored.txt b/checks/source-fixtures/selected-change/nested/specs/ignored.txt new file mode 100644 index 0000000..2df1c76 --- /dev/null +++ b/checks/source-fixtures/selected-change/nested/specs/ignored.txt @@ -0,0 +1 @@ +excluded one diff --git a/checks/source-fixtures/selected-change/other/specs/ignored.txt b/checks/source-fixtures/selected-change/other/specs/ignored.txt new file mode 100644 index 0000000..c03e796 --- /dev/null +++ b/checks/source-fixtures/selected-change/other/specs/ignored.txt @@ -0,0 +1 @@ +excluded two diff --git a/checks/source-fixtures/selected-change/specs b/checks/source-fixtures/selected-change/specs new file mode 100644 index 0000000..5da2b6b --- /dev/null +++ b/checks/source-fixtures/selected-change/specs @@ -0,0 +1 @@ +regular specs diff --git a/checks/source-fixtures/selected-change/specs2/value.txt b/checks/source-fixtures/selected-change/specs2/value.txt new file mode 100644 index 0000000..921019d --- /dev/null +++ b/checks/source-fixtures/selected-change/specs2/value.txt @@ -0,0 +1 @@ +specs2 keep diff --git a/checks/source-tests.nix b/checks/source-tests.nix new file mode 100644 index 0000000..b5e0255 --- /dev/null +++ b/checks/source-tests.nix @@ -0,0 +1,143 @@ +{ pkgs, flakeboxLib }: +let + inherit (pkgs) lib; + source = flakeboxLib.source; + fixture = ./source-fixtures; + + selectedPaths = [ + "Cargo.toml" + "nested" + "other" + "specs" + "specs2" + ]; + denySpecs = source.filters.excludeDirectoriesNamed [ "specs" ]; + + mkFixtureSource = + root: + source.fromPaths { + inherit root; + paths = selectedPaths; + filter = denySpecs; + }; + + baseline = mkFixtureSource (fixture + "/baseline"); + excludedChange = mkFixtureSource (fixture + "/excluded-change"); + selectedChange = mkFixtureSource (fixture + "/selected-change"); + + buildFiles = source.filesetFromPaths { + root = fixture + "/baseline"; + paths = [ + "Cargo.toml" + "nested" + "nested/keep" + "Cargo.toml" + ]; + }; + composed = source.fromFileset { + root = fixture + "/baseline"; + fileset = lib.fileset.union (lib.fileset.difference buildFiles ( + fixture + "/baseline/nested/specs" + )) (lib.fileset.maybeMissing (fixture + "/baseline/optional")); + }; + + filtered = source.fromPaths { + root = fixture + "/baseline"; + paths = selectedPaths; + filter = source.filters.all [ + denySpecs + (source.filters.not (path: type: type == "regular" && lib.hasSuffix "value.txt" path)) + ]; + }; + + globbed = source.fromGlobs { + root = fixture + "/baseline"; + patterns = [ + "Cargo.toml" + "nested/**" + "specs2/**" + ]; + filter = denySpecs; + }; + emptyGlob = source.fromGlobs { + root = fixture + "/baseline"; + patterns = [ "not-present/**" ]; + }; + + fails = value: !(builtins.tryEval (builtins.deepSeq value true)).success; +in +assert lib.assertMsg ( + toString baseline == toString excludedChange +) "changes to excluded content must preserve the source store path"; +assert lib.assertMsg ( + toString baseline != toString selectedChange +) "changes to selected content must alter the source store path"; +assert lib.assertMsg ( + !(source.filters.all [ + (_path: _type: false) + (_path: _type: throw "all did not short-circuit") + ] "unused" "regular") +) "source.filters.all must short-circuit"; +assert lib.assertMsg (source.filters.any [ + (_path: _type: true) + (_path: _type: throw "any did not short-circuit") +] "unused" "regular") "source.filters.any must short-circuit"; +assert lib.assertMsg (fails ( + source.filesetFromPaths { + root = fixture + "/baseline"; + paths = [ "" ]; + } +)) "empty subpaths must be rejected"; +assert lib.assertMsg (fails ( + source.filesetFromPaths { + root = fixture + "/baseline"; + paths = [ "../baseline" ]; + } +)) "parent subpaths must be rejected"; +assert lib.assertMsg (fails ( + source.filesetFromPaths { + root = fixture + "/baseline"; + paths = [ "/Cargo.toml" ]; + } +)) "absolute subpaths must be rejected"; +assert lib.assertMsg (fails ( + source.fromPaths { + root = fixture + "/baseline"; + paths = [ "missing" ]; + } +)) "missing selected paths must be rejected"; +assert lib.assertMsg (fails ( + source.fromFileset { + root = fixture + "/baseline"; + fileset = ./mergeArgs-tests.nix; + } +)) "filesets outside root must be rejected"; +pkgs.runCommand "source-tests" { } '' + test -f ${baseline}/Cargo.toml + test -f ${baseline}/nested/keep/value.txt + test -L ${baseline}/nested/manifest-link + test -f ${baseline}/specs + test -f ${baseline}/specs2/value.txt + test ! -e ${baseline}/nested/specs + test ! -e ${baseline}/other/specs + + test -f ${composed}/Cargo.toml + test -f ${composed}/nested/keep/value.txt + test -L ${composed}/nested/manifest-link + test ! -e ${composed}/nested/specs + + test -f ${filtered}/Cargo.toml + test ! -e ${filtered}/nested/keep/value.txt + test ! -e ${filtered}/specs2/value.txt + test ! -e ${filtered}/nested/specs + + test -f ${globbed}/Cargo.toml + test -f ${globbed}/nested/keep/value.txt + test -L ${globbed}/nested/manifest-link + test -f ${globbed}/specs2/value.txt + test ! -e ${globbed}/nested/specs + + test -z "$(find ${emptyGlob} -mindepth 1 -print -quit)" + + touch $out +'' diff --git a/checks/workspace-cross-compile/default.nix b/checks/workspace-cross-compile/default.nix index 451b7c6..d9174b1 100644 --- a/checks/workspace-cross-compile/default.nix +++ b/checks/workspace-cross-compile/default.nix @@ -6,16 +6,12 @@ let inherit (pkgs) lib; - rootDir = builtins.path { - name = "workspace-sanity"; - path = ./.; - }; + rootDir = ./.; # paths needed to build Rust code buildPaths = [ "Cargo.toml" "Cargo.lock" - ".cargo" "bin" "lib" "rocksdb" @@ -29,7 +25,7 @@ let multiOutput = (flakeboxLib.craneMultiBuild { }) ( craneLib': let - src = flakeboxLib.filterSubPaths { + src = flakeboxLib.source.fromPaths { root = rootDir; paths = buildPaths; }; @@ -88,7 +84,7 @@ let ./scripts/e2e-tests.sh ''; - src = flakeboxLib.filterSubPaths { + src = flakeboxLib.source.fromPaths { root = rootDir; paths = testPaths; }; diff --git a/checks/workspace-sanity/default.nix b/checks/workspace-sanity/default.nix index 95ac4e5..bdd2d92 100644 --- a/checks/workspace-sanity/default.nix +++ b/checks/workspace-sanity/default.nix @@ -2,16 +2,12 @@ let inherit (pkgs) lib; - rootDir = builtins.path { - name = "workspace-sanity"; - path = ./.; - }; + rootDir = ./.; # paths needed to build Rust code buildPaths = [ "Cargo.toml" "Cargo.lock" - ".cargo" "bin" "lib" ]; @@ -24,7 +20,7 @@ let multiOutput = (flakeboxLib.craneMultiBuild { }) ( craneLib': let - src = flakeboxLib.filterSubPaths { + src = flakeboxLib.source.fromPaths { root = rootDir; paths = buildPaths; }; @@ -78,7 +74,7 @@ let ./scripts/e2e-tests.sh ''; - src = flakeboxLib.filterSubPaths { + src = flakeboxLib.source.fromPaths { root = rootDir; paths = testPaths; }; diff --git a/docs/best-practices.md b/docs/best-practices.md index 602d31d..cea8ade 100644 --- a/docs/best-practices.md +++ b/docs/best-practices.md @@ -86,8 +86,8 @@ See [build profile Overrides section in The Cargo Book](https://doc.rust-lang.or Avoiding rebuilds is an important part of optimized Nix-based CI. It's especially important in slow to compile language like Rust. -Flakebox provides `flakeboxLib.filterSubPaths` function for convenient -source code filtering. +Flakebox provides `flakeboxLib.source.fromPaths` for validated, composable source filtering. +Pass a raw path such as `root = ./.` and list every build input explicitly. ### Use mold when possible diff --git a/docs/building-new-project.md b/docs/building-new-project.md index 8467b61..042bf6b 100644 --- a/docs/building-new-project.md +++ b/docs/building-new-project.md @@ -321,11 +321,8 @@ index cb41316..08c8f65 100644 pkgs = nixpkgs.legacyPackages.${system}; flakeboxLib = flakebox.lib.mkLib pkgs { }; + -+ rustSrc = flakeboxLib.filterSubPaths { -+ root = builtins.path { -+ name = "flakebox-tutorial"; -+ path = ./.; -+ }; ++ rustSrc = flakeboxLib.source.fromPaths { ++ root = ./.; + paths = [ + "Cargo.toml" + "Cargo.lock" @@ -434,11 +431,8 @@ This is where you can enable, disable, and configure various Flakebox features. The next binding is: ```nix - rustSrc = flakeboxLib.filterSubPaths { - root = builtins.path { - name = "flakebox-tutorial"; - path = ./.; - }; + rustSrc = flakeboxLib.source.fromPaths { + root = ./.; paths = [ "Cargo.toml" "Cargo.lock" @@ -448,7 +442,7 @@ The next binding is: }; ``` -This `filterSubPaths` is a function exposed by +This `source.fromPaths` function is exposed by `flakeboxLib` and is used for easy source code filtering. This is useful to avoid having to rebuild our Rust project when only irrelevant files changed. It's not strictly @@ -724,7 +718,7 @@ index a65ba7a..f4c64d7 100644 pkgs = nixpkgs.legacyPackages.${system}; flakeboxLib = flakebox.lib.mkLib pkgs { }; - rustSrc = flakeboxLib.filterSubPaths { + rustSrc = flakeboxLib.source.fromPaths { @@ -36,6 +41,10 @@ craneLib = (craneLib'.overrideArgs { pname = "flexbox-multibuild"; diff --git a/docs/flakeboxLib.md b/docs/flakeboxLib.md index 0830888..50991c6 100644 --- a/docs/flakeboxLib.md +++ b/docs/flakeboxLib.md @@ -1,83 +1,107 @@ # `flakeboxLib` functions -## `flakeboxLib.filterSubPaths` +## `flakeboxLib.source` -A recommended way to create a Nix path containing only selected list -of files and directories. +Use `flakeboxLib.source` to declare exactly which project files a build can +read. Excluding unrelated files prevents their changes from invalidating Rust +build derivations. -This is very important to avoid Nix unnecessarily recompiling Rust -code due to changes in irrelevant files. - -Since Nix does not understand -`cargo` it has to assume that change to any input (e.g. file) used in a build -step (derivation) could have caused the result to change. The only reason -to avoid it is to not pass such files (filter them out). - -Notably `craneLib` already performs certain filtering, in particular for -`*DepsOnly` functions, but for best results it's necessary to handle -it manually. - -While Nix includes functions for that purposes, we've found the `filterSubPaths` -an easy to use and maintain. +The normal literal-path API validates every path and imports the selected +files once: ```nix let - root = builtins.path { - name = "flakebox"; - path = ./.; - }; - src = flakeboxLib.filterSubPaths { - inherit root; - paths = [ - "Cargo.toml" - "Cargo.lock" - ".cargo" - "flakebox-bin" - ]; + buildPaths = [ + "Cargo.toml" + "Cargo.lock" + ".cargo" + "flakebox-bin" + ]; + + src = flakeboxLib.source.fromPaths { + root = ./.; + paths = buildPaths; }; in - ... +# ... ``` -It's a good practice to concatenate list of paths when chaining -post-build derivations. +`root` must be a Nix path value, not a store-path string produced by +`builtins.path`. Subpaths must be relative, non-empty, stay below `root`, and +exist. Filesets preserve symlinks but omit empty directories. + +Reuse path lists when later build stages need extra inputs: + +```nix +buildSrc = flakeboxLib.source.fromPaths { + root = ./.; + paths = buildPaths; +}; + +testSrc = flakeboxLib.source.fromPaths { + root = ./.; + paths = buildPaths ++ [ "scripts" ]; +}; +``` + +Changes under `scripts` then invalidate tests without invalidating the Rust +build. Include every build dependency, including build-script inputs, +`include_*` macro inputs, generated-code inputs, assets, schemas, and fixtures. + +### Fileset composition + +`source.filesetFromPaths` returns a nixpkgs file set for composition with +`pkgs.lib.fileset.union`, `unions`, `intersection`, and `difference`. +`source.fromFileset` performs the final source import: ```nix let - root = builtins.path { - name = "flakebox"; - path = ./.; - }; - buildPaths = [ - "Cargo.toml" - "Cargo.lock" - ".cargo" - "flakebox-bin" - ]; - src = flakeboxLib.filterSubPaths { - inherit root; + fs = pkgs.lib.fileset; + source = flakeboxLib.source; + buildFiles = source.filesetFromPaths { + root = ./.; paths = buildPaths; }; in { - #... - testXyz = craneLib.buildCommand { - cargoArtifacts = workspaceBuild; - src = flakeboxLib.filterSubPaths { - inherit root; - paths = buildPaths ++ [ "scripts" ]; - }; - - cmd = '' - patchShebangs ./scripts - ./scripts/test-xyz.sh - ''; + buildSrc = source.fromFileset { + root = ./.; + fileset = buildFiles; + }; + testSrc = source.fromFileset { + root = ./.; + fileset = fs.union buildFiles ./scripts; }; } ``` -Since `"scripts"` path is included only in the `src` -for tests, changes to test files will not cause -Rust code to rebuild. +Use `fs.maybeMissing` when a selected path is intentionally optional. +`fromFileset` rejects files outside `root`. + +### Directory-aware filters + +`fromPaths` and `fromFileset` accept a canonical Nix source predicate of +`absolutePathString: type: bool`. Returning false for a directory prunes its +whole subtree. Flakebox provides `source.filters.all`, `any`, `not`, and +`excludeDirectoriesNamed` for composition: + +```nix +buildSrc = flakeboxLib.source.fromPaths { + root = ./.; + paths = buildPaths; + filter = flakeboxLib.source.filters.excludeDirectoriesNamed [ "specs" ]; +}; +``` + +A regular file named `specs` is retained; directories with that basename are +pruned at any selected depth. + +### Glob filtering + +`source.fromGlobs` is an explicit native-traversal alternative for large trees. +It supports nixpkgs `sourceByGlobs` inclusion patterns such as `src/**`, `*.py`, +and `**/*.py`, and accepts the same optional `filter`. It retains matched empty +directories and requires nixpkgs 26.05 or newer. Prefer literal paths and +fileset composition unless measurements justify glob traversal. ## `flakeboxLib.craneMultiBuild ` diff --git a/docs/getting-started.md b/docs/getting-started.md index 5783b89..f96fe1d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -67,11 +67,8 @@ Following modifications to `flake.nix` are needed: ]; # Filter Rust source code - buildSrc = flakeboxLib.filterSubPaths { - root = builtins.path { - name = projectName; - path = ./.; - }; + buildSrc = flakeboxLib.source.fromPaths { + root = ./.; paths = buildPaths; }; diff --git a/flake.nix b/flake.nix index 53a5378..aad6c7a 100644 --- a/flake.nix +++ b/flake.nix @@ -139,11 +139,8 @@ }; }; - src = flakeboxLib.filterSubPaths { - root = builtins.path { - name = "flakebox"; - path = ./.; - }; + src = flakeboxLib.source.fromPaths { + root = ./.; paths = [ "Cargo.toml" "Cargo.lock" diff --git a/lib/default.nix b/lib/default.nix index 1a9d209..23d9611 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -112,9 +112,7 @@ lib.makeScope pkgs.newScope ( cargoCrap = callPackage ./pkgs/cargo-crap.nix { }; - filter = callPackage ./filter { }; - - filterSubPaths = self.filter.filterSubPaths; + source = callPackage ./source { }; pickBinary = callPackage ./pickBinary.nix { }; diff --git a/lib/filter/default.nix b/lib/filter/default.nix deleted file mode 100644 index 4269f37..0000000 --- a/lib/filter/default.nix +++ /dev/null @@ -1,74 +0,0 @@ -{ pkgs }: -let - lib = pkgs.lib; - cleanSourceWith = lib.cleanSourceWith; - - cleanSourceWithRel = - { - root, - filter, - trace ? false, - }: - let - baseStr = builtins.toString root; - in - lib.cleanSourceWith { - src = root; - filter = - path: type: - let - relPath = lib.removePrefix baseStr (toString path); - include = filter relPath type; - in - if trace then - (builtins.trace "${root}/${relPath}: ${lib.boolToString include}" include) - else - include; - }; - - # Filter `root` path, retaining only paths matching/included in elements of `paths` - # while correctly handling deeply nested `paths`. - filterSubPaths = - { - root, - paths, - trace ? false, - }: - cleanSourceWithRel { - inherit root trace; - filter = ( - relPath: type: - let - # since `/` can't appear in path elements, we can use it as a terminator to avoid - # elements that are just prefix matching full element filter - relPathTerm = relPath + "/"; - - isPrefix = builtins.any (dir: lib.hasPrefix relPathTerm ("/" + dir + "/")) paths; - isPrefixed = builtins.any (dir: lib.hasPrefix ("/" + dir + "/") relPathTerm) paths; - in - if type == "directory" then isPrefix || isPrefixed else isPrefixed - ); - }; - - filterSubdirs = - { - root, - dirs, - trace ? false, - }: - lib.warn - "`flakeboxLib.filter.filterSubdirs` is now `flakeboxLib.filterSubPaths` and argument `dirs` was renamed to `paths`" - filterSubPaths - { - inherit root trace; - paths = dirs; - }; -in -{ - inherit - cleanSourceWith - cleanSourceWithRel - filterSubdirs - filterSubPaths - ; -} diff --git a/lib/source/default.nix b/lib/source/default.nix new file mode 100644 index 0000000..5f2bcd9 --- /dev/null +++ b/lib/source/default.nix @@ -0,0 +1,90 @@ +{ pkgs }: +let + inherit (pkgs) lib; + fs = lib.fileset; + + filesetFromPaths = + { root, paths }: + fs.unions (map (lib.path.append root) paths); + + fromFileset = + { + root, + fileset, + filter ? null, + }: + let + allowed = + if filter == null then + null + else + fs.fromSource ( + lib.sources.cleanSourceWith { + src = root; + inherit filter; + } + ); + in + fs.toSource { + inherit root; + fileset = if allowed == null then fileset else fs.intersection fileset allowed; + }; + + fromPaths = + { + root, + paths, + filter ? null, + }: + fromFileset { + inherit root filter; + fileset = filesetFromPaths { inherit root paths; }; + }; + + fromGlobs = + { + root, + patterns, + filter ? null, + }: + if !(lib.sources ? sourceByGlobs) then + throw "flakeboxLib.source.fromGlobs requires nixpkgs with lib.sources.sourceByGlobs (26.05 or newer)" + else if !builtins.isPath root then + throw "flakeboxLib.source.fromGlobs: root must be a path value" + else + let + source = lib.sources.sourceByGlobs root patterns; + in + if filter == null then + source + else + # cleanSourceWith evaluates this filter before the source's glob filter, so + # rejecting a directory avoids both glob matching and subtree traversal. + lib.sources.cleanSourceWith { + src = source; + inherit filter; + }; +in +{ + inherit + filesetFromPaths + fromFileset + fromGlobs + fromPaths + ; + + filters = { + all = + filters: path: type: + lib.all (filter: filter path type) filters; + any = + filters: path: type: + lib.any (filter: filter path type) filters; + not = + filter: path: type: + !(filter path type); + excludeDirectoriesNamed = + names: path: type: + type != "directory" || !(lib.elem (builtins.baseNameOf path) names); + }; +} diff --git a/templates/default/flake.nix b/templates/default/flake.nix index 38ae855..83d511b 100644 --- a/templates/default/flake.nix +++ b/templates/default/flake.nix @@ -35,11 +35,8 @@ "src" ]; - buildSrc = flakeboxLib.filterSubPaths { - root = builtins.path { - name = projectName; - path = ./.; - }; + buildSrc = flakeboxLib.source.fromPaths { + root = ./.; paths = buildPaths; };