diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 8208d73f1..8b6e56dc2 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ | grep -v '^$' \ | xargs apt-get -y install \ clang \ + libomp-dev \ clang-tidy \ clang-format \ cmake \ diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 54a4f2b00..5dd25f9f9 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -69,8 +69,16 @@ runs: esac fi + # OpenMP is required. g++ ships libgomp; a Linux Clang needs the libomp of its own + # version, even when the runner already provides the compiler. + if [[ "$RUNNER_OS" != "macOS" && "${CXX:-}" == clang++* ]]; then + packages+=("libomp${CXX#clang++}-dev") + fi + if [[ "$RUNNER_OS" == "macOS" ]]; then brew install "${packages[@]}" + # Homebrew's libomp is keg-only; CMake's FindOpenMP searches OpenMP_ROOT. + echo "OpenMP_ROOT=$(brew --prefix libomp)" >> "$GITHUB_ENV" else sudo apt-get update sudo apt-get install -y "${packages[@]}" diff --git a/.github/tsan-libomp.supp b/.github/tsan-libomp.supp new file mode 100644 index 000000000..901c1fcc9 --- /dev/null +++ b/.github/tsan-libomp.supp @@ -0,0 +1,4 @@ +# ThreadSanitizer suppressions for LLVM's OpenMP runtime: its internal locking is not instrumented, so races +# reported from inside libomp itself are not evidence (docs/content/docs/building.mdx). Archer +# (OMP_TOOL_LIBRARIES=.../libarcher.so) supplies the OpenMP synchronization TSan needs for monoprop's own frames. +called_from_lib:libomp.so.5 diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 042c96d9c..4b61825af 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -15,8 +15,9 @@ on: type: string rungs: description: >- - One rung per line, ` | `. - `partitions` is an integer or `per-rank` for (physical cores / ranks). + One rung per line, ` | `. + `threads` (OpenMP threads, and so shards, per rank) is an integer or `per-rank` for + (physical cores / ranks). Blank lines and `#` comments are ignored. A name may carry a `.suffix`: the Bencher testbed is the part before the first `.`, so rungs sharing a prefix share a plot, which is how one shape measured by two processes keeps its rows @@ -47,7 +48,7 @@ jobs: with: # A cache filled by a non-MPI build would restore a monoprop without MPI, which every # rung below then shares. - cache-suffix: benchmark-mpi + cache-suffix: benchmark-mpi-sharded # The build below requires MPI::MPI_CXX, and the multi-rank rungs need mpiexec. mpi: "on" @@ -79,7 +80,7 @@ jobs: mkdir -p benches/results # fd 3, because mpiexec forwards its own stdin to rank 0 and would eat the rung list. while IFS='|' read -r head args <&3; do - IFS=$' \t' read -r rung ranks partitions rounds <<<"$head" + IFS=$' \t' read -r rung ranks threads rounds <<<"$head" case "${rung:-}" in '' | '#'*) continue ;; esac # The name reaches a result filename, a Bencher testbed and the prefix split # below, so hold it to what all three accept. A leading `.` would split to an @@ -94,9 +95,9 @@ jobs: # `0` divides by zero at per-rank and reaches `mpiexec -n 0` otherwise; and `08` is # an invalid octal literal there. Rejecting leading zeros keeps that arithmetic # base 10. - for field in ranks partitions rounds; do + for field in ranks threads rounds; do value="${!field:-}" - [[ "$field" == partitions && "$value" == per-rank ]] && continue + [[ "$field" == threads && "$value" == per-rank ]] && continue case "$value" in '') echo "::error::rung $rung has no $field"; exit 1 ;; *[!0-9]* | 0*) @@ -104,13 +105,13 @@ jobs: exit 1 ;; esac done - if [[ "$partitions" == per-rank ]]; then - partitions=$((BENCH_CORES / ranks)) + if [[ "$threads" == per-rank ]]; then + threads=$((BENCH_CORES / ranks)) fi # Oversubscription reports a plausible wall time and would silently corrupt the # testbed. - if ((partitions < 1 || ranks * partitions > BENCH_CORES)); then - echo "::error::rung $rung wants ${ranks}x${partitions} on $BENCH_CORES physical cores" + if ((threads < 1 || ranks * threads > BENCH_CORES)); then + echo "::error::rung $rung wants ${ranks}x${threads} on $BENCH_CORES physical cores" exit 1 fi # xargs honours quotes in `-k "a and b"` without evaluating the input. @@ -123,23 +124,25 @@ jobs: readarray -t argv <<< "$parsed" fi label="$BENCH_LABEL-$rung" - echo "::group::$rung: ${ranks}x${partitions}, $rounds rounds -> $label" - # `monoprop_PARTITIONS` also sizes the pinning below -- unpinned ranks vary by more + echo "::group::$rung: ${ranks}x${threads}, $rounds rounds -> $label" + # An exact team of T per rank, one thread per core of the rank's slot: placement is + # the launcher's and OpenMP's, never the library's. Unpinned threads vary by more # than the effects being tracked. - export monoprop_PARTITIONS="$partitions" monoprop_NUM_THREADS="$partitions" + export monoprop_NUM_THREADS="$threads" OMP_NUM_THREADS="$threads" OMP_DYNAMIC=FALSE + export OMP_PROC_BIND=close OMP_PLACES=cores export monoprop_BENCH_LABEL="$label" monoprop_BENCH_RESULTS=benches/results if ((ranks == 1)); then uv run --no-sync python -m pytest benches -o filterwarnings=default \ - --benchmark-json="benches/results/time-$label.json" \ + --runtime-shape=openmp --benchmark-json="benches/results/time-$label.json" \ -m "${monoprop_BENCH_MARKERS-not slow}" \ --bench-rounds "$rounds" "${argv[@]}" else uv run --no-sync mpiexec -n "$ranks" \ - --map-by "slot:PE=$monoprop_PARTITIONS" --bind-to core \ + --map-by "slot:PE=$threads" --bind-to core \ -x monoprop_BENCH_LABEL -x monoprop_BENCH_RESULTS \ - -x monoprop_PARTITIONS -x monoprop_NUM_THREADS \ + -x monoprop_NUM_THREADS -x OMP_NUM_THREADS -x OMP_DYNAMIC -x OMP_PROC_BIND -x OMP_PLACES \ python -m pytest benches -o filterwarnings=default \ - --benchmark-json="benches/results/time-$label.json" \ + --runtime-shape=openmp --benchmark-json="benches/results/time-$label.json" \ -m "${monoprop_BENCH_MARKERS-not slow}" \ --bench-rounds "$rounds" "${argv[@]}" fi @@ -148,7 +151,7 @@ jobs: # To the log too: a rung that OOMs the instance takes the artifact with it. cat "bmf-$rung.json" uv run --no-sync python .github/workflows/scripts/check_shape.py \ - "benches/results/$label.json" "$ranks" "$partitions" + "benches/results/$label.json" "$ranks" "$threads" echo "::endgroup::" done 3<<< "$RUNGS" diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index a3bde20fc..3b41f8854 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -16,6 +16,10 @@ on: - VERSION - pyproject.toml - CMakeLists.txt + - cmake/** + - cpp/**/CMakeLists.txt + - cpp/tests/find_package_smoke/** + - cpp/tests/link_export_probe/** - src/monoprop/bindings/CMakeLists.txt concurrency: @@ -67,10 +71,18 @@ jobs: ninja --version uv --version just --version - pkg-config --modversion hwloc mpicc --version ' + # Configures, links and runs the installed-package consumer, which instantiates the + # OpenMP worksharing template, with the shell's compiler and OpenMP runtime. + - name: Check the OpenMP consumer from the dev shell + run: | + nix develop --command bash -euo pipefail -c ' + monoprop_ENABLE_MPI=OFF just build --group workspace-test + just test-find-package + ' + flake-result: name: All flake checks if: always() diff --git a/.github/workflows/qa-analysis.yml b/.github/workflows/qa-analysis.yml index 946fb8bdf..b92d89c67 100644 --- a/.github/workflows/qa-analysis.yml +++ b/.github/workflows/qa-analysis.yml @@ -57,7 +57,7 @@ jobs: - name: Set up the build environment uses: ./.github/actions/setup with: - cache-suffix: coverage-mpi-${{ matrix.mpi }} + cache-suffix: coverage-mpi-${{ matrix.mpi }}-sharded mpi: ${{ matrix.mpi }} - name: Collect coverage @@ -142,7 +142,7 @@ jobs: - name: Set up the build environment uses: ./.github/actions/setup with: - cache-suffix: gcc-asan-ubsan + cache-suffix: gcc-asan-ubsan-sharded - name: Install package run: just build --group workspace-test @@ -157,16 +157,23 @@ jobs: if: failure() run: just sanitizer-reports - gcc-tsan: + clang-tsan: # TSan cannot load instrumented _core into stock CPython; Python coverage needs an - # instrumented interpreter. C++ tests cover the relevant partition and ShmComm paths. - name: GCC ThreadSanitizer (C++) + # instrumented interpreter. The C++ tests cover the sharded runtime's concurrency surface, + # whose OpenMP teams need LLVM's libomp with its Archer tool: GCC's libgomp is not + # TSan-aware, so its reports are not evidence either way (docs/content/docs/building.mdx). + name: Clang ThreadSanitizer with Archer (C++) runs-on: ubuntu-26.04 permissions: contents: read env: + CXX: clang++-18 + CC: clang-18 SKBUILD_CMAKE_BUILD_TYPE: "Tsan" SKBUILD_CMAKE_DEFINE: "monoprop_SANITIZER=tsan" + OMP_TOOL_LIBRARIES: /usr/lib/llvm-18/lib/libarcher.so + # libomp's own internal locking is not instrumented. + TSAN_OPTIONS: "suppressions=${{ github.workspace }}/.github/tsan-libomp.supp" steps: - uses: actions/checkout@v7.0.1 @@ -176,12 +183,12 @@ jobs: - name: Set up the build environment uses: ./.github/actions/setup with: - cache-suffix: gcc-tsan + cache-suffix: clang-tsan-sharded - name: Install package run: just build --group workspace-test - - name: Run C++ partition and ShmComm tests + - name: Run the sharded runtime's concurrency tests run: just test-cpp-tsan sonarqube-analysis: @@ -197,7 +204,7 @@ jobs: - name: Set up the build environment uses: ./.github/actions/setup with: - cache-suffix: sonarqube-mpi + cache-suffix: sonarqube-mpi-sharded mpi: "on" - name: Install package @@ -239,8 +246,11 @@ jobs: - name: Set up the build environment uses: ./.github/actions/setup + # The build below uses clang++, so the setup installs its matching libomp. + env: + CXX: clang++ with: - cache-suffix: clang-tidy-mpi + cache-suffix: clang-tidy-mpi-sharded mpi: "on" - name: Build (generates compile_commands.json) diff --git a/.github/workflows/scripts/check_shape.py b/.github/workflows/scripts/check_shape.py index 58996fe83..79fb2165c 100644 --- a/.github/workflows/scripts/check_shape.py +++ b/.github/workflows/scripts/check_shape.py @@ -14,12 +14,12 @@ """Fail a rung that did not run at the shape it was asked for. -A knob that fails to reach the ranks does not fail the run: it measures one partition per -rank at a plausible wall time. Neither does `partitions` above a rank's visible cores -- -that disables placement entirely, at 13.8x to 24.6x, warning only on C++ stderr, which -pytest's capture eats. +A knob that fails to reach the ranks does not fail the run: it measures a different team at a +plausible wall time. So the recorded rank count and per-rank thread count must be the rung's, +and the removed partition runtime's variable must not be set at all. Placement is the launcher's +and OpenMP's (`--bind-to core`, `OMP_PROC_BIND`/`OMP_PLACES`); its summary is printed for the log. -Usage: check_shape.py .json> +Usage: check_shape.py .json> """ from __future__ import annotations @@ -30,28 +30,26 @@ def main(argv: list[str]) -> str | None: - path, ranks, partitions = Path(argv[1]), int(argv[2]), argv[3] + path, ranks, threads = Path(argv[1]), int(argv[2]), argv[3] meta = json.loads(path.read_text())["meta"] pin = meta.get("pinning", {}) - threads = pin.get("threads", 0) - placed = pin.get("single_cpu_threads_min", 0) print( - f" ranks={meta['ranks']} partitions_env={meta['partitions_env']} " - f"threads={meta['monoprop_threads']} pinned>={placed} of {threads} " + f" ranks={meta['ranks']} threads={meta['monoprop_threads']} " + f"partitions_env={meta['partitions_env']} " + f"single-cpu threads>={pin.get('single_cpu_threads_min', 0)} of {pin.get('threads', 0)} " f"mask={pin.get('affinity_cpus_min')}..{pin.get('affinity_cpus_max')}" ) - if meta["ranks"] != ranks or meta["partitions_env"] != partitions: + if meta["ranks"] != ranks or meta["monoprop_threads"] != threads: return ( f"::error::{path.stem} ran at ranks={meta['ranks']} " - f"partitions_env={meta['partitions_env']}, asked for {ranks}x{partitions}" + f"threads={meta['monoprop_threads']}, asked for {ranks}x{threads}" ) - # An all-zero summary means /proc was unreadable, not that nothing was pinned. - if int(partitions) > 1 and threads and not placed: + if meta["partitions_env"] != "unset": return ( - f"::error::{path.stem} placed no thread on a CPU of its own. " - "partitions above the rank's visible cores return an empty placement order." + f"::error::{path.stem} ran with monoprop_PARTITIONS={meta['partitions_env']}; " + "the sharded runtime ignores it, so a rung must not set it" ) return None diff --git a/.github/workflows/scripts/resolve_cores.py b/.github/workflows/scripts/resolve_cores.py index 196ba958f..20037f824 100644 --- a/.github/workflows/scripts/resolve_cores.py +++ b/.github/workflows/scripts/resolve_cores.py @@ -48,9 +48,9 @@ def main() -> str | None: if not monoprop.has_mpi: return "monoprop was built without MPI; every rung shares this binary" - # PHYSICAL cores, and only the ones this process may run on. Physical, because that is what - # the engine enumerates and the logical count would cross the `partitions > visible cores` - # threshold on any SMT machine. Visible, because psutil counts the whole machine: under a + # PHYSICAL cores, and only the ones this process may run on. Physical, because the rungs size + # their thread counts in cores and an SMT sibling is not a second core; monoprop itself takes + # whatever OpenMP reports. Visible, because psutil counts the whole machine: under a # mask -- a cpuset, taskset, or a shared runner -- that number oversubscribes every rung and # names the testbed after cores the run never had. visible = os.sched_getaffinity(0) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8477b0af3..d58d28cae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,9 @@ jobs: uses: ./.github/actions/setup with: python-version: ${{ matrix.python-version }} - cache-suffix: ${{ matrix.compiler }}-mpi-${{ matrix.mpi }} + # The MPI-off and MPI builds are ABI-incompatible, and so is every build from before the partition runtime was + # removed: the `sharded` part retires the caches of the dual-runtime matrix. + cache-suffix: ${{ matrix.compiler }}-mpi-${{ matrix.mpi }}-sharded mpi: ${{ matrix.mpi }} - name: Install package @@ -87,6 +89,9 @@ jobs: - name: Get monoprop information run: just info + - name: Verify the runtime and MPI identity of the build + run: just check-runtime + - name: Verify that find_package(monoprop) works run: just test-find-package diff --git a/AGENTS.md b/AGENTS.md index 34cb2b266..af79766e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,8 +56,38 @@ just build-docs - Python reference data lives in `tests/reference_data.py` (`load_problem()` reads `tests/data/*.msgpack`); C++ uses `test_utils::load_case()` from `cpp/tests/TestData.h`. - Pytest's fd capture hides C++ stderr such as `COMMPROF`; rerun with `-s`. +- Benchmark evidence: `pytest benches --runtime-shape=... --build-mode=...` refuses contradictory + runs. `tools/benchmark-rank-local-openmp.py` has exactly three modes, `observe`, `validate` and + `compare` (see `docs/content/docs/benchmarks.mdx`). Launch MPI-off runs directly: `mpi4py` is + never imported when `monoprop.has_mpi` is false. - `uv sync` does not relink `bin/monoprop_unit_tests.x` -- compare mtimes and use the standalone recipe in `docs/content/docs/building.mdx`. +- OpenMP is a required build dependency in every configuration (`OpenMP::OpenMP_CXX`, linked PUBLIC on + both `monoprop-objs` and `monoprop`, `find_dependency` in the package config). The worksharing + primitive is `detail/parallel/Workshare.h::for_blocks`. Never call `omp_set_*` from the library. + The library calls the runtime (`ThreadBudget.cpp`), so `import monoprop` loads it: with `OMP_PLACES` + set and binding on, the importing thread is bound to the first place. +- One runtime: each rank's `MonomialPropagator` owns T shards (`detail/sharded/`), T captured once at + construction from `monoprop_NUM_THREADS` (strict parser; unset falls back to `omp_get_max_threads()`). + There is no partition runtime, selector or `monoprop_PARTITIONS` reader. Every MPI-using operation + enforces MPI's initializing thread; only that thread (the team's primary) calls MPI, so FUNNELED is + required. Every state operation separates validation from mutation (`run_operation_`): a failure after + mutation starts invalidates the object and aborts a multi-rank communicator (`mpi::operation_failed`). + Catch failures inside posted `Ticket`/`PendingAlltoallv`/physical-round lifetimes, never only at the + outermost level. `detail::build_layer` (with `run_exchange` and `mpi::begin_alltoallv`) is the low-level + one-owner-per-rank engine kept only as the tests' reference; production builds layers through + `detail/sharded/Construction.h`. +- Threaded kernels (`scale_cos_*`, `apply_fused_contract`, the `fused_find_and_collect` scan, the resolve + phases: incoming decode, `probe_frozen_positions`, certified-sink scatter) take a defaulted test-only + range observer; participation tests read it (`cpp/tests/KernelTestSupport.h`, `ScanTestSupport.h`). + Serial and threaded runs must agree bitwise. Scan workers only traverse frozen storage and merge in range + order; only typed cutoffs (`CutoffEvaluator::parallel_safe`) run on them. Probes read a frozen store; + received query streams are validated on the caller (`QueryWire::checked_extent`) before any decode; + miss IDs, insertion (`bulk_insert_hashed`) and inverted-index maintenance stay single-writer. A sink's + `on_resolved` runs on workers only with `static constexpr bool parallel_resolve = true`. TSan evidence + needs Clang + libomp + Archer; GCC's libgomp gives false reports (`building.mdx`). +- `cpp/tests/mpi_failure_driver.cpp` has its own `main()`, excluded from the unit-runner glob; + `ctest -L mpi_failure` runs its two-rank abort scenarios (30 s timeout; a timeout is a hang). - Slow CTest startup in MPI builds is `MPI_Init` fabric probing; see `monoprop_TEST_EXCLUDE_MPI_FABRIC` in `cpp/tests/CMakeLists.txt`. - Sanitizer rebuild and test: diff --git a/CMakeLists.txt b/CMakeLists.txt index 1442722c3..03c3403e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,10 @@ find_package( find_package(Threads REQUIRED) +# Required in every configuration, including MPI-off and single-thread builds: there is no +# OpenMP-disabled variant of the library. +find_package(OpenMP REQUIRED COMPONENTS CXX) + if(monoprop_ENABLE_MPI) find_package(MPI REQUIRED COMPONENTS CXX) endif() diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 000000000..258eac936 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,24 @@ +# Monoprop + +Terminology for retained operators and their numerical comparison. + +## Language + +**Propagator instance**: +An independent propagation context for an operator and its evolution. A copy is a separate instance with independent +operator state. +_Avoid_: Partition when referring to a copied propagator + +**Rank-local operator**: +The portion of one propagator instance's global retained operator assigned to one rank. +_Avoid_: The process's operator when several independent propagator instances are present + +**Global retained operator**: +The collection of retained term keys and associated coefficients for one propagated operator, independent of how those +terms are distributed. +_Avoid_: Energy or gradient when referring to the operator itself + +**Retained-operator equivalence**: +Identical global retained term keys, with corresponding coefficients equal within the applicable numerical tolerances. +Agreement of energies or gradients alone does not establish retained-operator equivalence. +_Avoid_: Observable agreement as a synonym diff --git a/README.md b/README.md index 4656ad524..2643be260 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,19 @@ GitHub Actions calls, so a lane can be reproduced locally. Full instructions — prerequisites, MPI options, and running the example executable — are in the [building guide](https://docs.monoprop.algorithmiq.tech/building). -In particular, from-source builds require `hwloc` and `pkg-config` so CMake can -locate `hwloc`. +In particular, from-source builds require OpenMP for C++ in every configuration, +including builds without MPI. GCC ships its OpenMP runtime; Clang needs the matching +`libomp`, and Apple Clang needs Homebrew's `libomp` with +`OpenMP_ROOT="$(brew --prefix libomp)"` set. MPI is optional and off by default. + +Every build runs the sharded OpenMP runtime: each MPI rank (or the one process +without MPI) owns one shard per OpenMP thread, with a fixed `monoprop_NUM_THREADS` +(and `OMP_NUM_THREADS`, `OMP_DYNAMIC=FALSE`) set before launch and import, the same on +every rank. There is no partition count or runtime selector, and placement belongs +to the launcher and OpenMP. With MPI, `MPI_THREAD_FUNNELED` is enough. The removal of +the partition runtime changed the C++ constructor and ABI, so rebuild C++ consumers +together with the library; see the +[parallelism guide](https://docs.monoprop.algorithmiq.tech/features/parallelism). ## Running the tests @@ -179,7 +190,7 @@ The repository is a [Nix flake](https://wiki.nixos.org/wiki/Flakes), so on Nix o NixOS none of the prerequisites have to be installed by hand: ```bash -nix develop # dev shell: C++ toolchain, hwloc, MPI, uv, just, node +nix develop # dev shell: C++ toolchain, OpenMP, MPI, uv, just, node nix build .#monoprop # build the package (`.#monoprop-mpi` for the MPI build) nix build .#monoprop-py313 # pick the interpreter: py312, py313, py314 (default) nix run # Python interpreter with monoprop importable diff --git a/benches/LADDER.md b/benches/LADDER.md index 36701a097..ad5a19ece 100644 --- a/benches/LADDER.md +++ b/benches/LADDER.md @@ -5,16 +5,18 @@ This benchmark separates kernel, shared-memory, process, network, and scaling ef - `N`: nodes - `C`: usable cores per node - `R`: MPI ranks per node -- `P`: in-process partitions per rank -- `P_total = N × R × P`: total partitions +- `P`: OpenMP threads, and with them shards, per rank +- `P_total = N × R × P`: total shards Choose `R` for the platform, normally one rank per NUMA domain, then set `P = C / R`. Keep `R` and `P` fixed wherever the ladder requires a direct comparison. > **Example platform.** All flags, term counts, timings, and memory figures below were measured on > code based on `main` commit `97f95f762dfc7174243ccd59dd4ecbbb9775b610`, on Deucalion x86 -> nodes with `C=128`, using `R=8` and `P=16` for MPI runs. They are calibration and job-sizing -> examples, not portable targets but memory should be relatively similar. +> nodes with `C=128`, using `R=8` and `P=16` for MPI runs, with the partition runtime that the +> sharded OpenMP runtime has since replaced (`P` partitions per rank then). They are historical +> calibration and job-sizing examples, not portable targets and not measurements of the sharded +> OpenMP runtime, but memory should be relatively similar. ## Ladder @@ -47,16 +49,21 @@ graph. Run `propagate` separately because it owns another operator. ## Declaring the shape -The launcher sets `N` and `R`; export `P` explicitly: +The launcher sets `N` and `R`. Declare `P` before the launch and the import, with exactly `P` threads per +rank and the placement left to the launcher and OpenMP: ```bash -export monoprop_PARTITIONS= -export monoprop_NUM_THREADS= +export monoprop_NUM_THREADS= OMP_NUM_THREADS= OMP_DYNAMIC=FALSE +export OMP_PLACES=cores OMP_PROC_BIND=close # one thread per physical core ``` -For multi-rank runs this is mandatory. The engine otherwise defaults to one partition per rank. -Before comparing results, check `ranks`, `nodes`, `ranks_per_node`, `partitions_env`, and -`monoprop_threads` in the report's Configuration table. +Pass `--runtime-shape=openmp` (and `--build-mode=mpi` or `--build-mode=mpi-off`) to have the suite +refuse a run whose environment, rank count or imported build contradicts that declaration; above one +rank the suite otherwise asks for the partition count of the removed runtime. Do not set +`monoprop_PARTITIONS`: the library ignores it, and the `openmp` shape refuses it. + +Before comparing results, check `ranks`, `nodes`, `ranks_per_node`, `partitions_env`, +`monoprop_threads`, `declared_shape` and `has_mpi` in the run metadata. ## Calibrating model size @@ -84,7 +91,7 @@ Examples 128-core platform. ### L1 example -Set `monoprop_PARTITIONS=1` and `monoprop_NUM_THREADS=1`. +Set `monoprop_NUM_THREADS=1` and `OMP_NUM_THREADS=1` (measured with the partition runtime at one partition). | row | flags | selector | terms | time | peak RSS | | --- | --- | --- | ---: | ---: | ---: | @@ -95,7 +102,7 @@ Set `monoprop_PARTITIONS=1` and `monoprop_NUM_THREADS=1`. ### L2a example -Set `monoprop_PARTITIONS=C` and `monoprop_NUM_THREADS=C`. On the example platform, `C=128`. These are medians of three repetitions. +Set `monoprop_NUM_THREADS=C` and `OMP_NUM_THREADS=C` (measured with the partition runtime at `C` partitions). On the example platform, `C=128`. These are medians of three repetitions. | row | flags | selector | terms | time | peak RSS | @@ -186,13 +193,12 @@ size sweep rather than treating this table as a target. ## Launch templates -For L1 and L2a, request one node and one rank. Set the core and partition counts from the granted -CPU affinity: +For L1 and L2a, request one node and one rank. Set the thread count from the granted CPU +affinity: ```bash -export monoprop_PARTITIONS="$C" -export monoprop_NUM_THREADS="$C" -just bench L2a-hubbard-branch --hubbard-cutoff=10 --hubbard-lower-atol=3.38e-06 \ +export monoprop_NUM_THREADS="$C" OMP_NUM_THREADS="$C" OMP_DYNAMIC=FALSE OMP_PLACES=cores OMP_PROC_BIND=close +just bench L2a-hubbard-branch --runtime-shape=openmp --hubbard-cutoff=10 --hubbard-lower-atol=3.38e-06 \ -k "test_model_propagate and hubbard" ``` @@ -200,13 +206,12 @@ For L2b, L3, and L4, request `R` ranks per node and `P` cores per rank. This Slu the scheduler variables have already been set consistently: ```bash -export monoprop_PARTITIONS="$P" -export monoprop_NUM_THREADS="$P" +export monoprop_NUM_THREADS="$P" OMP_NUM_THREADS="$P" OMP_DYNAMIC=FALSE OMP_PLACES=cores OMP_PROC_BIND=close export monoprop_BENCH_LABEL=L3-hubbard-branch export monoprop_BENCH_RESULTS=benches/results srun --ntasks-per-node="$R" --cpus-per-task="$P" --cpu-bind=cores \ - .venv/bin/python -m pytest benches -o filterwarnings=default \ + .venv/bin/python -m pytest benches -o filterwarnings=default --runtime-shape=openmp \ --benchmark-json="benches/results/time-$monoprop_BENCH_LABEL.json" \ -k "test_model_propagate and hubbard" \ --hubbard-cutoff=10 --hubbard-lower-atol=3.38e-06 @@ -221,7 +226,7 @@ each rank to avoid repeated dependency resolution on shared filesystems. - Repeat the rank count, cores per rank, and binding options on the launcher command; scheduler allocation directives may not propagate them. -- Keep `P` no larger than the CPUs visible to each rank. Otherwise thread placement is disabled. +- Keep `P` no larger than the CPUs visible to each rank: the team still has `P` threads, so they oversubscribe. - Use the scheduler affinity mask, such as `os.sched_getaffinity(0)`, to verify visible CPUs; `nproc` may honor unrelated thread limits. - Keep CPU binding enabled unless the target platform has been measured otherwise. diff --git a/benches/README.md b/benches/README.md index 852d1ac2f..1c19eef02 100644 --- a/benches/README.md +++ b/benches/README.md @@ -27,6 +27,11 @@ Everything reusable is in the `monoprop-bench-tools` package memory instrumentation, the model builders, and the two renderers that turn a run's artifacts into `REPORT.md` and Bencher Metric Format JSON. +`../tools/benchmark-rank-local-openmp.py` drives these benchmarks for the rank-local OpenMP parity +campaign. It runs a node, a separate whole-construction measurement and an untimed numerical +validation in fresh processes, then compares two arms against a frozen inventory. See the +Benchmarks documentation. + Cross-engine comparisons against other propagation libraries live in [`../packages/bench-third-party`](../packages/bench-third-party), a standalone uv project with its own lockfile. diff --git a/benches/conftest.py b/benches/conftest.py index 40ab612cd..7123dc4cb 100644 --- a/benches/conftest.py +++ b/benches/conftest.py @@ -25,9 +25,18 @@ so a change to ``_RESULTS`` has to land in both places. Recording is on only when the ``just bench`` recipe exports ``monoprop_BENCH_LABEL`` / ``monoprop_BENCH_RESULTS``. -The ``bench_comm`` fixture yields ``MPI.COMM_WORLD`` when ``mpi4py`` is available -(``None`` otherwise). Operations are barrier-wrapped so the timed cost reflects -the slowest rank, and only rank 0 writes results. +The ``bench_comm`` fixture yields ``MPI.COMM_WORLD`` when the imported extension was built +with MPI and ``mpi4py`` is available (``None`` otherwise). An MPI-off extension never imports +``mpi4py``: importing ``mpi4py.MPI`` initializes MPI, which an MPI-off measurement must not do. +Operations are barrier-wrapped so the timed cost reflects the slowest rank, and only rank 0 +writes results. + +Each memory window records whether it was exact on every rank, independently: +``opmemexact`` for the operation window and ``memhwmexact`` for the outer ``memhwm`` window. +``--runtime-shape`` (``partitions`` or ``openmp``) and ``--build-mode`` (``mpi`` or +``mpi-off``) declare the measured shape; the preflight in +:mod:`monoprop_bench_tools.preflight` checks them against the environment, the rank count and +the imported extension before anything is measured. """ from __future__ import annotations @@ -52,6 +61,13 @@ build_random_propagator, make_random_problem, ) +from monoprop_bench_tools.preflight import ( + PreflightError, + declared_shape, + import_mpi, + require_shape_runtime, + runtime_identity, +) import monoprop @@ -62,12 +78,13 @@ from monoprop import MajoranaPropagator -try: - from mpi4py import MPI -except (ImportError, OSError, RuntimeError): # pragma: no cover - optional MPI build - # mpi4py may be absent, or (the ABI wheel) present but unable to dlopen libmpi - # on a serial node with no MPI module loaded. - MPI = None +# The build mode of the extension actually imported; ``None`` if it cannot be read. +_HAS_MPI: bool | None = getattr(monoprop, "has_mpi", None) + +# Gated on the build mode before mpi4py is touched: an MPI-off run must not initialize MPI even +# when mpi4py is installed. For an MPI build, mpi4py may still be absent, or (the ABI wheel) +# present but unable to dlopen libmpi on a serial node; the run is then serial. +MPI = import_mpi(has_mpi=bool(_HAS_MPI)) def _rank() -> int: @@ -118,6 +135,11 @@ def _gather_lists(comm: Any, values: list[int]) -> list[list[int]]: return gathered if gathered is not None else [] +def _all_ranks(comm: Any, *, flag: bool) -> bool: + """Return whether ``flag`` holds on every rank. Collective; serial returns ``flag``.""" + return _reduce_min(comm, int(flag)) == 1 + + def _spread(comm: Any, value: int) -> dict[str, int]: """Reduce a per-rank number to ``sum`` (bounds the job) and ``max`` (bounds a node).""" return {"sum": _reduce_sum(comm, value), "max": _reduce_max(comm, value)} @@ -153,6 +175,9 @@ def _spread(comm: Any, value: int) -> dict[str, int]: "opmembase": {}, # node id -> that floor "opbytes": {}, # node id -> {"operator": n, "graph": n} "opmembreak": {}, # node id -> operator memory split (bytes) + # Per-window exactness, each true only when every rank's window was exact. + "opmemexact": {}, # node id -> the operation window (opmem*) + "memhwmexact": {}, # node id -> the outer window (memhwm, memhwm_max) } @@ -177,6 +202,20 @@ def pytest_addoption(parser: pytest.Parser) -> None: for name, kind, default, help_text in _RANDOM_OPTIONS: group.addoption(f"--{name}", type=kind, default=default, help=help_text) + shape = parser.getgroup("monoprop-shape", "monoprop measurement preflight") + shape.addoption( + "--runtime-shape", + choices=("partitions", "openmp"), + default=None, + help="Declared process layout, checked against the environment and rank count.", + ) + shape.addoption( + "--build-mode", + choices=("mpi", "mpi-off"), + default=None, + help="Required build mode, checked against the imported extension's has_mpi.", + ) + models = parser.getgroup("monoprop-models", "monoprop fixed-model overrides") for model, (config_cls, _builder, _steps) in MODELS.items(): for field in fields(config_cls): @@ -188,6 +227,15 @@ def pytest_addoption(parser: pytest.Parser) -> None: ) +def _core_sha256() -> str: + """Return the SHA-256 of the extension module this run imported (see :func:`_core_md5`).""" + try: + path = Path(monoprop._core.__file__) + return hashlib.sha256(path.read_bytes()).hexdigest() + except (AttributeError, OSError, TypeError): + return "unavailable" + + def _core_md5() -> str: """Return the md5 of the extension module this run imported. @@ -219,6 +267,12 @@ def _meta(nodes: int, ranks_per_node: int) -> dict[str, Any]: "hostname": socket.gethostname(), "monoprop_version": monoprop.__version__, "monoprop_core_md5": _core_md5(), + "monoprop_core_sha256": _core_sha256(), + "monoprop_core_path": str(getattr(monoprop._core, "__file__", "unavailable")), + "monoprop_runtime_identity": getattr( + monoprop._core, "__runtime_identity__", None + ), + "has_mpi": _HAS_MPI, "monoprop_variant": monoprop.__variant__, "monoprop_compiler_flags": monoprop.__compiler_flags__, "python_version": platform.python_version(), @@ -245,14 +299,44 @@ def _params(config: pytest.Config) -> dict[str, Any]: } -def _require_shape() -> None: - """Fail a multi-rank run that has not declared its partition count. +def _require_shape(config: pytest.Config) -> dict[str, Any] | None: + """Check the declared measurement shape, or the legacy guard when none is declared. - ``resolve_partition_count_`` defaults to ``ranks == 1 ? cores : 1``, so an unset knob - under MPI measures one partition per rank -- a single-threaded run at a plausible wall - time, with nothing in the timing to say so. Every rank raises, so no rank is left in a - collective. Serial runs take the engine default and are unaffected. + With ``--runtime-shape`` the shared preflight validates the declaration against the + environment, the observed rank count and the imported build mode (plus ``--build-mode``), + and returns it for the run metadata. Without one, the legacy guard remains: a multi-rank run + must declare its partition count, because ``resolve_partition_count_`` defaults to + ``ranks == 1 ? cores : 1`` and an unset knob under MPI measures one partition per rank -- a + single-threaded run at a plausible wall time, with nothing in the timing to say so. Every + rank raises, so no rank is left in a collective. Serial runs take the engine default. """ + runtime_shape = config.getoption("--runtime-shape") + build_mode = config.getoption("--build-mode") + expected_has_mpi = None if build_mode is None else build_mode == "mpi" + if expected_has_mpi is not None and _HAS_MPI is not expected_has_mpi: + msg = ( + f"--build-mode={build_mode} requires has_mpi={expected_has_mpi}, but the imported " + f"extension has has_mpi={_HAS_MPI}." + ) + raise pytest.UsageError(msg) + if runtime_shape is not None: + try: + shape = declared_shape( + runtime_shape, + os.environ, + ranks=_size(), + has_mpi=_HAS_MPI, + expected_has_mpi=expected_has_mpi, + ) + # The declared shape must be one the imported binary's own runtime serves. + identity = runtime_identity( + Path(monoprop._core.__file__), + getattr(monoprop._core, "__runtime_identity__", None), + ) + require_shape_runtime(runtime_shape, identity) + except PreflightError as exc: + raise pytest.UsageError(str(exc)) from exc + return {**shape, "runtime_identity": identity} if _size() > 1 and not os.environ.get("monoprop_PARTITIONS"): # noqa: SIM112 msg = ( "monoprop_PARTITIONS is unset on a run of " @@ -261,6 +345,7 @@ def _require_shape() -> None: "for 8 ranks per 128-core node." ) raise pytest.UsageError(msg) + return None @pytest.hookimpl(trylast=True) @@ -271,9 +356,10 @@ def pytest_configure(config: pytest.Config) -> None: ``trylast`` so the terminal reporter exists before non-root ranks unregister it. """ nodes, ranks_per_node = _nodes() # collective; every rank must call this - _require_shape() + shape = _require_shape(config) if _rank() == 0: _RESULTS["meta"] = _meta(nodes, ranks_per_node) + _RESULTS["meta"]["declared_shape"] = shape _RESULTS["params"] = _params(config) return @@ -418,6 +504,7 @@ def close(self, propagator: Any = None) -> None: _record("opmemdelta", self._key, _spread(self._comm, window.delta_bytes)) _record("opmempeak", self._key, _spread(self._comm, window.peak_bytes)) _record("opmembase", self._key, _spread(self._comm, window.baseline_bytes)) + _record("opmemexact", self._key, _all_ranks(self._comm, flag=window.exact)) _record_placement(self._comm) @@ -487,6 +574,8 @@ def record_memory(request: pytest.FixtureRequest, bench_comm: Any) -> Iterator[N key = request.node.nodeid.split("/")[-1] hwm = _reduce_sum(bench_comm, window.peak_bytes) hwm_max = _reduce_max(bench_comm, window.peak_bytes) + # Its own flag: an exact operation window inside it certifies nothing about this one. + _record("memhwmexact", key, _all_ranks(bench_comm, flag=window.exact)) if hwm: _record("memhwm", key, hwm) if hwm_max: diff --git a/cmake/compiler_flags/GNU.CXX.cmake b/cmake/compiler_flags/GNU.CXX.cmake index eaf9d99a1..80698d42d 100644 --- a/cmake/compiler_flags/GNU.CXX.cmake +++ b/cmake/compiler_flags/GNU.CXX.cmake @@ -6,9 +6,12 @@ if(CMAKE_CXX_COMPILER_ID MATCHES GNU) ) endif() + # GCC's default -ffp-contract=fast fuses multiplies and adds across statements, so two code paths that compute the + # same expression can round differently. `on` contracts within one expression only, which is Clang's default, and + # keeps the bitwise cross-path checks in the test suite meaningful. set( monoprop_CXX_FLAGS - "-Wall -Wno-unknown-pragmas -Wno-sign-compare -Woverloaded-virtual -Wwrite-strings -Wextra -Wconversion -Wnon-virtual-dtor -Wcast-align -Wunused-parameter -fdiagnostics-color=always -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer" + "-Wall -Wno-unknown-pragmas -Wno-sign-compare -Woverloaded-virtual -Wwrite-strings -Wextra -Wconversion -Wnon-virtual-dtor -Wcast-align -Wunused-parameter -fdiagnostics-color=always -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -ffp-contract=on" ) set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") diff --git a/cmake/monopropConfig.cmake.in b/cmake/monopropConfig.cmake.in index 89296ac2d..a4af9c79f 100644 --- a/cmake/monopropConfig.cmake.in +++ b/cmake/monopropConfig.cmake.in @@ -31,6 +31,13 @@ find_dependency( Threads REQUIRED ) +# Always required: the exported target links OpenMP::OpenMP_CXX in every configuration. +find_dependency( + OpenMP + REQUIRED + COMPONENTS + CXX +) set(monoprop_ENABLE_MPI "@monoprop_ENABLE_MPI@") if(monoprop_ENABLE_MPI) diff --git a/cpp/include/monoprop/Evolution.h b/cpp/include/monoprop/Evolution.h index 23f4d1902..b40aaaf8e 100644 --- a/cpp/include/monoprop/Evolution.h +++ b/cpp/include/monoprop/Evolution.h @@ -17,6 +17,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/parallel/Options.h" #include "monoprop/monopropExport.h" namespace monoprop { @@ -29,12 +30,17 @@ struct LayerAngle { double param = 0.0; ///< the optimizer parameter driving this layer }; +// Each entry below takes a trailing thread budget, `options`, for its local kernels; the default is serial. +// A rank-local failure after the entry has started communicating goes to mpi::operation_failed(): it aborts +// a multi-rank ordinary MPI communicator and rethrows the original exception otherwise. + /// Forward-evolve `op` through one layer; each rank owns its local coefficients, cross-rank cycles are communicated. monoprop_EXPORT auto evolve_step(VecD &op, const Layer &layer, double param, mpi::Comm comm, - const detail::LayerCosScale &cos_scale) -> void; + const detail::LayerCosScale &cos_scale, + detail::parallel::Options options = {}) -> void; /// Forward-evolve `op` through one layer of `graph`; `layer_idx` also selects the layer's cosine set. monoprop_EXPORT auto evolve_step(VecD &op, @@ -42,14 +48,16 @@ monoprop_EXPORT auto evolve_step(VecD &op, double param, size_t layer_idx, mpi::Comm comm, - const detail::LayerCosScale &cos_scale) -> void; + const detail::LayerCosScale &cos_scale, + detail::parallel::Options options = {}) -> void; /// Forward-evolve `coeffs` through every layer of `graph`, returning this rank's evolved coefficients. monoprop_EXPORT auto evolve_operator(VecD &&coeffs, const MPGraphView &graph, const VecD ¶ms, mpi::Comm comm, - const detail::LayerCosScale &cos_scale) -> VecD; + const detail::LayerCosScale &cos_scale, + detail::parallel::Options options = {}) -> VecD; /// Reverse-mode derivative of one layer: inverse-rotates (state, op) in place and returns the gradient term. /// `record` must be the pre-layer coefficients the forward pass kept for this same `layer_idx`. @@ -60,5 +68,6 @@ monoprop_EXPORT auto state_operator_derivative_local(VecD &state, LayerAngle angle, mpi::Comm comm, const detail::LayerCosAccumulate &cos_acc, - const detail::CosRecordView &record = {}) -> double; + const detail::CosRecordView &record = {}, + detail::parallel::Options options = {}) -> double; } // namespace monoprop diff --git a/cpp/include/monoprop/MPFunctions.h b/cpp/include/monoprop/MPFunctions.h index eb8bdd02b..d819d37f6 100644 --- a/cpp/include/monoprop/MPFunctions.h +++ b/cpp/include/monoprop/MPFunctions.h @@ -25,6 +25,7 @@ #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/parallel/Options.h" #include "monoprop/monopropExport.h" namespace monoprop { @@ -83,17 +84,19 @@ monoprop_EXPORT auto map_params(const VecD ¶meters, /// into it. Built at the call site and consumed there -- every member but `graph` is a non-owning /// reference, and `graph` is itself a view over layers the caller keeps alive. struct EvalRequest { - double e_core; ///< the identity term, added to the summed expectation value - const EvalState &state; ///< the contraction partner; see EvalState - const VecD &op; ///< un-evolved operator coefficients, one per term on this rank - const VecZ ¶meter_mapping; ///< optimizer order: which parameter drives graph layer i - const VecD &gen_coeffs; ///< optimizer order, parallel to parameter_mapping - MPGraphView graph; ///< replay window; MPGraph callers pass graph.replay_view() - const VecD ¶ms; ///< the optimizer's parameter vector, indexed by parameter_mapping + double e_core; ///< the identity term, added to the summed expectation value + const EvalState &state; ///< the contraction partner; see EvalState + const VecD &op; ///< un-evolved operator coefficients, one per term on this rank + const VecZ ¶meter_mapping; ///< optimizer order: which parameter drives graph layer i + const VecD &gen_coeffs; ///< optimizer order, parallel to parameter_mapping + MPGraphView graph; ///< replay window; MPGraph callers pass graph.replay_view() + const VecD ¶ms; ///< the optimizer's parameter vector, indexed by parameter_mapping + detail::parallel::Options parallel = {}; ///< the owning propagator's thread budget; serial by default }; /// Expectation value of the evolved operator against `request.state` plus `request.e_core`, summed over all /// ranks. `cos.scale` is required if `request.params` is non-empty; `cos.accumulate` is ignored here. +/// A rank-local failure past argument validation goes to mpi::operation_failed() before the allreduce. monoprop_EXPORT auto ev(const EvalRequest &request, mpi::Comm comm = MPI_COMM_WORLD, const detail::CosCallbacks &cos = {}) -> double; diff --git a/cpp/include/monoprop/MPGraph.h b/cpp/include/monoprop/MPGraph.h index e446c9d46..bbec599ef 100644 --- a/cpp/include/monoprop/MPGraph.h +++ b/cpp/include/monoprop/MPGraph.h @@ -100,6 +100,9 @@ class monoprop_EXPORT MPGraph { /// Non-owning replay view over the active layers, in build order. auto replay_view() const -> MPGraphView { return {layers_, active_begin_index(), layers(), false}; } + /// Non-owning view over the active layers, newest first: the window of an adjoint sweep. + auto reverse_replay_view() const -> MPGraphView { return {layers_, active_begin_index(), layers(), true}; } + auto is_schrodinger() const -> bool { return schrodinger_; } /// A normally-built layer stores no cosine set, so the companion cosine-index count cannot come from diff --git a/cpp/include/monoprop/MonomialPropagator.h b/cpp/include/monoprop/MonomialPropagator.h index ccd13c566..f6413500d 100644 --- a/cpp/include/monoprop/MonomialPropagator.h +++ b/cpp/include/monoprop/MonomialPropagator.h @@ -19,9 +19,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -44,15 +46,19 @@ #include "monoprop/detail/evolution/CosineRecompute.h" #include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/mpi/Routing.h" #include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/sharded/Exchange.h" +#include "monoprop/detail/sharded/RootObserver.h" +#include "monoprop/detail/sharded/State.h" namespace monoprop { namespace detail { struct FusedContract; -namespace partition { +// White-box test access; declared here and defined only by the test suite. template -class PartitionGroup; -} // namespace partition +struct PropagatorTestAccess; } // namespace detail /// A propagator setting is out of range, or inconsistent with another setting. @@ -63,8 +69,20 @@ class PropagatorConfigError : public std::runtime_error { using std::runtime_error::runtime_error; }; -/// A raw-layout accessor was called on a multi-partition propagator, which owns no operator or graph. -class MultiPartitionUnsupported : public std::runtime_error { +/// The propagator failed part-way through changing its state and can no longer be used. +// Thrown before any state is touched by every later state-consuming or mutating call, by copying or +// cloning it, and by the retained functionals it created. Nothing resets it: destroy the object and +// construct a new one, or keep using an independent copy made before the failure. +class InvalidPropagatorError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/// A raw-layout accessor (mp_op(), indexing(), graph(), graph_data()) was called on a propagator whose rank holds more +/// than one shard. The shard count is the thread budget T captured at construction, so the remedy is a launch with +/// monoprop_NUM_THREADS=1; the shard-transparent accessors, evolved_operator_terms(), +/// evolved_operator_coefficients() and term_expectation_values() work at every T. +class MultiShardUnsupported : public std::runtime_error { public: using std::runtime_error::runtime_error; }; @@ -72,8 +90,11 @@ class MultiPartitionUnsupported : public std::runtime_error { template class MonomialPropagator { public: - using PartitionChildFactory = std::function>(mpi::Comm)>; - + /// Collective over `comm`: every one of its P ranks constructs with the same arguments. Captures the thread budget + /// T once, here, from monoprop_NUM_THREADS (strictly parsed) or, when the variable is unset, from + /// omp_get_max_threads(); T is also this rank's shard count, and every rank must be launched with the same T. The + /// router's flat owner of shard t on rank r is r * T + t. There is no partition, thread or shard argument. + /// \throws PropagatorConfigError for an invalid budget or setting, before any shard is allocated. MonomialPropagator(const OperatorDict &initial_operator, unsigned int cutoff, const VecZ &initial_state, @@ -84,15 +105,14 @@ class MonomialPropagator { CutoffType cutoff_type = CutoffType::Length, std::optional> basis_change = std::nullopt, size_t logical_num_modes = NumModes, - Basis basis = Basis::Majorana, - size_t partitions = 0, - PartitionChildFactory child_factory = nullptr); + Basis basis = Basis::Majorana); - /// Out-of-line because partition_group_ is a unique_ptr to an incomplete type here. virtual ~MonomialPropagator(); - /// Deep copy: clones the operator store, shares the immutable graph cores, and clones the whole - /// partition group on a facade. The virtual destructor suppresses implicit moves, so a "move" deep-copies. + /// Deep copy: clones every shard's operator store on its owner and shares the immutable graph cores; the copy + /// starts with its own empty physical rounds. The virtual destructor suppresses implicit moves, so + /// a "move" deep-copies. The copy keeps the source's thread budget, and with it T: nothing is re-read. Throws + /// InvalidPropagatorError, before copying anything, if the source is invalid. MonomialPropagator(const MonomialPropagator &other); auto operator=(const MonomialPropagator &) -> MonomialPropagator & = delete; @@ -101,73 +121,61 @@ class MonomialPropagator { auto logical_num_modes() const -> size_t { return logical_num_modes_; } - /// Term count on this rank (allreduce for global). - auto size() const -> size_t { return partition_group_ ? partitioned_size_() : mp_op_.size(); } - - /// (cosine-only indices, cycles) on this rank; cosine-only = cos-scaled but not a rotation endpoint. - auto graph_size() const -> std::pair { - return partition_group_ ? partitioned_graph_size_() : std::pair{cos_index_count_(), graph_.total_cycles()}; + /// Term count on this rank, summed over its shards (allreduce for global). + auto size() const -> size_t { + require_valid_(); + return detail::sharded::total_size(shards_); } - /// Local to this rank. Single-partition only — see require_single_partition_. - auto graph() const -> const MPGraph & { - require_single_partition_("graph()"); - return graph_; - } + /// (cosine-only indices, cycles) on this rank, summed over its shards; cosine-only = cos-scaled but not a + /// rotation endpoint. + auto graph_size() const -> std::pair; - /// This rank's operator storage. Single-partition only — see require_single_partition_. - auto mp_op() -> detail::MPOperator & { - require_single_partition_("mp_op()"); - return mp_op_; - } - auto mp_op() const -> const detail::MPOperator & { - require_single_partition_("mp_op()"); - return mp_op_; - } + /// The sole shard's graph. Launch-time T = 1 only (any P) -- see sole_shard_. + auto graph() const -> const MPGraph & { return sole_shard_("graph()").graph; } + + /// The sole shard's operator storage. Launch-time T = 1 only (any P) -- see sole_shard_. + auto mp_op() -> detail::MPOperator & { return sole_shard_("mp_op()").op; } + auto mp_op() const -> const detail::MPOperator & { return sole_shard_("mp_op()").op; } - // The breakdown fields are additive over the disjoint hash partitions, so a facade sums them. + /// This rank's graph memory, summed over its disjoint shards (each shard's graph holds its own layer cores). auto graph_memory_usage() const -> GraphMemoryBreakdown { - if (partition_group_) { - return partitioned_graph_memory_usage_(); - } - return graph_.storage_memory_usage(); + require_valid_(); + return detail::sharded::graph_memory_usage(shards_); } + /// This rank's operator memory, summed over its disjoint shards (each including its own matched marks). auto operator_memory_usage() const -> detail::MPOperatorMemoryBreakdown { - if (partition_group_) { - return partitioned_operator_memory_usage_(); - } - // The stamp array is a member of THIS class, not of the operator, so the operator-side estimate - // leaves matched_scratch_bytes at 0 and only this level can fill it in. - auto breakdown = detail::estimate_memory_usage(mp_op_); - breakdown.matched_scratch_bytes = matched_scratch_.memory_bytes(); - return breakdown; + require_valid_(); + return detail::sharded::operator_memory_usage(shards_); } - auto graph_layers() const -> size_t { return partition_group_ ? partitioned_graph_layers_() : graph_.layers(); } + /// The number of graph layers; layer metadata is identical on every shard, so it is read once, never summed. + auto graph_layers() const -> size_t { + require_valid_(); + return shards_.front()->graph.layers(); + } /// A multi-term gate spans several layers, so n_gates() <= graph_layers(). auto n_gates() const -> size_t; /// Per-layer, in optimizer order (length = graph_layers()). - auto parameter_mapping() const -> VecZ { return graph_gate_arrays_().first; } + auto parameter_mapping() const -> VecZ { + require_valid_(); + return graph_gate_arrays_().first; + } /// Re-wire which parameter drives each graph layer, in place. Accepts a per-layer mapping (length /// graph_layers(), optimizer order) or a per-gate one (length n_gates()); on a tie, per-layer wins. auto set_parameter_mapping(const VecZ ¶meter_mapping) -> void; - /// This rank's monomial → coefficient index. Single-partition only — see require_single_partition_. - auto indexing() -> detail::OperatorIndex & { - require_single_partition_("indexing()"); - return *mp_op_.store; - } - auto indexing() const -> const detail::OperatorIndex & { - require_single_partition_("indexing()"); - return *mp_op_.store; - } + /// The sole shard's monomial → coefficient index. Launch-time T = 1 only (any P) -- see sole_shard_. + auto indexing() -> detail::OperatorIndex & { return *sole_shard_("indexing()").op.store; } + auto indexing() const -> const detail::OperatorIndex & { return *sole_shard_("indexing()").op.store; } - /// Per-layer (cos_inds, local_cycles, cross_rank_sin_send, cross_rank_sin_recv) for this - /// rank/partition. local_cycles is always empty: local cycles are folded into cross_rank[my_rank]. + /// Per-layer (cos_inds, local_cycles, cross_rank_sin_send, cross_rank_sin_recv) of the sole shard (launch-time + /// T = 1 only, any P -- see sole_shard_). local_cycles is always empty: local cycles are folded into + /// cross_rank[my_rank]. using LocalCycleData = std::tuple; using CrossRankData = std::tuple; // (indices, phases) using LayerData = @@ -175,6 +183,7 @@ class MonomialPropagator { auto graph_data() const -> std::vector; auto update_lower_atol(std::optional new_lower_atol) -> void { + require_valid_(); if (upper_atol_.has_value() && new_lower_atol.has_value() && (new_lower_atol.value() > upper_atol_.value())) { throw PropagatorConfigError( std::format("New lower_atol ({}) must be less than or equal to current upper_atol ({}).", @@ -185,6 +194,7 @@ class MonomialPropagator { } auto update_upper_atol(std::optional new_upper_atol) -> void { + require_valid_(); if (lower_atol_.has_value() && new_upper_atol.has_value() && new_upper_atol.value() < lower_atol_.value()) { throw PropagatorConfigError( std::format("New upper_atol ({}) must be greater than or equal to current lower_atol ({}).", @@ -203,6 +213,7 @@ class MonomialPropagator { } auto update_cutoff_type(CutoffType new_cutoff_type) -> void { + require_valid_(); validate_cutoff_config_(new_cutoff_type, basis_change_); update_setting_([&](MonomialPropagator &p) { p.cutoff_type_ = new_cutoff_type; @@ -212,6 +223,7 @@ class MonomialPropagator { /// The basis the cutoff is measured in; nullopt ⇒ the native basis. auto update_basis_change(std::optional> new_basis_change) -> void { + require_valid_(); validate_cutoff_config_(cutoff_type_, new_basis_change); update_setting_([&](MonomialPropagator &p) { p.basis_change_ = new_basis_change; @@ -223,7 +235,11 @@ class MonomialPropagator { auto basis() const -> Basis { return basis_; } - auto core_term() const -> double { return partition_group_ ? partitioned_core_term_() : core_term_; } + /// The initial operator's identity coefficient: rank-level metadata, replicated, never seeded into a shard. + auto core_term() const -> double { + require_valid_(); + return core_term_; + } auto cutoff() const -> unsigned int { return cutoff_; } @@ -235,7 +251,7 @@ class MonomialPropagator { auto basis_change() const -> std::optional> { return basis_change_; } - /// MPI_COMM_SELF on a partition, which trades over an in-process comm instead. + /// The ordinary communicator this propagator was constructed on. auto comm() const -> MPI_Comm { return comm_.mpi; } /// Build the propagation graph, one layer per generator, recording each layer's gate info @@ -274,14 +290,15 @@ class MonomialPropagator { /// Contract the graph into the operator (Heisenberg) or state (Schrodinger). `inplace` consumes the /// graph and updates internal state; otherwise nothing is mutated. Core term excluded either way. - /// Coefficients are positioned by the owning partition's indexing(), so on a facade the result is - /// the per-partition blocks concatenated in partition order: the same multiset as an unpartitioned - /// run, but not positionally stable across partition counts — and the count is auto-picked from the - /// host's core count unless pinned. Use evolved_operator_terms() when positions must mean something. + /// Rank-local: each store's coefficients are positioned by its own index, and the result concatenates this + /// rank's blocks in ascending shard order. The order is stable for a fixed launch + /// geometry (P, T), but positions are not comparable across geometries -- changing T changes which shard owns a + /// term. Use evolved_operator_terms() when positions must mean something. auto contract_partially(const VecD ¶meters, bool inplace) -> VecD; - /// Decoded (indices, coefficient) terms with |coeff| >= atol, gathered across all partitions. - /// Contracts non-inplace. Core term excluded. + /// Decoded (indices, coefficient) terms with |coeff| >= atol (atol = 0 keeps stored zeros), over every store of + /// this rank, each decoded by its own index; rank-local, so the global operator is the union over ranks. + /// Coefficients are rounded to 1e-12. Contracts non-inplace. Core term excluded. auto evolved_operator_terms(const VecD ¶meters, double atol) -> std::vector>>; @@ -291,7 +308,8 @@ class MonomialPropagator { /// The expectation value of each of `terms` under the truncated evolution, in query order. If `terms` /// are the initial operator's terms, expectation_value() is the sum of their coefficients times these - /// values. All values come from one adjoint replay of the graph. Keys must be canonical. + /// values. All values come from one adjoint replay of the graph over every shard. Keys must be canonical. + /// Collective, and the result is global: every rank returns every key's value. /// /// In Heisenberg, only initial-operator terms give their true expectation value: a row created /// mid-circuit gives the derivative of expectation_value() with respect to its coefficient, and a term @@ -324,56 +342,8 @@ class MonomialPropagator { auto apply_initial_operator_(const OperatorDict &op_dict) -> std::pair, VecD>; bool schrodinger_; - mpi::Comm comm_; // real MPI across nodes, or an in-process comm across partitions + mpi::Comm comm_; // the ordinary communicator of the P ranks CutoffFn cutoff_fn_; - detail::MPOperator mp_op_; - MPGraph graph_; - // Per-gate layer-build scratch, reused across gates; carries no state between them. - detail::MatchedEpochSet matched_scratch_; - - // A perf hint, never a correctness constraint: overflow spills losslessly. Sized to the cutoff's - // structural position bound when it has one. - auto packed_inline_width_() const -> size_t; - - // `requested` 0 ⇒ env/auto. Returns 1 for the ordinary single-partition path. - static auto resolve_partition_count_(size_t requested, mpi::Comm comm) -> size_t; - - // Partition fan-out vocabulary. Every one of these is facade-only: partition_group_ != nullptr is a precondition. - // - // `for_each_partition_` / `map_partitions_` / `concat_partitions_` dispatch to the partitions' own pinned master - // threads, which is mandatory for anything that touches partition state: the partitions trade over an - // in-process comm and their collectives are barrier-synced, so all S must run together. - // `fold_partitions_` / `sum_partitions_` / `first_partition_` instead read quiescent partitions straight from the - // facade thread, which is safe precisely because they mutate nothing. - - auto for_each_partition_(const std::function &fn) -> void; - - // One result per partition, in partition order. - template > - auto map_partitions_(Fn fn) -> std::vector; - - // Concatenated in partition order. The partitions are disjoint, so the result enumerates the whole - // operator (deterministic for a fixed partition count). - template > - auto concat_partitions_(Fn fn) -> R; - - // Sequential fold of `proj(partition)`; `accumulate(total, value)` combines. - template > - auto fold_partitions_(Proj proj, Accumulate accumulate) const -> R; - - // fold_partitions_ for the additive breakdowns: the fields sum over the disjoint hash partitions. - template > - auto sum_partitions_(Proj proj) const -> R; - - // Partition 0 for values that are not partitioned: the graph structure and gate info are identical on - // every partition, the core (identity) term is replicated on all of them, and an allreduced scalar is - // already global on each. Anything hash-partitioned must go through sum_/concat_partitions_ instead. - auto first_partition_() const -> const MonomialPropagator &; - - auto is_partition_facade() const -> bool { return static_cast(partition_group_); } - - template > - auto map_partitions_indexed_(Fn fn) -> std::vector; private: unsigned int cutoff_; @@ -400,42 +370,40 @@ class MonomialPropagator { // Immutable after construction. Basis basis_{Basis::Majorana}; - // Intra-process partition runtime. Null ⇒ ordinary single-partition propagator; non-null ⇒ a partition facade - // whose own mp_op_/graph_ are unused and every method fans out to the S partition propagators. - std::unique_ptr> partition_group_; - // PartitionGroup rebinds a cloned partition's comm_ to its own transport during a deep copy. - friend class detail::partition::PartitionGroup; - - // A facade's own graph_/mp_op_ are never populated, so handing them out would return plausible-looking - // empty state; there is no meaningful merge either, since the callers want one partition's raw layout. - auto require_single_partition_(const char *what) const -> void { - if (partition_group_) { - throw MultiPartitionUnsupported( - std::format("{} is not available on a multi-partition propagator: the facade owns " - "no operator or graph of its own. Use the partition-transparent " - "accessors (size(), graph_size(), evolved_operator_terms(), ...) or " - "construct with partitions=1.", - what)); - } - } + friend struct detail::PropagatorTestAccess; - auto partitioned_size_() const -> size_t; - auto partitioned_graph_size_() const -> std::pair; - auto partitioned_graph_layers_() const -> size_t; - auto partitioned_core_term_() const -> double; - auto partitioned_operator_memory_usage_() const -> detail::MPOperatorMemoryBreakdown; - auto partitioned_graph_memory_usage_() const -> GraphMemoryBreakdown; + // This object's thread budget, captured once at construction and kept by copies and retained functionals; its + // thread count is the shard count T. + detail::parallel::Options parallel_{}; + // Set when an operation fails after it started changing state; never cleared (see require_valid_). + bool invalid_{false}; - auto cos_index_count_() const -> size_t; + /// Throw InvalidPropagatorError if a failed operation left this object's state unusable. + auto require_valid_() const -> void; - // Validation stays at the call site, so a rejected value throws before anything is mutated, and - // the partitions do not re-validate what the facade already checked against identical field values. - auto update_setting_(const std::function &mutate) -> void { - mutate(*this); - if (partition_group_) { - for_each_partition_(mutate); - } - } + /// Mark this object unusable. Called from a failure path, before the failure is reported or rethrown. + auto invalidate_() noexcept -> void { invalid_ = true; } + + /// Entry check for a state operation: the initializing-thread rule (MPI-using operations; a wrong thread fails + /// fast, locally, before any MPI call), then validity. + auto enter_operation_(bool uses_mpi) const -> void; + + /// A failure after mutation started: invalidate, then rethrow the original on one rank; above one rank abort the + /// communicator (mpi::operation_failed) from MPI's initializing thread, or stop this process from any other + /// thread (mpi::require_initializing_thread). + [[noreturn]] auto mutation_failed_(std::exception_ptr error) -> void; + + /// Run one state operation. `body(mutation_started)` validates first and sets `mutation_started` + /// immediately before its first change to any cache, state or workspace. A failure before that + /// rethrows and leaves the object usable; a failure after it goes to mutation_failed_(). + template + auto run_operation_(bool uses_mpi, Body &&body) -> decltype(auto); + + /// Reject generator indices outside the system before any gate runs. + auto validate_generators_(const std::vector &majoranas) const -> void; + + /// `other`, once checked valid; used first in the copy constructor's member initializers. + static auto checked_source_(const MonomialPropagator &other) -> const MonomialPropagator &; auto regenerate_cutoff_fn_() -> void; @@ -443,74 +411,106 @@ class MonomialPropagator { auto validate_cutoff_config_(CutoffType cutoff_type, const std::optional> &basis_change) const -> void; - auto initialize_operator_caches_() -> void; - - auto current_picture_coeffs_() -> const VecD &; + // Warns once if this call's generator shifts do not span all log2(R) rank bits. + auto report_routing_coverage_(const std::vector &majoranas) -> void; - auto extend_coeffs_from_current_picture_if_needed_(VecD &coeffs) -> void; + // Reconstruct the optimizer-order (parameter_mapping, gen_coeffs) arrays from the layers' gate info. + auto graph_gate_arrays_() const -> std::pair; - auto evolve_mode_build_graph_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - std::optional only_rotate_len_k) -> void; + /// graph_data() of one store: its graph's layers, with cosine sets read from the stored masks or folded over + /// the store's inverted index. + static auto graph_data_of_(const MPGraph &graph, const detail::MPOperator &op, Basis basis) + -> std::vector; - // Returns {build_angle, apply_angle}; apply is the build angle, negated in Schrödinger. - auto gate_angle_(const VecD &mapped_params, size_t i, size_t majoranas_size) const -> std::pair { - const size_t idx = schrodinger_ ? i : majoranas_size - 1 - i; - const double build_angle = mapped_params[idx]; - return {build_angle, schrodinger_ ? -build_angle : build_angle}; - } + /// Cosine-only index count of one store's graph; see graph_size(). + static auto cos_index_count_of_(const MPGraph &graph, const detail::MPOperator &op, Basis basis) + -> size_t; - auto evolve_mode_graph_with_coeffs_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - const VecD ¶meters, - const VecD &operator_coeffs, - std::optional only_rotate_len_k) -> void; + //! Tag of the private constructor PropagatorTestAccess uses to observe construction. + struct ObservedTag {}; - auto evolve_mode_contract_immediately_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecD ¶meters, - std::optional only_rotate_len_k) -> void; + /// The public constructor's body, with a test-only observer (null from the public constructor). + MonomialPropagator(ObservedTag tag, + const detail::sharded::RootObserver *observer, + const OperatorDict &initial_operator, + unsigned int cutoff, + const VecZ &initial_state, + std::optional schrodinger_cutoff, + mpi::Comm comm, + std::optional lower_atol, + std::optional upper_atol, + CutoffType cutoff_type, + std::optional> basis_change, + size_t logical_num_modes, + Basis basis); + + // Flat-owner routing at geometry (P, T): P = comm_'s size, T = the captured budget. Prepared once on the caller at + // construction and used for seeding, construction and initial-operator updates. + std::optional router_; + // This rank among comm_'s P, with the communicator-agreed replay transport; read once, on the caller, at + // construction. Copies keep it: they share comm_. + detail::sharded::PhysicalWorld world_{}; + // This owner's physical rounds, reused by every multi-rank operation (construction, evaluation, replay) and by its + // functionals, so their staging persists across calls. Never shared: a copy starts with its own empty set. + std::unique_ptr rounds_ = std::make_unique(); + // Test-only: null in production, set only through PropagatorTestAccess. Copies inherit it. + const detail::sharded::RootObserver *observer_ = nullptr; + // The T shard states of this rank, shard t at flat owner t; one per worker of the captured budget. Declared + // last, so a copy validates its source (checked_source_, first initializer) before allocating any shard. + detail::sharded::Shards shards_; + + /// The only shard, for the raw-layout accessors. + /// \throws InvalidPropagatorError if invalid (checked first); MultiShardUnsupported unless launched with T = 1: + /// there is no single store to hand out, and shard 0 alone is not this rank's operator. + auto sole_shard_(const char *what) const -> const detail::sharded::ShardState &; + auto sole_shard_(const char *what) -> detail::sharded::ShardState &; + + // Settings are root-only: the shards hold mathematical state, and every construction reads the root's values. + auto update_setting_(const std::function &mutate) -> void { + require_valid_(); + try { + mutate(*this); + } + catch (...) { + mutation_failed_(std::current_exception()); + } + } - // Warns once if this call's generator shifts do not span all log2(R) rank bits. - auto report_routing_coverage_(const std::vector &majoranas) -> void; + /// Converted generators, after validate_generators_(). + auto generators_(const std::vector &majoranas) const -> MonomialList; - template - auto run_gate_loop_(const std::vector &majoranas, - std::optional only_rotate_len_k, - EvolutionFunc evolution_func) -> void; - - auto propagate_one_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs = std::nullopt, - std::optional param = std::nullopt, - size_t param_index = 0, - double gen_coeff = 0.0, - size_t gate_index = 0) -> void; - - // fused_scale_coeffs (ContractImmediately only): the picture's mutable coeff vector for the uncapped - // fused cos sweep; the taken decision is reported via fused_scale so the apply matches. See build_layer. - auto build_evolve_result_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs = std::nullopt, - std::optional param = std::nullopt, - CosMask *out_cos = nullptr, - detail::FusedContract *fused_contract = nullptr, - VecD *fused_scale_coeffs = nullptr, - bool *fused_scale = nullptr) -> std::shared_ptr; - - template > - auto make_functional_(Fn &&func, std::optional pare_threshold) -> std::function; + /// The construction seam's read-only rank-level configuration. + auto construction_context_() const -> detail::sharded::ConstructionContext; - // Reconstruct the optimizer-order (parameter_mapping, gen_coeffs) arrays from the layers' gate info. - auto graph_gate_arrays_() const -> std::pair; + /// Transfer a construction outcome to run_operation_: mutation flag first, then the original error. + static auto finish_construction_(const detail::sharded::ConstructionOutcome &outcome, bool &mutation_started) + -> void; - auto evolve_operator_with_recompute_(VecD &&coeffs, const MPGraphView &graph, const VecD ¶ms) -> VecD; + /// Run `body(shard)` once on every owner, in one team of the captured budget, after reporting `work`. + /// \return The lowest failing owner's original exception, after the join. + template + auto run_owner_phase_(detail::sharded::RootWork work, Body &&body) -> std::exception_ptr; + + /// Snapshot, optionally pare and build callbacks for every shard (prepare_retained), transferring the outcome. + auto prepare_retained_(std::optional pare_threshold, bool &mutation_started) + -> detail::sharded::RetainedEvaluation; + + /// One evaluation of `retained` at `params`, already validated: one team, then the ordered fold, the physical + /// reduction and the identity once. A failure before the team (request views, the evaluator's argument checks) + /// propagates and mutates nothing. A failure from the team on -- a phase error, the fold, the reduction -- + /// follows the post-mutation policy, because the team touched the owners' lazy caches: inside an operation + /// (`mutation_started` non-null) the flag is set and the error rethrown for run_operation_; from a functional + /// (null) it goes to mutation_failed_() here. + auto evaluate_retained_(const detail::sharded::RetainedEvaluation &retained, + const VecD ¶ms, + bool gradient, + bool *mutation_started) -> std::pair; + + /// Per-shard evolved coefficients at validated `parameters` (contract_partially's blocks, in shard order). + /// Sets `mutation_started` before the first cache or graph change; inplace consumes the graph and stores the + /// blocks only once concatenation can no longer fail. + auto contract_blocks_(const VecD ¶meters, bool inplace, bool &mutation_started, VecD *concatenated) + -> std::vector; }; } // namespace monoprop diff --git a/cpp/monoprop/CMakeLists.txt b/cpp/monoprop/CMakeLists.txt index e2bb624ce..142dfb8a2 100644 --- a/cpp/monoprop/CMakeLists.txt +++ b/cpp/monoprop/CMakeLists.txt @@ -1,13 +1,6 @@ find_package(Boost 1.85 CONFIG REQUIRED) message(STATUS "Using Boost: ${Boost_DIR} (version ${Boost_VERSION})") -find_package(PkgConfig REQUIRED QUIET) -pkg_check_modules(HWLOC REQUIRED QUIET IMPORTED_TARGET GLOBAL "hwloc>=2.9") -message( - STATUS - "Using hwloc: ${HWLOC_LINK_LIBRARIES} (version ${HWLOC_VERSION})" -) - target_sources( monoprop-objs PRIVATE @@ -17,6 +10,8 @@ target_sources( Validation.cpp ) +# PUBLIC/INTERFACE, never private: the macro changes template definitions and class layout, so the bindings, the tests +# and installed consumers must see exactly the library's value. target_compile_definitions( monoprop-objs PUBLIC @@ -43,15 +38,16 @@ target_include_directories( $ ) +# OpenMP is linked on both targets: the shared library is assembled from TARGET_OBJECTS and does not +# inherit monoprop-objs' link requirements, so it needs its own link against the runtime. target_link_libraries( monoprop-objs PUBLIC monoprop-sanitizers Boost::boost Threads::Threads + OpenMP::OpenMP_CXX $<$:MPI::MPI_CXX> - PRIVATE - PkgConfig::HWLOC ) set_target_properties( @@ -105,6 +101,7 @@ target_compile_features(monoprop INTERFACE cxx_std_23) target_link_libraries( monoprop PUBLIC + OpenMP::OpenMP_CXX $<$:MPI::MPI_CXX> INTERFACE Boost::boost @@ -112,7 +109,6 @@ target_link_libraries( PRIVATE # Keep the non-exported sanitizer target out of monopropTargets. $ - PkgConfig::HWLOC ) add_subdirectory(algebra) diff --git a/cpp/monoprop/Evolution.cpp b/cpp/monoprop/Evolution.cpp index b477c1316..0b00e8d1f 100644 --- a/cpp/monoprop/Evolution.cpp +++ b/cpp/monoprop/Evolution.cpp @@ -24,36 +24,17 @@ #include "monoprop/MPGraph.h" #include "monoprop/TypeAliases.h" #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" +#include "monoprop/detail/evolution/LayerReplay.h" #include "monoprop/detail/mpi/Exchange.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/OperationFailure.h" namespace monoprop { namespace { -// Endpoint accumulators, in pre-layer units like the sum they are subtracted from. -struct EndpointContrib { - double cos_terms = 0.0; - double sin_terms = 0.0; -}; - -struct TrigValues { - double cos_val; - double sin_val; - double sec_val; - double g_val; // 2·gen_coeff - - explicit TrigValues(double param, double gen_coeff = 1.0) { - const double g = 2.0 * gen_coeff; - cos_val = std::cos(g * param); - sin_val = std::sin(g * param); - sec_val = 1.0 / cos_val; - g_val = g; - } -}; - -auto combine_endpoint_contrib(const EndpointContrib &a, const EndpointContrib &b) -> EndpointContrib { - return {.cos_terms = a.cos_terms + b.cos_terms, .sin_terms = a.sin_terms + b.sin_terms}; -} +using detail::replay::DerivativeSnapshotScratch; +using detail::replay::EndpointContrib; +using detail::replay::TrigValues; struct FlatExchangeBuffers { VecD send_buffer; @@ -163,41 +144,21 @@ inline auto finish_layer_exchange(InFlightExchange &in_flight, Apply apply) .my_rank = in_flight.my_rank}); } -// Per-thread pre-cos snapshot buffers, reused across layers, indexed by occupied position. Passed -// whole rather than re-split at each hand-off, which is how a send buffer becomes a recv one. -struct DerivativeSnapshotScratch { - std::vector sin_send_state; - std::vector sin_send_op; - std::vector sin_recv_state; - std::vector sin_recv_op; - VecD self_recv_op; ///< self-slot entry op, kept only where a record overwrites op before it is read -}; - auto derivative_snapshot_scratch() -> DerivativeSnapshotScratch & { static thread_local DerivativeSnapshotScratch scratch; return scratch; } // Pack sin_send entries from the pre-cos snapshots; the live state/op there are clobbered by the cos pass. +// The send layout stays dense in the world; only the snapshot is indexed by position. void pack_cross_rank_derivative_payload_impl(const DerivativeSnapshotScratch &snap, const LayerTraversal &layer, int my_rank, const LayerExchangeLayout &layout, VecD &send_buffer) { - layer.for_each_occupied_slot( - [my_rank, &layout, &snap, &send_buffer](size_t pos, size_t rank, const detail::CrossRankSlotView &slot) { - if (std::cmp_equal(rank, my_rank)) { - return; - } - // The send layout stays dense in the world; only the snapshot is indexed by position. - const auto base = static_cast(layout.displs[rank]); - const auto &bs = snap.sin_send_state[pos]; - const auto &bh = snap.sin_send_op[pos]; - for (size_t k = 0; k < slot.sin_send_count; ++k) { - send_buffer[base + (2 * k)] = bs[k]; - send_buffer[base + (2 * k) + 1] = bh[k]; - } - }); + detail::replay::pack_derivative_payload(snap, layer, static_cast(my_rank), [&](size_t rank) { + return send_buffer.data() + layout.displs[rank]; + }); } // Remote endpoint pass: own pre-cos values come from the sin_recv snapshots, partner values from the @@ -208,31 +169,14 @@ auto apply_cross_rank_derivative_exchange_impl(VecD &state, const DerivativeSnapshotScratch &snap, const TrigValues &trig, const ExchangePayload &payload) -> EndpointContrib { - EndpointContrib local{}; - layer.for_each_occupied_slot( - [&payload, &snap, &trig, &op, &state, &local](size_t pos, size_t rank, const detail::CrossRankSlotView &slot) { - if (std::cmp_equal(rank, payload.my_rank)) { - return; - } - const auto *rv = payload.recv_buffer.data() + payload.recv_displs[rank]; - const auto &ds = snap.sin_recv_state[pos]; - const auto &dh = snap.sin_recv_op[pos]; - for (size_t k = 0; k < slot.sin_send_count; ++k) { - const size_t i = detail::slot_sin_recv_index(slot, k); - const auto phi = static_cast(detail::slot_sin_recv_phase(slot, k)); - // Inverse-rotation write-back (−sin): un-evolves state/op for the next reverse layer. - const double ps = -trig.sin_val * phi; - const double s_old = ds[k]; - const double h_old = dh[k]; - const double s_p = rv[2 * k]; - const double h_p = rv[(2 * k) + 1]; - local.cos_terms += s_old * op[i]; // pre-layer, matching what the cos pass added to A - local.sin_terms += phi * s_old * h_p; - op[i] = (h_old * trig.cos_val) + (ps * h_p); - state[i] = (s_old * trig.cos_val) + (ps * s_p); - } - }); - return local; + return detail::replay::apply_derivative_payload( + state, + op, + layer, + snap, + trig, + static_cast(payload.my_rank), + [&](size_t rank) { return payload.recv_buffer.data() + payload.recv_displs[rank]; }); } // Pack + Ialltoallv fire up front so the transfer overlaps the cos pass and the self-slot. @@ -270,32 +214,17 @@ void pack_cross_rank_evolution_payload_impl(VecD &op, int my_rank, const LayerExchangeLayout &layout, VecD &send_buffer) { - layer.for_each_occupied_slot( - [my_rank, &layout, &send_buffer, &op](size_t rank, const detail::CrossRankSlotView &slot) { - if (std::cmp_equal(rank, my_rank)) { - return; - } - const auto base = static_cast(layout.displs[rank]); - for (size_t k = 0; k < slot.sin_send_count; ++k) { - send_buffer[base + k] = op[detail::slot_sin_send_index(slot, k)]; - } - }); + detail::replay::pack_evolution_payload(op, layer, static_cast(my_rank), [&](size_t rank) { + return send_buffer.data() + layout.displs[rank]; + }); } void apply_cross_rank_evolution_exchange_impl(VecD &op, const LayerTraversal &layer, double sin_val, const ExchangePayload &payload) { - // op[i] is already cos-scaled, so only the sine term is added; rv[k] is the partner's pre-cos value. - layer.for_each_occupied_slot([&payload, sin_val, &op](size_t rank, const detail::CrossRankSlotView &slot) { - if (std::cmp_equal(rank, payload.my_rank)) { - return; - } - const auto *rv = payload.recv_buffer.data() + payload.recv_displs[rank]; - for (size_t k = 0; k < slot.sin_send_count; ++k) { - const size_t i = detail::slot_sin_recv_index(slot, k); - op[i] += sin_val * static_cast(detail::slot_sin_recv_phase(slot, k)) * rv[k]; - } + detail::replay::apply_evolution_payload(op, layer, sin_val, static_cast(payload.my_rank), [&](size_t rank) { + return payload.recv_buffer.data() + payload.recv_displs[rank]; }); } @@ -322,99 +251,6 @@ inline auto finish_cross_rank_evolution_exchange(VecD &op, }); } -// Self-slot endpoint pass: sin_recv entries k and k+pairs are the two endpoints of one rotation, so reading -// both before writing either avoids the read-after-write hazard. `h_pre` carries the entry op where a -// record overwrote it, and is null where op still holds it. -auto apply_self_slot_derivative_paired(VecD &state, - VecD &op, - const LayerTraversal &layer, - const TrigValues &trig, - const double *h_pre) -> EndpointContrib { - const auto slot = layer.cross_rank_self_slot(); - const size_t self_d_count = slot.sin_send_count; - if (self_d_count == 0) { - return {}; - } - const auto pairs = self_d_count / 2; - EndpointContrib local{}; - for (size_t k = 0; k < pairs; ++k) { - const size_t i1 = detail::slot_sin_recv_index(slot, k); - const auto phi1 = static_cast(detail::slot_sin_recv_phase(slot, k)); - const size_t i2 = detail::slot_sin_recv_index(slot, k + pairs); - const auto phi2 = static_cast(detail::slot_sin_recv_phase(slot, k + pairs)); - // Recover pre-cos values. - const double s1 = state[i1] * trig.sec_val; - const double s2 = state[i2] * trig.sec_val; - const double h1 = h_pre != nullptr ? h_pre[k] : op[i1] * trig.cos_val; - const double h2 = h_pre != nullptr ? h_pre[k + pairs] : op[i2] * trig.cos_val; - local.cos_terms += (s1 * op[i1]) + (s2 * op[i2]); // pre-layer, as in A - local.sin_terms += (phi1 * s1 * h2) + (phi2 * s2 * h1); - // Inverse-rotation write-back (−sin); see apply_cross_rank_derivative_exchange_impl. - const double ps1 = -trig.sin_val * phi1; - const double ps2 = -trig.sin_val * phi2; - op[i1] = (h1 * trig.cos_val) + (ps1 * h2); - state[i1] = (s1 * trig.cos_val) + (ps1 * s2); - op[i2] = (h2 * trig.cos_val) + (ps2 * h1); - state[i2] = (s2 * trig.cos_val) + (ps2 * s1); - } - return local; -} - -// Emptied, not filled: the self slot recovers live, and a previous layer's snapshot must not be read. -void clear_slot_snapshot(DerivativeSnapshotScratch &snap, size_t pos) { - snap.sin_send_state[pos].clear(); - snap.sin_send_op[pos].clear(); - snap.sin_recv_state[pos].clear(); - snap.sin_recv_op[pos].clear(); -} - -void fill_slot_snapshot(DerivativeSnapshotScratch &snap, - size_t pos, - const VecD &state, - const VecD &op, - const detail::CrossRankSlotView &slot) { - const size_t count = slot.sin_send_count; - auto &bs = snap.sin_send_state[pos]; - auto &bh = snap.sin_send_op[pos]; - auto &ds = snap.sin_recv_state[pos]; - auto &dh = snap.sin_recv_op[pos]; - bs.resize(count); - bh.resize(count); - ds.resize(count); - dh.resize(count); - for (size_t k = 0; k < count; ++k) { - const size_t bi = detail::slot_sin_send_index(slot, k); - bs[k] = state[bi]; - bh[k] = op[bi]; - const size_t di = detail::slot_sin_recv_index(slot, k); - ds[k] = state[di]; - dh[k] = op[di]; - } -} - -// Snapshot pre-cos (state, op) at every remote rank's sin_send/sin_recv endpoints before the cos pass -// clobbers them. The self slot needs none — it recovers live — so it is cleared. -void snapshot_remote_endpoints(const VecD &state, - const VecD &op, - const LayerTraversal &layer, - size_t my_rank, - DerivativeSnapshotScratch &snap) { - // Grow-only: shrinking would free the allocations this scratch exists to reuse. - const size_t occupied = layer.occupied_slot_count(); - snap.sin_send_state.resize(std::max(snap.sin_send_state.size(), occupied)); - snap.sin_send_op.resize(std::max(snap.sin_send_op.size(), occupied)); - snap.sin_recv_state.resize(std::max(snap.sin_recv_state.size(), occupied)); - snap.sin_recv_op.resize(std::max(snap.sin_recv_op.size(), occupied)); - layer.for_each_occupied_slot( - [&snap, my_rank, &state, &op](size_t pos, size_t r, const detail::CrossRankSlotView &slot) { - if (r == my_rank) { - clear_slot_snapshot(snap, pos); - return; - } - fill_slot_snapshot(snap, pos, state, op, slot); - }); -} - } // namespace auto state_operator_derivative_local(VecD &state, @@ -424,44 +260,45 @@ auto state_operator_derivative_local(VecD &state, LayerAngle angle, mpi::Comm comm, const detail::LayerCosAccumulate &cos_acc, - const detail::CosRecordView &record) -> double { + const detail::CosRecordView &record, + [[maybe_unused]] detail::parallel::Options options) -> double { const TrigValues trig(angle.param, angle.gen_coeff); const auto layer = graph.get_layer_traversal(layer_idx); const auto my_rank = static_cast(mpi::rank(comm)); const size_t R = layer.cross_rank_rank_count(); auto &snap = derivative_snapshot_scratch(); - snapshot_remote_endpoints(state, op, layer, my_rank, snap); - - // The self slot reads its entry op off the post-cos slots, which a record overwrites first. - const bool self_pre = record.count > 0 && my_rank < R; - if (self_pre) { - const auto slot = layer.cross_rank_self_slot(); - snap.self_recv_op.resize(slot.sin_send_count); - for (size_t k = 0; k < slot.sin_send_count; ++k) { - snap.self_recv_op[k] = op[detail::slot_sin_recv_index(slot, k)]; + // Snapshots and packing may throw on this rank alone before it posts: guarded, since peers post regardless. + auto in_flight = mpi::guard_distributed(comm, [&] { + detail::replay::snapshot_remote_endpoints(state, op, layer, my_rank, snap); + // The self slot reads its entry op off the post-cos slots, which a record overwrites first. + if (record.count > 0 && my_rank < R) { + detail::replay::snapshot_self_recv_op(op, layer, snap); } - } - - // No-op at single rank; the transfer touches only buffers, so the cos pass below may mutate state/op. - auto in_flight = begin_cross_rank_derivative_exchange(snap, layer, comm); - - // A = Σ s_old·h_pre over all anticommuting indices, endpoints included — hence the subtraction below. - // Pre-dividing lets the kernel's own ×sec land back on the recorded value; restore fixes up the rest. - detail::predivide_cos_record(op.data(), record, trig.cos_val); - const double A = cos_acc(layer_idx, state.data(), op.data(), trig.cos_val, trig.sec_val) * trig.sec_val; - detail::restore_cos_record(op.data(), record); + // No-op at single rank; the transfer touches only buffers, so the cos pass below may mutate state/op. + return begin_cross_rank_derivative_exchange(snap, layer, comm); + }); + const bool self_pre = record.count > 0 && my_rank < R; + // Caught here, while in_flight still owns posted requests: unwinding past it would wait on peers. EndpointContrib ep; - if (my_rank < R) { - ep = apply_self_slot_derivative_paired(state, op, layer, trig, self_pre ? snap.self_recv_op.data() : nullptr); - } + const double A = mpi::guard_distributed(comm, [&] { + const double a = detail::replay::accumulate_layer(state, op, layer_idx, trig, record, cos_acc); + if (my_rank < R) { + ep = detail::replay::apply_self_slot_derivative_paired(state, + op, + layer, + trig, + self_pre ? snap.self_recv_op.data() : nullptr); + } + return a; + }); const auto remote = finish_cross_rank_derivative_exchange(state, op, layer, snap, trig, in_flight); - ep = combine_endpoint_contrib(ep, remote); + ep = detail::replay::combine_endpoint_contrib(ep, remote); - // dE/dθ = −g·(sin·(A − ep.cos_terms) − ep.sin_terms); note the plus sign. Both sums are in pre-layer - // units, so the cancellation happens before any ×sec rather than after it. - return -trig.g_val * (trig.sin_val * (A - ep.cos_terms) - ep.sin_terms); + // Note the plus sign in layer_derivative. Both sums are in pre-layer units, so the cancellation happens + // before any ×sec rather than after it. + return detail::replay::layer_derivative(trig, A, ep); } namespace { @@ -478,39 +315,41 @@ auto evolve_step_traversal_impl(VecD &op, // This rank's own sin_send values before the cos pass. Unconditional, since the remote pack skips // the self slot, so single-rank works. - const auto self_slot = layer.cross_rank_self_slot(); - const size_t self_b_count = self_slot.sin_send_count; VecD self_b_snapshot; - self_b_snapshot.resize(self_b_count); - for (size_t k = 0; k < self_b_count; ++k) { - self_b_snapshot[k] = op[detail::slot_sin_send_index(self_slot, k)]; - } - // Pack + start the exchange before the cos scan so partner values are pre-cos and the transfer overlaps. - auto in_flight = begin_cross_rank_evolution_exchange(op, layer, comm); - cos_scale(layer_idx, op_data, cos_val); + // A throw before posting is guarded too: the peers post and wait regardless. + auto in_flight = mpi::guard_distributed(comm, [&] { + detail::replay::snapshot_self_sources(op, layer, self_b_snapshot); + return begin_cross_rank_evolution_exchange(op, layer, comm); + }); + // Caught here, while in_flight still owns posted requests: unwinding past it would wait on peers. + mpi::guard_distributed(comm, [&] { cos_scale(layer_idx, op_data, cos_val); }); finish_cross_rank_evolution_exchange(op, layer, sin_val, in_flight); // Self-slot sin_recv entries: op[i] is already cos-scaled, so only the sine term is added. - for (size_t k = 0; k < self_b_count; ++k) { - const size_t i = detail::slot_sin_recv_index(self_slot, k); - op[i] += sin_val * static_cast(detail::slot_sin_recv_phase(self_slot, k)) * self_b_snapshot[k]; - } + detail::replay::apply_self_sources(op, layer, sin_val, self_b_snapshot); } } // namespace +// The layer's cosine scaling runs threaded through the budget captured in `cos_scale`; the endpoint work +// below it is still serial, so `options` itself is not yet consumed at this level. auto evolve_step(VecD &op, const MPGraphView &graph, double param, size_t layer_idx, mpi::Comm comm, - const detail::LayerCosScale &cos_scale) -> void { + const detail::LayerCosScale &cos_scale, + [[maybe_unused]] detail::parallel::Options options) -> void { evolve_step_traversal_impl(op, graph.get_layer_traversal(layer_idx), param, layer_idx, comm, cos_scale); } // A standalone Layer replays as a one-layer graph, so layer_idx 0 is the only cosine set to select. -auto evolve_step(VecD &op, const Layer &layer, double param, mpi::Comm comm, const detail::LayerCosScale &cos_scale) - -> void { +auto evolve_step(VecD &op, + const Layer &layer, + double param, + mpi::Comm comm, + const detail::LayerCosScale &cos_scale, + [[maybe_unused]] detail::parallel::Options options) -> void { evolve_step_traversal_impl(op, layer.traversal(), param, 0, comm, cos_scale); } @@ -518,11 +357,12 @@ auto evolve_operator(VecD &&coeffs, const MPGraphView &graph, const VecD ¶ms, mpi::Comm comm, - const detail::LayerCosScale &cos_scale) -> VecD { + const detail::LayerCosScale &cos_scale, + detail::parallel::Options options) -> VecD { // Evolved in place in the caller's moved-from vector, then handed back: no per-layer copy. VecD evolved = std::move(coeffs); for (size_t i = 0; i < graph.layers(); ++i) { - evolve_step(evolved, graph, params[i], i, comm, cos_scale); + evolve_step(evolved, graph, params[i], i, comm, cos_scale, options); } return evolved; } diff --git a/cpp/monoprop/MPFunctions.cpp b/cpp/monoprop/MPFunctions.cpp index 57c74ae1a..f2d4a9da6 100644 --- a/cpp/monoprop/MPFunctions.cpp +++ b/cpp/monoprop/MPFunctions.cpp @@ -22,38 +22,22 @@ #include "monoprop/Evolution.h" #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" +#include "monoprop/detail/evolution/LayerReplay.h" +#include "monoprop/detail/mpi/OperationFailure.h" namespace monoprop { -// A per-layer cosine callback this evaluation path needs was left empty. -class MissingLayerCallback : public std::invalid_argument { -public: - using std::invalid_argument::invalid_argument; -}; - -// A state's rows and values disagree in length, a sparse row names a slot outside the state, or the -// operator being contracted is shorter than the state. One type because every case is a caller-supplied -// length or index that does not fit the rest of the call. -class EvalStateArgumentError : public std::invalid_argument { -public: - using std::invalid_argument::invalid_argument; -}; +// MissingLayerCallback and EvalStateArgumentError live in LayerReplay.h, shared with the sharded evaluator. namespace { -// What a layer's forward pass keeps for the reverse pass, as flags on cos_wanted. -constexpr uint8_t kRecordRotationsBelow = 1; ///< the coefficients the layer below rotates -constexpr uint8_t kRecordCosineSet = 2; ///< its own cosine set, which a vanishing cosine destroys - // Per-thread scratch so repeated ev/grad calls stay allocation-light. struct EvalScratch { VecD state; VecD op; VecD mapped_params; VecD gradient; - std::vector cos_indices; ///< recorded coefficient indices, layer slices back to back - VecD cos_values; ///< the pre-layer coefficient at each of them - VecZ cos_offset; ///< per layer: start of its slice, plus a tail entry + detail::replay::CosRecords records; ///< the selective pre-layer records of the reverse pass std::vector cos_wanted; ///< per layer: which records the parameters earn it }; @@ -62,73 +46,15 @@ auto eval_scratch() -> EvalScratch & { return scratch; } -// Flag the layers whose record can change an answer, and return whether any needs its cosine set. -// Layer j records what layer j-1 rotates, so it earns its keep only once the layers already reversed can -// amplify an error there by a full significand -- a bound read off the parameters, before any coefficient -// is touched. +// Flag the layers whose record can change an answer and empty the records; see plan_cos_records. auto layout_cos_records(size_t layers, EvalScratch &scratch) -> bool { - scratch.cos_wanted.assign(layers, 0); - scratch.cos_offset.assign(layers + 1, 0); - scratch.cos_indices.clear(); - scratch.cos_values.clear(); - bool any_cosine_set = false; - double spread = 0.0; - for (size_t j = layers; j-- > 0;) { - const double cos_val = std::cos(2 * scratch.mapped_params[j]); - const double own = -std::log2(std::abs(cos_val)); - if (j > 0 && spread + own >= detail::kRecordSpreadBits) { - scratch.cos_wanted[j] |= kRecordRotationsBelow; - } - if (std::abs(cos_val) < detail::kVanishingCos) { - scratch.cos_wanted[j] |= kRecordCosineSet; - any_cosine_set = true; - } - spread += own; - } + const bool any_cosine_set = + detail::replay::plan_cos_records(std::span(scratch.mapped_params.data(), layers), + scratch.cos_wanted); + detail::replay::reset_records(scratch.records, layers); return any_cosine_set; } -// Open `layer_idx`'s slice and fill it from the not-yet-stepped `op`. Called in layer order, so each -// slice starts where the previous one ended. -auto record_pre_layer(EvalScratch &scratch, - const MPGraphView &graph, - const detail::LayerCosIndices &cos_inds, - size_t layer_idx) -> void { - scratch.cos_offset[layer_idx] = scratch.cos_values.size(); - const uint8_t wanted = scratch.cos_wanted[layer_idx]; - if (wanted == 0) { - return; - } - const size_t begin = scratch.cos_indices.size(); - if ((wanted & kRecordRotationsBelow) != 0U) { - const auto below = graph.get_layer_traversal(layer_idx - 1); - const auto mark = [&scratch](size_t, size_t i, auto) { - scratch.cos_indices.push_back(static_cast(i)); - }; - for (size_t r = 0; r < below.cross_rank_rank_count(); ++r) { - below.for_each_cross_rank_sin_recv_range(r, 0, below.cross_rank_sin_recv_size(r), mark); - } - } - if ((wanted & kRecordCosineSet) != 0U) { - cos_inds(layer_idx, scratch.cos_indices); - } - const size_t end = scratch.cos_indices.size(); - scratch.cos_values.resize(end); - for (size_t k = begin; k < end; ++k) { - scratch.cos_values[k] = scratch.op[scratch.cos_indices[k]]; - } -} - -// What the reverse pass reads back for `layer_idx`. -auto layer_record_in(const EvalScratch &scratch, size_t layer_idx) -> detail::CosRecordView { - const size_t begin = scratch.cos_offset[layer_idx]; - const size_t count = scratch.cos_offset[layer_idx + 1] - begin; - if (count == 0) { - return {}; - } - return {.indices = scratch.cos_indices.data() + begin, .values = scratch.cos_values.data() + begin, .count = count}; -} - // Graph is traversed in simulation order but parameter_mapping is stored in optimizer order; write the // mapped coefficients forward or reversed accordingly. auto fill_mapped_params(VecD &result, @@ -154,22 +80,35 @@ auto prepare_evolved_operator(const EvalRequest &request, mpi::Comm comm, const } auto &scratch = eval_scratch(); fill_mapped_params(scratch.mapped_params, request.params, request.parameter_mapping, request.gen_coeffs, 1.0, true); - scratch.op = request.op; - if (!record) { - scratch.op = evolve_operator(std::move(scratch.op), request.graph, scratch.mapped_params, comm, cos.scale); - return; - } - - // Step layer by layer so each record is taken while op still holds that layer's pre-layer values. + // Decided from the replicated parameters alone, so every rank rejects a missing callback together. const size_t layers = request.graph.layers(); - if (layout_cos_records(layers, scratch) && !cos.indices) { + if (record && layout_cos_records(layers, scratch) && !cos.indices) { throw MissingLayerCallback("A layer whose cosine vanishes requires a cos_indices (reverse) callback."); } - for (size_t i = 0; i < layers; ++i) { - record_pre_layer(scratch, request.graph, cos.indices, i); - evolve_step(scratch.op, request.graph, scratch.mapped_params[i], i, comm, cos.scale); - } - scratch.cos_offset[layers] = scratch.cos_values.size(); + // Local work between the per-layer exchanges; each exchange guards its own in-flight phase. + mpi::guard_distributed(comm, [&] { + scratch.op = request.op; + if (!record) { + scratch.op = evolve_operator(std::move(scratch.op), + request.graph, + scratch.mapped_params, + comm, + cos.scale, + request.parallel); + return; + } + // Step layer by layer so each record is taken while op still holds that layer's pre-layer values. + for (size_t i = 0; i < layers; ++i) { + detail::replay::record_pre_layer(scratch.records, + scratch.cos_wanted, + request.graph, + cos.indices, + i, + scratch.op); + evolve_step(scratch.op, request.graph, scratch.mapped_params[i], i, comm, cos.scale, request.parallel); + } + detail::replay::close_records(scratch.records, layers); + }); } } // namespace @@ -285,20 +224,23 @@ auto map_params(const VecD ¶meters, } auto ev(const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) -> double { - // The allreduce is unconditional -- ShmComm's is barrier-synced, so short-circuiting an empty local - // sum past it would deadlock every peer. + // The allreduce is unconditional: it is collective, so short-circuiting an empty local sum past it would + // deadlock every peer. if (request.params.empty()) { - return request.e_core + mpi::allreduce_sum(request.state.dot(request.op), comm); + const double local = mpi::guard_distributed(comm, [&] { return request.state.dot(request.op); }); + return request.e_core + mpi::allreduce_sum(local, comm); } prepare_evolved_operator(request, comm, cos, /*record=*/false); - return request.e_core + mpi::allreduce_sum(request.state.dot(eval_scratch().op), comm); + const double local = mpi::guard_distributed(comm, [&] { return request.state.dot(eval_scratch().op); }); + return request.e_core + mpi::allreduce_sum(local, comm); } auto ev_and_grad(const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) -> std::pair { if (request.params.empty()) { - return {request.e_core + mpi::allreduce_sum(request.state.dot(request.op), comm), VecD(0)}; + const double local = mpi::guard_distributed(comm, [&] { return request.state.dot(request.op); }); + return {request.e_core + mpi::allreduce_sum(local, comm), VecD(0)}; } // cos.scale is checked in prepare_evolved_operator, shared by both paths; cos.accumulate is this path's own. @@ -310,7 +252,7 @@ auto ev_and_grad(const EvalRequest &request, mpi::Comm comm, const detail::CosCa // this path needs it dense. Densifying into the shared scratch keeps energy-only runs from ever // building one. auto &scratch = eval_scratch(); - request.state.scatter_into(scratch.state); + mpi::guard_distributed(comm, [&] { request.state.scatter_into(scratch.state); }); prepare_evolved_operator(request, comm, cos, /*record=*/true); auto &state_ = scratch.state; @@ -318,20 +260,24 @@ auto ev_and_grad(const EvalRequest &request, mpi::Comm comm, const detail::CosCa const auto expectation_value = mpi::allreduce_sum(inner_product(state_, op_), comm); const auto ¶meter_mapping = request.parameter_mapping; - scratch.gradient.assign(request.params.size(), 0.0); - for (size_t i = 0; i < parameter_mapping.size(); ++i) { - const auto idx = parameter_mapping.size() - 1 - i; - const auto param_ind = parameter_mapping[i]; - scratch.gradient[param_ind] += - state_operator_derivative_local(state_, - op_, - request.graph, - idx, - {.gen_coeff = request.gen_coeffs[i], .param = request.params[param_ind]}, - comm, - cos.accumulate, - layer_record_in(scratch, idx)); - } + // Local work between the per-layer exchanges and before the gradient allreduce. + mpi::guard_distributed(comm, [&] { + scratch.gradient.assign(request.params.size(), 0.0); + for (size_t i = 0; i < parameter_mapping.size(); ++i) { + const auto idx = parameter_mapping.size() - 1 - i; + const auto param_ind = parameter_mapping[i]; + scratch.gradient[param_ind] += state_operator_derivative_local( + state_, + op_, + request.graph, + idx, + {.gen_coeff = request.gen_coeffs[i], .param = request.params[param_ind]}, + comm, + cos.accumulate, + detail::replay::layer_record_in(scratch.records, idx), + request.parallel); + } + }); mpi::allreduce_sum_inplace(scratch.gradient, comm); return {request.e_core + expectation_value, scratch.gradient}; diff --git a/cpp/monoprop/TypeAliases.h b/cpp/monoprop/TypeAliases.h index 684cc6b33..06cbbc3de 100644 --- a/cpp/monoprop/TypeAliases.h +++ b/cpp/monoprop/TypeAliases.h @@ -35,7 +35,7 @@ using VecI = std::vector; using VecZ = std::vector; -using TermIndex = std::uint32_t; // Max number of terms per partition +using TermIndex = std::uint32_t; // Max number of terms per shard // resize() leaves new trivial elements uninitialized; see the default_init_allocator caveat. template diff --git a/cpp/monoprop/algebra/AlgebraCommon.h b/cpp/monoprop/algebra/AlgebraCommon.h index 927acf0b9..2c1333ca4 100644 --- a/cpp/monoprop/algebra/AlgebraCommon.h +++ b/cpp/monoprop/algebra/AlgebraCommon.h @@ -287,6 +287,16 @@ class CutoffEvaluator { auto support_cutoff() const -> const SupportCutoff * { return support_cutoff_; } + /*! + * \brief Whether scan workers may evaluate this cutoff concurrently. + * + * True only when the predicate is one of the typed built-in cutoffs (LengthCutoff, SupportCutoff), + * whose evaluation reads only their own immutable fields. Any other predicate is opaque, even a + * stateless lambda: the evaluator cannot inspect it, and a const std::function says nothing about what + * its target does. Opaque predicates, including the basis-change closures, keep the serial scan. + */ + auto parallel_safe() const noexcept -> bool { return length_cutoff_ != nullptr || support_cutoff_ != nullptr; } + auto operator()(const Monomial &mono) const -> bool { if (length_cutoff_ != nullptr) { return (*length_cutoff_)(mono); diff --git a/cpp/monoprop/algebra/MajoranaAlgebra.h b/cpp/monoprop/algebra/MajoranaAlgebra.h index 81ffca05a..0175d2328 100644 --- a/cpp/monoprop/algebra/MajoranaAlgebra.h +++ b/cpp/monoprop/algebra/MajoranaAlgebra.h @@ -167,7 +167,7 @@ inline auto paired_op_size(size_t max_ones, size_t logical_num_modes) -> size_t // The emission order is a contract. Ascending pair count, then, within a pair count, ascending // lexicographic order of the selected modes' index tuple. A distributed propagator turns this sequence // into row indices by keeping the entries it owns and numbering them 0, 1, 2, ..., so reordering the walk -// renumbers every stored row and stops two runs at one (rank, partition) count from being bit-identical. +// renumbers every stored row and stops two runs at one (rank, shard) count from being bit-identical. template auto for_each_paired_monomial(size_t max_ones, size_t logical_num_modes, F &&f) -> void { // Clamp in pairs, not bits: max_ones counts pairs and bounds the fill over `selector`, one slot per mode. diff --git a/cpp/monoprop/detail/CMakeLists.txt b/cpp/monoprop/detail/CMakeLists.txt index f75f51c56..805e964e3 100644 --- a/cpp/monoprop/detail/CMakeLists.txt +++ b/cpp/monoprop/detail/CMakeLists.txt @@ -13,5 +13,6 @@ add_subdirectory(graph_encoding) add_subdirectory(monomial_propagator) add_subdirectory(mpi) add_subdirectory(operator) +add_subdirectory(parallel) add_subdirectory(pare) -add_subdirectory(partition) +add_subdirectory(sharded) diff --git a/cpp/monoprop/detail/EnvConfig.h b/cpp/monoprop/detail/EnvConfig.h index fd433b9fa..8ac7070c6 100644 --- a/cpp/monoprop/detail/EnvConfig.h +++ b/cpp/monoprop/detail/EnvConfig.h @@ -23,14 +23,19 @@ #include #include -// Single home for runtime environment configuration, read once per process. -// -// monoprop_NUM_THREADS positive int (1..1e6), else ignored → num_threads -// monoprop_PARTITIONS int N | "auto" | "off"; parsed where it is used (resolve_partition_count_) -// monoprop_ROUTING "splitmix" | "linear" → routing_mode -// monoprop_ROUTE_SEED decimal uint64 basis seed → route_seed -// -// Both routing knobs throw on a malformed value: a silent default would change the transport unseen. +/* + * Process-cached runtime configuration: the routing knobs, read once per process. + * + * monoprop_ROUTING "splitmix" | "linear" → routing_mode + * monoprop_ROUTE_SEED decimal uint64 basis seed → route_seed + * + * Both routing knobs throw on a malformed value: a silent default would change the transport unseen. + * + * The thread budget is not read here. Every propagator captures monoprop_NUM_THREADS when it is constructed, through + * the strict parallel::capture_thread_budget() (ThreadBudget.h), which rejects a malformed value and falls back to + * omp_get_max_threads() only when the variable is unset. No partition count is read at all: the removed partition + * runtime's monoprop_PARTITIONS is ignored. + */ namespace monoprop::config { @@ -43,21 +48,6 @@ enum class RoutingMode : std::uint8_t { Splitmix, Linear }; namespace detail { -inline auto parse_positive_int(const char *text) -> std::optional { - if (text == nullptr) { - return std::nullopt; - } - char *end = nullptr; - const long value = std::strtol(text, &end, 10); - if (end == text || *end != '\0') { - return std::nullopt; - } - if (value <= 0 || value > 1'000'000) { - return std::nullopt; - } - return static_cast(value); -} - [[noreturn]] inline auto reject_env(std::string_view name, const char *text, std::string_view expected) -> void { throw EnvConfigError(std::string{name} + "=\"" + text + "\" is not " + std::string{expected} + "; correct it or leave the variable unset."); @@ -95,7 +85,6 @@ inline auto parse_routing_mode(std::string_view name, const char *text) -> std:: } // namespace detail struct Settings { - std::optional num_threads; std::optional routing_mode; std::optional route_seed; }; @@ -104,7 +93,6 @@ struct Settings { inline auto get() -> const Settings & { static const Settings settings = [] { Settings s; - s.num_threads = detail::parse_positive_int(std::getenv("monoprop_NUM_THREADS")); s.routing_mode = detail::parse_routing_mode("monoprop_ROUTING", std::getenv("monoprop_ROUTING")); s.route_seed = detail::parse_uint64("monoprop_ROUTE_SEED", std::getenv("monoprop_ROUTE_SEED")); return s; diff --git a/cpp/monoprop/detail/evolution/CMakeLists.txt b/cpp/monoprop/detail/evolution/CMakeLists.txt index 637fdab43..5fa191cf8 100644 --- a/cpp/monoprop/detail/evolution/CMakeLists.txt +++ b/cpp/monoprop/detail/evolution/CMakeLists.txt @@ -10,4 +10,8 @@ target_sources( "LayerBuilder.h" ) +# Library-internal: included only by compiled library sources (so the library's floating-point flags apply to every +# instantiation), never by an installed header, and therefore not installed. +target_sources(monoprop-objs PRIVATE LayerReplay.h) + add_subdirectory(layer_build) diff --git a/cpp/monoprop/detail/evolution/CosineRecompute.h b/cpp/monoprop/detail/evolution/CosineRecompute.h index aab034d08..cde799338 100644 --- a/cpp/monoprop/detail/evolution/CosineRecompute.h +++ b/cpp/monoprop/detail/evolution/CosineRecompute.h @@ -29,16 +29,24 @@ #include #include #include +#include #include #include +#include "monoprop/MPGraph.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" +#include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/operator/InvertedIndex.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/Workshare.h" namespace monoprop::detail { +//! Stored mask blocks (one 64-bit word each) per scale_cos_mask range: as many words as a fold block. +inline constexpr size_t cos_mask_range_blocks = kColumnBlockWords; + // Reconstruct a layer's generator Monomial from the raw words stored on its LayerCore. template inline auto generator_from_words(const std::vector &gw) -> Monomial { @@ -51,7 +59,7 @@ inline auto generator_from_words(const std::vector &gw) -> Monomial [](size_t, size_t) {}); } +/* + * A layer's cosine fold words in [wlo, whi), ascending, as for_each_fold_word() visits them (zero words skipped), for + * the per-call kernels. A fold of one block takes the direct word sweep: the posting choice pays only across many + * words, and for_each_fold_word()'s setup (cursors, a lower_bound per column, posting counts) dominated a small fold + * (T = 1 tiny energies took 40 % longer per call). Both give the same words. `row_parity` is fold_row_parity(), + * fetched by the caller; the fold block is the executing thread's own. + */ +template +inline auto for_each_cos_fold_word(const InvertedIndex &sc, + const LazyFold &r, + size_t wlo, + size_t whi, + const uint64_t *row_parity, + WordOp &&on_word) -> void { + const std::span columns{r.columns.data(), r.columns.size()}; + if (r.fold.mask_words <= kColumnBlockWords) { + uint64_t *const blk = column_block_scratch().data(); + combine_columns_block(sc, columns, blk, wlo, whi); + for (size_t wi = wlo; wi < whi; ++wi) { + if (const uint64_t bits = apply_fold_mask(blk[wi - wlo], wi, r.fold, row_parity); bits != 0) { + on_word(wi, bits); + } + } + return; + } + for_each_fold_word(sc, + columns, + wlo, + whi, + r.fold.last_word, + r.fold.last_mask, + row_parity, + on_word, + [](size_t, size_t) {}); +} + // Append a layer's cosine-set indices to `out`, ascending. template auto cos_indices_lazy(const InvertedIndex &sc, const LazyFold &r, std::vector &out) @@ -194,17 +238,64 @@ auto cos_indices_lazy(const InvertedIndex &sc, const LazyFold +auto cos_count_lazy(const InvertedIndex &sc, const LazyFold &r) -> size_t { + size_t count = 0; + for_each_lazy_fold_word(sc, r, [&count](size_t /*wi*/, uint64_t bits) { + count += static_cast(std::popcount(bits)); + }); + return count; +} + +// The number of indices cos_indices_mask() appends. +inline auto cos_count_mask(const CosMask &cos) -> size_t { + size_t count = 0; + for (const auto &block : cos.blocks) { + count += static_cast(std::popcount(block.second)); + } + return count; +} + inline auto cos_indices_mask(const CosMask &cos, std::vector &out) -> void { for (const auto &[base, bits] : cos.blocks) { for_each_cos_index(base, bits, [&out](size_t i) { out.push_back(static_cast(i)); }); } } -template -auto scale_cos_lazy(const InvertedIndex &sc, const LazyFold &r, double *coeff, double cos_val) - -> void { - for_each_lazy_fold_word(sc, r, [&](size_t wi, uint64_t bits) { - for_each_cos_index(wi * 64, bits, [&](size_t i) { coeff[i] *= cos_val; }); +/*! + * \brief Multiply every coefficient in a layer's recomputed cosine set by `cos_val`. + * + * Each kColumnBlockWords fold block is one logical range, so the ranges touch disjoint coefficients. A range folds + * its own words (for_each_cos_fold_word()) with the executing thread's own scratch; no caller is inside another fold. + * + * \param sc The inverted index the fold reads; must not change during the call. + * \param r The layer's fold recipe. + * \param coeff Coefficients, covering at least `r.fold.mask_words * 64` rows or the index rows. + * \param cos_val The factor. + * \param options Thread budget. + * \param observer Test seam; see NoRangeObserver. + */ +template +auto scale_cos_lazy(const InvertedIndex &sc, + const LazyFold &r, + double *coeff, + double cos_val, + parallel::Options options = {}, + const Observer &observer = {}) -> void { + const size_t mask_words = r.fold.mask_words; + // Fetched here, on the caller, for this call only: row_parity_words() builds its cache lazily, so it must + // never run in a worker, and the pointer moves whenever the index grows. + const uint64_t *const row_parity = fold_row_parity(sc, r.fold); + const size_t ranges = logical_ranges(mask_words, kColumnBlockWords); + observer.prepare(KernelRange::cos_lazy, ranges); + parallel::for_blocks(ranges, options, [&](size_t range) { + observer.visit(KernelRange::cos_lazy, range); + const size_t bb = range * kColumnBlockWords; + const size_t be = std::min(bb + kColumnBlockWords, mask_words); + for_each_cos_fold_word(sc, r, bb, be, row_parity, [&](size_t wi, uint64_t bits) { + for_each_cos_index(wi * 64, bits, [&](size_t i) { coeff[i] *= cos_val; }); + }); }); } @@ -215,9 +306,11 @@ auto accumulate_cos_lazy(const InvertedIndex &sc, double *state, double *ham, double cos_val, - double sec_val) -> double { + double sec_val, + [[maybe_unused]] parallel::Options options = {}) -> double { double loc = 0.0; - for_each_lazy_fold_word(sc, r, [&](size_t wi, uint64_t bits) { + const uint64_t *const row_parity = fold_row_parity(sc, r.fold); + for_each_cos_fold_word(sc, r, 0, r.fold.mask_words, row_parity, [&](size_t wi, uint64_t bits) { for_each_cos_index(wi * 64, bits, [&](size_t i) { loc += state[i] * ham[i]; ham[i] *= sec_val; @@ -227,15 +320,42 @@ auto accumulate_cos_lazy(const InvertedIndex &sc, return loc; } -inline auto scale_cos_mask(double *coeff, const CosMask &cos, double cos_val) -> void { +/*! + * \brief Multiply every coefficient in a stored cosine mask by `cos_val`, leaving all others untouched. + * + * Each cos_mask_range_blocks run of mask blocks is one logical range. CosMask blocks are ascending with + * distinct word-aligned bases, so the ranges touch disjoint coefficients. + * + * \param coeff Coefficients, covering every index in `cos`. + * \param cos The mask. + * \param cos_val The factor. + * \param options Thread budget. + * \param observer Test seam; see NoRangeObserver. + */ +template +auto scale_cos_mask(double *coeff, + const CosMask &cos, + double cos_val, + parallel::Options options = {}, + const Observer &observer = {}) -> void { const size_t n = cos.blocks.size(); - for (size_t k = 0; k < n; ++k) { - const auto [base, bits] = cos.blocks[k]; - for_each_cos_index(base, bits, [&](size_t i) { coeff[i] *= cos_val; }); - } + const size_t ranges = logical_ranges(n, cos_mask_range_blocks); + observer.prepare(KernelRange::cos_mask, ranges); + parallel::for_blocks(ranges, options, [&](size_t range) { + observer.visit(KernelRange::cos_mask, range); + const size_t hi = std::min(n, (range + 1) * cos_mask_range_blocks); + for (size_t k = range * cos_mask_range_blocks; k < hi; ++k) { + const auto [base, bits] = cos.blocks[k]; + for_each_cos_index(base, bits, [&](size_t i) { coeff[i] *= cos_val; }); + } + }); } -inline auto accumulate_cos_mask(double *state, double *ham, const CosMask &cos, double cos_val, double sec_val) - -> double { +inline auto accumulate_cos_mask(double *state, + double *ham, + const CosMask &cos, + double cos_val, + double sec_val, + [[maybe_unused]] parallel::Options options = {}) -> double { const size_t n = cos.blocks.size(); double loc = 0.0; for (size_t k = 0; k < n; ++k) { @@ -249,9 +369,11 @@ inline auto accumulate_cos_mask(double *state, double *ham, const CosMask &cos, return loc; } +// Fold into `c`, replacing its contents; reuses its capacity. template -inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { - CosMask c; +inline auto fold_to_cos_mask_into(const FoldCache &p, CosMask &c) -> void { + c.blocks.clear(); + c.total_count = 0; for (size_t wi = 0; wi < p.fold.mask_words; ++wi) { const uint64_t b = fold_word(p, wi); if (b) { @@ -259,6 +381,12 @@ inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { c.total_count += static_cast(std::popcount(b)); } } +} + +template +inline auto fold_to_cos_mask(const FoldCache &p) -> CosMask { + CosMask c; + fold_to_cos_mask_into(p, c); return c; } // Cos-index count without materialising the blocks; for diagnostics (graph_size). @@ -280,4 +408,111 @@ inline auto fold_to_indices(const FoldCache &p) -> VecZ { return inds; } +/*! + * \brief A layer's full cosine set (see full_cos_mask()) folded into `out`, replacing its contents and reusing its + * capacity: a caller that folds layer after layer into one buffer allocates it once instead of once per layer. + */ +template +auto full_cos_mask_into(const InvertedIndex &inverted_index, + const LayerTraversal &layer, + Basis basis, + CosMask &out) -> void { + const auto gen = generator_from_words(layer.generator_words()); + const auto combined = make_fold_cache(inverted_index, gen, layer.scaled_count(), basis); + fold_to_cos_mask_into(combined, out); +} + +/*! + * \brief A layer's full cosine set, folded from `inverted_index` and materialized as a mask; the paring input. + * \param inverted_index The index of the store the layer's rows belong to. + * \param layer The layer; its generator words and scaled_count select the fold. + * \param basis The coefficient encoding. + */ +template +auto full_cos_mask(const InvertedIndex &inverted_index, const LayerTraversal &layer, Basis basis) -> CosMask { + CosMask mask; + full_cos_mask_into(inverted_index, layer, basis, mask); + return mask; +} + +/*! + * \brief Per-layer cosine callbacks replaying `graph` against `inverted_index`. + * + * A pared layer replays its stored mask (an empty stored mask replays nothing); every other layer recomputes its + * fold from the index. Each closure captures `options` for its kernels. The closures keep raw pointers into the index + * and into the layers' stored masks, so they must outlive neither the index nor the graph owning those layers. Row + * parity words are fetched inside each call, never cached, since they move whenever the store grows. The result is + * marked CosCallbacks::owner_parallel. Named apart from monoprop::build_cos_callbacks, + * which delegates here, so argument-dependent lookup never makes an unqualified call ambiguous: the closures touch only + * this index, these layers and thread-local scratch. + * + * \param inverted_index The index of the store whose rows the graph's endpoints name. + * \param graph The replay window; its layers must outlive the callbacks. + * \param basis The coefficient encoding (Pauli folds the generator's J image). + * \param options The kernels' thread budget; serial by default. + */ +template +auto make_cos_callbacks(const InvertedIndex &inverted_index, + const MPGraphView &graph, + Basis basis = Basis::Majorana, + parallel::Options options = {}) -> CosCallbacks { + struct LayerCos { + bool recomputes_cos = false; + LazyFold recipe{}; // used iff recomputes_cos + const CosMask *filtered = nullptr; // points into a pruned layer's stored cos + }; + auto cache = std::make_shared>(); + cache->reserve(graph.layers()); + for (size_t i = 0; i < graph.layers(); ++i) { + const auto &layer = graph.get_layer(i); + LayerCos entry; + if (const CosMask *pruned = layer.pruned_cos(); pruned != nullptr) { + entry.recomputes_cos = false; + entry.filtered = pruned; + } + else { + entry.recomputes_cos = true; + const auto t = layer.traversal(); + const auto gen = generator_from_words(t.generator_words()); + entry.recipe = make_lazy_fold(inverted_index, gen, t.scaled_count(), basis); + } + cache->push_back(std::move(entry)); + } + + const auto *sc = &inverted_index; + LayerCosScale cos_scale = [cache, sc, options](size_t i, double *c, double v) { + const auto &e = (*cache)[i]; + if (!e.recomputes_cos) { + scale_cos_mask(c, *e.filtered, v, options); + } + else { + scale_cos_lazy(*sc, e.recipe, c, v, options); + } + }; + LayerCosAccumulate cos_acc = [cache, sc, options](size_t i, double *s, double *h, double v, double sec) { + const auto &e = (*cache)[i]; + if (!e.recomputes_cos) { + return accumulate_cos_mask(s, h, *e.filtered, v, sec, options); + } + return accumulate_cos_lazy(*sc, e.recipe, s, h, v, sec, options); + }; + LayerCosIndices cos_inds = [cache, sc](size_t i, std::vector &out) { + const auto &e = (*cache)[i]; + if (!e.recomputes_cos) { + cos_indices_mask(*e.filtered, out); + return; + } + cos_indices_lazy(*sc, e.recipe, out); + }; + LayerCosCount cos_count = [cache, sc](size_t i) -> size_t { + const auto &e = (*cache)[i]; + return e.recomputes_cos ? cos_count_lazy(*sc, e.recipe) : cos_count_mask(*e.filtered); + }; + return {.scale = std::move(cos_scale), + .accumulate = std::move(cos_acc), + .indices = std::move(cos_inds), + .count = std::move(cos_count), + .owner_parallel = true}; +} + } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/CosineRecomputeCallbacks.h b/cpp/monoprop/detail/evolution/CosineRecomputeCallbacks.h index 8604c179a..a28add5ab 100644 --- a/cpp/monoprop/detail/evolution/CosineRecomputeCallbacks.h +++ b/cpp/monoprop/detail/evolution/CosineRecomputeCallbacks.h @@ -62,11 +62,21 @@ using LayerCosAccumulate = std::function; /// Appends one layer's cosine-set indices to `out`; the gradient calls it only where the cosine vanishes. using LayerCosIndices = std::function &out)>; +/// How many indices LayerCosIndices appends for `layer`, without materialising them. +using LayerCosCount = std::function; struct CosCallbacks { LayerCosScale scale; ///< forward path; required whenever the parameters are non-empty LayerCosAccumulate accumulate; ///< reverse path; required by the gradient only LayerCosIndices indices; ///< reverse path; required by the gradient only + /// Optional: exactly what `indices` appends per layer. Lets records reserve once; only memory depends on it. + LayerCosCount count; + /// Provenance, not a property of the closures: set only by detail::make_cos_callbacks() (behind + /// build_cos_callbacks()), whose closures touch only their own inverted index (including its lazy caches), graph + /// layers and the calling thread's scratch. Distinct owners' sets marked this way may then run concurrently, each + /// on one thread at a time. Anything else is opaque and runs one owner at a time, on the primary thread of a + /// sharded evaluation. + bool owner_parallel = false; }; } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/LayerReplay.h b/cpp/monoprop/detail/evolution/LayerReplay.h new file mode 100644 index 000000000..d1b02f6bf --- /dev/null +++ b/cpp/monoprop/detail/evolution/LayerReplay.h @@ -0,0 +1,556 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +/* + * Owner-local layer replay and reverse-derivative kernels, shared by the low-level single-store evaluator + * (Evolution.cpp, MPFunctions.cpp) and the sharded evaluator (detail/sharded/Evaluation.cpp). + * + * Library-internal: included only by compiled library sources, never by an installed header, so every + * instantiation is compiled with the library's own floating-point flags. The kernels perform no MPI and no + * synchronization. Where a kernel reads a partner owner's endpoint values it takes a `block_of(rank)` callable + * returning that partner's block for this owner: the low-level path points into a received MPI buffer, the sharded + * path into the partner's published snapshot. Slots are visited in ascending order, the self slot is skipped, + * and each block is read at the same positions, so both paths perform the same operations in the same order. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/MPGraph.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" + +namespace monoprop { + +/*! + * \brief A per-layer cosine callback an evaluation path needs was left empty. + */ +class MissingLayerCallback : public std::invalid_argument { +public: + using std::invalid_argument::invalid_argument; +}; + +/*! + * \brief A state's rows and values disagree in length, a sparse row names a slot outside the state, or the operator + * being contracted is shorter than the state. + * + * One type because every case is a caller-supplied length or index that does not fit the rest of the call. + */ +class EvalStateArgumentError : public std::invalid_argument { +public: + using std::invalid_argument::invalid_argument; +}; + +} // namespace monoprop + +namespace monoprop::detail::replay { + +//! Endpoint accumulators, in pre-layer units like the sum they are subtracted from. +struct EndpointContrib { + double cos_terms = 0.0; //!< Sum of s_old * h over the rotation endpoints. + double sin_terms = 0.0; //!< Sum of phase * s_old * h_partner over the rotation endpoints. +}; + +//! One layer's trigonometric factors for the reverse pass. +struct TrigValues { + double cos_val; //!< cos(g * param) + double sin_val; //!< sin(g * param) + double sec_val; //!< 1 / cos_val + double g_val; //!< 2 * gen_coeff + + /*! + * \brief Factors of the rotation by `2 * gen_coeff * param`. + * \param param The optimizer parameter. + * \param gen_coeff The generator coefficient. + */ + explicit TrigValues(double param, double gen_coeff = 1.0) { + const double g = 2.0 * gen_coeff; + cos_val = std::cos(g * param); + sin_val = std::sin(g * param); + sec_val = 1.0 / cos_val; + g_val = g; + } +}; + +//! The two endpoint sums added componentwise, self first. +inline auto combine_endpoint_contrib(const EndpointContrib &a, const EndpointContrib &b) -> EndpointContrib { + return {.cos_terms = a.cos_terms + b.cos_terms, .sin_terms = a.sin_terms + b.sin_terms}; +} + +//! dE/dθ = −g·(sin·(A − ep.cos_terms) − ep.sin_terms); both sums are in pre-layer units. +inline auto layer_derivative(const TrigValues &trig, double A, const EndpointContrib &ep) -> double { + return -trig.g_val * (trig.sin_val * (A - ep.cos_terms) - ep.sin_terms); +} + +/*! + * \brief layer_derivative() from the raw cosine accumulation (the kernel's return value, before its ×sec). + * + * For an evaluator that finishes a layer in a later phase than it accumulates it. The low-level evaluator computes + * `A = raw * sec` and the derivative inline in one function, where the compiler may contract the product into the + * subtraction `A - ep.cos_terms`; keeping `A` as its own statement here, from the unrounded `raw`, gives the compiler + * the same dataflow. Out of line on purpose: the low-level evaluator adds the derivative to the gradient across a + * translation-unit boundary, so it never contracts with that addition either. + */ +[[gnu::noinline]] inline auto layer_derivative_from_raw(const TrigValues &trig, double raw, const EndpointContrib &ep) + -> double { + const double A = raw * trig.sec_val; + return -trig.g_val * (trig.sin_val * (A - ep.cos_terms) - ep.sin_terms); +} + +// --- Forward replay ------------------------------------------------------------------------------------------- + +/*! + * \brief Snapshot this owner's own sin_send values before the cosine pass overwrites them. + * \param op The coefficients about to be stepped. + * \param layer The layer. + * \param out Resized to the self slot's endpoint count and overwritten. + */ +inline auto snapshot_self_sources(const VecD &op, const LayerTraversal &layer, VecD &out) -> void { + const auto self_slot = layer.cross_rank_self_slot(); + const size_t count = self_slot.sin_send_count; + out.resize(count); + for (size_t k = 0; k < count; ++k) { + out[k] = op[detail::slot_sin_send_index(self_slot, k)]; + } +} + +/*! + * \brief Pack every partner slot's pre-cosine sin_send values into its block. + * \param block_of Callable `block_of(rank) -> double *`, the destination of partner `rank`'s block. + */ +template +auto pack_evolution_payload(const VecD &op, const LayerTraversal &layer, size_t my_rank, BlockOf &&block_of) -> void { + layer.for_each_occupied_slot([&](size_t rank, const detail::CrossRankSlotView &slot) { + if (rank == my_rank) { + return; + } + double *const dst = block_of(rank); + for (size_t k = 0; k < slot.sin_send_count; ++k) { + dst[k] = op[detail::slot_sin_send_index(slot, k)]; + } + }); +} + +/*! + * \brief Add every partner's pre-cosine values into this owner's already cos-scaled sin_recv targets. + * \param block_of Callable `block_of(rank) -> const double *`, partner `rank`'s block for this owner. + */ +template +auto apply_evolution_payload(VecD &op, const LayerTraversal &layer, double sin_val, size_t my_rank, BlockOf &&block_of) + -> void { + // op[i] is already cos-scaled, so only the sine term is added; rv[k] is the partner's pre-cos value. + layer.for_each_occupied_slot([&](size_t rank, const detail::CrossRankSlotView &slot) { + if (rank == my_rank) { + return; + } + const double *const rv = block_of(rank); + for (size_t k = 0; k < slot.sin_send_count; ++k) { + const size_t i = detail::slot_sin_recv_index(slot, k); + op[i] += sin_val * static_cast(detail::slot_sin_recv_phase(slot, k)) * rv[k]; + } + }); +} + +//! Add the self slot's snapshotted sources into its already cos-scaled sin_recv targets. +inline auto apply_self_sources(VecD &op, const LayerTraversal &layer, double sin_val, const VecD &self_sources) + -> void { + const auto self_slot = layer.cross_rank_self_slot(); + for (size_t k = 0; k < self_slot.sin_send_count; ++k) { + const size_t i = detail::slot_sin_recv_index(self_slot, k); + op[i] += sin_val * static_cast(detail::slot_sin_recv_phase(self_slot, k)) * self_sources[k]; + } +} + +// --- Reverse derivatives -------------------------------------------------------------------------------------- + +/*! + * \brief Pre-cosine snapshot buffers of one owner, reused across layers, indexed by occupied position. + * + * Passed whole rather than re-split at each hand-off, which is how a send buffer becomes a receive one. + */ +struct DerivativeSnapshotScratch { + std::vector sin_send_state; //!< Per occupied position: state at the sin_send endpoints. + std::vector sin_send_op; //!< Per occupied position: op at the sin_send endpoints. + std::vector sin_recv_state; //!< Per occupied position: state at the sin_recv endpoints. + std::vector sin_recv_op; //!< Per occupied position: op at the sin_recv endpoints. + VecD self_recv_op; //!< Self-slot entry op, kept only where a record overwrites op before it is read. +}; + +// Emptied, not filled: the self slot recovers live, and a previous layer's snapshot must not be read. +inline auto clear_slot_snapshot(DerivativeSnapshotScratch &snap, size_t pos) -> void { + snap.sin_send_state[pos].clear(); + snap.sin_send_op[pos].clear(); + snap.sin_recv_state[pos].clear(); + snap.sin_recv_op[pos].clear(); +} + +inline auto fill_slot_snapshot(DerivativeSnapshotScratch &snap, + size_t pos, + const VecD &state, + const VecD &op, + const detail::CrossRankSlotView &slot) -> void { + const size_t count = slot.sin_send_count; + auto &bs = snap.sin_send_state[pos]; + auto &bh = snap.sin_send_op[pos]; + auto &ds = snap.sin_recv_state[pos]; + auto &dh = snap.sin_recv_op[pos]; + bs.resize(count); + bh.resize(count); + ds.resize(count); + dh.resize(count); + for (size_t k = 0; k < count; ++k) { + const size_t bi = detail::slot_sin_send_index(slot, k); + bs[k] = state[bi]; + bh[k] = op[bi]; + const size_t di = detail::slot_sin_recv_index(slot, k); + ds[k] = state[di]; + dh[k] = op[di]; + } +} + +/*! + * \brief Snapshot pre-cosine (state, op) at every partner slot's sin_send/sin_recv endpoints. + * + * The self slot needs none -- it recovers live -- so it is cleared. + */ +inline auto snapshot_remote_endpoints(const VecD &state, + const VecD &op, + const LayerTraversal &layer, + size_t my_rank, + DerivativeSnapshotScratch &snap) -> void { + // Grow-only: shrinking would free the allocations this scratch exists to reuse. Each size is tested here, not by + // resize(): once grown, the call is out of line and costs a full frame at every reverse step. Per vector, because + // a failed growth may leave them at different sizes in a scratch that outlives the failure. + const size_t occupied = layer.occupied_slot_count(); + for (std::vector *scratch : + {&snap.sin_send_state, &snap.sin_send_op, &snap.sin_recv_state, &snap.sin_recv_op}) { + if (scratch->size() < occupied) { + scratch->resize(occupied); + } + } + layer.for_each_occupied_slot( + [&snap, my_rank, &state, &op](size_t pos, size_t r, const detail::CrossRankSlotView &slot) { + if (r == my_rank) { + clear_slot_snapshot(snap, pos); + return; + } + fill_slot_snapshot(snap, pos, state, op, slot); + }); +} + +/*! + * \brief Keep the self slot's entry op at its sin_recv endpoints, which a record overwrites before the self pass. + */ +inline auto snapshot_self_recv_op(const VecD &op, const LayerTraversal &layer, DerivativeSnapshotScratch &snap) + -> void { + const auto slot = layer.cross_rank_self_slot(); + snap.self_recv_op.resize(slot.sin_send_count); + for (size_t k = 0; k < slot.sin_send_count; ++k) { + snap.self_recv_op[k] = op[detail::slot_sin_recv_index(slot, k)]; + } +} + +/*! + * \brief Pack sin_send (state, op) pairs from the pre-cosine snapshots into every partner slot's block. + * \param block_of Callable `block_of(rank) -> double *`, the destination of partner `rank`'s block. + */ +template +auto pack_derivative_payload(const DerivativeSnapshotScratch &snap, + const LayerTraversal &layer, + size_t my_rank, + BlockOf &&block_of) -> void { + layer.for_each_occupied_slot([&](size_t pos, size_t rank, const detail::CrossRankSlotView &slot) { + if (rank == my_rank) { + return; + } + double *const dst = block_of(rank); + const auto &bs = snap.sin_send_state[pos]; + const auto &bh = snap.sin_send_op[pos]; + for (size_t k = 0; k < slot.sin_send_count; ++k) { + dst[2 * k] = bs[k]; + dst[(2 * k) + 1] = bh[k]; + } + }); +} + +/*! + * \brief Remote endpoint pass: own pre-cosine values from the sin_recv snapshots, partner values from its block. + * + * Must run after the cosine pass -- the ordering is floating-point significant. + * + * \param block_of Callable `block_of(rank) -> const double *`, partner `rank`'s (state, op) pairs for this owner. + * \return The endpoint sums over every partner slot, accumulated in ascending slot order from zero. + */ +template +auto apply_derivative_payload(VecD &state, + VecD &op, + const LayerTraversal &layer, + const DerivativeSnapshotScratch &snap, + const TrigValues &trig, + size_t my_rank, + BlockOf &&block_of) -> EndpointContrib { + EndpointContrib local{}; + layer.for_each_occupied_slot([&](size_t pos, size_t rank, const detail::CrossRankSlotView &slot) { + if (rank == my_rank) { + return; + } + const double *const rv = block_of(rank); + const auto &ds = snap.sin_recv_state[pos]; + const auto &dh = snap.sin_recv_op[pos]; + for (size_t k = 0; k < slot.sin_send_count; ++k) { + const size_t i = detail::slot_sin_recv_index(slot, k); + const auto phi = static_cast(detail::slot_sin_recv_phase(slot, k)); + // Inverse-rotation write-back (−sin): un-evolves state/op for the next reverse layer. + const double ps = -trig.sin_val * phi; + const double s_old = ds[k]; + const double h_old = dh[k]; + const double s_p = rv[2 * k]; + const double h_p = rv[(2 * k) + 1]; + local.cos_terms += s_old * op[i]; // pre-layer, matching what the cos pass added to A + local.sin_terms += phi * s_old * h_p; + op[i] = (h_old * trig.cos_val) + (ps * h_p); + state[i] = (s_old * trig.cos_val) + (ps * s_p); + } + }); + return local; +} + +/*! + * \brief Self-slot endpoint pass over whole rotation pairs. + * + * sin_recv entries k and k + count/2 are the two endpoints of one rotation, so reading both before writing either + * avoids the read-after-write hazard. `h_pre` carries the entry op where a record overwrote it, and is null where + * op still holds it. + */ +inline auto apply_self_slot_derivative_paired(VecD &state, + VecD &op, + const LayerTraversal &layer, + const TrigValues &trig, + const double *h_pre) -> EndpointContrib { + const auto slot = layer.cross_rank_self_slot(); + const size_t self_d_count = slot.sin_send_count; + if (self_d_count == 0) { + return {}; + } + const auto pairs = self_d_count / 2; + EndpointContrib local{}; + for (size_t k = 0; k < pairs; ++k) { + const size_t i1 = detail::slot_sin_recv_index(slot, k); + const auto phi1 = static_cast(detail::slot_sin_recv_phase(slot, k)); + const size_t i2 = detail::slot_sin_recv_index(slot, k + pairs); + const auto phi2 = static_cast(detail::slot_sin_recv_phase(slot, k + pairs)); + // Recover pre-cos values. + const double s1 = state[i1] * trig.sec_val; + const double s2 = state[i2] * trig.sec_val; + const double h1 = h_pre != nullptr ? h_pre[k] : op[i1] * trig.cos_val; + const double h2 = h_pre != nullptr ? h_pre[k + pairs] : op[i2] * trig.cos_val; + local.cos_terms += (s1 * op[i1]) + (s2 * op[i2]); // pre-layer, as in A + local.sin_terms += (phi1 * s1 * h2) + (phi2 * s2 * h1); + // Inverse-rotation write-back (−sin); see apply_derivative_payload. + const double ps1 = -trig.sin_val * phi1; + const double ps2 = -trig.sin_val * phi2; + op[i1] = (h1 * trig.cos_val) + (ps1 * h2); + state[i1] = (s1 * trig.cos_val) + (ps1 * s2); + op[i2] = (h2 * trig.cos_val) + (ps2 * h1); + state[i2] = (s2 * trig.cos_val) + (ps2 * s1); + } + return local; +} + +/*! + * \brief The layer's cosine accumulation A in pre-layer units, with the record stood in and put back. + * + * A = Σ s_old·h_pre over all anticommuting indices, endpoints included -- hence the later subtraction. Pre-dividing + * lets the kernel's own ×sec land back on the recorded value; the restore fixes up the rest. + */ +inline auto accumulate_layer(VecD &state, + VecD &op, + size_t layer_idx, + const TrigValues &trig, + const detail::CosRecordView &record, + const detail::LayerCosAccumulate &cos_acc) -> double { + detail::predivide_cos_record(op.data(), record, trig.cos_val); + const double a = cos_acc(layer_idx, state.data(), op.data(), trig.cos_val, trig.sec_val) * trig.sec_val; + detail::restore_cos_record(op.data(), record); + return a; +} + +/*! + * \brief As accumulate_layer(), but returns the kernel's raw accumulation, before its ×sec. + * + * The sharded evaluator's form; see layer_derivative_from_raw(). + */ +inline auto accumulate_layer_raw(VecD &state, + VecD &op, + size_t layer_idx, + const TrigValues &trig, + const detail::CosRecordView &record, + const detail::LayerCosAccumulate &cos_acc) -> double { + detail::predivide_cos_record(op.data(), record, trig.cos_val); + const double raw = cos_acc(layer_idx, state.data(), op.data(), trig.cos_val, trig.sec_val); + detail::restore_cos_record(op.data(), record); + return raw; +} + +// --- Selective pre-layer records ------------------------------------------------------------------------------ + +//! Record flag: the coefficients the layer below rotates. +inline constexpr uint8_t kRecordRotationsBelow = 1; +//! Record flag: the layer's own cosine set, which a vanishing cosine destroys. +inline constexpr uint8_t kRecordCosineSet = 2; + +/*! + * \brief Flag the layers whose record can change an answer; return whether any needs its cosine set. + * + * Layer j records what layer j-1 rotates, so it earns its keep only once the layers already reversed can amplify an + * error there by a full significand -- a bound read off the parameters, before any coefficient is touched. + * + * \param mapped_params The replay angles, in layer order. + * \param wanted Overwritten with one flag set per layer. + */ +inline auto plan_cos_records(std::span mapped_params, std::vector &wanted) -> bool { + const size_t layers = mapped_params.size(); + wanted.assign(layers, 0); + bool any_cosine_set = false; + double spread = 0.0; + for (size_t j = layers; j-- > 0;) { + const double cos_val = std::cos(2 * mapped_params[j]); + const double own = -std::log2(std::abs(cos_val)); + if (j > 0 && spread + own >= detail::kRecordSpreadBits) { + wanted[j] |= kRecordRotationsBelow; + } + if (std::abs(cos_val) < detail::kVanishingCos) { + wanted[j] |= kRecordCosineSet; + any_cosine_set = true; + } + spread += own; + } + return any_cosine_set; +} + +//! One owner's selective pre-layer records: layer slices back to back. +struct CosRecords { + std::vector indices; //!< Recorded coefficient indices, layer slices back to back. + VecD values; //!< The pre-layer coefficient at each of them. + VecZ offset; //!< Per layer: start of its slice, plus a tail entry. +}; + +//! Empty the records for a replay of `layers` layers, keeping their capacity. +inline auto reset_records(CosRecords &records, size_t layers) -> void { + records.offset.assign(layers + 1, 0); + records.indices.clear(); + records.values.clear(); +} + +/*! + * \brief Reserve `records` for one replay with exactly what record_pre_layer() appends. + * + * Per selected layer: the sin_recv entries of the layer below, plus the layer's cosine set as `count` reports it. + * Reserving once replaces the geometric growth whose freed blocks the allocator may keep resident. Without `count`, + * only the rotations below are reserved and the cosine sets grow as before. Only capacity depends on this; the + * contents are unchanged. + * + * \param records The owner's records, after reset_records(). + * \param wanted The per-layer record flags of plan_cos_records(). + * \param graph The owner's graph; `wanted` has one entry per layer. + * \param count Optional, CosCallbacks::count: exactly what the indices callback appends per layer. Called on the + * calling thread, so it must be safe to call there. + * \throws std::length_error if the total overflows `size_t`. + * \throws std::bad_alloc if the reservation fails. + */ +inline auto reserve_records(CosRecords &records, + std::span wanted, + const MPGraphView &graph, + const detail::LayerCosCount &count) -> void { + size_t total = 0; + const auto add = [&total](size_t entries) { + if (entries > std::numeric_limits::max() - total) { + throw std::length_error("reserve_records: the record size overflows size_t"); + } + total += entries; + }; + for (size_t layer_idx = 0; layer_idx < wanted.size(); ++layer_idx) { + const uint8_t want = wanted[layer_idx]; + if ((want & kRecordRotationsBelow) != 0U) { + const auto below = graph.get_layer_traversal(layer_idx - 1); + for (size_t r = 0; r < below.cross_rank_rank_count(); ++r) { + add(below.cross_rank_sin_recv_size(r)); + } + } + if ((want & kRecordCosineSet) != 0U && count) { + add(count(layer_idx)); + } + } + records.indices.reserve(total); + records.values.reserve(total); +} + +/*! + * \brief Open `layer_idx`'s slice and fill it from the not-yet-stepped `op`. + * + * Called in layer order, so each slice starts where the previous one ended. + */ +inline auto record_pre_layer(CosRecords &records, + std::span wanted, + const MPGraphView &graph, + const detail::LayerCosIndices &cos_inds, + size_t layer_idx, + const VecD &op) -> void { + records.offset[layer_idx] = records.values.size(); + const uint8_t want = wanted[layer_idx]; + if (want == 0) { + return; + } + const size_t begin = records.indices.size(); + if ((want & kRecordRotationsBelow) != 0U) { + const auto below = graph.get_layer_traversal(layer_idx - 1); + const auto mark = [&records](size_t, size_t i, auto) { records.indices.push_back(static_cast(i)); }; + for (size_t r = 0; r < below.cross_rank_rank_count(); ++r) { + below.for_each_cross_rank_sin_recv_range(r, 0, below.cross_rank_sin_recv_size(r), mark); + } + } + if ((want & kRecordCosineSet) != 0U) { + cos_inds(layer_idx, records.indices); + } + const size_t end = records.indices.size(); + records.values.resize(end); + for (size_t k = begin; k < end; ++k) { + records.values[k] = op[records.indices[k]]; + } +} + +//! Close the last slice after the forward pass over `layers` layers. +inline auto close_records(CosRecords &records, size_t layers) -> void { + records.offset[layers] = records.values.size(); +} + +//! What the reverse pass reads back for `layer_idx`. +inline auto layer_record_in(const CosRecords &records, size_t layer_idx) -> detail::CosRecordView { + const size_t begin = records.offset[layer_idx]; + const size_t count = records.offset[layer_idx + 1] - begin; + if (count == 0) { + return {}; + } + return {.indices = records.indices.data() + begin, .values = records.values.data() + begin, .count = count}; +} + +} // namespace monoprop::detail::replay diff --git a/cpp/monoprop/detail/evolution/layer_build/Common.h b/cpp/monoprop/detail/evolution/layer_build/Common.h index 53ebd7581..04d28a4f2 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Common.h +++ b/cpp/monoprop/detail/evolution/layer_build/Common.h @@ -28,6 +28,39 @@ namespace monoprop::detail { +/*! + * \brief The kinds of logical work range a threaded kernel reports to its range observer (see NoRangeObserver). + */ +enum class KernelRange { + cos_lazy, //!< scale_cos_lazy: one range per kColumnBlockWords fold words. + cos_mask, //!< scale_cos_mask: one range per cos_mask_range_blocks stored mask blocks. + fused_gather, //!< apply_fused_contract insert snapshots: one range per fused_record_range records. + fused_apply, //!< apply_fused_contract rotations: one range per fused_record_range records. + scan, //!< fused_find_and_collect: one range per contiguous run of whole fold blocks (scan_ranges). + decode, //!< probe_incoming_queries record decode: one range per probe_block_queries queries. + incoming_probe, //!< probe_frozen_positions over incoming queries: one range per probe_block_queries queries. + self_probe, //!< probe_frozen_positions over one self window: one range per probe_block_queries queries. + scatter, //!< resolve_incoming on_resolved scatter: one range per resolve_scatter_block queries. +}; + +/*! + * \brief The observer production kernels use: it does nothing and compiles away. + * + * A kernel calls `prepare(kind, ranges)` once on the calling thread before any range runs, then + * `visit(kind, range)` at the start of each logical range, from whichever thread owns that range. Tests + * substitute an observer that records the worker of each range; that seam is the only reason the kernels + * take an observer, and no production caller passes one. + */ +struct NoRangeObserver { + auto prepare(KernelRange /*kind*/, size_t /*ranges*/) const noexcept -> void {} + auto visit(KernelRange /*kind*/, size_t /*range*/) const noexcept -> void {} +}; + +//! Number of `per_range`-sized logical ranges covering `items`. +inline constexpr auto logical_ranges(size_t items, size_t per_range) -> size_t { + return (items + per_range - 1) / per_range; +} + // Sentinel for "no index resolved yet" in the resolve slots. Deliberately equal to // OperatorIndex::kNotFound, so one `>= size` bound check covers a miss and an unresolved slot alike. inline constexpr size_t kMissingIndex = std::numeric_limits::max(); diff --git a/cpp/monoprop/detail/evolution/layer_build/Engine.h b/cpp/monoprop/detail/evolution/layer_build/Engine.h index 99fde5cbe..fe1a7a374 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Engine.h +++ b/cpp/monoprop/detail/evolution/layer_build/Engine.h @@ -18,11 +18,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -36,8 +38,10 @@ #include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/graph_encoding/MPGraphEncodingStorage.h" #include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/mpi/OperationFailure.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" +#include "monoprop/detail/parallel/Options.h" namespace monoprop::detail { @@ -73,6 +77,9 @@ inline auto append_inserted_endpoints(CosMask &cos_all, size_t combined_size, co template struct GraphSink { static constexpr bool wants_values = false; + // on_resolved for distinct queries may run concurrently after prepare(): each writes only its own + // acc[slot].in_entries element, which prepare() sized. Certifies nothing about the other methods. + static constexpr bool parallel_resolve = true; [[nodiscard]] auto incoming_form() const -> QueryForm { return QueryForm::Plain; } [[nodiscard]] auto querier_form() const -> QueryForm { return QueryForm::Plain; } using Response = TermIndex; @@ -127,15 +134,18 @@ struct GraphSink { size_t q, size_t ip, const IncomingProbe &pr, - const mpi::WindowVec & /*incoming*/) -> Response { + typename QueryWire::WireView /*stream*/) -> Response { acc[pr.window.slot(s)].in_entries[in_base_[s] + q] = {ip, pr.phase_of[g]}; return static_cast(ip); } - auto process_reserve(const mpi::WindowVec> & /*inc_r*/, size_t /*my_rank*/) -> void {} + template + auto process_reserve(const mpi::WindowVec & /*inc_r*/, size_t /*my_rank*/) -> void {} + // `resp`, `srcs` and `qbuf` are views of the resolver's published answers and this owner's own sources and + // plain queries; all three stay unchanged until this returns. auto on_response_block(size_t r, - const std::vector &resp, - const std::vector &srcs, - const VecZ &qbuf) -> void { + std::span resp, + std::span srcs, + typename QueryWire::WireView qbuf) -> void { auto &out = acc[r].out_entries; const size_t base = out.size(); const size_t nq = resp.size(); @@ -194,6 +204,10 @@ struct GraphSink { template struct ContractSink { static constexpr bool wants_values = true; + // on_resolved for distinct queries may run concurrently after prepare() in Heisenberg: each writes only its + // own cross_half element and reads the pre-insert coefficients. Schrodinger scoring stays serial, see + // parallel_resolve_enabled(). Certifies nothing about the other methods (on_response_block appends). + static constexpr bool parallel_resolve = true; // This rank receives fused records, but on_response_block is handed its own plain queries_r; // reading the wrong form there decodes a neighbouring record's phase, i.e. a coefficient sign flip. [[nodiscard]] auto incoming_form() const -> QueryForm { return QueryForm::Fused; } @@ -216,6 +230,8 @@ struct ContractSink { // No constructor on purpose: as an aggregate the call site names each field, so the two adjacent // bools cannot be swapped silently. GraphSink keeps its ctor because it sizes `acc` from R. + // At most `queries` self hits follow: reserved once, instead of a reallocation chain as the hits arrive. + auto reserve_self_hits(size_t queries) -> void { fc.hits.reserve(fc.hits.size() + queries); } // Self-resolve hit: both endpoints are local. [[gnu::always_inline]] auto self_hit(size_t src, size_t found, int phase, double v_src) -> void { const double v_tgt = fused_scale ? op_coeffs[found] * inv_cos : op_coeffs[found]; @@ -230,6 +246,9 @@ struct ContractSink { fc.inserts[def_base_ + k] = RotationRec{src, idx, v_src, /*v_tgt=*/0.0, static_cast(phase)}; } + //! The Schrodinger fresh-insert arm scores the state through mono_at; it keeps the serial scatter. + [[nodiscard]] auto parallel_resolve_enabled() const -> bool { return !schrodinger; } + // Cross-rank (R>1). Send buffer = queries interleaved with their v_src stream into `scratch` // (combined_qv_), so one alltoallv carries query + value. auto send_buffer(mpi::WindowVec &queries, @@ -249,12 +268,13 @@ struct ContractSink { cross_base_ = fc.cross_half.size(); fc.cross_half.resize(cross_base_ + pr.nq_total); } + // `stream` is sender s's fused query stream, a view the caller keeps unchanged until the scatter returns. auto on_resolved(size_t g, - mpi::WindowIndex s, + mpi::WindowIndex /*s*/, size_t /*q*/, size_t ip, const IncomingProbe &pr, - const mpi::WindowVec &incoming) -> Response { + typename QueryWire::WireView stream) -> Response { double v_tgt; if (ip < pr.base) { v_tgt = fused_scale ? op_coeffs[ip] * inv_cos : op_coeffs[ip]; @@ -266,12 +286,13 @@ struct ContractSink { v_tgt = 0.0; // Heisenberg fresh insert } fc.cross_half[cross_base_ + g] = HalfRotationRec{ip, - QueryWire::value_at(incoming[s], pr.off_of[g]), + QueryWire::value_at(stream, pr.off_of[g]), static_cast(pr.phase_of[g]), /*is_insert=*/ip >= pr.base}; return v_tgt; } - auto process_reserve(const mpi::WindowVec> &inc_r, size_t my_rank_) -> void { + template + auto process_reserve(const mpi::WindowVec &inc_r, size_t my_rank_) -> void { const mpi::SlotWindow w = inc_r.window(); size_t incoming = 0; for (const auto wi : w.indices()) { @@ -283,9 +304,9 @@ struct ContractSink { } // A querier half always writes a pre-gate term the cos sweep already covered ⇒ is_insert=false. auto on_response_block(size_t /*r*/, - const std::vector &rval, - const std::vector &srcs, - const VecZ &qbuf) -> void { + std::span rval, + std::span srcs, + typename QueryWire::WireView qbuf) -> void { const size_t nq = rval.size(); const QueryForm form = querier_form(); size_t off = 0; @@ -308,10 +329,38 @@ struct ContractSink { } }; -// Owns build_layer's machinery over a compile-time Sink policy. combined_size = the pre-layer operator size. -template +/* + * Owns one layer's partner resolution over a compile-time Sink policy for one owner: the flat routing slot + * `my_rank` out of `R` (a physical rank of the low-level one-owner-per-rank engine, or rank * T + shard of a sharded + * root), its store, its + * matched marks and its sink. combined_size = the pre-layer operator size, the source boundary. + * + * Each pass (leaders, then followers) is three owner-local phases with a handoff between them: + * + * prepare_exchange adopt the traversal's outgoing streams, drop followers a leader already matched, resolve + * same-owner queries against the frozen store (their misses are deferred), and expose the + * outgoing payload. + * resolve_published decode, probe and answer the queries other owners published to this owner, in ascending + * sender order, then insert their misses and publish them in the index. + * consume_published fold the answers to this owner's own queries into its sink, in window and query order. + * + * finish() then inserts the deferred same-owner misses (leaders, then followers) and finalizes the sink. The phases + * perform no MPI, no barrier and no nested worksharing beyond the serial-or-budgeted kernels `options` selects; the + * caller owns every handoff. run_exchange() is the low-level engine's MPI transport adapter over the same phases. + * + * Handoff lifetimes: the payload prepare_exchange returns is read by its destinations until they have resolved it, + * and consume_published still reads this owner's own sources and plain queries, so neither may change until this + * owner consumed its answers and every destination resolved the payload. The next prepare_exchange replaces them. + * + * `Observer` is the test seam of the threaded kernels (see NoRangeObserver). It may also provide + * `exchange_prepared(leaders, payload, sources, values)`, `exchange_resolved(leaders, responses)` and + * `exchange_consumed(leaders, responses)`, which the phases call with what they produced or consumed; production + * observers provide none of them. + */ +template struct LayerBuildEngine { using RowPosT = typename OperatorIndex::PosT; + using Response = typename Sink::Response; //!< One answer per resolved query. // A miss keeps its decoded positions (pos_at indexes deferred_pos_flat_) and the probe's hash. struct DeferredSelfMiss { @@ -323,14 +372,13 @@ struct LayerBuildEngine { double v_src = 0.0; // ContractSink only: op_pre[src] captured at scan emit; 0 for GraphSink }; MPOperator &local_op; // scanned, looked up, and grown by the inserts - mpi::Comm comm; - size_t R; - size_t my_rank; + size_t R; // flat owners in the routing world + size_t my_rank; // this owner's flat routing slot // Follower-matched set over [0, combined_size), caller-owned (see MatchedEpochSet). Distinct leaders // → distinct found, so each slot is marked once. MatchedEpochSet &matched; size_t combined_size; - // `plan`'s window for my_rank: every per-slot array below is sized to it. + // The traversal's window for my_rank: every per-slot array below is sized to it. mpi::SlotWindow window; mpi::WindowVec queries_r; mpi::WindowVec> src_idx_r; @@ -344,28 +392,29 @@ struct LayerBuildEngine { mpi::WindowVec> src_val_r; // Fused query+value send scratch (ContractSink, R>1): shared by a gate's two exchange passes. mpi::WindowVec combined_qv_; - // This gate's destination ranks (mpi::PeerPlan), derived once in build_layer. - mpi::PeerPlan plan; Sink sink; + // Budget forwarded to the within-owner kernels; a sharded caller passes the serial default. + parallel::Options options; + Observer observer; LayerBuildEngine(MPOperator &local_op_, - mpi::Comm comm_, size_t R_, size_t my_rank_, MatchedEpochSet &matched_scratch, size_t combined_size_, Sink &&sink_, - mpi::PeerPlan plan_, - mpi::SlotWindow window_) + mpi::SlotWindow window_, + parallel::Options options_ = {}, + const Observer &observer_ = {}) : local_op(local_op_), - comm(comm_), R(R_), my_rank(my_rank_), matched(matched_scratch), combined_size(combined_size_), window(window_), - plan(plan_), - sink(std::move(sink_)) { + sink(std::move(sink_)), + options(options_), + observer(observer_) { assert(window.stop() <= R && window.count != 0); queries_r.reset(window); src_idx_r.reset(window); @@ -394,40 +443,155 @@ struct LayerBuildEngine { } } - // One partner-resolution pass over the given query streams, which it takes ownership of. Round 1 - // carries the queries (the sink may fuse the v_src stream into them) and the resolver inserts absent - // partners in that same round; round 2 returns the answers. Taking the streams here rather than having - // the caller assign the members first is what makes the two-pass protocol unmissable — the follower - // pass must also drop the queries a leader already matched, and that only holds once the leader pass - // has run. - auto run_exchange(bool is_leader_pass, - mpi::WindowVec &&queries, - mpi::WindowVec> &&src_idx, - mpi::WindowVec> &&src_val, - SelfQueryStage &&self_stage) -> void { - assert(queries.window() == window && src_idx.window() == window); - assert(src_val.size() == 0 || src_val.window() == window); // empty under GraphSink + /*! + * \brief Adopt one pass's outgoing streams, resolve its same-owner queries and expose the outgoing payload. + * + * Taking the streams here rather than having the caller assign the members first is what makes the two-pass + * protocol unmissable: the follower pass must drop the queries a leader already matched, which holds only once + * this owner has resolved every leader query addressed to it. Follower filtering is stable, so kept queries keep + * their order. Same-owner queries are probed against the frozen store; their misses are deferred to finish(). + * + * \param leaders Whether this is the leader pass. + * \param queries Plain query streams per window slot, from the traversal; the self slot is empty. + * \param sources Source rows parallel to `queries`, the self slot parallel to `self_stage`. + * \param values Captured source values parallel to `sources` (ContractSink), or empty. + * \param self_stage Same-owner queries as unencoded positions. + * \return The payload to publish, one block per destination slot: the plain queries (GraphSink) or the fused + * query/value stream (ContractSink). It stays valid and unchanged until the next prepare_exchange() + * or the engine's destruction; the self slot is empty. With a single owner nothing is published. + */ + auto prepare_exchange(bool leaders, + mpi::WindowVec &&queries, + mpi::WindowVec> &&sources, + mpi::WindowVec> &&values, + SelfQueryStage &&self_stage) -> const mpi::WindowVec & { + assert(queries.window() == window && sources.window() == window); + assert(values.size() == 0 || values.window() == window); // empty under GraphSink + leaders_pass_ = leaders; queries_r = std::move(queries); - src_idx_r = std::move(src_idx); + src_idx_r = std::move(sources); self_stage_ = std::move(self_stage); - // src_val is empty unless Sink::wants_values, so the move is a no-op under GraphSink. - src_val_r = std::move(src_val); - if (!is_leader_pass && R > 1) { + // values is empty unless Sink::wants_values, so the move is a no-op under GraphSink. + src_val_r = std::move(values); + if (!leaders && R > 1) { drop_matched_cross_rank_followers(); } - resolve_self_queries(is_leader_pass); + resolve_self_queries(leaders); + const mpi::WindowVec *payload = &queries_r; + if (R > 1) { + payload = &sink.send_buffer(queries_r, src_val_r, combined_qv_); + } + if constexpr (requires { observer.exchange_prepared(leaders, *payload, src_idx_r, src_val_r); }) { + observer.exchange_prepared(leaders, *payload, src_idx_r, src_val_r); + } + return *payload; + } + + /*! + * \brief Resolve the queries other owners published to this owner, then insert and publish its misses. + * + * Senders are consumed in ascending window order, each in its own query order, so row IDs do not depend on when a + * sender produced its block. Every stream is validated before any is decoded; probes read the frozen store; the + * scatter reads pre-insert coefficients; misses are then inserted and indexed on this owner only. + * + * \param incoming One read-only view per window slot of this owner's window: the block the slot's owner + * published for this owner. The viewed buffers must stay unchanged until this returns. + * \param leaders Whether this is the leader pass; leader hits mark their target rows as matched. + * \return One answer per received query, per sender slot, in query order. + * \throws std::invalid_argument if `incoming` does not cover this owner's window, before anything is read. + * \throws As resolve_incoming(); a failure after insertion began leaves the store partly grown. + */ + auto resolve_published(const mpi::WindowVec> &incoming, bool leaders) + -> mpi::WindowVec> { + if (incoming.window() != window) { + throw std::invalid_argument(std::format("LayerBuildEngine::resolve_published: incoming slots [{}, {}) do " + "not match the owner's window [{}, {})", + incoming.window().base, + incoming.window().stop(), + window.base, + window.stop())); + } + auto responses = + resolve_incoming(incoming, local_op, leaders, matched, combined_size, sink, options, observer); + if constexpr (requires { observer.exchange_resolved(leaders, responses); }) { + observer.exchange_resolved(leaders, responses); + } + return responses; + } + + /*! + * \brief Fold the answers to this owner's published queries into its sink. + * + * Reads this owner's own sources and plain queries, which prepare_exchange() retained, so each answer meets the + * query it answers. The self slot is skipped: prepare_exchange() resolved it. + * + * \param responses One read-only view per window slot: the answers the slot's owner published for this owner's + * queries, in query order. The viewed buffers must stay unchanged until this returns. + * \throws std::invalid_argument if `responses` does not cover this owner's window, and std::length_error if a + * block does not hold exactly one answer per query; both before any record is written. + */ + auto consume_published(const mpi::WindowVec> &responses) -> void { + if (responses.window() != window) { + throw std::invalid_argument(std::format("LayerBuildEngine::consume_published: response slots [{}, {}) do " + "not match the owner's window [{}, {})", + responses.window().base, + responses.window().stop(), + window.base, + window.stop())); + } + for (const auto wi : window.indices()) { + if (window.slot(wi) != my_rank && responses[wi].size() != src_idx_r[wi].size()) { + throw std::length_error(std::format("LayerBuildEngine::consume_published: slot {} answered {} of {} " + "queries", + window.slot(wi), + responses[wi].size(), + src_idx_r[wi].size())); + } + } + if constexpr (requires { observer.exchange_consumed(leaders_pass_, responses); }) { + observer.exchange_consumed(leaders_pass_, responses); + } + process_responses(responses, src_idx_r, queries_r, my_rank, sink); + } + + /*! + * \brief The low-level engine's transport adapter: one pass over the same phases, with MPI rounds over `comm`. + * + * Round 1 carries the payload, and the resolver inserts absent partners in that same round; round 2 returns the + * answers. Each round completes inside its handle's lifetime and under the distributed guard, so a failure after + * posting never reaches a handle's draining destructor while a peer is still inside the round. + * + * \param comm The communicator whose flat slots are the routing owners. + * \param plan The generator's peer plan; its window is this engine's window. + */ + auto run_exchange(mpi::Comm comm, + mpi::PeerPlan plan, + bool is_leader_pass, + mpi::WindowVec &&queries, + mpi::WindowVec> &&src_idx, + mpi::WindowVec> &&src_val, + SelfQueryStage &&self_stage) -> void { + const mpi::WindowVec &send = prepare_exchange(is_leader_pass, + std::move(queries), + std::move(src_idx), + std::move(src_val), + std::move(self_stage)); if (R <= 1) { return; } - mpi::WindowVec &send = sink.send_buffer(queries_r, src_val_r, combined_qv_); mpi::WindowVec inc_q; - mpi::begin_alltoallv(send, comm, /*skip_self=*/false, /*known_recv_counts=*/nullptr, plan).wait_into(inc_q); - auto resp = resolve_incoming(inc_q, local_op, is_leader_pass, matched, combined_size, sink); + auto query_round = mpi::begin_alltoallv(send, comm, /*skip_self=*/false, /*known_recv_counts=*/nullptr, plan); + mpi::guard_distributed(comm, [&] { query_round.wait_into(inc_q); }); + // The peers may already be inside the response round, so a failure here (a malformed stream, a + // decode/probe/scatter worker, publication) aborts rather than unwinds. + auto resp = + mpi::guard_distributed(comm, [&] { return resolve_published(mpi::views_of(inc_q), is_leader_pass); }); std::vector resp_recv = response_recv_counts(); - mpi::WindowVec> inc_r; + mpi::WindowVec> inc_r; // Answers retrace the queries over an XOR involution, so the same plan holds. - mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv, plan).wait_into(inc_r); - process_responses(inc_r, src_idx_r, queries_r, my_rank, sink); + auto response_round = mpi::begin_alltoallv(resp, comm, /*skip_self=*/false, &resp_recv, plan); + mpi::guard_distributed(comm, [&] { response_round.wait_into(inc_r); }); + consume_published(mpi::views_of(inc_r)); } // Followers a leader already matched must not be re-resolved over the wire, so compact them out. @@ -488,7 +652,8 @@ struct LayerBuildEngine { std::span(deferred_pos_flat_).subspan(m.pos_at, m.k)); sink.emit_deferred(k, base + k, m.src, m.phase, m.v_src); } - local_op.store->bulk_insert_hashed(n_miss, base, [&](size_t j) { return deferred_self_misses[j].hash; }); + local_op.store + ->bulk_insert_hashed(n_miss, base, [&](size_t j) { return deferred_self_misses[j].hash; }, options); local_op.reindex_after_growth(base, n_miss); } @@ -510,35 +675,57 @@ struct LayerBuildEngine { return counts; } - // Batched self-resolve over the index's group-prefetch find_batch; hits/misses are emitted to the sink - // in query order. `lv` is the per-query v_src array parallel to `ls` (read only when Sink::wants_values). - static constexpr size_t kResolveBatch = 64; + bool leaders_pass_ = true; // the pass the retained streams belong to, as prepare_exchange() adopted them + + // Self-resolve scratch for one window of at most self_probe_window queries, reused across windows and both + // passes: gathered offsets (absolute into the stage), counts, results, hashes and the emission payload. + std::vector self_pos_off_; + std::vector self_k_of_; + std::vector self_found_; + std::vector self_hash_; + std::vector self_phase_; + std::vector self_src_; + std::vector self_val_; // ContractSink only + + // Self-resolve in bounded windows. For each window, on the caller: stable follower filtering (leaders skip + // it) and a gather into the scratch; then probe_frozen_positions (KernelRange::self_probe) on workers over + // at most 16 blocks; then, on the caller, hit/miss emission in original order. Nothing is inserted between + // windows or passes: misses keep their positions and hashes for insert_deferred_self_misses. `lv` is the + // per-query v_src array parallel to `ls` (read only when Sink::wants_values). auto resolve_range_(std::vector &ls, [[maybe_unused]] std::vector *lv, bool is_leader_pass) -> void { const size_t op_size = local_op.store->size(); - // Gathered per batch because a matched follower is skipped; offsets stay absolute into pos_flat. - std::array pos_off; - std::array k_of; - std::array hashes; - std::array phases; - std::array srcs; - std::array vals; - std::array found; const size_t hi = self_stage_.size(); + const size_t cap = std::min(hi, self_probe_window); + if constexpr (requires { sink.reserve_self_hits(hi); }) { + sink.reserve_self_hits(hi); + } + if (self_pos_off_.size() < cap) { + self_pos_off_.resize(cap); + self_k_of_.resize(cap); + self_found_.resize(cap); + self_hash_.resize(cap); + self_phase_.resize(cap); + self_src_.resize(cap); + if constexpr (Sink::wants_values) { + self_val_.resize(cap); + } + } + const std::span stage_pos(self_stage_.pos_flat.data(), self_stage_.positions()); size_t q = 0; while (q < hi) { size_t m = 0; - for (; q < hi && m < kResolveBatch; ++q) { + for (; q < hi && m < self_probe_window; ++q) { const size_t src = ls[q]; if (!is_leader_pass && matched.is_marked(src)) { continue; // follower already matched by a leader → not an independent rotation } - pos_off[m] = self_stage_.pos_off[q]; - k_of[m] = self_stage_.k_of[q]; - phases[m] = self_stage_.phase_of[q]; - srcs[m] = src; + self_pos_off_[m] = self_stage_.pos_off[q]; + self_k_of_[m] = self_stage_.k_of[q]; + self_phase_[m] = self_stage_.phase_of[q]; + self_src_[m] = src; if constexpr (Sink::wants_values) { - vals[m] = (*lv)[q]; + self_val_[m] = (*lv)[q]; } ++m; } @@ -546,39 +733,45 @@ struct LayerBuildEngine { break; } // The hashes come back because a miss needs one at insert, folded from these same positions. - local_op.store->find_batch_positions(std::span(self_stage_.pos_flat), - std::span(pos_off).first(m), - std::span(k_of).first(m), - std::span(found).first(m), - std::span(hashes).first(m)); + probe_frozen_positions(*local_op.store, + stage_pos, + std::span(self_pos_off_).first(m), + std::span(self_k_of_).first(m), + std::span(self_found_).first(m), + std::span(self_hash_).first(m), + options, + observer, + KernelRange::self_probe); // Overlap the batch's scattered hit-coefficient misses; smaller operators are cache-resident. if constexpr (requires { sink.op_coeffs[size_t{}]; }) { constexpr auto prefetch_hit_min_rows = size_t{1} << 21; for (auto j = size_t{0}; op_size >= prefetch_hit_min_rows && j < m; ++j) { - if (found[j] < op_size) { - __builtin_prefetch(&sink.op_coeffs[found[j]]); + if (self_found_[j] < op_size) { + __builtin_prefetch(&sink.op_coeffs[self_found_[j]]); } } } for (size_t j = 0; j < m; ++j) { double v_src = 0.0; if constexpr (Sink::wants_values) { - v_src = vals[j]; + v_src = self_val_[j]; } + const size_t found = self_found_[j]; // kNotFound == kMissingIndex == size_t max, so one bound check covers both. - if (found[j] < op_size) { + if (found < op_size) { // Freshly inserted partners (found >= combined_size) skip the mark: combined_size bounds it. - if (is_leader_pass && found[j] < combined_size) { - matched.mark(found[j]); + if (is_leader_pass && found < combined_size) { + matched.mark(found); } - sink.self_hit(srcs[j], found[j], phases[j], v_src); + sink.self_hit(self_src_[j], found, self_phase_[j], v_src); } else { // The stage dies with this pass and the misses are flushed after both, so copy now. const size_t at = deferred_pos_flat_.size(); - const auto *const first = self_stage_.pos_flat.data() + pos_off[j]; - deferred_pos_flat_.insert(deferred_pos_flat_.end(), first, first + k_of[j]); - deferred_self_misses.push_back({at, k_of[j], hashes[j], srcs[j], phases[j], v_src}); + const auto *const first = self_stage_.pos_flat.data() + self_pos_off_[j]; + deferred_pos_flat_.insert(deferred_pos_flat_.end(), first, first + self_k_of_[j]); + deferred_self_misses.push_back( + {at, self_k_of_[j], self_hash_[j], self_src_[j], self_phase_[j], v_src}); } } } @@ -590,37 +783,59 @@ static inline auto empty_coeffs() -> const VecD & { return coeffs; } -// Primary-path layer builder: one fused scan, then two resolve passes into the chosen sink. See LayerBuilder.h. +/*! + * \brief One owner's traversal of one gate: its outgoing streams, cosine set and the gate-wide decisions. + * + * Produced by scan_gate() and consumed by a LayerBuildEngine: the streams move into its two passes, the cosine set + * into finish(). + */ template -auto build_layer(MPOperator &local_op, - const Monomial &gen, - const CutoffFn &cutoff_fn, - const std::optional &atol, - std::optional> local_coeffs, - const std::optional &upper_atol, - const std::optional ¶m, - std::optional only_rotate_len_k, - MatchedEpochSet &matched_scratch, - mpi::Comm comm, - CosMask *out_cos = nullptr, - FusedContract *fused_contract = nullptr, - bool schrodinger = false, - VecD *fused_scale_coeffs = nullptr, - bool *fused_scale_out = nullptr, - Basis basis = Basis::Majorana) -> std::shared_ptr { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); - const size_t my_rank = static_cast(mpi::rank(comm)); - const size_t R = static_cast(mpi::size(comm)); - // R is the flat world (ranks x partitions). - const routing::Router router = router_for(comm); - assert(router.flat_world() == R); +struct GateScan { + FusedScanResult streams; //!< Leader/follower queries, sources, values and self stages. + CosMask cos_all; //!< Anticommuting rows, ascending and disjoint; empty for the fused sweep. + mpi::PeerPlan plan; //!< The generator's peer plan (the low-level engine's transport only). + mpi::SlotWindow window; //!< The owner's destination window: every per-slot stream is sized to it. + size_t combined_size = 0; //!< Pre-layer store size: the source boundary and the matched-set bound. + bool identity = false; //!< Identity generator: nothing anticommutes and nothing is exchanged. + bool fused_scale = false; //!< The fused cosine sweep scaled the coefficients during the traversal. + double inv_cos = 1.0; //!< Pre-cosine recovery factor for hit targets under the fused sweep. +}; + +/*! + * \brief Owner traversal and source capture for one gate: the partner queries of every anticommuting row. + * + * The first phase of build_layer(), extracted so that a sharded caller can run it per owner. Reads `op` and its lazy + * caches, which it may build, and under the fused sweep writes `*fused_scale_coeffs` (captured values are taken + * before each overwrite); it grows no store and performs no MPI. With a serial `options` it runs entirely on the + * calling thread. An opaque cutoff predicate is evaluated here, on whichever thread calls it. + * + * \param router Flat-owner routing; `my_rank` is this owner's flat slot and `op` holds only rows it owns. + * \param use_fused Whether the caller contracts immediately (ContractSink): values are captured and the fused + * cosine sweep may run. + * \return The streams, cosine set and decisions; `fused_scale` is the single authority for the apply. + */ +template +auto scan_gate(MPOperator &local_op, + const Monomial &gen, + const CutoffFn &cutoff_fn, + const std::optional &atol, + std::optional> local_coeffs, + const std::optional &upper_atol, + const std::optional ¶m, + std::optional only_rotate_len_k, + const routing::Router &router, + size_t my_rank, + bool use_fused, + VecD *fused_scale_coeffs, + Basis basis, + parallel::Options options = {}, + const Observer &observer = {}) -> GateScan { + GateScan scan; // Under linear routing every query for this generator goes to rank my_rank ^ rank_shift(gen). const size_t gen_shift = router.rank_shift(gen); - const auto plan = mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(gen_shift)}; + scan.plan = mpi::PeerPlan{.sparse = router.is_linear(), .shift = static_cast(gen_shift)}; // S of the P slots under linear routing, all P otherwise; every per-slot array is sized to it. - const mpi::SlotWindow scan_window = plan.window(my_rank, router.ranks(), router.partitions()); - // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). - const bool use_fused = (fused_contract != nullptr); + scan.window = scan.plan.window(my_rank, router.ranks(), router.shards_per_rank()); const auto cut_st = build_majorana_evolution_cutoff_state(atol, local_coeffs, upper_atol, param); const auto &coeffs = local_coeffs.value_or(empty_coeffs()).get(); const CutoffEvaluator cut_eval{cutoff_fn}; @@ -630,86 +845,165 @@ auto build_layer(MPOperator &local_op, // recovery is impossible; two-pass fallback). cos is even, so the sweep's cos(2·build_angle) matches // the apply's cos(2·apply_angle) bit-for-bit. const double cos_build = (use_fused && param.has_value()) ? std::cos(2.0 * param.value()) : 1.0; - const bool fused_scale = use_fused && !only_rotate_len_k.has_value() && fused_scale_coeffs != nullptr - && param.has_value() && cos_build != 0.0; - // build_layer is the single authority for this decision; the fused caller must drive its apply from it. - if (fused_scale_out != nullptr) { - *fused_scale_out = fused_scale; - } + scan.fused_scale = use_fused && !only_rotate_len_k.has_value() && fused_scale_coeffs != nullptr && param.has_value() + && cos_build != 0.0; + scan.inv_cos = scan.fused_scale ? 1.0 / cos_build : 1.0; assert(fused_scale_coeffs == nullptr || (local_coeffs && &local_coeffs->get() == fused_scale_coeffs)); // An identity generator anticommutes with nothing, so its exchange would be empty collectives. The - // generator list is replicated, so every rank skips it together. - const bool identity_gen = !gen.any(); - - FusedScanResult fused; - CosMask cos_all; - if (!identity_gen) { - double *const sweep_ptr = fused_scale ? fused_scale_coeffs->data() : nullptr; + // generator list is replicated, so every owner skips it together. + scan.identity = !gen.any(); + if (!scan.identity) { + double *const sweep_ptr = scan.fused_scale ? fused_scale_coeffs->data() : nullptr; + auto &fused = scan.streams; fused = with_algebra(basis, [&]() { - return fused_find_and_collect(local_op, - gen, - cut_eval, - cut_st, - coeffs, - only_rotate_len_k, - scan_window, - my_rank, - router, - gen_shift, - /*capture_values=*/use_fused, - sweep_ptr, - cos_build); + return fused_find_and_collect(local_op, + gen, + cut_eval, + cut_st, + coeffs, + only_rotate_len_k, + scan.window, + my_rank, + router, + gen_shift, + /*capture_values=*/use_fused, + sweep_ptr, + cos_build, + options, + observer); }); + auto &cos_all = scan.cos_all; if (fused.cos_blocks.size() == 1) { // The serial scan produces a single cosine block set — take it wholesale. cos_all = std::move(fused.cos_blocks[0]); } else { - // Cosine block sets are disjoint and ascending; concatenate in order. + // A threaded scan produces one set per word range, in range order: the ranges own disjoint + // ascending words, so concatenating in order keeps the blocks ascending and disjoint. + size_t blocks = 0; for (const auto &block : fused.cos_blocks) { + blocks += block.blocks.size(); + } + cos_all.blocks.reserve(blocks); + for (auto &block : fused.cos_blocks) { + assert(block.blocks.empty() || cos_all.blocks.empty() + || cos_all.blocks.back().first < block.blocks.front().first); cos_all.total_count += block.total_count; cos_all.blocks.insert(cos_all.blocks.end(), block.blocks.begin(), block.blocks.end()); + CosMask{}.blocks.swap(block.blocks); // release each range's copy as soon as it is appended } } fused.cos_blocks = std::vector{}; } + scan.combined_size = local_op.store->size(); + return scan; +} + +/*! + * \brief Stamp a finished layer's recompute metadata: its generator words and the post-insertion store size. + * + * scaled_count is the store size after this layer's inserts, not the pre-layer source boundary: the fold truncated + * to it reproduces the "all anticommuting" cos bit-for-bit with no stored bitmap. Fused mode has no LayerCore. + */ +template +auto stamp_layer_metadata(LayerCore &storage, const Monomial &gen, const MPOperator &local_op) + -> void { + storage.generator_words.assign(gen.data(), gen.data() + mpi_detail::kWords); + storage.scaled_count = static_cast(local_op.store->size()); +} + +// Primary-path layer builder: one fused scan, then two resolve passes into the chosen sink. See LayerBuilder.h. +// The scan (fused_find_and_collect), the incoming decode and frozen probes, the self probe and a certified +// sink's scatter may run on workers; validation, missing-ID assignment, insertion, publication, inverted-index +// maintenance and graph packing stay on the caller. `observer` is the test seam of those phases (NoRangeObserver). +template +auto build_layer(MPOperator &local_op, + const Monomial &gen, + const CutoffFn &cutoff_fn, + const std::optional &atol, + std::optional> local_coeffs, + const std::optional &upper_atol, + const std::optional ¶m, + std::optional only_rotate_len_k, + MatchedEpochSet &matched_scratch, + mpi::Comm comm, + CosMask *out_cos = nullptr, + FusedContract *fused_contract = nullptr, + bool schrodinger = false, + VecD *fused_scale_coeffs = nullptr, + bool *fused_scale_out = nullptr, + Basis basis = Basis::Majorana, + parallel::Options options = {}, + const Observer &observer = {}) -> std::shared_ptr { + validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); + const size_t my_rank = static_cast(mpi::rank(comm)); + const size_t R = static_cast(mpi::size(comm)); + // R is the flat world: one owner per rank. + const routing::Router router = router_for(comm); + assert(router.flat_world() == R); + // Fused contraction runs at all rank counts (R>1 via the cross-rank half-rotation exchange). + const bool use_fused = (fused_contract != nullptr); + auto scan = scan_gate(local_op, + gen, + cutoff_fn, + atol, + local_coeffs, + upper_atol, + param, + only_rotate_len_k, + router, + my_rank, + use_fused, + fused_scale_coeffs, + basis, + options, + observer); + // scan_gate is the single authority for this decision; the fused caller must drive its apply from it. + if (fused_scale_out != nullptr) { + *fused_scale_out = scan.fused_scale; + } auto run = [&](Sink sink) -> std::shared_ptr { - LayerBuildEngine eng(local_op, - comm, - R, - my_rank, - matched_scratch, - /*combined_size=*/local_op.store->size(), - std::move(sink), - plan, - scan_window); - if (!identity_gen) { - eng.run_exchange(/*is_leader_pass=*/true, + LayerBuildEngine eng(local_op, + R, + my_rank, + matched_scratch, + scan.combined_size, + std::move(sink), + scan.window, + options, + observer); + if (!scan.identity) { + auto &fused = scan.streams; + eng.run_exchange(comm, + scan.plan, + /*is_leader_pass=*/true, std::move(fused.leader_queries), std::move(fused.leader_src), std::move(fused.leader_val), std::move(fused.leader_self)); - eng.run_exchange(/*is_leader_pass=*/false, + eng.run_exchange(comm, + scan.plan, + /*is_leader_pass=*/false, std::move(fused.follower_queries), std::move(fused.follower_src), std::move(fused.follower_val), std::move(fused.follower_self)); } - return eng.finish(std::move(cos_all), out_cos); + return eng.finish(std::move(scan.cos_all), out_cos); }; std::shared_ptr storage; if (use_fused) { - const double inv_cos = fused_scale ? 1.0 / cos_build : 1.0; // pre-cos recovery factor for hit v_tgt + const auto &coeffs = local_coeffs.value_or(empty_coeffs()).get(); storage = run(ContractSink{.R = R, .my_rank = my_rank, .fc = *fused_contract, .op_coeffs = coeffs, - .fused_scale = fused_scale, - .inv_cos = inv_cos, + .fused_scale = scan.fused_scale, + .inv_cos = scan.inv_cos, .schrodinger = schrodinger, .basis = basis}); } @@ -717,12 +1011,9 @@ auto build_layer(MPOperator &local_op, storage = run(GraphSink{R, my_rank}); } - // Recompute metadata rides with the layer so it survives every graph transform. scaled_count is the - // post-insert operator size: the fold truncated to it reproduces the "all anticommuting" cos - // bit-for-bit with no stored bitmap. Fused mode has no LayerCore to stamp. + // Recompute metadata rides with the layer so it survives every graph transform. if (storage != nullptr) { - storage->generator_words.assign(gen.data(), gen.data() + mpi_detail::kWords); - storage->scaled_count = static_cast(local_op.store->size()); + stamp_layer_metadata(*storage, gen, local_op); } return storage; diff --git a/cpp/monoprop/detail/evolution/layer_build/FusedApply.h b/cpp/monoprop/detail/evolution/layer_build/FusedApply.h index fdd002de5..e22b4997f 100644 --- a/cpp/monoprop/detail/evolution/layer_build/FusedApply.h +++ b/cpp/monoprop/detail/evolution/layer_build/FusedApply.h @@ -14,51 +14,63 @@ #pragma once +#include +#include #include +#include +#include #include "monoprop/TypeAliases.h" #include "monoprop/detail/evolution/CosineRecompute.h" #include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/Workshare.h" namespace monoprop::detail { -// The drain paired with build_layer's fused emission: complete each rotation by adding its sine term -// directly to op_coeffs (the ContractImmediately forward path at all rank counts). The gate's cosine -// scale reaches the coefficients two ways: -// • fused_scale (no length cap, default): the scan already scaled every anticommuting coeff, so no cos pass runs -// here; slots born after that sweep (fresh inserts) fold cos in via their apply arm below. -// • two-pass (length cap / cos==0 fallback): scale_cos_mask runs here, then every arm is a plain add. -// At R>1 each rank applies only the add to the slot it owns (half rotations in fc.cross_half). -inline auto apply_fused_contract(FusedContract &fc, - VecD &op_coeffs, - const CosMask &cos, - double param, - bool schrodinger, - bool fused_scale) -> void { - // (1) insert records: v_tgt is the freshly-inserted term's pre-cos coeff, readable only now op_coeffs - // is extended. Needed only in Schrödinger — a Heisenberg fresh insert has coeff 0, so skip the gather. - if (schrodinger) { - for (size_t k = 0; k < fc.inserts.size(); ++k) { - fc.inserts[k].v_tgt = op_coeffs[fc.inserts[k].tgt]; +//! Records per apply_fused_contract logical range, for both the insert gather and the rotation pass. +inline constexpr size_t fused_record_range = 1024; + +/*! + * \brief Whether every coefficient slot the contract writes has exactly one writing record. + * + * The slots are the sources and targets of `hits` and `inserts` and the local slots of `cross_half`. + * apply_fused_contract's unsynchronized parallel pass relies on this; debug builds assert it on every + * call. Sorts a transient copy of the slot IDs: O(records log records) time and O(records) memory. + */ +inline auto fused_add_owners_unique(const FusedContract &fc) -> bool { + std::vector slots; + slots.reserve(2 * (fc.hits.size() + fc.inserts.size()) + fc.cross_half.size()); + for (const auto *records : {&fc.hits, &fc.inserts}) { + for (const RotationRec &r : *records) { + slots.push_back(r.src); + slots.push_back(r.tgt); } } - - // (2) Two-pass mode only: cos scale over all anticommuting endpoints (inserts included). - const double cos_val = std::cos(2 * param); - const double sin_val = std::sin(2 * param); - double *const c = op_coeffs.data(); - if (!fused_scale) { - scale_cos_mask(c, cos, cos_val); + for (const HalfRotationRec &h : fc.cross_half) { + slots.push_back(h.local_idx); } + std::ranges::sort(slots); + return std::ranges::adjacent_find(slots) == slots.end(); +} - // (3) One pass over hits ++ inserts ++ cross_half. Each op slot is touched by exactly one add (pivot - // split + ⊕G-injective targets + drop_matched_cross_rank_followers), which is what makes the plain += - // safe. In fused_scale mode a slot born after the sweep (insert targets, resolver miss halves) folds - // the gate's cos in here (c = cos·c + sin). +/*! + * \brief Apply records `[lo, hi)` of hits ++ inserts ++ cross_half, each its complete rotation formula. + * + * Out of line on purpose: the serial and every threaded range call this one compiled body, so however the + * compiler contracts `cos·c + sin·φ·v` into a fused multiply-add, the choice is the same at every budget + * and serial and threaded results stay bitwise equal. The cost is one call per fused_record_range records. + */ +[[gnu::noinline]] inline auto apply_fused_record_range(const FusedContract &fc, + double *c, + size_t lo, + size_t hi, + double cos_val, + double sin_val, + bool fused_scale) -> void { const size_t n_hit = fc.hits.size(); const size_t n_full = n_hit + fc.inserts.size(); - const size_t n_cross = fc.cross_half.size(); - for (size_t k = 0; k < n_full + n_cross; ++k) { + for (size_t k = lo; k < hi; ++k) { if (k < n_full) { const bool is_insert = k >= n_hit; const RotationRec &r = is_insert ? fc.inserts[k - n_hit] : fc.hits[k]; @@ -82,4 +94,78 @@ inline auto apply_fused_contract(FusedContract &fc, } } +/*! + * \brief The drain paired with build_layer's fused emission: complete each rotation by adding its sine term + * directly to op_coeffs (the ContractImmediately forward path at all rank counts). + * + * The gate's cosine scale reaches the coefficients two ways: + * - fused_scale (no length cap, default): the scan already scaled every anticommuting coeff, so no cos pass + * runs here; slots born after that sweep (fresh inserts) fold cos in via their apply arm below. + * - two-pass (length cap / cos==0 fallback): scale_cos_mask runs here, then every arm is a plain add. + * At R>1 each rank applies only the add to the slot it owns (half rotations in fc.cross_half). + * + * The caller has already sized and extended op_coeffs. The three phases below run in order, each joined + * before the next, and each is split into fixed logical ranges that run on up to `options.threads` workers. + * + * \param observer Test seam; see NoRangeObserver. + */ +template +auto apply_fused_contract(FusedContract &fc, + VecD &op_coeffs, + const CosMask &cos, + double param, + bool schrodinger, + bool fused_scale, + parallel::Options options = {}, + const Observer &observer = {}) -> void { + double *const c = op_coeffs.data(); + + // (1) insert records: v_tgt is the freshly-inserted term's pre-cos coeff, readable only now op_coeffs + // is extended. Needed only in Schrödinger — a Heisenberg fresh insert has coeff 0, so skip the gather. + if (schrodinger) { + RotationRec *const inserts = fc.inserts.data(); + const size_t n_inserts = fc.inserts.size(); + const size_t ranges = logical_ranges(n_inserts, fused_record_range); + observer.prepare(KernelRange::fused_gather, ranges); + // Each range writes only its own records' snapshot field and reads coefficients nothing writes yet. + parallel::for_blocks(ranges, options, [&](size_t range) { + observer.visit(KernelRange::fused_gather, range); + const size_t hi = std::min(n_inserts, (range + 1) * fused_record_range); + for (size_t k = range * fused_record_range; k < hi; ++k) { + inserts[k].v_tgt = c[inserts[k].tgt]; + } + }); + } + + // (2) Two-pass mode only: cos scale over all anticommuting endpoints (inserts included). + const double cos_val = std::cos(2 * param); + const double sin_val = std::sin(2 * param); + if (!fused_scale) { + scale_cos_mask(c, cos, cos_val, options, observer); + } + + // (3) One pass over hits ++ inserts ++ cross_half. Each op slot is touched by exactly one add (pivot + // split + ⊕G-injective targets + drop_matched_cross_rank_followers), which is what makes the plain += + // safe. In fused_scale mode a slot born after the sweep (insert targets, resolver miss halves) folds + // the gate's cos in here (c = cos·c + sin). + assert(fused_add_owners_unique(fc)); + const size_t n_records = fc.hits.size() + fc.inserts.size() + fc.cross_half.size(); + const size_t ranges = logical_ranges(n_records, fused_record_range); + observer.prepare(KernelRange::fused_apply, ranges); + // Distinct records write distinct slots (the one-add-owner property above), and every value a record + // reads is its own snapshot or its own slot, so the ranges need no synchronization. Never paper over a + // violation with atomics: it would mean the scan emitted a slot twice. + parallel::for_blocks(ranges, options, [&](size_t range) { + observer.visit(KernelRange::fused_apply, range); + const size_t lo = range * fused_record_range; + apply_fused_record_range(fc, + c, + lo, + std::min(n_records, lo + fused_record_range), + cos_val, + sin_val, + fused_scale); + }); +} + } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/evolution/layer_build/QueryWire.h b/cpp/monoprop/detail/evolution/layer_build/QueryWire.h index 7f296d8a3..8c93bc42c 100644 --- a/cpp/monoprop/detail/evolution/layer_build/QueryWire.h +++ b/cpp/monoprop/detail/evolution/layer_build/QueryWire.h @@ -19,8 +19,11 @@ #include #include #include +#include #include #include +#include +#include #include #include @@ -32,6 +35,14 @@ namespace monoprop::detail { /*! @brief `Fused` records carry a trailing value word after the positions; `Plain` ones do not. */ enum class QueryForm { Plain, Fused }; +/*! @brief A received query stream that no QueryWire::push sequence could have produced: a truncated or + * noncanonical record, an out-of-range header field, or positions outside the term's width. + */ +class MalformedQueryStream : public std::invalid_argument { +public: + using std::invalid_argument::invalid_argument; +}; + /*! @brief One term's wire record for a cross-rank query: its ascending set-bit positions, * gap-coded, plus a phase. * @@ -89,14 +100,18 @@ struct QueryWire { int phase; //!< the record's ternary phase }; - /*! @brief Bit-packs variable-width fields into `buf`, one 64-bit word at a time. */ + /*! @brief Bit-packs variable-width fields into `buf`, one 64-bit word at a time. + * + * put() and flush() append to `buf`, so they may throw std::bad_alloc; they are deliberately not noexcept, + * which would turn an allocation failure while encoding a query into std::terminate. + */ struct Writer { VecZ &buf; uint64_t cur = 0; size_t nbits = 0; //!< bits held in cur, always less than 64 size_t words = 0; - [[gnu::always_inline]] auto put(uint64_t v, size_t width) noexcept -> void { + [[gnu::always_inline]] auto put(uint64_t v, size_t width) -> void { if (width == 0) { return; } @@ -114,7 +129,7 @@ struct QueryWire { nbits = nbits + width - 64U; } - auto flush() noexcept -> void { + auto flush() -> void { if (nbits != 0) { buf.push_back(static_cast(cur)); ++words; @@ -186,6 +201,51 @@ struct QueryWire { return words_of_header(header_at(buf, off)); } + /*! @brief A validated record's position count and its words, the fused value word included. */ + struct RecordExtent { + size_t k; //!< positions the record decodes to + size_t words; //!< words from the record start to the next record + }; + + /*! @brief Validates the record at `off` in `buf` without reading past the buffer, and returns its extent. + * + * Checks, in order: `off` names a word of `buf`; the phase field is ternary; k <= kMaxPositions; gw <= + * kPosBits; the header is canonical (`h.bits == header_bits_for(h.k)`, so an escaped count is at least + * kKEscape); and the whole record, value word included for `Fused`, fits the rest of the buffer. A + * noncanonical escape would otherwise make words_of_header() undercount what read_positions() consumes. + * Positions themselves are checked by read_positions(), which decodes them. + * + * \throws MalformedQueryStream naming the failed check. + */ + [[nodiscard]] static auto checked_extent(WireView buf, QueryForm form, size_t off) -> RecordExtent { + if (off >= buf.size()) { + throw MalformedQueryStream( + std::format("QueryWire: no record at word {} of a {}-word stream", off, buf.size())); + } + const Header h = header_at(buf, off); + const auto malformed = [&](std::string_view what) { + return MalformedQueryStream(std::format("QueryWire: malformed record at word {}: {}", off, what)); + }; + if (h.phase > 1) { + throw malformed("the phase field is not ternary"); + } + if (h.k > kMaxPositions) { + throw malformed(std::format("{} positions exceed the width's {}", h.k, kMaxPositions)); + } + if (h.gw > kPosBits) { + throw malformed(std::format("gap width {} exceeds {}", h.gw, kPosBits)); + } + if (h.bits != header_bits_for(h.k)) { + throw malformed(std::format("a noncanonical escaped count {}", h.k)); + } + // Every term fits kMaxPositions <= 65535 positions of at most 16 bits, so gap_bits cannot wrap. + const size_t words = words_of_header(h) + (form == QueryForm::Fused ? 1U : 0U); + if (words > buf.size() - off) { + throw malformed(std::format("{} words needed, {} remain", words, buf.size() - off)); + } + return {h.k, words}; + } + [[nodiscard]] static auto k_at(WireView buf, size_t off) noexcept -> size_t { return header_at(buf, off).k; } [[nodiscard]] static auto phase_at(WireView buf, size_t off) noexcept -> int { return header_at(buf, off).phase; } @@ -238,18 +298,32 @@ struct QueryWire { /*! @brief Decodes the record at `off` into `out`, whose element type is deduced so the resolve * path can decode straight into the store's narrower position width. * - * `Decoded::next` names the word just past the positions, excluding any value word. + * The record must lie within `buf` (checked_extent() establishes that for received streams) and `out` + * must hold its k positions. Each cumulative position is checked below kBits as a size_t, before it is + * narrowed to the output type: that also bounds the next gap's addition, so the positions are strictly + * ascending and cannot wrap. `Decoded::next` names the word just past the positions, excluding any value + * word. + * + * \throws MalformedQueryStream if a position is outside [0, kBits); `out` is then partly written. */ template static auto read_positions(WireView buf, size_t off, Out &&out) -> Decoded { using OutT = std::ranges::range_value_t; const Header h = header_at(buf, off); Reader r{buf, off, h.bits}; + const auto check = [&](size_t pos) { + if (pos >= kBits) { + throw MalformedQueryStream( + std::format("QueryWire: record at word {} decodes position {} of a {}-bit term", off, pos, kBits)); + } + }; if (h.k != 0) { auto prev = static_cast(r.get(kPosBits)); + check(prev); out[0] = static_cast(prev); for (size_t j = 1; j < h.k; ++j) { prev += static_cast(r.get(h.gw)) + 1U; + check(prev); out[j] = static_cast(prev); } } diff --git a/cpp/monoprop/detail/evolution/layer_build/Resolve.h b/cpp/monoprop/detail/evolution/layer_build/Resolve.h index ac06dc6c1..856f27c14 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Resolve.h +++ b/cpp/monoprop/detail/evolution/layer_build/Resolve.h @@ -14,10 +14,15 @@ #pragma once +#include #include +#include #include +#include +#include #include #include +#include #include #include "monoprop/TypeAliases.h" @@ -28,9 +33,79 @@ #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/operator/MPOperator.h" #include "monoprop/detail/operator/RowAccess.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/Workshare.h" namespace monoprop::detail { +//! Queries per logical block of the frozen probe and the incoming decode: a function of the query count only. +inline constexpr size_t probe_block_queries = 256; +//! Queries per logical block of the incoming on_resolved scatter. +inline constexpr size_t resolve_scatter_block = 1024; +//! Most self queries probed per window; bounds the gathered scratch (at most 16 probe blocks per window). +inline constexpr size_t self_probe_window = 4096; + +/*! + * \brief Look up `out.size()` position-list queries in a frozen store, on up to `options.threads` workers. + * + * Query `q` is `pos_flat[pos_off[q], pos_off[q] + k_of[q])`; offsets are absolute into the one shared buffer. + * The queries split into logical blocks of probe_block_queries, each one OperatorIndex::find_batch_positions + * call over its slice of `pos_off`, `k_of`, `out` and `hash_out`, so the batched pipeline, the exact collision + * check and the cached hashes are the serial ones and the results are identical at every budget and team + * size. `out[q]` is the row ID or OperatorIndex::kNotFound; `hash_out`, when not empty, receives each query's + * fold hash. A single block runs on the caller. + * + * The store must stay frozen -- no row, overflow, table or index mutation -- until this returns; it returns + * only after every worker has joined. + * + * \param observer Test seam; see NoRangeObserver. \param kind The KernelRange it reports. + * \throws std::invalid_argument unless `pos_off`, `k_of` and `out` have equal lengths, `hash_out` is empty or of + * that length, and every query's span lies inside `pos_flat`; all checked before any block runs. + */ +template +auto probe_frozen_positions(const OperatorIndex &store, + std::span::PosT> pos_flat, + std::span pos_off, + std::span k_of, + std::span out, + std::span hash_out, + parallel::Options options = {}, + const Observer &observer = {}, + KernelRange kind = KernelRange::incoming_probe) -> void { + const size_t n = out.size(); + if (pos_off.size() != n || k_of.size() != n || (!hash_out.empty() && hash_out.size() != n)) { + throw std::invalid_argument(std::format("probe_frozen_positions: {} offsets, {} counts, {} outputs and {} " + "hashes; the spans must have equal lengths (hashes may be empty)", + pos_off.size(), + k_of.size(), + n, + hash_out.size())); + } + for (size_t q = 0; q < n; ++q) { + if (pos_off[q] > pos_flat.size() || k_of[q] > pos_flat.size() - pos_off[q]) { + throw std::invalid_argument(std::format("probe_frozen_positions: query {} spans [{}, +{}) outside the " + "{}-position buffer", + q, + pos_off[q], + k_of[q], + pos_flat.size())); + } + } + const size_t blocks = logical_ranges(n, probe_block_queries); + observer.prepare(kind, blocks); + // Each block writes only its own slice of `out` and `hash_out`; the store and pos_flat are read-only. + parallel::for_blocks(blocks, options, [&](size_t block) { + observer.visit(kind, block); + const size_t lo = block * probe_block_queries; + const size_t m = std::min(probe_block_queries, n - lo); + store.find_batch_positions(pos_flat, + pos_off.subspan(lo, m), + k_of.subspan(lo, m), + out.subspan(lo, m), + hash_out.empty() ? std::span{} : hash_out.subspan(lo, m)); + }); +} + // Picture-independent probe/insert machinery shared by resolve_incoming and its callers. Each miss takes // the next index base+j in (sender,record) order, so the assignment (and multi-rank bit-exactness) cannot // drift between resolvers. Queries are source⊕G over globally-distinct sources, ⊕G injective ⇒ queries @@ -48,11 +123,14 @@ struct IncomingProbe { // g → word offset of that query inside its sender's buffer; a query ordinal names no position. DefaultInitVector off_of; DefaultInitVector idx_of; // g → resolved index (hit: < base; miss: base+j) - std::vector miss_g; // j → the g that became miss j (Phase 4 reads the key of miss_g[j]) - size_t base = 0; // op size before the miss inserts (the miss-index base) + // j → the query ordinal g that became miss j (Phase 4 reads its key). Ordinals, not TermIndex values: + // their range is the query count, not the store's. + std::vector miss_g; + size_t base = 0; // op size before the miss inserts (the miss-index base) size_t nq_total = 0; - // The queries as they arrived, flat: query g owns pos_flat[pos_off[g] .. pos_off[g] + k_of[g]). + // The queries as they arrived, flat: query g owns pos_flat[pos_off[g] .. pos_off[g] + k_of[g]). Every + // per-query array holds exactly nq_total elements, pos_flat exactly the decoded positions. DefaultInitVector pos_flat; DefaultInitVector pos_off; DefaultInitVector k_of; @@ -83,90 +161,189 @@ struct IncomingProbe { } }; -// Read-only phases 1-2 of the exchange: `form` says whether the incoming records are fused -// (ContractSink) or plain (GraphSink). The caller runs phase 3, then insert_incoming_misses. -template -auto probe_incoming_queries(const mpi::WindowVec &incoming, // serialized, one VecZ per sender slot +namespace resolve_detail { + +// Refuses a length the vector type cannot hold, before any allocation. +template +auto check_fits(const Vec &v, size_t n, const char *what) -> void { + if (n > v.max_size()) { + throw std::length_error(std::format("probe_incoming_queries: {} {} exceed the vector limit", n, what)); + } +} + +} // namespace resolve_detail + +/*! + * \brief Read-only phases 1-2 of the exchange: validate, decode and probe the received queries, then assign + * each miss its predicted row ID. + * + * `incoming` holds one serialized stream per sender window slot: owning `VecZ` blocks as a transport unpacked them, or + * read-only views of buffers another owner published (QueryWire::WireView); the streams are only read and must stay + * unchanged until this returns. `form` says whether the records are fused (ContractSink) or plain (GraphSink). In + * order: + * 1. On the caller, every sender stream is walked in window order with QueryWire::checked_extent, so a + * malformed header, a noncanonical escape or a truncated record is rejected before any payload is read. + * The query and position prefixes, sender narrowing and vector limits are checked, every per-query array + * is allocated once at its exact size, and the offsets (absolute into `pos_flat`), counts and senders are + * filled in canonical sender-window/query order. + * 2. Workers decode disjoint blocks of records into their prepared position spans; read_positions rejects + * positions outside the term's width, and each record must end exactly at its prepared boundary. + * 3. After that join, probe_frozen_positions looks every query up in the frozen store, keeping its hash. + * 4. After that join, the caller checks that base + misses stays below the row-ID ceiling, then assigns miss + * j the ID base + j in ascending sender-window/query order. + * A failure in any phase leaves `op` untouched. The caller runs phase 3 (the scatter), then + * insert_incoming_misses. + * + * \param observer Test seam reporting KernelRange::decode and KernelRange::incoming_probe; see NoRangeObserver. + * \throws MalformedQueryStream for a malformed stream (from the caller before decoding, or from a decode worker + * for invalid positions, rethrown after the join and before probing). + * \throws TermIndexCeilingReached if the misses would pass the row-ID ceiling. + */ +template + requires std::convertible_to::WireView> +auto probe_incoming_queries(const mpi::WindowVec &incoming, // serialized, one stream per sender slot MPOperator &op, - QueryForm form) -> IncomingProbe { + QueryForm form, + parallel::Options options = {}, + const Observer &observer = {}) -> IncomingProbe { using QW = QueryWire; using PosT = typename IncomingProbe::PosT; IncomingProbe pr; pr.window = incoming.window(); + pr.base = op.store->size(); const size_t senders = pr.window.count; + if (senders > size_t{std::numeric_limits::max()} + 1U) { + throw std::length_error( + std::format("probe_incoming_queries: {} senders exceed the uint32 sender index", senders)); + } + // 1a. Validate every stream and count its queries and positions, before any decode. + resolve_detail::check_fits(pr.goff, senders + 1, "sender offsets"); pr.goff.assign(senders + 1, 0); + size_t positions = 0; for (size_t k = 0; k < senders; ++k) { - const size_t nq = QW::count_queries(incoming[mpi::WindowIndex{k}], form); + const typename QW::WireView buf = incoming[mpi::WindowIndex{k}]; + size_t nq = 0; + for (size_t off = 0; off < buf.size();) { + const auto ext = QW::checked_extent(buf, form, off); + if (ext.k > std::numeric_limits::max() - positions) { + throw std::length_error("probe_incoming_queries: the decoded position count overflows"); + } + positions += ext.k; + off += ext.words; // checked_extent guarantees off + words <= buf.size() + ++nq; + } + // nq <= buf.size() words, but the running sum over senders is still checked. + if (nq > std::numeric_limits::max() - pr.goff[k]) { + throw std::length_error("probe_incoming_queries: the query count overflows"); + } pr.goff[k + 1] = pr.goff[k] + nq; } pr.nq_total = pr.goff[senders]; - if (pr.nq_total == 0) { - return pr; - } + const size_t n = pr.nq_total; + + // 1b. Allocate every output once, at its exact size; workers never resize. + resolve_detail::check_fits(pr.sender_wi, n, "queries"); + resolve_detail::check_fits(pr.phase_of, n, "queries"); + resolve_detail::check_fits(pr.off_of, n, "queries"); + resolve_detail::check_fits(pr.idx_of, n, "queries"); + resolve_detail::check_fits(pr.pos_off, n, "queries"); + resolve_detail::check_fits(pr.k_of, n, "queries"); + resolve_detail::check_fits(pr.hash_of, n, "queries"); + resolve_detail::check_fits(pr.pos_flat, positions, "positions"); + pr.sender_wi.resize(n); + pr.phase_of.resize(n); + pr.off_of.resize(n); + pr.idx_of.resize(n); + pr.pos_off.resize(n); + pr.k_of.resize(n); + pr.hash_of.resize(n); + pr.pos_flat.resize(positions); - pr.sender_wi.resize(pr.nq_total); + // 1c. Canonical sender-window/query order: record offsets, counts and absolute position offsets. The headers + // were validated above, so reading them again cannot leave the buffer. + size_t pos_at = 0; for (size_t k = 0; k < senders; ++k) { - std::fill(pr.sender_wi.begin() + static_cast(pr.goff[k]), - pr.sender_wi.begin() + static_cast(pr.goff[k + 1]), - static_cast(k)); - } - - // Phase 1 (read-only): deserialize, then probe with the group-prefetch batch find. One walk per sender. - pr.phase_of.resize(pr.nq_total); - pr.off_of.resize(pr.nq_total); - pr.idx_of.resize(pr.nq_total); - pr.pos_off.resize(pr.nq_total); - pr.k_of.resize(pr.nq_total); - pr.hash_of.resize(pr.nq_total); - pr.pos_flat.clear(); - // A hint only, so this stays one allocation for the common case. - pr.pos_flat.reserve(pr.nq_total * QW::kReservePositionsPerQuery); - for (size_t si = 0; si < senders; ++si) { - const VecZ &buf = incoming[mpi::WindowIndex{si}]; + const typename QW::WireView buf = incoming[mpi::WindowIndex{k}]; size_t off = 0; - for (size_t g = pr.goff[si]; g < pr.goff[si + 1]; ++g) { - const size_t k = QW::k_at(buf, off); - const size_t at = pr.pos_flat.size(); - pr.pos_flat.resize(at + k); // default-init grow: read_query writes every element - pr.pos_off[g] = at; - pr.k_of[g] = static_cast(k); + for (size_t g = pr.goff[k]; g < pr.goff[k + 1]; ++g) { + const auto h = QW::header_at(buf, off); + pr.sender_wi[g] = static_cast(k); pr.off_of[g] = off; - const auto d = QW::read_query(buf, form, off, std::span(pr.pos_flat).subspan(at, k)); - pr.phase_of[g] = d.phase; - off = d.next; + pr.k_of[g] = static_cast(h.k); // k <= kMaxPositions <= 65535 + pr.pos_off[g] = pos_at; + pos_at += h.k; + off += QW::words_of_header(h) + (form == QueryForm::Fused ? 1U : 0U); } } - { - const size_t op_size = op.store->size(); - // The vectors are sized to capacity, not to nq_total, so every span is trimmed explicitly. - op.store->find_batch_positions(std::span(pr.pos_flat), - std::span(pr.pos_off).first(pr.nq_total), - std::span(pr.k_of).first(pr.nq_total), - std::span(pr.idx_of).first(pr.nq_total), - std::span(pr.hash_of).first(pr.nq_total)); - for (size_t g = 0; g < pr.nq_total; ++g) { - if (pr.idx_of[g] >= op_size) { // kNotFound is size_t max → also lands here - pr.idx_of[g] = kMissingIndex; + assert(pos_at == positions); + + // 2. Decode disjoint records into their prepared spans. + const size_t blocks = logical_ranges(n, probe_block_queries); + observer.prepare(KernelRange::decode, blocks); + parallel::for_blocks(blocks, options, [&](size_t block) { + observer.visit(KernelRange::decode, block); + const size_t hi = std::min(n, (block + 1) * probe_block_queries); + for (size_t g = block * probe_block_queries; g < hi; ++g) { + const size_t k = pr.sender_wi[g]; + const typename QW::WireView buf = incoming[mpi::WindowIndex{k}]; + const auto d = QW::read_query(buf, + form, + pr.off_of[g], + std::span(pr.pos_flat).subspan(pr.pos_off[g], pr.k_of[g])); + const size_t boundary = (g + 1 < pr.goff[k + 1]) ? pr.off_of[g + 1] : buf.size(); + if (d.next != boundary) { + throw MalformedQueryStream( + std::format("probe_incoming_queries: query {} ends at word {}, not at its prepared boundary {}", + g, + d.next, + boundary)); } + pr.phase_of[g] = d.phase; } - } + }); - // Phase 2 ((sender,query) prefix order): each miss takes the next index base+j. - pr.base = op.store->size(); - for (size_t g = 0; g < pr.nq_total; ++g) { - if (pr.idx_of[g] == kMissingIndex) { - pr.idx_of[g] = pr.base + pr.miss_g.size(); - pr.miss_g.push_back(static_cast(g)); + // 3. Frozen probe: nothing mutates op until insert_incoming_misses. + probe_frozen_positions(*op.store, + std::span(pr.pos_flat), + std::span(pr.pos_off), + std::span(pr.k_of), + std::span(pr.idx_of), + std::span(pr.hash_of), + options, + observer, + KernelRange::incoming_probe); + + // 4. Phase 2 ((sender,query) prefix order): each miss takes the next index base+j. The row-ID limit is + // checked before any predicted ID is written. + size_t n_miss = 0; + for (size_t g = 0; g < n; ++g) { + n_miss += static_cast(pr.idx_of[g] >= pr.base); // kNotFound is size_t max, so it lands here too + } + if (n_miss > OperatorIndex::kIndexCeiling - pr.base) { + throw TermIndexCeilingReached(std::format("probe_incoming_queries: {} new terms on a store of {} would pass " + "the 2^32 TermIndex ceiling", + n_miss, + pr.base)); + } + pr.miss_g.resize(n_miss); + size_t j = 0; + for (size_t g = 0; g < n; ++g) { + if (pr.idx_of[g] >= pr.base) { + pr.idx_of[g] = pr.base + j; + pr.miss_g[j++] = g; } } return pr; } // Phase 4 (bulk insert of the distinct absent terms) into op slots [base, base+n_miss). Call after the -// caller's Phase-3 scatter, which reads pre-insert op_coeffs for hits and needs base == op.size(). +// caller's Phase-3 scatter, which reads pre-insert op_coeffs for hits and needs base == op.size(). Publication +// is single-writer: grow, set_positions, bulk_insert_hashed with the probe's cached hashes (`options` is its +// publication seam, which the packed index ignores), then the inverted-index maintenance. template -auto insert_incoming_misses(MPOperator &op, const IncomingProbe &pr) -> void { +auto insert_incoming_misses(MPOperator &op, const IncomingProbe &pr, parallel::Options options = {}) + -> void { const size_t n_miss = pr.miss_g.size(); if (n_miss == 0) { return; @@ -174,14 +351,59 @@ auto insert_incoming_misses(MPOperator &op, const IncomingProbegrow_rows_geometric(n_miss); + assert(base == pr.base); for (size_t j = 0; j < n_miss; ++j) { const size_t g = pr.miss_g[j]; op.store->set_positions(base + j, pr.positions_at(g)); } - op.store->bulk_insert_hashed(n_miss, base, [&](size_t j) { return pr.hash_of[pr.miss_g[j]]; }); + op.store->bulk_insert_hashed(n_miss, base, [&](size_t j) { return pr.hash_of[pr.miss_g[j]]; }, options); op.reindex_after_growth(base, n_miss); } +/*! + * \brief Whether the leader hits of `pr` (rows below `combined_size`) are pairwise distinct, so their matched + * marks are writes to distinct elements. + * + * Holds in production because queries are source XOR G over globally distinct sources; the parallel scatter + * relies on it and debug builds assert it. O(hits log hits); for assertions and tests only. + */ +template +auto leader_marks_unique(const IncomingProbe &pr, size_t combined_size) -> bool { + std::vector hits; + for (size_t g = 0; g < pr.nq_total; ++g) { + if (pr.idx_of[g] < combined_size) { + hits.push_back(pr.idx_of[g]); + } + } + std::ranges::sort(hits); + return std::ranges::adjacent_find(hits) == hits.end(); +} + +/*! + * \brief Whether `sink` certifies that its on_resolved calls for distinct queries may run concurrently. + * + * A sink opts in with `static constexpr bool parallel_resolve = true`, which certifies only independent + * on_resolved writes after prepare(), never its other methods. A sink may further veto it at run time with + * `parallel_resolve_enabled()`. A missing or false capability keeps the serial scatter. + */ +template +auto sink_resolves_in_parallel(const Sink &sink) -> bool { + if constexpr (requires { Sink::parallel_resolve; }) { + if constexpr (static_cast(Sink::parallel_resolve)) { + if constexpr (requires { + { sink.parallel_resolve_enabled() } -> std::convertible_to; + }) { + return sink.parallel_resolve_enabled(); + } + else { + return true; + } + } + } + (void)sink; + return false; +} + // resolve_incoming / process_responses are the picture-independent cross-rank exchange skeletons; what // each resolved query records and answers with is supplied by a compile-time sink. The two concrete sinks // (GraphSink / ContractSink) live in Engine.h, each also carrying the self-resolve + finalize policy. @@ -189,46 +411,79 @@ auto insert_incoming_misses(MPOperator &op, const IncomingProbe -auto resolve_incoming(const mpi::WindowVec &incoming, // serialized, one VecZ per sender slot +// Threading: decode and probe as probe_incoming_queries; the scatter runs on workers only for a sink that +// certifies it (sink_resolves_in_parallel), otherwise on the caller; publication stays on the caller, after +// the scatter, which reads pre-insert coefficients. `observer` is the test seam (see NoRangeObserver). +// +// `incoming` is as for probe_incoming_queries: owning blocks or read-only views of published buffers, unchanged until +// this returns. The sink's on_resolved receives the sender's stream as a QueryWire::WireView. +template + requires std::convertible_to::WireView> +auto resolve_incoming(const mpi::WindowVec &incoming, // serialized, one stream per sender slot MPOperator &op, bool is_leader_pass, MatchedEpochSet &matched, size_t combined_size, // pre-layer op size: bounds the matched set - Sink &sink) -> mpi::WindowVec> { + Sink &sink, + parallel::Options options = {}, + const Observer &observer = {}) -> mpi::WindowVec> { using Resp = typename Sink::Response; - const IncomingProbe pr = probe_incoming_queries(incoming, op, sink.incoming_form()); + const IncomingProbe pr = + probe_incoming_queries(incoming, op, sink.incoming_form(), options, observer); // The pairing is an XOR involution, so the response window is the query window. mpi::WindowVec> responses(pr.window); for (const auto wi : pr.window.indices()) { responses[wi].assign(pr.goff[wi.value + 1] - pr.goff[wi.value], Sink::init_response()); } if (pr.nq_total == 0) { + observer.prepare(KernelRange::scatter, 0); return responses; } // Phase 3 (scatter): responses + sink records + matched-follower marks. Freshly inserted partners - // (ip ≥ combined_size) skip the mark. + // (ip ≥ combined_size) skip the mark. prepare() sizes every sink container on the caller first. sink.prepare(pr, op, responses); - for (size_t g = 0; g < pr.nq_total; ++g) { - const mpi::WindowIndex s = pr.sender_index(g); - const size_t q = g - pr.goff[s.value]; - const size_t ip = pr.idx_of[g]; - responses[s][q] = sink.on_resolved(g, s, q, ip, pr, incoming); - if (is_leader_pass && ip < combined_size) { - matched.mark(ip); + const size_t n = pr.nq_total; + const size_t blocks = logical_ranges(n, resolve_scatter_block); + const auto scatter = [&](size_t block) { + observer.visit(KernelRange::scatter, block); + const size_t hi = std::min(n, (block + 1) * resolve_scatter_block); + for (size_t g = block * resolve_scatter_block; g < hi; ++g) { + const mpi::WindowIndex s = pr.sender_index(g); + const size_t q = g - pr.goff[s.value]; + const size_t ip = pr.idx_of[g]; + responses[s][q] = sink.on_resolved(g, s, q, ip, pr, typename QueryWire::WireView(incoming[s])); + if (is_leader_pass && ip < combined_size) { + matched.mark(ip); + } + } + }; + observer.prepare(KernelRange::scatter, blocks); + if (sink_resolves_in_parallel(sink)) { + // Each query writes its own response and its own sink record; a leader hit marks its own row, distinct + // across queries (leader_marks_unique), and misses (ip >= base >= combined_size) mark nothing. The + // marks are distinct uint16 elements, so no synchronization; never paper over a duplicate with atomics. + assert(!is_leader_pass || leader_marks_unique(pr, combined_size)); + parallel::for_blocks(blocks, options, scatter); + } + else { + for (size_t block = 0; block < blocks; ++block) { + scatter(block); } } - insert_incoming_misses(op, pr); + insert_incoming_misses(op, pr, options); return responses; } // Querier rank (any cross-rank sink): fold each resolver response into a querier-side record. The self/ // local slot was already resolved inline, so it is skipped here. inc_r[k][q] answers query q sent to -// the window's k-th slot. -template -auto process_responses(const mpi::WindowVec> &inc_r, +// the window's k-th slot. `inc_r` holds owning response blocks or read-only views of blocks the resolver +// published; either way the sink receives each block, its sources and its plain query stream as views, and +// records them in window order, preserving each block's query order. +template + requires std::convertible_to> +auto process_responses(const mpi::WindowVec &inc_r, const mpi::WindowVec> &src_idx, const mpi::WindowVec &queries, // serialized query buffers (for phase recovery) size_t my_rank, @@ -241,7 +496,10 @@ auto process_responses(const mpi::WindowVec if (r == my_rank) { continue; } - sink.on_response_block(r, inc_r[wi], src_idx[wi], queries[wi]); + sink.on_response_block(r, + std::span(inc_r[wi]), + std::span(src_idx[wi]), + typename QueryWire::WireView(queries[wi])); } } diff --git a/cpp/monoprop/detail/evolution/layer_build/Scan.h b/cpp/monoprop/detail/evolution/layer_build/Scan.h index 17ed75619..5f551138b 100644 --- a/cpp/monoprop/detail/evolution/layer_build/Scan.h +++ b/cpp/monoprop/detail/evolution/layer_build/Scan.h @@ -21,8 +21,11 @@ #include #include #include +#include +#include #include #include +#include #include #include #include @@ -40,6 +43,8 @@ #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/operator/InvertedIndex.h" #include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/Workshare.h" namespace monoprop::detail { @@ -258,287 +263,573 @@ struct FusedScanResult { SelfQueryStage follower_self; }; -// Classify, cut off and emit in one pass over the anticommuting terms. Queries go to the owner of -// M'=M⊕G (routing::Router; self at R==1) in ascending source-index order, so resolve and index assignment are -// deterministic. `fused_scale_coeffs` (no length cap only; must alias coeffs.data()) scales every anticommuting -// coeff in place by `fused_scale_cos`=cos(2·build_angle), so no cosine set is built and a hit's stored -// value is post-cos (resolve recovers it via 1/cos). -// -// `gen_shift` must be router.rank_shift(gen) and `op` hold only terms `my_rank` owns, so the owner of -// M⊕G is rank(M) ^ gen_shift (asserted against dest()). `window` is the plan's window for `my_rank`. +/*! + * \brief Number of logical word ranges a threaded scan of `words` inverted-index words uses. + * + * `min(options.threads, ceil(words / kColumnBlockWords))`: a function of the word count and the budget + * only, never of the team the runtime actually provides, so range IDs and the merged order are fixed. + * + * \throws std::invalid_argument if `options.threads` is not positive. + */ +inline auto scan_ranges(size_t words, parallel::Options options) -> size_t { + if (options.threads < 1) { + throw std::invalid_argument( + std::format("fused_find_and_collect: the thread budget must be positive, got {}", options.threads)); + } + return std::min(static_cast(options.threads), logical_ranges(words, kColumnBlockWords)); +} + +/*! + * \brief Words `[first, second)` of logical scan range `range` out of `ranges` over `words` words. + * + * Ranges are contiguous runs of whole kColumnBlockWords fold blocks, ascending with the range ID and + * covering every word; each is nonempty when `ranges` does not exceed the block count (scan_ranges). + */ +inline auto scan_range_words(size_t range, size_t ranges, size_t words) -> std::pair { + assert(range < ranges && ranges <= logical_ranges(words, kColumnBlockWords)); + // Rows are uint32 TermIndex values, so words <= 2^26, blocks <= 2^16 and these products stay below 2^32. + const size_t blocks = logical_ranges(words, kColumnBlockWords); + const size_t lo = (range * blocks / ranges) * kColumnBlockWords; + const size_t hi = std::min(((range + 1) * blocks / ranges) * kColumnBlockWords, words); + return {lo, hi}; +} + +/*! + * \brief Scan inverted-index words `[wlo, whi)` of `op` for terms anticommuting with `gen`, and emit their + * partner queries and cosine words into a fresh result. + * + * This is the whole serial scan restricted to one word range; fused_find_and_collect runs it once over + * every word, or once per logical range on scan workers and merges the pieces in range order. It reads only + * frozen storage and never calls a lazy accessor: `index` must be `op.inverted_index()` and `row_parity` + * its `row_parity_words()` (null unless the fold needs the odd-|G| correction), both prepared on the caller. + * The tail mask comes from the index's global last word, so only the range holding that word masks it. All + * scratch is private to the invocation or thread_local on the executing thread. With `fused_scale_coeffs` + * it writes the coefficients of anticommuting sources in its own words only, each once, after reading the + * pre-scale value its cutoff decisions and captured value use. It resolves no target, grows no store and + * calls no MPI. + * + * The remaining parameters are fused_find_and_collect's; see there. + */ template -auto fused_find_and_collect(const MPOperator &op, - const Monomial &gen, - const CutoffEvaluator &cutoff_eval, - const CutoffContext &cut_st, - const VecD &coeffs, - std::optional only_rotate_len_k, - mpi::SlotWindow window, - size_t my_rank, - const routing::Router &router, - size_t gen_shift, - bool capture_values = false, - double *fused_scale_coeffs = nullptr, - double fused_scale_cos = 1.0) -> FusedScanResult { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); +auto fused_find_and_collect_range(const MPOperator &op, + const Monomial &gen, + const CutoffEvaluator &cutoff_eval, + const CutoffContext &cut_st, + const VecD &coeffs, + std::optional only_rotate_len_k, + mpi::SlotWindow window, + size_t my_rank, + const routing::Router &router, + size_t gen_shift, + const InvertedIndex &index, + const uint64_t *row_parity, + size_t wlo, + size_t whi, + bool capture_values, + double *fused_scale_coeffs, + double fused_scale_cos) -> FusedScanResult { const size_t gen_pop = gen.count(); const size_t rank_count = router.flat_world(); const auto ectx = A::make_gen_context(gen); - assert(window.stop() <= rank_count && window.count != 0); FusedScanResult res; res.leader_queries.reset(window); res.leader_src.reset(window); res.follower_queries.reset(window); res.follower_src.reset(window); - // Sized even on the early returns below, so the fused engine's src_val_r access stays in bounds. if (capture_values) { res.leader_val.reset(window); res.follower_val.reset(window); } - { - // The anticommutation fold runs over G's inverted-index columns (Majorana: G; Pauli: J(G) = - // pair_swap(G), so pauli_anticommutes = parity(|M ∩ J(G)|), and Pauli never needs the odd-|G| - // correction since parity(|G ∩ J(G)|)=0). The pivot splitting each pair is a set bit of the real G - // (gen.find_first()), not J(G) — A and A⊕G differ exactly on G's bits. - const Monomial fold_gen = A::fold_generator(gen); - const bool g_odd = A::fold_needs_odd_correction(gen); - const auto gen_columns = build_even_parity_generator_columns(fold_gen); - if (gen_columns.count == 0) { - return res; + // The anticommutation fold runs over G's inverted-index columns (Majorana: G; Pauli: J(G) = + // pair_swap(G), so pauli_anticommutes = parity(|M ∩ J(G)|), and Pauli never needs the odd-|G| + // correction since parity(|G ∩ J(G)|)=0). The pivot splitting each pair is a set bit of the real G + // (gen.find_first()), not J(G) — A and A⊕G differ exactly on G's bits. + const Monomial fold_gen = A::fold_generator(gen); + const bool g_odd = A::fold_needs_odd_correction(gen); + const auto gen_columns = build_even_parity_generator_columns(fold_gen); + const std::span gen_cols(gen_columns.indices.data(), gen_columns.count); + assert(index.rows() == op.store->size() && (!g_odd || row_parity != nullptr)); + const size_t n = op.store->size(); + const size_t last_word = index.words() - 1; + const uint64_t last_word_mask = (n % 64 == 0) ? ~uint64_t{0} : ((uint64_t{1} << (n % 64)) - 1); + + auto &lq = res.leader_queries; + auto &ls = res.leader_src; + auto &lv = res.leader_val; + auto &fq = res.follower_queries; + auto &fs = res.follower_src; + auto &fv = res.follower_val; + + const OperatorIndex &ham = *op.store; + using RowPosT = typename OperatorIndex::PosT; + + // The generator's positions, once per range: the merge's second input. + std::vector gen_pos; + gen_pos.reserve(gen_pop); + for (size_t b = gen.find_first(); b < gen.size(); b = gen.find_next(b)) { + gen_pos.push_back(static_cast(b)); + } + // pbuf capacity is 2*NumModes: the partner's positions are distinct, so this always suffices. + std::vector pbuf(2 * NumModes); + + auto push = [&](const Monomial &dense, + std::span pos, + int phase, + size_t i, + double v_src, + bool is_follower) { + // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. + // Must agree with find_rank (both go through routing::Router), or a row duplicates silently. + size_t r_prime = my_rank; + if (rank_count != 1) { + r_prime = router.dest_from_shift(dense, my_rank, gen_shift); + assert(r_prime == router.dest(dense)); // an identity, not an approximation } - const auto &inverted_index = op.inverted_index(); - const size_t word_count = inverted_index.words(); - if (word_count == 0) { - return res; + if (r_prime == my_rank) { + (is_follower ? res.follower_self : res.leader_self).push(pos, phase); } - const uint64_t *const row_parity_ptr = g_odd ? inverted_index.row_parity_words() : nullptr; - const size_t n = op.store->size(); - // The fused sweep writes fused_scale_coeffs[i] for every anticommuting i < n, so it must be the - // very array the reads come from and cover the full operator — a violation corrupts 1/cos recovery. - assert(fused_scale_coeffs == nullptr || (fused_scale_coeffs == coeffs.data() && coeffs.size() >= n)); - - const size_t last_word = word_count - 1; - const uint64_t last_word_mask = (n % 64 == 0) ? ~uint64_t{0} : ((uint64_t{1} << (n % 64)) - 1); - const std::span gen_cols(gen_columns.indices.data(), gen_columns.count); - - // The scan can be skipped only when every fold column is empty; a dense column always holds ≥1 - // posting, so any dense column makes it non-empty. - bool fold_cols_empty = true; - for (size_t ci = 0; ci < gen_columns.count; ++ci) { - const size_t c = gen_columns.indices[ci]; - if (inverted_index.column_is_dense(c)) { - fold_cols_empty = false; - } - else if (!inverted_index.sparse_column_rows(c).empty()) { - fold_cols_empty = false; - } + else { + QueryWire::push(is_follower ? fq.at_slot(r_prime) : lq.at_slot(r_prime), pos, phase); } - // Zero-postings early-out: no term touches a fold column ⇒ nothing anticommutes, so skip pass 1. - // g_odd guard is load-bearing: an odd Majorana generator anticommutes with disjoint odd-weight terms. - const bool skip_scan = !g_odd && fold_cols_empty; - - auto &lq = res.leader_queries; - auto &ls = res.leader_src; - auto &lv = res.leader_val; - auto &fq = res.follower_queries; - auto &fs = res.follower_src; - auto &fv = res.follower_val; - - const OperatorIndex &ham = *op.store; - using RowPosT = typename OperatorIndex::PosT; - - // The generator's positions, once per gate: the merge's second input. - std::vector gen_pos; - gen_pos.reserve(gen_pop); - for (size_t b = gen.find_first(); b < gen.size(); b = gen.find_next(b)) { - gen_pos.push_back(static_cast(b)); + (is_follower ? fs : ls).at_slot(r_prime).push_back(i); + if (capture_values) { + (is_follower ? fv : lv).at_slot(r_prime).push_back(v_src); } - // pbuf capacity is 2*NumModes: the partner's positions are distinct, so this always suffices. - std::vector pbuf(2 * NumModes); - - auto push = [&](const Monomial &dense, - std::span pos, - int phase, - size_t i, - double v_src, - bool is_follower) { - // Single rank: every partner is self-owned, skip the O(W) hash; multi-rank routes by owner. - // Must agree with find_rank (both go through routing::Router), or a row duplicates silently. - size_t r_prime = my_rank; - if (rank_count != 1) { - r_prime = router.dest_from_shift(dense, my_rank, gen_shift); - assert(r_prime == router.dest(dense)); // an identity, not an approximation - } - if (r_prime == my_rank) { - (is_follower ? res.follower_self : res.leader_self).push(pos, phase); - } - else { - QueryWire::push(is_follower ? fq.at_slot(r_prime) : lq.at_slot(r_prime), pos, phase); - } - (is_follower ? fs : ls).at_slot(r_prime).push_back(i); - if (capture_values) { - (is_follower ? fv : lv).at_slot(r_prime).push_back(v_src); - } - }; - - // Single rank, built-in cutoff: build the partner from positions, with no O(W) work per term. - constexpr bool kWideEnough = Monomial::num_words() >= kPositionsEmitMinWords; - [[maybe_unused]] const bool emit_positions_only = - kWideEnough && rank_count == 1 && cutoff_eval.has_positions_form() - && cutoff_eval.max_slot_bound().value_or(~size_t{0}) - <= kPositionsEmitMaxSlotsPerWord * Monomial::num_words(); - const std::span gen_pos_span(gen_pos); - const std::span pbuf_span(pbuf); - - // The dynamic gate runs before emit_term_products, so a gate-rejected term computes no products. - // abs_c/v_src come from the caller's coeff read, not re-read. - auto emit = [&](size_t mono_pop, size_t i, double abs_c, double v_src, bool is_follower) { - if (!rotation_dynamic_gate(only_rotate_len_k, mono_pop, cut_st, abs_c)) { - return; - } - if constexpr (kWideEnough) { - if (emit_positions_only) { - if (const auto src = ham.row_positions(i); src.inlined()) { - const auto merged = merge_partner_positions_paired(src.pos, gen_pos_span, pbuf_span); - const auto partner = std::span(pbuf).first(merged.count); - if (!cutoff_eval.passes_positions(partner, merged.pairs) && !cut_st.is_above_upper(abs_c)) { - return; - } - const int phase = - A::emit_phase(A::rotation_sign_positions(ectx, src.pos), mono_pop, gen_pop, merged.overlap); - (is_follower ? res.follower_self : res.leader_self).push(partner, phase); - (is_follower ? fs : ls).at_slot(my_rank).push_back(i); - if (capture_values) { - (is_follower ? fv : lv).at_slot(my_rank).push_back(v_src); - } + }; + + // One flat owner (one rank, T = 1), built-in cutoff: build the partner from positions, with no O(W) work per + // term. With several owners the partner's owner hash reads the dense form. + constexpr bool kWideEnough = Monomial::num_words() >= kPositionsEmitMinWords; + [[maybe_unused]] const bool emit_positions_only = + kWideEnough && rank_count == 1 && cutoff_eval.has_positions_form() + && cutoff_eval.max_slot_bound().value_or(~size_t{0}) + <= kPositionsEmitMaxSlotsPerWord * Monomial::num_words(); + const std::span gen_pos_span(gen_pos); + const std::span pbuf_span(pbuf); + + // The dynamic gate runs before emit_term_products, so a gate-rejected term computes no products. + // abs_c/v_src come from the caller's coeff read, not re-read. + auto emit = [&](size_t mono_pop, size_t i, double abs_c, double v_src, bool is_follower) { + if (!rotation_dynamic_gate(only_rotate_len_k, mono_pop, cut_st, abs_c)) { + return; + } + if constexpr (kWideEnough) { + if (emit_positions_only) { + if (const auto src = ham.row_positions(i); src.inlined()) { + const auto merged = merge_partner_positions_paired(src.pos, gen_pos_span, pbuf_span); + const auto partner = std::span(pbuf).first(merged.count); + if (!cutoff_eval.passes_positions(partner, merged.pairs) && !cut_st.is_above_upper(abs_c)) { return; } + const int phase = + A::emit_phase(A::rotation_sign_positions(ectx, src.pos), mono_pop, gen_pop, merged.overlap); + (is_follower ? res.follower_self : res.leader_self).push(partner, phase); + (is_follower ? fs : ls).at_slot(my_rank).push_back(i); + if (capture_values) { + (is_follower ? fv : lv).at_slot(my_rank).push_back(v_src); + } + return; } } - const auto p = emit_term_products(ham, - i, - ectx, - std::span(gen_pos), - std::span(pbuf)); - // Structural cutoff on the partner M⊕G, unless upper_atol rescues it (CutoffContext::is_above_upper). - const bool struct_pass = cutoff_eval.passes_with_popcount(p.new_mono, p.k); - if (!struct_pass && !cut_st.is_above_upper(abs_c)) { - return; - } - const int phase = A::emit_phase(p.phase_factor, mono_pop, gen_pop, p.overlap); - push(p.new_mono, std::span(pbuf).first(p.k), phase, i, v_src, is_follower); - }; - - // Pass 1 and pass 2 stay fused over `nz`: splitting them regressed measurably, as `nz` spills L1 - // between them. `nz` is thread_local so each partition master reuses its capacity across gates. - thread_local std::vector nz; - size_t n_anti = 0; - size_t n_foll = 0; - if (skip_scan) { - nz.clear(); // pass 1 clears it on entry; the skip must too (thread_local reuse) } - else { - even_parity_scan_pass1(inverted_index, - gen_cols, - gen.find_first(), - /*wlo=*/0, - /*whi=*/word_count, - last_word, - last_word_mask, - g_odd, - row_parity_ptr, - nz, - n_anti, - n_foll); + const auto p = emit_term_products(ham, i, ectx, gen_pos_span, pbuf_span); + // Structural cutoff on the partner M⊕G, unless upper_atol rescues it (CutoffContext::is_above_upper). + const bool struct_pass = cutoff_eval.passes_with_popcount(p.new_mono, p.k); + if (!struct_pass && !cut_st.is_above_upper(abs_c)) { + return; } - if (rank_count == 1) { - // A hint only; wider terms grow the buffer as needed. - const size_t pq = QueryWire::kReservePositionsPerQuery; - res.leader_self.reserve(n_anti - n_foll, pq); - ls.at_slot(my_rank).reserve(n_anti - n_foll); - res.follower_self.reserve(n_foll, pq); - fs.at_slot(my_rank).reserve(n_foll); + const int phase = A::emit_phase(p.phase_factor, mono_pop, gen_pop, p.overlap); + push(p.new_mono, std::span(pbuf).first(p.k), phase, i, v_src, is_follower); + }; + + // Pass 1 and pass 2 stay fused over `nz`: splitting them regressed measurably, as `nz` spills L1 + // between them. `nz` is thread_local, acquired here on the executing thread, so a scan worker reuses its + // own capacity across gates and never shares the caller's. + thread_local std::vector nz; + size_t n_anti = 0; + size_t n_foll = 0; + even_parity_scan_pass1(index, + gen_cols, + gen.find_first(), + wlo, + whi, + last_word, + last_word_mask, + g_odd, + row_parity, + nz, + n_anti, + n_foll); + if (rank_count == 1) { + // A hint only; wider terms grow the buffer as needed. + const size_t pq = QueryWire::kReservePositionsPerQuery; + res.leader_self.reserve(n_anti - n_foll, pq); + ls.at_slot(my_rank).reserve(n_anti - n_foll); + res.follower_self.reserve(n_foll, pq); + fs.at_slot(my_rank).reserve(n_foll); + } + auto derive_coeff = [&](size_t i) -> std::pair { + if (capture_values) { + const double v_src = (i < coeffs.size()) ? coeffs[i] : 0.0; + return {v_src, cut_st.use_coeff_checks ? std::abs(v_src) : 0.0}; } - auto derive_coeff = [&](size_t i) -> std::pair { - if (capture_values) { - const double v_src = (i < coeffs.size()) ? coeffs[i] : 0.0; - return {v_src, cut_st.use_coeff_checks ? std::abs(v_src) : 0.0}; - } - return {0.0, cut_st.abs_coeff_for(i, coeffs)}; - }; - const bool word_aligned_cos = !only_rotate_len_k.has_value(); - CosineWordBuilder cos_b; - // In large sparse scans, prefetch rows and coefficients kPrefetchWords nonzero words ahead. - constexpr size_t kPrefetchWords = 8; - constexpr size_t kPrefetchMinRows = size_t{1} << 21; - const double *const prefetch_coeffs = fused_scale_coeffs != nullptr ? fused_scale_coeffs : coeffs.data(); - const bool prefetch_ahead = ham.size() >= kPrefetchMinRows && n_anti * 16 < ham.size() - && (fused_scale_coeffs != nullptr || coeffs.size() >= ham.size()); - const std::span nz_words(nz); - for (size_t kw = 0; kw < nz_words.size(); ++kw) { - const auto &w = nz_words[kw]; - if (prefetch_ahead && kw + kPrefetchWords < nz_words.size()) { - const auto &ahead = nz_words[kw + kPrefetchWords]; - for (uint64_t m = ahead.overlap; m; m &= m - 1) { - const size_t i = ahead.base + static_cast(std::countr_zero(m)); - ham.prefetch_row(i); - __builtin_prefetch(&prefetch_coeffs[i]); - } + return {0.0, cut_st.abs_coeff_for(i, coeffs)}; + }; + const bool word_aligned_cos = !only_rotate_len_k.has_value(); + CosineWordBuilder cos_b; + // In large sparse scans, prefetch rows and coefficients kPrefetchWords nonzero words ahead. + constexpr size_t kPrefetchWords = 8; + constexpr size_t kPrefetchMinRows = size_t{1} << 21; + const double *const prefetch_coeffs = fused_scale_coeffs != nullptr ? fused_scale_coeffs : coeffs.data(); + const bool prefetch_ahead = ham.size() >= kPrefetchMinRows && n_anti * 16 < ham.size() + && (fused_scale_coeffs != nullptr || coeffs.size() >= ham.size()); + // Bound once: each access to the thread_local `nz` would otherwise go through the TLS lookup. + const std::span nz_words(nz); + for (size_t kw = 0; kw < nz_words.size(); ++kw) { + const auto &w = nz_words[kw]; + if (prefetch_ahead && kw + kPrefetchWords < nz_words.size()) { + const auto &ahead = nz_words[kw + kPrefetchWords]; + for (uint64_t m = ahead.overlap; m; m &= m - 1) { + const size_t i = ahead.base + static_cast(std::countr_zero(m)); + ham.prefetch_row(i); + __builtin_prefetch(&prefetch_coeffs[i]); } - if (word_aligned_cos && fused_scale_coeffs != nullptr) { - // Fused cos sweep: scaling in place here is what replaces building a cosine set. - for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; - const double v_src = fused_scale_coeffs[i]; - fused_scale_coeffs[i] = v_src * fused_scale_cos; - const double abs_c = std::abs(v_src); - if (cut_st.is_below_sin(abs_c)) { - continue; - } - const size_t mono_pop = op.store->popcount(i); - const bool is_follower = (w.foll >> tz) & 1u; - emit(mono_pop, i, abs_c, v_src, is_follower); + } + if (word_aligned_cos && fused_scale_coeffs != nullptr) { + // Fused cos sweep: scaling in place here is what replaces building a cosine set. + for (uint64_t m = w.overlap; m; m &= m - 1) { + const size_t tz = static_cast(std::countr_zero(m)); + const size_t i = w.base + tz; + const double v_src = fused_scale_coeffs[i]; + fused_scale_coeffs[i] = v_src * fused_scale_cos; + const double abs_c = std::abs(v_src); + if (cut_st.is_below_sin(abs_c)) { + continue; } + const size_t mono_pop = op.store->popcount(i); + const bool is_follower = (w.foll >> tz) & 1u; + emit(mono_pop, i, abs_c, v_src, is_follower); } - else if (word_aligned_cos) { - // No orbital gate: record the whole word in the cosine set, then per bit apply the atol - // gate before the popcount row read — deferring popcount saves random packed-row loads. - cos_b.push_word(w.base, w.overlap); - for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; - const auto [v_src, abs_c] = derive_coeff(i); - if (cut_st.is_below_sin(abs_c)) { - continue; - } - const size_t mono_pop = op.store->popcount(i); - const bool is_follower = (w.foll >> tz) & 1u; - emit(mono_pop, i, abs_c, v_src, is_follower); + } + else if (word_aligned_cos) { + // No orbital gate: record the whole word in the cosine set, then per bit apply the atol + // gate before the popcount row read — deferring popcount saves random packed-row loads. + cos_b.push_word(w.base, w.overlap); + for (uint64_t m = w.overlap; m; m &= m - 1) { + const size_t tz = static_cast(std::countr_zero(m)); + const size_t i = w.base + tz; + const auto [v_src, abs_c] = derive_coeff(i); + if (cut_st.is_below_sin(abs_c)) { + continue; } + const size_t mono_pop = op.store->popcount(i); + const bool is_follower = (w.foll >> tz) & 1u; + emit(mono_pop, i, abs_c, v_src, is_follower); } - else { - // Orbital gate active: it needs mono_pop, and the per-index cosine push covers only - // orbital-passing terms, so the popcount row read must precede both. - for (uint64_t m = w.overlap; m; m &= m - 1) { - const size_t tz = static_cast(std::countr_zero(m)); - const size_t i = w.base + tz; - const size_t mono_pop = op.store->popcount(i); - if (mono_pop > static_cast(*only_rotate_len_k)) { - continue; - } - cos_b.push_index(i); - const auto [v_src, abs_c] = derive_coeff(i); - const bool is_follower = (w.foll >> tz) & 1u; - emit(mono_pop, i, abs_c, v_src, is_follower); + } + else { + // Orbital gate active: it needs mono_pop, and the per-index cosine push covers only + // orbital-passing terms, so the popcount row read must precede both. + for (uint64_t m = w.overlap; m; m &= m - 1) { + const size_t tz = static_cast(std::countr_zero(m)); + const size_t i = w.base + tz; + const size_t mono_pop = op.store->popcount(i); + if (mono_pop > static_cast(*only_rotate_len_k)) { + continue; } + cos_b.push_index(i); + const auto [v_src, abs_c] = derive_coeff(i); + const bool is_follower = (w.foll >> tz) & 1u; + emit(mono_pop, i, abs_c, v_src, is_follower); } } - res.cos_blocks.push_back(cos_b.finish()); } + res.cos_blocks.push_back(cos_b.finish()); return res; } +// `total + add`, refusing a sum a merged scan buffer could not hold. +inline auto checked_scan_total(size_t total, size_t add, size_t max_size, const char *what) -> size_t { + if (add > max_size || total > max_size - add) { + throw std::length_error(std::format("fused_find_and_collect: the merged {} exceed {}", what, max_size)); + } + return total + add; +} + +// Concatenates one stream across the pieces in range order into a single buffer and releases every piece's +// buffer. Moves a piece's buffer instead of copying it when it holds the whole stream or already has room +// for the total; otherwise reserves the exact total once. +template +auto merge_scan_stream(Pieces &pieces, Get &&get, const char *what) -> Vec { + constexpr size_t npos = std::numeric_limits::max(); + size_t total = 0; + size_t first = npos; + size_t nonempty = 0; + for (size_t k = 0; k < pieces.size(); ++k) { + const Vec &v = get(pieces[k]); + total = checked_scan_total(total, v.size(), v.max_size(), what); + if (!v.empty()) { + first = (first == npos) ? k : first; + ++nonempty; + } + } + Vec out; + if (first != npos) { + Vec &lead = get(pieces[first]); + size_t next = first; + if (nonempty == 1 || lead.capacity() >= total) { + out = std::move(lead); + ++next; + } + else { + out.reserve(total); + } + for (size_t k = next; k < pieces.size(); ++k) { + const Vec &v = get(pieces[k]); + out.insert(out.end(), v.begin(), v.end()); + Vec{}.swap(get(pieces[k])); // release now: clear() would keep the capacity + } + } + // Empty pieces may still hold a reservation. + for (auto &piece : pieces) { + Vec{}.swap(get(piece)); + } + return out; +} + +// Appends each piece's self-stage queries in range order through SelfQueryStage::push, which rebases the +// position offsets; only the logical queries/positions are read, never a capacity tail. +template +auto merge_self_stage(std::vector> &pieces, + SelfQueryStage FusedScanResult::*member) -> SelfQueryStage { + using Stage = SelfQueryStage; + using PosT = typename Stage::PosT; + constexpr size_t npos = std::numeric_limits::max(); + size_t queries = 0; + size_t positions = 0; + size_t first = npos; + size_t nonempty = 0; + for (size_t k = 0; k < pieces.size(); ++k) { + const Stage &st = pieces[k].*member; + queries = checked_scan_total(queries, st.size(), st.pos_off.max_size(), "self queries"); + positions = checked_scan_total(positions, st.positions(), st.pos_flat.max_size(), "self positions"); + if (st.size() != 0) { + first = (first == npos) ? k : first; + ++nonempty; + } + } + Stage out; + if (first != npos) { + Stage &lead = pieces[first].*member; + size_t next = first; + if (nonempty == 1 || (lead.pos_off.size() >= queries && lead.pos_flat.size() >= positions)) { + out = std::move(lead); + ++next; + } + else { + out.reserve(queries, 0); + // pos_flat is sized to capacity, not filled (see SelfQueryStage), so this is the exact reservation. + out.pos_flat.resize(positions); + } + for (size_t k = next; k < pieces.size(); ++k) { + const Stage &st = pieces[k].*member; + for (size_t q = 0; q < st.size(); ++q) { + out.push(std::span(st.pos_flat.data() + st.pos_off[q], st.k_of[q]), st.phase_of[q]); + } + pieces[k].*member = Stage{}; + } + assert(out.size() == queries && out.positions() == positions); + } + for (auto &piece : pieces) { + piece.*member = Stage{}; + } + return out; +} + +/*! + * \brief Merge the per-range results of one threaded scan into the result the serial scan produces. + * + * `pieces[r]` must be range `r`'s result over `window`, ranges ascending in source words. Each stream is + * concatenated in range order, so every destination's leader and follower queries, sources and values keep + * their source-order subsequence; wire records are whole words and are copied, never re-encoded. The two + * self stages merge separately through SelfQueryStage::push. The cosine sets are moved in range order and + * stay ascending and disjoint. Value streams exist only when `capture_values`. Every piece's buffers are + * released as soon as they are drained. + * + * \throws std::length_error if a merged stream would exceed its container's max_size. + */ +template +auto merge_scan_ranges(std::vector> &pieces, + mpi::SlotWindow window, + [[maybe_unused]] size_t my_rank, + bool capture_values) -> FusedScanResult { + using Result = FusedScanResult; + Result out; + const auto merge_window = [&](mpi::WindowVec Result::*member, const char *what) { + mpi::WindowVec &merged = out.*member; + merged.reset(window); + for (const auto wi : window.indices()) { + merged[wi] = merge_scan_stream(pieces, [&](Result &p) -> T & { return (p.*member)[wi]; }, what); + } + }; + merge_window(&Result::leader_queries, "leader queries"); + merge_window(&Result::leader_src, "leader sources"); + merge_window(&Result::follower_queries, "follower queries"); + merge_window(&Result::follower_src, "follower sources"); + if (capture_values) { + merge_window(&Result::leader_val, "leader values"); + merge_window(&Result::follower_val, "follower values"); + } + out.leader_self = merge_self_stage(pieces, &Result::leader_self); + out.follower_self = merge_self_stage(pieces, &Result::follower_self); + size_t masks = 0; + for (const auto &piece : pieces) { + masks = checked_scan_total(masks, piece.cos_blocks.size(), out.cos_blocks.max_size(), "cosine sets"); + } + out.cos_blocks.reserve(masks); + for (auto &piece : pieces) { + for (auto &mask : piece.cos_blocks) { + out.cos_blocks.push_back(std::move(mask)); + } + std::vector{}.swap(piece.cos_blocks); + } +#ifndef NDEBUG + if (window.contains(my_rank)) { + assert(out.leader_queries.at_slot(my_rank).empty() && out.follower_queries.at_slot(my_rank).empty()); + assert(out.leader_self.size() == out.leader_src.at_slot(my_rank).size()); + assert(out.follower_self.size() == out.follower_src.at_slot(my_rank).size()); + assert(!capture_values || out.leader_self.size() == out.leader_val.at_slot(my_rank).size()); + assert(!capture_values || out.follower_self.size() == out.follower_val.at_slot(my_rank).size()); + } + else { + assert(out.leader_self.size() == 0 && out.follower_self.size() == 0); + } +#endif + return out; +} + +// Classify, cut off and emit in one pass over the anticommuting terms. Queries go to the owner of +// M'=M⊕G (routing::Router; self at R==1) in ascending source-index order, so resolve and index assignment are +// deterministic. `fused_scale_coeffs` (no length cap only; must alias coeffs.data()) scales every anticommuting +// coeff in place by `fused_scale_cos`=cos(2·build_angle), so no cosine set is built and a hit's stored +// value is post-cos (resolve recovers it via 1/cos). +// +// `gen_shift` must be router.rank_shift(gen) and `op` hold only terms `my_rank` owns, so the owner of +// M⊕G is rank(M) ^ gen_shift (asserted against dest()). `window` is the plan's window for `my_rank`. +// +// Threading: with a parallel-safe cutoff (CutoffEvaluator::parallel_safe) and at least two logical ranges +// (scan_ranges), the words split into ranges that scan workers traverse with fused_find_and_collect_range, +// each into a private result, and merge_scan_ranges joins them in range order: the output is identical to +// the serial scan's at every budget and team size. Otherwise the serial scan runs on the caller. Lazy +// index/parity data is prepared here, on the caller, before any worker starts. `observer` is a test seam +// (see NoRangeObserver) reporting the KernelRange::scan ranges. +template +auto fused_find_and_collect(const MPOperator &op, + const Monomial &gen, + const CutoffEvaluator &cutoff_eval, + const CutoffContext &cut_st, + const VecD &coeffs, + std::optional only_rotate_len_k, + mpi::SlotWindow window, + size_t my_rank, + const routing::Router &router, + size_t gen_shift, + bool capture_values = false, + double *fused_scale_coeffs = nullptr, + double fused_scale_cos = 1.0, + parallel::Options options = {}, + const Observer &observer = {}) -> FusedScanResult { + validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); + (void)scan_ranges(0, options); // validates the budget before anything runs + assert(window.stop() <= router.flat_world() && window.count != 0); + + // Sized even on the early returns below, so the fused engine's src_val_r access stays in bounds. + const auto empty_result = [&] { + FusedScanResult res; + res.leader_queries.reset(window); + res.leader_src.reset(window); + res.follower_queries.reset(window); + res.follower_src.reset(window); + if (capture_values) { + res.leader_val.reset(window); + res.follower_val.reset(window); + } + return res; + }; + + const Monomial fold_gen = A::fold_generator(gen); + const bool g_odd = A::fold_needs_odd_correction(gen); + const auto gen_columns = build_even_parity_generator_columns(fold_gen); + if (gen_columns.count == 0) { + observer.prepare(KernelRange::scan, 0); + return empty_result(); + } + // Both lazy accessors may build caches, so they run here and never in a worker. + const auto &index = op.inverted_index(); + const size_t word_count = index.words(); + if (word_count == 0) { + observer.prepare(KernelRange::scan, 0); + return empty_result(); + } + const uint64_t *const row_parity = g_odd ? index.row_parity_words() : nullptr; + // The fused sweep writes fused_scale_coeffs[i] for every anticommuting i < n, so it must be the + // very array the reads come from and cover the full operator — a violation corrupts 1/cos recovery. + assert(fused_scale_coeffs == nullptr || (fused_scale_coeffs == coeffs.data() && coeffs.size() >= op.store->size())); + + // The scan can be skipped only when every fold column is empty; a dense column always holds ≥1 + // posting, so any dense column makes it non-empty. + bool fold_cols_empty = true; + for (size_t ci = 0; ci < gen_columns.count; ++ci) { + const size_t c = gen_columns.indices[ci]; + if (index.column_is_dense(c) || !index.sparse_column_rows(c).empty()) { + fold_cols_empty = false; + } + } + // Zero-postings early-out: no term touches a fold column ⇒ nothing anticommutes, so skip pass 1. + // g_odd guard is load-bearing: an odd Majorana generator anticommutes with disjoint odd-weight terms. + if (!g_odd && fold_cols_empty) { + observer.prepare(KernelRange::scan, 0); + auto res = empty_result(); + res.cos_blocks.push_back(CosMask{}); + return res; + } + + const auto scan_words = [&](size_t wlo, size_t whi) { + return fused_find_and_collect_range(op, + gen, + cutoff_eval, + cut_st, + coeffs, + only_rotate_len_k, + window, + my_rank, + router, + gen_shift, + index, + row_parity, + wlo, + whi, + capture_values, + fused_scale_coeffs, + fused_scale_cos); + }; + // Opaque cutoffs and small scans keep the serial scan: one range on the caller, no merge. + const size_t ranges = cutoff_eval.parallel_safe() ? scan_ranges(word_count, options) : 1; + observer.prepare(KernelRange::scan, ranges); + if (ranges <= 1) { + observer.visit(KernelRange::scan, 0); + return scan_words(0, word_count); + } + // One private result per logical range, never per fold block: O(ranges * window.count) metadata. + std::vector> pieces(ranges); + parallel::for_blocks(ranges, options, [&](size_t range) { + observer.visit(KernelRange::scan, range); + const auto [wlo, whi] = scan_range_words(range, ranges, word_count); + pieces[range] = scan_words(wlo, whi); + }); + return merge_scan_ranges(pieces, window, my_rank, capture_values); +} + } // namespace monoprop::detail diff --git a/cpp/monoprop/detail/graph/MPGraphViews.h b/cpp/monoprop/detail/graph/MPGraphViews.h index 80df2672e..09cb26e9c 100644 --- a/cpp/monoprop/detail/graph/MPGraphViews.h +++ b/cpp/monoprop/detail/graph/MPGraphViews.h @@ -52,7 +52,7 @@ struct GraphMemoryBreakdown final { + exchange_layout_bytes; } - // Lets a partitioned propagator aggregate its per-partition graph breakdowns. + // Lets a sharded propagator aggregate its per-shard graph breakdowns. auto operator+=(const GraphMemoryBreakdown &o) -> GraphMemoryBreakdown & { layer_descriptor_bytes += o.layer_descriptor_bytes; layer_storage_object_bytes += o.layer_storage_object_bytes; diff --git a/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp b/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp index 5a7e56e6f..cf32d0bac 100644 --- a/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp +++ b/cpp/monoprop/detail/graph_encoding/MPGraphEncoding.cpp @@ -47,7 +47,7 @@ static auto checked_world_slot(size_t rank) -> uint32_t { auto checked_term_index(size_t value, const char *what) -> TermIndex { if (value > static_cast(std::numeric_limits::max())) { throw std::overflow_error( - std::format("{} {} exceeds the TermIndex ceiling {}; raise the partition or rank count.", + std::format("{} {} exceeds the TermIndex ceiling {}; raise the rank or thread (shard) count.", what, value, std::numeric_limits::max())); diff --git a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt index 6afe6999f..093828315 100644 --- a/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt +++ b/cpp/monoprop/detail/monomial_propagator/CMakeLists.txt @@ -6,4 +6,5 @@ target_sources( FILES "MonomialPropagatorCommon.h" "MonomialPropagator.inl" + "ShardedPropagator.inl" ) diff --git a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl index f9a09fe15..709af0e5b 100644 --- a/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl +++ b/cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl @@ -14,25 +14,11 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include #include -#include -#include -#include #include +#include #include "monoprop/Validation.h" #include "monoprop/algebra/Algebra.h" @@ -41,17 +27,14 @@ #include "monoprop/detail/evolution/LayerBuilder.h" #include "monoprop/detail/evolution/layer_build/FusedApply.h" #include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" -#include "monoprop/detail/partition/PartitionGroup.h" +#include "monoprop/detail/mpi/OperationFailure.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/Construction.h" +#include "monoprop/detail/sharded/Evaluation.h" +#include "monoprop/detail/sharded/State.h" namespace monoprop { -// The ranks disagree on S. The count comes from partitions= or the environment on -// every rank independently, so the fix is to the launch, and it may belong to a different rank. -class PartitionCountMismatch : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - // The requested operation does not agree with the graph this propagator currently holds -- either it // requires no stored graph, or its parameter_mapping matches neither the stored layer nor gate count. // The caller recovers by contracting or rebuilding the graph, not by fixing an isolated argument. @@ -74,372 +57,110 @@ public: using std::invalid_argument::invalid_argument; }; -template -MonomialPropagator::MonomialPropagator(const OperatorDict &initial_operator, - unsigned int cutoff, - const VecZ &initial_state, - std::optional schrodinger_cutoff, - mpi::Comm comm, - std::optional lower_atol, - std::optional upper_atol, - CutoffType cutoff_type, - std::optional> basis_change, - size_t logical_num_modes, - Basis basis, - size_t partitions, - PartitionChildFactory child_factory) - : schrodinger_{schrodinger_cutoff.has_value()}, - comm_{comm}, - mp_op_{}, - graph_(schrodinger_cutoff.has_value()), - cutoff_{cutoff}, - lower_atol_{lower_atol}, - upper_atol_{upper_atol}, - logical_num_modes_{logical_num_modes}, - cutoff_type_{cutoff_type}, - basis_change_{basis_change}, - basis_{basis} { - if (logical_num_modes_ == 0 || logical_num_modes_ > NumModes) { - throw PropagatorConfigError( - std::format("logical_num_modes ({}) must be in the range [1, {}].", logical_num_modes_, NumModes)); - } - - validate_cutoff_config_(cutoff_type_, basis_change_); - - mp_op_.basis = basis_; - - if (upper_atol.has_value() && lower_atol.has_value() && (upper_atol.value() < lower_atol.value())) { - throw PropagatorConfigError(std::format("upper_atol ({}) must be greater than or equal to lower_atol ({}).", - upper_atol.value(), - lower_atol.value())); - } - - const size_t n_partitions = resolve_partition_count_(partitions, comm); - // The R ranks x S partitions form one flat P = R*S SPMD world, so a mismatch across ranks would - // deadlock at the first hybrid collective. - if (comm.kind == mpi::Comm::Kind::Mpi && mpi::size(comm) > 1 - && mpi::allreduce_sum(n_partitions, comm) != n_partitions * static_cast(mpi::size(comm))) { - throw PartitionCountMismatch( - "Partition count differs across MPI ranks — every rank must resolve the same " - "partitions= / monoprop_PARTITIONS / monoprop_NUM_THREADS so R*S is a consistent world."); - } - - // Before the partition workers start: a child throwing after its siblings enter a collective - // poisons the shared-memory transport and masks this error. - size_t max_pairs = 0; - size_t expected_schrodinger_local_terms = 1; - if (schrodinger_) { - const auto sc = std::min(*schrodinger_cutoff, static_cast(2 * logical_num_modes_)); - max_pairs = sc / 2 + sc % 2; - const size_t global_terms = paired_op_size(max_pairs, logical_num_modes_); - // paired_op_size saturates rather than wrapping, so this is "too large to count", not a size. - const bool uncountable = global_terms == std::numeric_limits::max(); - const size_t world_slots = n_partitions * static_cast(mpi::size(comm)); - const size_t share = global_terms / std::max(1, world_slots); - if (uncountable || share >= detail::OperatorIndex::kIndexCeiling) { - const auto how_many = uncountable ? std::string("more than 2^64") : std::format("{}", global_terms); - const auto per_slot = uncountable ? std::string("as many") : std::format("{}", share); - throw PropagatorConfigError( - std::format("schrodinger_cutoff ({}) admits {} paired basis terms over {} active modes, " - "about {} per world slot — more than can be walked or addressed. Lower " - "schrodinger_cutoff, or raise the rank x partition count (currently {}).", - *schrodinger_cutoff, - how_many, - logical_num_modes_, - per_slot, - world_slots)); - } - expected_schrodinger_local_terms = std::max(1, share); - } - - if (n_partitions > 1) { - PartitionChildFactory factory = - child_factory ? std::move(child_factory) : PartitionChildFactory{[=](mpi::Comm partition_comm) { - return std::make_unique>(initial_operator, - cutoff, - initial_state, - schrodinger_cutoff, - partition_comm, - lower_atol, - upper_atol, - cutoff_type, - basis_change, - logical_num_modes, - basis, - /*partitions=*/1); - }}; - partition_group_ = std::make_unique>(static_cast(n_partitions), - factory, - comm); - return; - } - - const size_t my_rank = static_cast(mpi::rank(comm_)); - check_routing_agreement(comm_); // fail here rather than hang the first exchange - const routing::Router router = router_for(comm_); // geometry() can hit MPI: never per term - MonomialList local_heisenberg_terms; - - double core_term = 0.0; - for (const auto &[indices, coefficient] : initial_operator) { - const auto majorana_bitset = indices_to_bitset_checked(indices, 2 * logical_num_modes_); - const auto encoded_coeff = algebra_encode_coeff(basis_, coefficient, majorana_bitset); - - // Store the core term separately as it is orders of magnitude larger than the other terms - if (indices.empty()) { - core_term = encoded_coeff; - continue; - } - if (my_rank == find_rank(majorana_bitset, router)) { - mp_op_.init_op_map[majorana_bitset] = encoded_coeff; - local_heisenberg_terms.push_back(majorana_bitset); - } - } - - // Schrodinger's initial rows are the global paired basis, hash-partitioned over the P = R*S world - // slots; Heisenberg's local_heisenberg_terms was already filtered to this slot's share above. Hence the - // two reserves: a global count needs its 1/P share, a local one already is the share. - const size_t expected_local_terms = - schrodinger_ ? expected_schrodinger_local_terms : std::max(1, local_heisenberg_terms.size()); - - // Must run before the store: packed_inline_width_() derives the packed-row width from cutoff_fn_. - regenerate_cutoff_fn_(); - mp_op_.store = std::make_unique>(packed_inline_width_()); - mp_op_.store->reserve(expected_local_terms); - // Store replaced: drop the stale lazy inverted index so it rebuilds against the new store. - mp_op_.inverted_index_.reset(); - - size_t i = 0; - // The initial monomials are distinct, so emplace (insert-if-absent) is an assigning insert here. A row - // index is a position in the kept subsequence, so the enumeration order below is load-bearing. - const auto keep_if_owned = [&](const Monomial &mono) { - if (my_rank == find_rank(mono, router)) { - mp_op_.append_term(mono); - mp_op_.store->emplace(mono, i++); - } - }; - if (schrodinger_) { - // PartitionGroup constructs the S partitions concurrently on their own - // masters, so a list here would hold P full copies of the global basis at one instant. - for_each_paired_monomial(max_pairs, logical_num_modes_, keep_if_owned); - } - else { - for (const auto &mono : local_heisenberg_terms) { // already this slot's share; the test is a no-op - keep_if_owned(mono); - } - } - - mp_op_.initial_state = initial_state; - core_term_ = core_term; - - initialize_operator_caches_(); -} - template MonomialPropagator::~MonomialPropagator() = default; template -MonomialPropagator::MonomialPropagator(const MonomialPropagator &other) - : schrodinger_(other.schrodinger_), - comm_(other.comm_), - cutoff_fn_(other.cutoff_fn_), - mp_op_(other.mp_op_), - graph_(other.graph_), - matched_scratch_(other.matched_scratch_), - cutoff_(other.cutoff_), - lower_atol_(other.lower_atol_), - upper_atol_(other.upper_atol_), - core_term_(other.core_term_), - initial_operator_epoch_(other.initial_operator_epoch_), - heisenberg_gates_absorbed_(other.heisenberg_gates_absorbed_), - routing_coverage_reported_(other.routing_coverage_reported_), - logical_num_modes_(other.logical_num_modes_), - cutoff_type_(other.cutoff_type_), - basis_change_(other.basis_change_), - basis_(other.basis_), - partition_group_(other.partition_group_ - ? std::make_unique>(*other.partition_group_) - : nullptr) {} - -template -auto MonomialPropagator::resolve_partition_count_(size_t requested, mpi::Comm comm) -> size_t { - if (requested >= 1) { - return requested; - } - // One serial partition per physical core, capped by monoprop_NUM_THREADS. On a multi-rank comm this - // engages only when threads were explicitly requested, so a pure-MPI run is not oversubscribed. - const auto compute_auto = [&]() -> size_t { - const int ranks = mpi::size(comm); - size_t cores = detail::partition::enumerate_physical_cores().size(); - if (cores == 0) { - // Topology unreadable: use half the hardware threads so smt siblings aren't counted as cores. - cores = std::max(1, static_cast(std::thread::hardware_concurrency()) / 2); - } - const auto num_threads = config::get().num_threads; - const size_t budget = - num_threads.has_value() ? static_cast(*num_threads) : (ranks == 1 ? cores : size_t{1}); - return std::max(1, std::min(budget, cores)); - }; - if (const char *env = std::getenv("monoprop_PARTITIONS")) { - const std::string_view v(env); - if (v == "auto") { - return compute_auto(); - } - if (v == "off") { - return 1; - } - char *end = nullptr; - const long n = std::strtol(env, &end, 10); - if (end != env && *end == '\0' && n >= 1) { - return static_cast(n); - } - } - return compute_auto(); -} - -// Partition fan-out vocabulary; the declarations record which helper is legal where. - -template -auto MonomialPropagator::for_each_partition_(const std::function &fn) -> void { - partition_group_->run_on_all([&](int r) { fn(partition_group_->partition(r)); }); +auto MonomialPropagator::checked_source_(const MonomialPropagator &other) -> const MonomialPropagator & { + other.require_valid_(); + return other; } template -template -auto MonomialPropagator::map_partitions_(Fn fn) -> std::vector { - return detail::partition::map_partitions(*partition_group_, fn); +auto MonomialPropagator::require_valid_() const -> void { + if (invalid_) { + throw InvalidPropagatorError( + "This propagator is no longer usable: an earlier operation failed after it had started changing the " + "operator, graph or cached state, so that state is incomplete. Construct a new propagator (or use a " + "copy made before the failure); the failed object can only be destroyed."); + } } template -template -auto MonomialPropagator::map_partitions_indexed_(Fn fn) -> std::vector { - return detail::partition::collect_on_all(*partition_group_, - [&](int r) -> R { return fn(r, partition_group_->partition(r)); }); +auto MonomialPropagator::enter_operation_(bool uses_mpi) const -> void { + // At every P and T: the caller becomes the primary of the operation's team, which alone makes its MPI calls. + if (uses_mpi) { + mpi::require_initializing_thread(); + } + require_valid_(); } template -template -auto MonomialPropagator::concat_partitions_(Fn fn) -> R { - const auto per_partition = map_partitions_(fn); - size_t total = 0; - for (const auto &v : per_partition) { - total += v.size(); +template +auto MonomialPropagator::run_operation_(bool uses_mpi, Body &&body) -> decltype(auto) { + enter_operation_(uses_mpi); + bool mutation_started = false; + try { + return std::forward(body)(mutation_started); } - R merged; - merged.reserve(total); - for (const auto &v : per_partition) { - merged.insert(merged.end(), v.begin(), v.end()); + catch (...) { + if (!mutation_started) { + throw; + } + mutation_failed_(std::current_exception()); } - return merged; } template -template -auto MonomialPropagator::fold_partitions_(Proj proj, Accumulate accumulate) const -> R { - R total{}; - for (int r = 0; r < partition_group_->partition_count(); ++r) { - accumulate(total, proj(partition_group_->partition(r))); +auto MonomialPropagator::validate_generators_(const std::vector &majoranas) const -> void { + for (const auto &gate : majoranas) { + (void)indices_to_bitset_checked(gate, 2 * logical_num_modes_); } - return total; -} - -template -template -auto MonomialPropagator::sum_partitions_(Proj proj) const -> R { - return fold_partitions_(proj, [](R &total, const R &value) { total += value; }); -} - -template -auto MonomialPropagator::first_partition_() const -> const MonomialPropagator & { - return partition_group_->partition(0); -} - -template -auto MonomialPropagator::partitioned_size_() const -> size_t { - return sum_partitions_([](const MonomialPropagator &s) { return s.size(); }); -} - -template -auto MonomialPropagator::partitioned_graph_size_() const -> std::pair { - // One pass: graph_size() recomputes the cosine-only count, so it must not be projected twice. - return fold_partitions_([](const MonomialPropagator &s) { return s.graph_size(); }, - [](std::pair &total, const std::pair &value) { - total.first += value.first; - total.second += value.second; - }); -} - -template -auto MonomialPropagator::partitioned_graph_layers_() const -> size_t { - return first_partition_().graph_layers(); -} - -template -auto MonomialPropagator::partitioned_core_term_() const -> double { - return first_partition_().core_term(); -} - -template -auto MonomialPropagator::partitioned_operator_memory_usage_() const - -> detail::MPOperatorMemoryBreakdown { - return sum_partitions_([](const MonomialPropagator &s) { return s.operator_memory_usage(); }); -} - -template -auto MonomialPropagator::partitioned_graph_memory_usage_() const -> GraphMemoryBreakdown { - return sum_partitions_([](const MonomialPropagator &s) { return s.graph_memory_usage(); }); } template -auto MonomialPropagator::packed_inline_width_() const -> size_t { - constexpr size_t kMax = detail::OperatorIndex::kMaxInlinePositions; - constexpr size_t kDefault = detail::OperatorIndex::kDefaultInlinePositions; - if (schrodinger_) { - return kDefault; +auto MonomialPropagator::validate_cutoff_config_(CutoffType cutoff_type, + const std::optional> &basis_change) const + -> void { + with_algebra(basis_, [&]() { + if (A::requires_support_cutoff && cutoff_type != CutoffType::Support) { + throw CutoffConfigError("Pauli basis requires cutoff_type == Support " + "(Length has no Pauli-weight meaning under the Pauli encoding)."); + } + if (!A::allows_basis_change && basis_change.has_value()) { + throw CutoffConfigError("Pauli basis does not accept a basis_change " + "(the encoding is already the Jordan-Wigner image)."); + } + }); + // regenerate_cutoff_fn_ indexes rows [0, 2*logical_num_modes) unconditionally, so a short + // basis_change is an out-of-bounds read. + if (basis_change.has_value() && basis_change->size() != 2 * logical_num_modes_) { + throw CutoffConfigError(std::format("basis_change must have exactly 2*logical_num_modes ({}) rows; got {}.", + 2 * logical_num_modes_, + basis_change->size())); } - // The bound is already in physical slots (CutoffEvaluator::max_slot_bound), so nothing to scale. - const auto bound = detail::CutoffEvaluator(cutoff_fn_).max_slot_bound(); - if (!bound) { - return kDefault; + // regenerate_cutoff_fn_ would reject these too, but only after an update started changing settings. + if (basis_change.has_value()) { + for (const auto &row : *basis_change) { + (void)indices_to_bitset_checked(row, 2 * logical_num_modes_); + } } - return std::min(*bound, kMax); } template -auto MonomialPropagator::apply_initial_operator_(const OperatorDict &op_dict) - -> std::pair, VecD> { - ++initial_operator_epoch_; - if (partition_group_) { - // The facade holds no local terms of its own, so the return is empty. - for_each_partition_([&](MonomialPropagator &s) { s.update_initial_operator(op_dict); }); - return {}; - } - const routing::Router router = router_for(comm_); // hoisted, never per term - const size_t my_rank = static_cast(mpi::rank(comm_)); - - OperatorDict new_op; - for (const auto &[ind, coeff] : op_dict) { - const auto mono = indices_to_bitset_checked(ind, 2 * logical_num_modes_); - if (ind.empty()) { // Core term, store in all - core_term_ = algebra_encode_coeff(basis_, coeff, mono); - continue; - } - if (my_rank == find_rank(mono, router)) { - const auto mono_indices = bitset_to_indices(mono); - new_op[mono_indices] = coeff; +auto MonomialPropagator::regenerate_cutoff_fn_() -> void { + if (basis_change_.has_value()) { + MonomialList basis; + basis.reserve(2 * logical_num_modes_); + for (size_t i = 0; i < 2 * logical_num_modes_; ++i) { + basis.push_back(indices_to_bitset_checked(basis_change_.value()[i], 2 * logical_num_modes_)); } + cutoff_fn_ = detail::cutoff_function_basis_change(cutoff_type_, cutoff_, basis, logical_num_modes_); + } + else { + cutoff_fn_ = detail::cutoff_function(cutoff_type_, cutoff_, logical_num_modes_); } - - return mp_op_.update_initial_operator(new_op, schrodinger_); } template -auto MonomialPropagator::graph_data() const -> std::vector { - require_single_partition_("graph_data()"); +auto MonomialPropagator::graph_data_of_(const MPGraph &graph, + const detail::MPOperator &op, + Basis basis) -> std::vector { std::vector layers; - const auto num_layers = graph_.layers(); + const auto num_layers = graph.layers(); layers.reserve(num_layers); for (size_t i = 0; i < num_layers; ++i) { - const auto traversal = graph_.get_layer_traversal(i); + const auto traversal = graph.get_layer_traversal(i); const size_t rank_count = traversal.cross_rank_rank_count(); // Always empty: local cycles are folded into cross_rank[my_rank]. @@ -473,7 +194,7 @@ auto MonomialPropagator::graph_data() const -> std::vector } else if (const auto &gw = traversal.generator_words(); !gw.empty()) { const auto gen = detail::generator_from_words(gw); - auto p = detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); + auto p = detail::make_fold_cache(op.inverted_index(), gen, traversal.scaled_count(), basis); cos_inds = detail::fold_to_indices(p); } layers.emplace_back(std::move(cos_inds), std::move(local_cyc_data), std::move(b_data), std::move(d_data)); @@ -482,13 +203,15 @@ auto MonomialPropagator::graph_data() const -> std::vector } template -auto MonomialPropagator::cos_index_count_() const -> size_t { +auto MonomialPropagator::cos_index_count_of_(const MPGraph &graph, + const detail::MPOperator &op, + Basis basis) -> size_t { // Only a pared layer stores a cosine set; otherwise recompute the fold here. Cosine-only = cos-scaled // minus the rotation endpoints, saturating at 0. size_t total = 0; - const auto num_layers = graph_.layers(); + const auto num_layers = graph.layers(); for (size_t i = 0; i < num_layers; ++i) { - const auto traversal = graph_.get_layer_traversal(i); + const auto traversal = graph.get_layer_traversal(i); size_t cos_total = 0; if (traversal.has_stored_cos()) { cos_total = traversal.num_cos_inds(); @@ -496,7 +219,7 @@ auto MonomialPropagator::cos_index_count_() const -> size_t { else if (const auto &gw = traversal.generator_words(); !gw.empty()) { const auto gen = detail::generator_from_words(gw); const auto fold = - detail::make_fold_cache(mp_op_.inverted_index(), gen, traversal.scaled_count(), basis_); + detail::make_fold_cache(op.inverted_index(), gen, traversal.scaled_count(), basis); cos_total = detail::fold_popcount(fold); } const size_t endpoints = traversal.total_rotation_endpoints(); @@ -505,859 +228,22 @@ auto MonomialPropagator::cos_index_count_() const -> size_t { return total; } -template -auto MonomialPropagator::validate_cutoff_config_(CutoffType cutoff_type, - const std::optional> &basis_change) const - -> void { - with_algebra(basis_, [&]() { - if (A::requires_support_cutoff && cutoff_type != CutoffType::Support) { - throw CutoffConfigError("Pauli basis requires cutoff_type == Support " - "(Length has no Pauli-weight meaning under the Pauli encoding)."); - } - if (!A::allows_basis_change && basis_change.has_value()) { - throw CutoffConfigError("Pauli basis does not accept a basis_change " - "(the encoding is already the Jordan-Wigner image)."); - } - }); - // regenerate_cutoff_fn_ indexes rows [0, 2*logical_num_modes) unconditionally, so a short - // basis_change is an out-of-bounds read. - if (basis_change.has_value() && basis_change->size() != 2 * logical_num_modes_) { - throw CutoffConfigError(std::format("basis_change must have exactly 2*logical_num_modes ({}) rows; got {}.", - 2 * logical_num_modes_, - basis_change->size())); - } -} - -template -auto MonomialPropagator::regenerate_cutoff_fn_() -> void { - if (basis_change_.has_value()) { - MonomialList basis; - basis.reserve(2 * logical_num_modes_); - for (size_t i = 0; i < 2 * logical_num_modes_; ++i) { - basis.push_back(indices_to_bitset_checked(basis_change_.value()[i], 2 * logical_num_modes_)); - } - cutoff_fn_ = detail::cutoff_function_basis_change(cutoff_type_, cutoff_, basis, logical_num_modes_); - } - else { - cutoff_fn_ = detail::cutoff_function(cutoff_type_, cutoff_, logical_num_modes_); - } -} - -template -auto MonomialPropagator::initialize_operator_caches_() -> void { - (void)mp_op_.get_operator(); - // Heisenberg warms the sparse state only; densifying here would defeat it. Schrödinger's dense vector - // IS the live evolved vector. - if (schrodinger_) { - (void)mp_op_.dense_state(); - } - else { - (void)mp_op_.sparse_state(); - } - (void)mp_op_.inverted_index(); - mp_op_.op_coeffs.shrink_to_fit(); - mp_op_.shrink_state_to_fit(); -} - -template -auto MonomialPropagator::current_picture_coeffs_() -> const VecD & { - return schrodinger_ ? mp_op_.dense_state() : mp_op_.get_operator(); -} - -template -auto MonomialPropagator::extend_coeffs_from_current_picture_if_needed_(VecD &coeffs) -> void { - if (coeffs.size() >= mp_op_.size()) { - return; - } - - const auto ¤t = current_picture_coeffs_(); - if (&coeffs == ¤t) { - return; - } - - if (coeffs.size() < current.size()) { - coeffs.insert(coeffs.end(), current.begin() + static_cast(coeffs.size()), current.end()); - } - coeffs.resize(mp_op_.size(), 0.0); -} - -template -auto MonomialPropagator::evolve_mode_build_graph_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - std::optional only_rotate_len_k) -> void { - const auto majoranas_size = majoranas.size(); - run_gate_loop_(majoranas, - only_rotate_len_k, - [this, ¶meter_mapping, &gen_coeffs, &gate_indices, majoranas_size](const VecZ &mono, - std::optional rot_len, - size_t i) { - const auto idx = !schrodinger_ ? majoranas_size - 1 - i : i; - propagate_one_(mono, - rot_len, - std::nullopt, - std::nullopt, - parameter_mapping[idx], - gen_coeffs[idx], - gate_indices[idx]); - }); -} - -template -auto MonomialPropagator::evolve_mode_graph_with_coeffs_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecZ &gate_indices, - const VecD ¶meters, - const VecD &operator_coeffs, - std::optional only_rotate_len_k) -> void { - auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0); - auto coeffs = operator_coeffs; - const auto majoranas_size = majoranas.size(); - - run_gate_loop_(majoranas, - only_rotate_len_k, - [this, ¶meter_mapping, &gen_coeffs, &gate_indices, &mapped_params, &coeffs, majoranas_size]( - const VecZ &mono, - std::optional rot_len, - size_t i) { - const auto idx = !schrodinger_ ? majoranas_size - 1 - i : i; - const auto [build_angle, apply_angle] = gate_angle_(mapped_params, i, majoranas_size); - // The cos word list is not persisted on the layer; the builder moves it out transiently. - auto cos = std::make_shared(); - auto storage = build_evolve_result_(mono, rot_len, std::cref(coeffs), build_angle, cos.get()); - graph_.append(storage, parameter_mapping[idx], gen_coeffs[idx], gate_indices[idx]); - - extend_coeffs_from_current_picture_if_needed_(coeffs); - - Layer layer(std::move(storage)); - detail::LayerCosScale cos_scale = [cos](size_t, double *c, double v) { - detail::scale_cos_mask(c, *cos, v); - }; - evolve_step(coeffs, layer, apply_angle, comm_, cos_scale); - }); -} - -template -auto MonomialPropagator::evolve_mode_contract_immediately_(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecD ¶meters, - std::optional only_rotate_len_k) -> void { - auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0); - // Called for the side effect alone: it returns a reference to the very vector selected below. - (void)current_picture_coeffs_(); - VecD *op_coeffs = schrodinger_ ? &mp_op_.state_coeffs : &mp_op_.op_coeffs; - const auto majoranas_size = majoranas.size(); - run_gate_loop_( - majoranas, - only_rotate_len_k, - [this, &mapped_params, op_coeffs, majoranas_size](const VecZ &mono, std::optional rot_len, size_t i) { - const auto [build_angle, apply_angle] = gate_angle_(mapped_params, i, majoranas_size); - // extend_coeffs must run after build_evolve_result_'s self-rank grow and before the apply. - CosMask cos; - detail::FusedContract fc; - bool fused_scale = false; - build_evolve_result_(mono, rot_len, std::cref(*op_coeffs), build_angle, &cos, &fc, op_coeffs, &fused_scale); - extend_coeffs_from_current_picture_if_needed_(*op_coeffs); - detail::apply_fused_contract(fc, *op_coeffs, cos, apply_angle, schrodinger_, fused_scale); - }); -} - -template -auto MonomialPropagator::build_graph(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - std::optional gate_indices, - std::optional parameters, - std::optional only_rotate_len_k) -> void { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); - if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { - s.build_graph(majoranas, parameter_mapping, gen_coeffs, gate_indices, parameters, only_rotate_len_k); - }); - return; - } - if (majoranas.empty()) { - return; - } - validate_coefficient_lengths(parameter_mapping, gen_coeffs); - - VecZ local_gates; - if (gate_indices.has_value()) { - local_gates = std::move(*gate_indices); - } - else { - local_gates.resize(majoranas.size()); - std::iota(local_gates.begin(), local_gates.end(), size_t{0}); - } - validate_gate_indices(local_gates, majoranas.size()); - const size_t gate_offset = n_gates(); - for (auto &g : local_gates) { - g += gate_offset; - } - - if (!parameters.has_value()) { - evolve_mode_build_graph_(majoranas, parameter_mapping, gen_coeffs, local_gates, only_rotate_len_k); - } - else { - // map_params() indexes `parameters` by parameter_mapping, so a too-short vector reads out of bounds. - validate_parameters_length(*parameters, parameter_mapping); - // Coefficient-informed build: seed by contracting the existing graph so atol truncation sees - // realistic coefficients. That graph covers the parameter prefix [0, m). - VecD seed; - if (graph_layers() > 0) { - const auto existing = graph_gate_arrays_(); - const size_t m = expected_num_params(existing.first); - // The per-mapping check above only covers this call's indices, which may all sit above the - // prefix the stored graph needs. Truncating instead would replay the existing graph at a silently - // different point on the axis, and map_params would fail one layer down on the sliced vector. - if (parameters->size() < m) { - throw SeedParametersTooShort( - std::format("Coefficient-informed build_graph() needs at least {} parameter value(s) to replay the " - "existing {}-layer graph as a seed, but got {}.", - m, - graph_layers(), - parameters->size())); - } - const VecD existing_params(parameters->begin(), parameters->begin() + static_cast(m)); - seed = contract_partially(existing_params, false); - } - else { - seed = current_picture_coeffs_(); - } - evolve_mode_graph_with_coeffs_(majoranas, - parameter_mapping, - gen_coeffs, - local_gates, - *parameters, - seed, - only_rotate_len_k); - } -} - -template -auto MonomialPropagator::propagate(const std::vector &majoranas, - const VecZ ¶meter_mapping, - const VecD &gen_coeffs, - const VecD ¶meters, - std::optional only_rotate_len_k) -> void { - validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); - if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { - s.propagate(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); - }); - return; - } - if (majoranas.empty()) { - return; - } - validate_coefficient_lengths(parameter_mapping, gen_coeffs); - validate_parameters_length(parameters, parameter_mapping); - if (graph_layers() > 0) { - throw GraphStateConflict(std::format("Cannot propagate() on top of a non-empty graph of {} layer(s): " - "propagate() evolves and contracts in place and assumes no stored graph. " - "Call contract_partially() to fold the existing graph first, or use " - "build_graph() to extend it.", - graph_layers())); - } - evolve_mode_contract_immediately_(majoranas, parameter_mapping, gen_coeffs, parameters, only_rotate_len_k); - if (!schrodinger_) { - heisenberg_gates_absorbed_ = true; - } -} - -template -auto MonomialPropagator::report_routing_coverage_(const std::vector &majoranas) -> void { - // One report per rank, so only its partition 0 speaks. - if (routing_coverage_reported_ || comm_.shm_rank != 0) { - return; - } - const routing::Router router = router_for(comm_); - if (!router.is_linear()) { - routing_coverage_reported_ = true; - return; // splitmix: no subspace to fall short of - } - std::vector shifts; - shifts.reserve(majoranas.size()); - for (const auto &gate : majoranas) { - // Out-of-range indices are build_evolve_result_'s to reject; leave the latch clear for a retry. - if (std::ranges::any_of(gate, [this](size_t i) { return i >= 2 * logical_num_modes_; })) { - return; - } - shifts.push_back(static_cast(router.rank_shift(indices_to_bitset(gate)))); - } - routing_coverage_reported_ = true; - std::ranges::sort(shifts); - shifts.erase(std::ranges::unique(shifts).begin(), shifts.end()); - const size_t span = routing::gf2_rank(shifts); - if (span >= router.linear_bits()) { - return; - } - // A warning: ownership is still correct, the ranks are just under-used. Reports capacity (cosets), - // not idle ranks, which also depend on how the initial operator straddles them. - const auto line = - std::format("COMMROUTE rank={} linear_bits={} shift_rank={} shifts={} ranks_per_coset={} rank_cosets={}\n", - static_cast(mpi::rank(comm_)) / router.partitions(), - router.linear_bits(), - span, - shifts.size(), - size_t{1} << span, - router.ranks() >> span); - std::fputs(line.c_str(), stderr); - std::fflush(stderr); -} - -template -template -auto MonomialPropagator::run_gate_loop_(const std::vector &majoranas, - std::optional only_rotate_len_k, - EvolutionFunc evolution_func) -> void { - report_routing_coverage_(majoranas); - // Serial per partition; parallelism comes from partitioning the operator across cores. - for (size_t i = 0; i < majoranas.size(); ++i) { - const auto idx = !schrodinger_ ? majoranas.size() - 1 - i : i; - const auto &mono = majoranas[idx]; - evolution_func(mono, only_rotate_len_k, i); - } - - initialize_operator_caches_(); -} - -template -auto MonomialPropagator::build_evolve_result_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs, - std::optional param, - CosMask *out_cos, - detail::FusedContract *fused_contract, - VecD *fused_scale_coeffs, - bool *fused_scale) -> std::shared_ptr { - // The only place a gate generator's indices are bounds-checked: nothing between the public entry - // points and here constrains them. - const auto gen_mono = indices_to_bitset_checked(gen_vec, 2 * logical_num_modes_); - - // The cos-recompute metadata is written onto the returned LayerCore. - return detail::build_layer(mp_op_, - gen_mono, - cutoff_fn_, - lower_atol_, - coeffs, - upper_atol_, - param, - only_rotate_len_k, - matched_scratch_, - comm_, - out_cos, - fused_contract, - schrodinger_, - fused_scale_coeffs, - fused_scale, - basis_); -} - -template -auto MonomialPropagator::propagate_one_(const VecZ &gen_vec, - std::optional only_rotate_len_k, - std::optional> coeffs, - std::optional param, - size_t param_index, - double gen_coeff, - size_t gate_index) -> void { - graph_.append(build_evolve_result_(gen_vec, only_rotate_len_k, coeffs, param), param_index, gen_coeff, gate_index); -} - +/// Per-layer cosine callbacks replaying `graph` against `inverted_index`; each closure captures `options` +/// for its kernels. The closures keep raw pointers into the index, so they must not outlive it. template auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, const MPGraphView &graph, - Basis basis = Basis::Majorana) -> detail::CosCallbacks; - -template -auto MonomialPropagator::evolve_operator_with_recompute_(VecD &&coeffs, - const MPGraphView &graph, - const VecD ¶ms) -> VecD { - const auto &inverted_index = mp_op_.inverted_index(); - // Only the scale side is consumed; build both through the shared builder for consistency. - auto cos_scale = build_cos_callbacks(inverted_index, graph, basis_).scale; - return evolve_operator(std::move(coeffs), graph, params, comm_, cos_scale); -} - -template -auto MonomialPropagator::n_gates() const -> size_t { - if (partition_group_) { - return first_partition_().n_gates(); - } - const size_t count = graph_.layers(); - size_t max_gate = 0; - bool any = false; - for (size_t layer = 0; layer < count; ++layer) { - const size_t g = graph_.get_layer_traversal(layer).gate_index(); - max_gate = any ? std::max(max_gate, g) : g; - any = true; - } - return any ? max_gate + 1 : 0; -} - -template -auto MonomialPropagator::set_parameter_mapping(const VecZ ¶meter_mapping) -> void { - if (partition_group_) { - for_each_partition_([&](MonomialPropagator &s) { s.set_parameter_mapping(parameter_mapping); }); - return; - } - const size_t count = graph_.layers(); - const size_t gates = n_gates(); - - // The LayerCore is shared and immutable, so relabelling copies it and replaces the layer's core. - auto relabel = [this](size_t layer, size_t new_param_index) { - auto &target = graph_.get_layer(layer); - auto new_core = std::make_shared(target.core()); - new_core->param_index = new_param_index; - if (const CosMask *pruned = target.pruned_cos()) { - target = Layer(std::move(new_core), *pruned); - } - else { - target = Layer(std::move(new_core)); - } - }; - - if (parameter_mapping.size() == count) { - // Per-layer mapping in optimizer order. - for (size_t layer = 0; layer < count; ++layer) { - relabel(layer, parameter_mapping[count - 1 - layer]); - } - } - else if (parameter_mapping.size() == gates) { - // Per-gate mapping, indexed by absolute gate index. - for (size_t layer = 0; layer < count; ++layer) { - relabel(layer, parameter_mapping[graph_.get_layer_traversal(layer).gate_index()]); - } - } - else { - throw GraphStateConflict(std::format("parameter_mapping has {} entries; expected {} (per graph " - "layer) or {} (per gate).", - parameter_mapping.size(), - count, - gates)); - } -} - -template -auto MonomialPropagator::graph_gate_arrays_() const -> std::pair { - if (partition_group_) { - return first_partition_().graph_gate_arrays_(); - } - const size_t count = graph_.layers(); - VecZ parameter_mapping(count); - VecD gen_coeffs(count); - // Layers store gate info in simulation order; the evaluation machinery expects optimizer order (the reverse). - for (size_t layer = 0; layer < count; ++layer) { - const auto traversal = graph_.get_layer_traversal(layer); - const size_t optimizer_index = count - 1 - layer; - parameter_mapping[optimizer_index] = traversal.param_index(); - gen_coeffs[optimizer_index] = traversal.gen_coeff(); - } - return {std::move(parameter_mapping), std::move(gen_coeffs)}; -} + Basis basis = Basis::Majorana, + detail::parallel::Options options = {}) -> detail::CosCallbacks; template -auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, const MPGraphView &graph, Basis basis) - -> detail::CosCallbacks { - struct LayerCos { - bool recomputes_cos = false; - detail::LazyFold recipe{}; // used iff recomputes_cos - const CosMask *filtered = nullptr; // points into a pruned layer's stored cos - }; - auto cache = std::make_shared>(); - cache->reserve(graph.layers()); - for (size_t i = 0; i < graph.layers(); ++i) { - const auto &layer = graph.get_layer(i); - LayerCos entry; - if (const CosMask *pruned = layer.pruned_cos(); pruned != nullptr) { - entry.recomputes_cos = false; - entry.filtered = pruned; - } - else { - entry.recomputes_cos = true; - const auto t = layer.traversal(); - const auto gen = detail::generator_from_words(t.generator_words()); - entry.recipe = detail::make_lazy_fold(inverted_index, gen, t.scaled_count(), basis); - } - cache->push_back(std::move(entry)); - } - - const auto *sc = &inverted_index; - detail::LayerCosScale cos_scale = [cache, sc](size_t i, double *c, double v) { - const auto &e = (*cache)[i]; - if (!e.recomputes_cos) { - detail::scale_cos_mask(c, *e.filtered, v); - } - else { - detail::scale_cos_lazy(*sc, e.recipe, c, v); - } - }; - detail::LayerCosAccumulate cos_acc = [cache, sc](size_t i, double *s, double *h, double v, double sec) { - const auto &e = (*cache)[i]; - if (!e.recomputes_cos) { - return detail::accumulate_cos_mask(s, h, *e.filtered, v, sec); - } - return detail::accumulate_cos_lazy(*sc, e.recipe, s, h, v, sec); - }; - detail::LayerCosIndices cos_inds = [cache, sc](size_t i, std::vector &out) { - const auto &e = (*cache)[i]; - if (!e.recomputes_cos) { - detail::cos_indices_mask(*e.filtered, out); - return; - } - detail::cos_indices_lazy(*sc, e.recipe, out); - }; - return {.scale = std::move(cos_scale), .accumulate = std::move(cos_acc), .indices = std::move(cos_inds)}; -} - -template -template -auto MonomialPropagator::make_functional_(Fn &&func, std::optional pare_threshold) - -> std::function { - auto gate_arrays = graph_gate_arrays_(); - auto parameter_mapping = std::move(gate_arrays.first); - auto gen_coeffs = std::move(gate_arrays.second); - const auto num_params = expected_num_params(parameter_mapping); - - // Nothing here needs a dense state: energy only dots it against the evolved operator, and the gradient - // scatters it into its own thread-local scratch before back-evolving. So Heisenberg hands over just the - // sparse scores; Schrödinger's state is the live evolved vector, snapshotted whole. - // EvalState owns its rows and snapshots the term count -- a later append push_backs onto the operator's - // sparse rows, which would both dangle a view and outrun the `op` captured below. - const auto num_terms = mp_op_.size(); - auto state = [&] { - if (schrodinger_) { - return EvalState::dense(mp_op_.dense_state()); - } - const auto sparse = mp_op_.sparse_state(); - return EvalState::sparse(num_terms, sparse.rows, sparse.values); - }(); - VecD op = mp_op_.get_operator(); - const auto core_term = this->core_term(); - const auto comm = comm_; - - const auto expected_layers = graph_layers(); - // Aliased rather than copied: the check below needs the live counter, like graph->layers(). - const auto *epoch = &initial_operator_epoch_; - const auto expected_epoch = initial_operator_epoch_; - const auto &inverted_index = mp_op_.inverted_index(); - - // One owning handle either way: pare hands back a heap-owned MPGraph the functional must keep alive - // (build_cos_callbacks holds pointers into its layers' stored cos); non-pare aliases graph_. - std::shared_ptr graph; - if (pare_threshold.has_value()) { - auto full_cos_of_layer = [this, &inverted_index](size_t i) -> CosMask { - const auto layer = graph_.get_layer_traversal(i); - const auto gen = detail::generator_from_words(layer.generator_words()); - const auto combined = detail::make_fold_cache(inverted_index, gen, layer.scaled_count(), basis_); - return detail::fold_to_cos_mask(combined); - }; - // Threshold the picture's driving vector: the Hamiltonian in Schrödinger, the state otherwise. - const auto keep = schrodinger_ ? indices_above(op, *pare_threshold) : state.indices_above(*pare_threshold); - const auto count = schrodinger_ ? op.size() : state.length(); - graph = - std::make_shared(pare_graph(graph_, keep, count, schrodinger_, comm_, full_cos_of_layer)); - } - else { - graph = std::shared_ptr(std::shared_ptr{}, &graph_); - } - - // The folds keep raw column pointers into this propagator's inverted index, so the returned callable - // must not outlive the propagator. - auto cos = build_cos_callbacks(inverted_index, graph->replay_view(), basis_); - - return [func = std::move(func), - core_term, - state = std::move(state), - op = std::move(op), - graph = std::move(graph), - parameter_mapping, - gen_coeffs, - num_params, - epoch, - expected_epoch, - expected_layers, - cos = std::move(cos), - comm](const VecD ¶ms) -> R { - validate_expected_initial_operator(*epoch, expected_epoch); - validate_functional_call(params, num_params); - validate_expected_graph_layers(graph->layers(), expected_layers); - return func(EvalRequest{.e_core = core_term, - .state = state, - .op = op, - .parameter_mapping = parameter_mapping, - .gen_coeffs = gen_coeffs, - .graph = graph->replay_view(), - .params = params}, - comm, - cos); - }; -} - -template -auto MonomialPropagator::expectation_value_functional(std::optional pare_threshold) - -> std::function { - if (partition_group_) { - // Each partition allreduces internally, so partition 0 is the global value. The group is captured by - // raw pointer, so the returned callable must not outlive this propagator. - auto fns = std::make_shared>>( - map_partitions_([&](MonomialPropagator &s) { return s.expectation_value_functional(pare_threshold); })); - auto *grp = partition_group_.get(); - return [grp, fns](const VecD ¶ms) -> double { - return detail::partition::collect_on_all(*grp, - [&](int r) { return (*fns)[static_cast(r)](params); })[0]; - }; - } - return make_functional_(ev_fn, pare_threshold); -} - -template -auto MonomialPropagator::expectation_value_and_gradient_functional(std::optional pare_threshold) - -> std::function(const VecD &)> { - if (partition_group_) { - auto fns = std::make_shared(const VecD &)>>>(map_partitions_( - [&](MonomialPropagator &s) { return s.expectation_value_and_gradient_functional(pare_threshold); })); - auto *grp = partition_group_.get(); - return [grp, fns](const VecD ¶ms) -> std::pair { - return detail::partition::collect_on_all(*grp, - [&](int r) { return (*fns)[static_cast(r)](params); })[0]; - }; - } - return make_functional_(ev_and_grad_fn, pare_threshold); -} - -template -auto MonomialPropagator::expectation_value(const VecD ¶meters) -> double { - if (partition_group_) { - // Each partition allreduces internally, so every partition returns the global value; take partition 0. - return map_partitions_([&](MonomialPropagator &s) { return s.expectation_value(parameters); })[0]; - } - return expectation_value_functional(std::nullopt)(parameters); -} - -template -auto MonomialPropagator::expectation_value_and_gradient(const VecD ¶meters) -> std::pair { - if (partition_group_) { - // As in expectation_value(): the gradient is allreduced inside each partition. - return map_partitions_([&](MonomialPropagator &s) { return s.expectation_value_and_gradient(parameters); })[0]; - } - return expectation_value_and_gradient_functional(std::nullopt)(parameters); -} - -template -auto MonomialPropagator::contract_partially(const VecD ¶meters, bool inplace) -> VecD { - if (partition_group_) { - return concat_partitions_([&](MonomialPropagator &s) { return s.contract_partially(parameters, inplace); }); - } - const auto gate_arrays = graph_gate_arrays_(); - const auto ¶meter_mapping = gate_arrays.first; - const auto &gen_coeffs = gate_arrays.second; - validate_parameters_length(parameters, parameter_mapping); - - if (parameters.empty()) { - return current_picture_coeffs_(); - } - - const size_t num_majoranas = parameter_mapping.size(); - // Inplace slicing produces an owned MPGraph that must be bound to a named local before viewing - // (never view a temporary); slice_view() views this graph's still-live layers directly. - if (schrodinger_) { - const auto &state = mp_op_.dense_state(); - const auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, -1.0); - VecD evolved_state; - if (inplace) { - const MPGraph sliced = graph_.slice_graph(num_majoranas, true); - evolved_state = evolve_operator_with_recompute_(VecD(state), sliced.replay_view(), mapped_params); - mp_op_.state_coeffs = evolved_state; - } - else { - evolved_state = - evolve_operator_with_recompute_(VecD(state), graph_.slice_view(num_majoranas), mapped_params); - } - return evolved_state; - } - - const auto &op = mp_op_.get_operator(); - const auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0, true); - VecD evolved_op; - if (inplace) { - const MPGraph sliced = graph_.slice_graph(num_majoranas, true); - evolved_op = evolve_operator_with_recompute_(VecD(op), sliced.replay_view(), mapped_params); - mp_op_.op_coeffs = evolved_op; - heisenberg_gates_absorbed_ = true; - } - else { - evolved_op = evolve_operator_with_recompute_(VecD(op), graph_.slice_view(num_majoranas), mapped_params); - } - return evolved_op; -} - -template -auto MonomialPropagator::evolved_operator_terms(const VecD ¶meters, double atol) - -> std::vector>> { - using Term = std::pair>; - // `p` is always unpartitioned here (a partition, or *this), so indexing() is available. - const auto collect = [&](MonomialPropagator &p) -> std::vector { - std::vector terms; - const VecD evolved = p.contract_partially(parameters, false); - p.indexing().for_each([&](const auto &mono, size_t idx) { - if (idx >= evolved.size()) { - return; - } - const double coeff = evolved[idx]; - if (std::abs(coeff) < atol) { - return; - } - // Round to drop anti-hermitian numerical noise (Majorana un-applies the Hermitian phase). - const auto decoded = algebra_decode_coeff(basis_, coeff, mono); - const std::complex rounded(std::round(decoded.real() * 1e12) / 1e12, - std::round(decoded.imag() * 1e12) / 1e12); - terms.emplace_back(bitset_to_indices(mono), rounded); - }); - return terms; - }; - if (!partition_group_) { - return collect(*this); - } - return concat_partitions_(collect); -} - -namespace detail { - -/// Coefficients of `keys` in `p`'s evolved operator, in query order; a key `p` does not carry yields 0. -/// `p` must be unpartitioned -- a partition, or a propagator that was never partitioned -- so that -/// indexing() is available. Non-inplace. Rank-local. -template -auto probe_coefficients(MonomialPropagator &p, - const VecD ¶meters, - const std::vector> &keys) -> std::vector> { - const VecD evolved = p.contract_partially(parameters, false); - std::vector rows(keys.size()); - p.indexing().find_batch(keys.data(), keys.size(), rows.data()); - std::vector> found(keys.size()); - for (size_t q = 0; q < keys.size(); ++q) { - if (rows[q] >= evolved.size()) { // kNotFound is size_t max, so this covers a miss too - continue; - } - found[q] = algebra_decode_coeff(p.basis(), evolved[rows[q]], keys[q]); - } - return found; -} - -} // namespace detail - -template -auto MonomialPropagator::evolved_operator_coefficients(const VecD ¶meters, const std::vector &terms) - -> std::vector> { - // Checked: these are user-supplied indices. Past the storage width the unchecked encode's bit - // position underflows into an out-of-bounds write; between the logical and storage widths it - // encodes a monomial outside this propagator's system, which reads back 0 instead of raising. - auto keys = terms | std::views::transform([this](const VecZ &term) { - return indices_to_bitset_checked(term, 2 * logical_num_modes_); - }) - | std::ranges::to>>(); - - // map_partitions_ takes a callable over MonomialPropagator &, so the free function is bound to - // this query rather than called directly. - const auto probe = [¶meters, &keys](MonomialPropagator &p) { - return detail::probe_coefficients(p, parameters, keys); - }; - - std::vector> out(terms.size()); - if (!partition_group_) { - out = probe(*this); - } - else { - // The partitions are disjoint, so at most one contributes per key. - for (const auto &partition_found : map_partitions_(probe)) { - for (size_t q = 0; q < out.size(); ++q) { - out[q] += partition_found[q]; - } - } - } - - // Round off the anti-hermitian numerical noise. - for (auto &coeff : out) { - coeff = {std::round(coeff.real() * 1e12) / 1e12, std::round(coeff.imag() * 1e12) / 1e12}; - } - - // Heisenberg only: the empty monomial is diverted to core_term_ instead of indexed, so the probe - // cannot have found it. - if (!schrodinger_) { - const auto core = core_term(); - for (size_t q = 0; q < terms.size(); ++q) { - if (terms[q].empty()) { - out[q] = {core, 0.0}; - } - } - } - return out; -} - -template -auto MonomialPropagator::term_expectation_values(const VecD ¶meters, const std::vector &terms) - -> std::vector> { - // Absorbed gates are no longer in the graph, so the replay would silently leave them out of every value. - // The facade never sets its own flag, so check a partition's (they all agree) before fanning out. - if (partition_group_ ? first_partition_().heisenberg_gates_absorbed_ : heisenberg_gates_absorbed_) { - throw GraphStateConflict("term_expectation_values() is unavailable once propagate() or an in-place " - "contract_partially() has absorbed gates into a Heisenberg operator, because " - "the values would miss those gates. Call it after build_graph() and before " - "contracting in place."); - } - if (partition_group_) { - // Each partition allreduces internally, so every partition returns the global values. - return map_partitions_([&](MonomialPropagator &s) { return s.term_expectation_values(parameters, terms); })[0]; - } - // Checked: these are user-supplied indices (see evolved_operator_coefficients()). - const auto keys = terms | std::views::transform([this](const VecZ &term) { - return indices_to_bitset_checked(term, 2 * logical_num_modes_); - }) - | std::ranges::to>>(); - std::vector rows(keys.size()); - mp_op_.store->find_batch(keys.data(), keys.size(), rows.data()); - - const auto ev_op_gradient_fn = [](const EvalRequest &request, mpi::Comm comm, const detail::CosCallbacks &cos) { - return ev_op_gradient(request, comm, cos); - }; - const VecD values = make_functional_(ev_op_gradient_fn, std::nullopt)(parameters); - - // Each key lives on one rank, so one allreduce gathers each key's value and a found flag: values - // first, then flags. - const size_t count = keys.size(); - VecD gathered(2 * count, 0.0); - for (size_t q = 0; q < count; ++q) { - if (rows[q] < values.size()) { // kNotFound is size_t max, so this covers a miss too - gathered[q] = values[rows[q]]; - gathered[count + q] = 1.0; - } - } - mpi::allreduce_sum_inplace(gathered, comm_); - - std::vector> out(count); - for (size_t q = 0; q < count; ++q) { - if (terms[q].empty()) { // the core term is a fixed identity, which every state reads as 1 - out[q] = 1.0; - continue; - } - if (gathered[count + q] == 0.0) { - if (!schrodinger_) { - throw detail::OperatorTermNotFound( - std::format("Operator term [{}] not found in the evolved operator. To evaluate a term outside " - "the initial operator, add it with coefficient 0 before building the graph.", - join_with_separator(terms[q], ", "))); - } - continue; - } - // The engine stores row M as the Hermitian h·M, with h = 1 or i, so gathered[q] is ⟨h·M⟩. - // To recover ⟨M⟩ we divide by h, which is equivalent to conjugation. - out[q] = std::conj(algebra_decode_coeff(basis_, gathered[q], keys[q])); - } - return out; +auto build_cos_callbacks(const detail::InvertedIndex &inverted_index, + const MPGraphView &graph, + Basis basis, + detail::parallel::Options options) -> detail::CosCallbacks { + return detail::make_cos_callbacks(inverted_index, graph, basis, options); } } // namespace monoprop + +#include "monoprop/detail/monomial_propagator/ShardedPropagator.inl" diff --git a/cpp/monoprop/detail/monomial_propagator/ShardedPropagator.inl b/cpp/monoprop/detail/monomial_propagator/ShardedPropagator.inl new file mode 100644 index 000000000..bf863412a --- /dev/null +++ b/cpp/monoprop/detail/monomial_propagator/ShardedPropagator.inl @@ -0,0 +1,1028 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// Included only by MonomialPropagator.inl. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/OperationFailure.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/Construction.h" +#include "monoprop/detail/sharded/Evaluation.h" +#include "monoprop/detail/sharded/RootObserver.h" +#include "monoprop/detail/sharded/State.h" +#include "monoprop/detail/sharded/Team.h" + +/* + * The sharded OpenMP root: MonomialPropagator over the T shard states of one rank. + * + * Ownership. The root owns configuration, the ordinary communicator of P ranks, the captured budget (T threads = T + * shards), the prepared (P, T) router, the replicated identity coefficient, the initial-operator epoch, validity, and + * this rank's T shard states (flat owners rank * T + t). It holds no operator, graph or matched marks of its own and + * never dispatches to child propagators. Every operation that touches substantial shard state runs it on the shard's + * owner in one operation-scoped team: seeding and copying (State.h), construction and propagation (Construction.h), + * retained preparation, evaluation and replay (Evaluation.h), and the root's own owner phases (initial-operator + * updates, remapping, picture copies). The caller performs only rank-level metadata work and small per-shard + * view/callback setup before a team. + * + * Failure. Arguments are validated before any team. A seam returns its error after its team has joined, with + * whether mutation began; the root transfers the mutation flag before rethrowing under run_operation_, so a + * post-mutation failure invalidates this object (mutation_failed_) and reaches mpi::operation_failed() before any + * result reduction, while a pre-mutation failure leaves it usable. There is no rollback. + * + * Ranks. Every rank constructs and calls the root collectively, with the same arguments. Rank-local results + * (exports, contraction blocks, size() and the memory aggregates) cover this rank's shards; energies and gradients + * are reduced over the communicator after the ascending-shard fold, with the identity once. Inside a team only the + * primary, which is MPI's initializing thread, makes MPI calls, so MPI_THREAD_FUNNELED suffices. + * + * Launch contract (not checked): exactly T workers per team, OMP_DYNAMIC=FALSE and no limit below T, the same T on + * every rank; public calls outside any OpenMP region. Checked: every MPI-using public call comes from MPI's + * initializing thread (a wrong thread fails fast, locally), and initialized MPI provides at least FUNNELED. + */ + +namespace monoprop { + +template +MonomialPropagator::MonomialPropagator(const OperatorDict &initial_operator, + unsigned int cutoff, + const VecZ &initial_state, + std::optional schrodinger_cutoff, + mpi::Comm comm, + std::optional lower_atol, + std::optional upper_atol, + CutoffType cutoff_type, + std::optional> basis_change, + size_t logical_num_modes, + Basis basis) + : MonomialPropagator(ObservedTag{}, + nullptr, + initial_operator, + cutoff, + initial_state, + schrodinger_cutoff, + comm, + lower_atol, + upper_atol, + cutoff_type, + std::move(basis_change), + logical_num_modes, + basis) {} + +template +MonomialPropagator::MonomialPropagator(ObservedTag /*tag*/, + const detail::sharded::RootObserver *observer, + const OperatorDict &initial_operator, + unsigned int cutoff, + const VecZ &initial_state, + std::optional schrodinger_cutoff, + mpi::Comm comm, + std::optional lower_atol, + std::optional upper_atol, + CutoffType cutoff_type, + std::optional> basis_change, + size_t logical_num_modes, + Basis basis) + : schrodinger_{schrodinger_cutoff.has_value()}, + comm_{comm}, + cutoff_{cutoff}, + lower_atol_{lower_atol}, + upper_atol_{upper_atol}, + logical_num_modes_{logical_num_modes}, + cutoff_type_{cutoff_type}, + basis_change_{std::move(basis_change)}, + basis_{basis}, + observer_{observer} { + // The geometry is the launch's: P ranks of comm x the captured budget T. No argument or other environment variable + // selects or checks it (the removed partition runtime's monoprop_PARTITIONS is never read). + // Before any MPI query or collective in this constructor. + mpi::require_initializing_thread(); + try { + mpi::require_thread_support(); + } + catch (...) { + // A rank that threw here would strand its peers in the routing agreement below. + mpi::operation_failed(comm, std::current_exception()); + } + const auto ranks = static_cast(mpi::size(comm)); + try { + parallel_ = detail::parallel::capture_thread_budget(); + } + catch (const std::invalid_argument &e) { + throw PropagatorConfigError(e.what()); + } + const auto threads = static_cast(parallel_.threads); + + if (logical_num_modes_ == 0 || logical_num_modes_ > NumModes) { + throw PropagatorConfigError( + std::format("logical_num_modes ({}) must be in the range [1, {}].", logical_num_modes_, NumModes)); + } + validate_cutoff_config_(cutoff_type_, basis_change_); + if (upper_atol.has_value() && lower_atol.has_value() && (upper_atol.value() < lower_atol.value())) { + throw PropagatorConfigError(std::format("upper_atol ({}) must be greater than or equal to lower_atol ({}).", + upper_atol.value(), + lower_atol.value())); + } + + std::optional paired; + if (schrodinger_) { + // The basis is shared by the P * T flat owners. + paired = detail::sharded::paired_basis_bounds(*schrodinger_cutoff, logical_num_modes_, ranks * threads); + if (!paired->countable() || paired->share >= detail::OperatorIndex::kIndexCeiling) { + const auto how_many = + paired->countable() ? std::format("{}", paired->global_terms) : std::string("more than 2^64"); + const auto per_shard = paired->countable() ? std::format("{}", paired->share) : std::string("as many"); + throw PropagatorConfigError( + std::format("schrodinger_cutoff ({}) admits {} paired basis terms over {} active modes, about {} per " + "shard -- more than can be walked or addressed. Lower schrodinger_cutoff, or raise the " + "rank x thread count P * T (currently {}).", + *schrodinger_cutoff, + how_many, + logical_num_modes_, + per_shard, + ranks * threads)); + } + } + + // Routing mode and seed agree across the communicator (collective; every rank got here on replicated input). A + // linear router then rejects a non-power-of-two P on every rank together. + check_routing_agreement(comm_); + world_ = detail::sharded::PhysicalWorld::of(comm_); + router_.emplace(routing::make_router(ranks, threads)); + + const double core_term = + detail::sharded::validate_initial_operator(initial_operator, basis_, logical_num_modes_); + // Before seeding: the packed-row width derives from the cutoff predicate. + regenerate_cutoff_fn_(); + const auto seed = detail::sharded::OperatorSeed{ + .initial_operator = initial_operator, + .initial_state = initial_state, + .router = *router_, + .basis = basis_, + .logical_num_modes = logical_num_modes_, + .paired = paired, + .inline_width = detail::sharded::packed_inline_width(schrodinger_, cutoff_fn_)}; + shards_ = detail::sharded::seed_shards(parallel_, + seed, + world_.rank, + detail::sharded::RootShardObserver{observer_}); + core_term_ = core_term; +} + +template +MonomialPropagator::MonomialPropagator(const MonomialPropagator &other) + : schrodinger_(checked_source_(other).schrodinger_), + comm_(other.comm_), + cutoff_fn_(other.cutoff_fn_), + cutoff_(other.cutoff_), + lower_atol_(other.lower_atol_), + upper_atol_(other.upper_atol_), + core_term_(other.core_term_), + initial_operator_epoch_(other.initial_operator_epoch_), + heisenberg_gates_absorbed_(other.heisenberg_gates_absorbed_), + routing_coverage_reported_(other.routing_coverage_reported_), + logical_num_modes_(other.logical_num_modes_), + cutoff_type_(other.cutoff_type_), + basis_change_(other.basis_change_), + basis_(other.basis_), + parallel_(other.parallel_), + router_(other.router_), + world_(other.world_), + observer_(other.observer_), + // The source's budget, not a new capture: a copy keeps T. + shards_(detail::sharded::copy_shards(other.parallel_, + other.shards_, + detail::sharded::RootShardObserver{other.observer_})) {} + +template +auto MonomialPropagator::mutation_failed_(std::exception_ptr error) -> void { + invalidate_(); + // A rank-local operation (settings, the parameter mapping) may run on any thread, so no MPI call is made before + // the thread is known: one rank has no peer to release and rethrows; above one rank only MPI's initializing thread + // may abort the communicator, and any other thread stops this process locally. + if (world_.ranks == 1) { + std::rethrow_exception(std::move(error)); + } + mpi::require_initializing_thread(); + mpi::operation_failed(comm_, std::move(error)); +} + +template +auto MonomialPropagator::sole_shard_(const char *what) const + -> const detail::sharded::ShardState & { + require_valid_(); + if (shards_.size() != 1) { + throw MultiShardUnsupported(std::format( + "{} is only available when this rank holds one shard, but it holds {} (one per thread of the " + "budget captured at construction), and no single store represents the rank. Launch with " + "monoprop_NUM_THREADS=1 (on every rank) to inspect the raw layout, or use the shard-transparent " + "accessors (size(), graph_size(), the memory breakdowns, evolved_operator_terms(), ...).", + what, + shards_.size())); + } + return *shards_.front(); +} + +template +auto MonomialPropagator::sole_shard_(const char *what) -> detail::sharded::ShardState & { + static_cast(std::as_const(*this).sole_shard_(what)); // the checks + return *shards_.front(); +} + +template +auto MonomialPropagator::graph_size() const -> std::pair { + require_valid_(); + auto total = std::pair{0, 0}; + for (const auto &state : shards_) { + // May rebuild a stale inverted index (a lazy cache); construction leaves every index current. + total.first += cos_index_count_of_(state->graph, state->op, basis_); + total.second += state->graph.total_cycles(); + } + return total; +} + +template +auto MonomialPropagator::graph_data() const -> std::vector { + const auto &state = sole_shard_("graph_data()"); + return graph_data_of_(state.graph, state.op, basis_); +} + +template +auto MonomialPropagator::n_gates() const -> size_t { + require_valid_(); + // Layer metadata is identical on every shard. + const MPGraph &graph = shards_.front()->graph; + const size_t count = graph.layers(); + size_t max_gate = 0; + bool any = false; + for (size_t layer = 0; layer < count; ++layer) { + const size_t g = graph.get_layer_traversal(layer).gate_index(); + max_gate = any ? std::max(max_gate, g) : g; + any = true; + } + return any ? max_gate + 1 : 0; +} + +template +auto MonomialPropagator::graph_gate_arrays_() const -> std::pair { + const MPGraph &graph = shards_.front()->graph; + const size_t count = graph.layers(); + VecZ parameter_mapping(count); + VecD gen_coeffs(count); + // Layers store gate info in simulation order; the evaluation machinery expects optimizer order (the reverse). + for (size_t layer = 0; layer < count; ++layer) { + const auto traversal = graph.get_layer_traversal(layer); + const size_t optimizer_index = count - 1 - layer; + parameter_mapping[optimizer_index] = traversal.param_index(); + gen_coeffs[optimizer_index] = traversal.gen_coeff(); + } + return {std::move(parameter_mapping), std::move(gen_coeffs)}; +} + +template +auto MonomialPropagator::report_routing_coverage_(const std::vector &majoranas) -> void { + if (routing_coverage_reported_) { + return; + } + const routing::Router &router = *router_; + if (!router.is_linear()) { + // One rank (or splitmix): no rank subspace to fall short of. + routing_coverage_reported_ = true; + return; + } + std::vector shifts; + shifts.reserve(majoranas.size()); + for (const auto &gate : majoranas) { + shifts.push_back(static_cast(router.rank_shift(indices_to_bitset(gate)))); + } + routing_coverage_reported_ = true; + std::ranges::sort(shifts); + shifts.erase(std::ranges::unique(shifts).begin(), shifts.end()); + const size_t span = routing::gf2_rank(shifts); + if (span >= router.linear_bits()) { + return; + } + const auto line = + std::format("COMMROUTE rank={} linear_bits={} shift_rank={} shifts={} ranks_per_coset={} rank_cosets={}\n", + mpi::rank(comm_), + router.linear_bits(), + span, + shifts.size(), + size_t{1} << span, + router.ranks() >> span); + std::fputs(line.c_str(), stderr); + std::fflush(stderr); +} + +template +auto MonomialPropagator::generators_(const std::vector &majoranas) const -> MonomialList { + MonomialList generators; + generators.reserve(majoranas.size()); + for (const auto &gate : majoranas) { + generators.push_back(indices_to_bitset_checked(gate, 2 * logical_num_modes_)); + } + return generators; +} + +template +auto MonomialPropagator::construction_context_() const -> detail::sharded::ConstructionContext { + return detail::sharded::ConstructionContext{.cutoff_fn = cutoff_fn_, + .router = *router_, + .lower_atol = lower_atol_, + .upper_atol = upper_atol_, + .basis = basis_, + .schrodinger = schrodinger_, + .world = world_, + .rounds = rounds_.get()}; +} + +template +auto MonomialPropagator::finish_construction_(const detail::sharded::ConstructionOutcome &outcome, + bool &mutation_started) -> void { + mutation_started = mutation_started || outcome.mutation_started; + if (outcome.error) { + std::rethrow_exception(outcome.error); + } +} + +template +template +auto MonomialPropagator::run_owner_phase_(detail::sharded::RootWork work, Body &&body) -> std::exception_ptr { + return detail::sharded::run_team(parallel_, [&](size_t shard, detail::sharded::TeamFailure &failure) noexcept { + // The only phase: its decision ends the sequence either way. + static_cast(detail::sharded::phase(failure, shard, [&] { + detail::sharded::notify_root(observer_, work, shard); + body(shard); + })); + }); +} + +template +auto MonomialPropagator::build_graph(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + std::optional gate_indices, + std::optional parameters, + std::optional only_rotate_len_k) -> void { + run_operation_(true, [&](bool &mutation_started) { + validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); + if (majoranas.empty()) { + return; + } + validate_coefficient_lengths(parameter_mapping, gen_coeffs); + validate_generators_(majoranas); + VecZ local_gates; + if (gate_indices.has_value()) { + local_gates = *gate_indices; + } + else { + local_gates.resize(majoranas.size()); + std::iota(local_gates.begin(), local_gates.end(), size_t{0}); + } + validate_gate_indices(local_gates, majoranas.size()); + VecD mapped_params; + // Replay angles of the existing layers, exactly as contract_partially() computes them, for the seed. + std::optional seed_params; + if (parameters.has_value()) { + validate_parameters_length(*parameters, parameter_mapping); + if (graph_layers() > 0) { + const auto [existing_mapping, existing_gen_coeffs] = graph_gate_arrays_(); + const size_t m = expected_num_params(existing_mapping); + if (parameters->size() < m) { + throw SeedParametersTooShort(std::format( + "Coefficient-informed build_graph() needs at least {} parameter value(s) to replay the " + "existing {}-layer graph as a seed, but got {}.", + m, + graph_layers(), + parameters->size())); + } + const VecD prefix(parameters->begin(), parameters->begin() + static_cast(m)); + seed_params = schrodinger_ ? map_params(prefix, existing_mapping, existing_gen_coeffs, -1.0) + : map_params(prefix, existing_mapping, existing_gen_coeffs, 1.0, true); + } + mapped_params = map_params(*parameters, parameter_mapping, gen_coeffs, 1.0); + } + const size_t gate_offset = n_gates(); + for (auto &g : local_gates) { + g += gate_offset; + } + const auto generators = generators_(majoranas); + report_routing_coverage_(majoranas); + const auto ctx = construction_context_(); + const auto circuit = detail::sharded::GraphCircuit{.generators = generators, + .parameter_mapping = parameter_mapping, + .gen_coeffs = gen_coeffs, + .gate_indices = local_gates, + .only_rotate_len_k = only_rotate_len_k}; + const auto observer = detail::sharded::RootConstructionObserver{observer_}; + if (!parameters.has_value()) { + finish_construction_(detail::sharded::build_graph(parallel_, shards_, ctx, circuit, observer), + mutation_started); + return; + } + const auto informed = detail::sharded::InformedCircuit{ + .graph = circuit, + .mapped_params = mapped_params, + .seed_params = seed_params ? std::optional>(*seed_params) : std::nullopt}; + finish_construction_( + detail::sharded::build_graph_informed(parallel_, shards_, ctx, informed, observer), + mutation_started); + }); +} + +template +auto MonomialPropagator::propagate(const std::vector &majoranas, + const VecZ ¶meter_mapping, + const VecD &gen_coeffs, + const VecD ¶meters, + std::optional only_rotate_len_k) -> void { + run_operation_(true, [&](bool &mutation_started) { + validate_only_rotate_len_k_(only_rotate_len_k, 2 * logical_num_modes_); + if (majoranas.empty()) { + return; + } + validate_coefficient_lengths(parameter_mapping, gen_coeffs); + validate_parameters_length(parameters, parameter_mapping); + validate_generators_(majoranas); + if (graph_layers() > 0) { + throw GraphStateConflict( + std::format("Cannot propagate() on top of a non-empty graph of {} layer(s): " + "propagate() evolves and contracts in place and assumes no stored graph. " + "Call contract_partially() to fold the existing graph first, or use " + "build_graph() to extend it.", + graph_layers())); + } + const auto mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, 1.0); + const auto generators = generators_(majoranas); + report_routing_coverage_(majoranas); + const auto ctx = construction_context_(); + const auto circuit = detail::sharded::PropagationCircuit{.generators = generators, + .mapped_params = mapped_params, + .only_rotate_len_k = only_rotate_len_k}; + finish_construction_(detail::sharded::propagate(parallel_, + shards_, + ctx, + circuit, + detail::sharded::RootConstructionObserver{observer_}), + mutation_started); + if (!schrodinger_) { + heisenberg_gates_absorbed_ = true; + } + }); +} + +template +auto MonomialPropagator::set_parameter_mapping(const VecZ ¶meter_mapping) -> void { + run_operation_(false, [&](bool &mutation_started) { + const size_t count = graph_layers(); + const size_t gates = n_gates(); + const bool per_layer = parameter_mapping.size() == count; + if (!per_layer && parameter_mapping.size() != gates) { + throw GraphStateConflict(std::format("parameter_mapping has {} entries; expected {} (per graph " + "layer) or {} (per gate).", + parameter_mapping.size(), + count, + gates)); + } + mutation_started = true; + // The LayerCore is shared and immutable, so relabelling copies it and replaces the layer's core; each owner + // copies its own shard's cores. + const auto error = run_owner_phase_(detail::sharded::RootWork::remap, [&](size_t shard) { + MPGraph &graph = shards_[shard]->graph; + for (size_t layer = 0; layer < count; ++layer) { + // Per-layer mapping in optimizer order; per-gate mapping indexed by absolute gate index. + const size_t new_param_index = per_layer + ? parameter_mapping[count - 1 - layer] + : parameter_mapping[graph.get_layer_traversal(layer).gate_index()]; + auto &target = graph.get_layer(layer); + auto new_core = std::make_shared(target.core()); + new_core->param_index = new_param_index; + if (const CosMask *pruned = target.pruned_cos()) { + target = Layer(std::move(new_core), *pruned); + } + else { + target = Layer(std::move(new_core)); + } + } + }); + if (error) { + std::rethrow_exception(error); + } + }); +} + +template +auto MonomialPropagator::apply_initial_operator_(const OperatorDict &op_dict) + -> std::pair, VecD> { + return run_operation_(true, [&](bool &mutation_started) -> std::pair, VecD> { + // Read-only pass on the caller: every index is checked and routed with the shards' own (P, T) ownership + // before anything, the epoch included, changes. Each rank applies its own shards' share only. + const routing::Router &router = *router_; + const size_t first_local = world_.rank * shards_.size(); + std::vector shares(shards_.size()); + std::optional new_core_term; + for (const auto &[ind, coeff] : op_dict) { + const auto mono = indices_to_bitset_checked(ind, 2 * logical_num_modes_); + if (ind.empty()) { // the replicated identity is rank-level metadata + new_core_term = algebra_encode_coeff(basis_, coeff, mono); + continue; + } + const size_t owner = find_rank(mono, router); + if (owner >= first_local && owner - first_local < shares.size()) { + shares[owner - first_local][bitset_to_indices(mono)] = coeff; + } + } + // A term the operator lacks is an argument error: reject it before the epoch moves, as one shard alone would. + for (size_t shard = 0; shard < shares.size(); ++shard) { + shards_[shard]->op.require_updatable_terms(shares[shard], schrodinger_); + } + + mutation_started = true; + ++initial_operator_epoch_; + if (new_core_term) { + core_term_ = *new_core_term; + } + std::vector, VecD>> updated(shards_.size()); + const auto error = run_owner_phase_(detail::sharded::RootWork::initial_operator, [&](size_t shard) { + updated[shard] = shards_[shard]->op.update_initial_operator(shares[shard], schrodinger_); + }); + if (error) { + std::rethrow_exception(error); + } + // This rank's new (terms, coefficients), shard blocks in shard order. + std::pair, VecD> merged; + for (auto &[terms, coeffs] : updated) { + merged.first.insert(merged.first.end(), terms.begin(), terms.end()); + merged.second.insert(merged.second.end(), coeffs.begin(), coeffs.end()); + } + return merged; + }); +} + +template +auto MonomialPropagator::prepare_retained_(std::optional pare_threshold, bool &mutation_started) + -> detail::sharded::RetainedEvaluation { + auto [parameter_mapping, gen_coeffs] = graph_gate_arrays_(); + const auto context = detail::sharded::RetainedContext{.core_term = core_term_, + .parameter_mapping = std::move(parameter_mapping), + .gen_coeffs = std::move(gen_coeffs), + .pare_threshold = pare_threshold, + .basis = basis_, + .schrodinger = schrodinger_, + .rank = world_.rank}; + auto outcome = detail::sharded::prepare_retained(parallel_, shards_, context, observer_); + mutation_started = mutation_started || outcome.mutation_started; + if (outcome.error) { + std::rethrow_exception(outcome.error); + } + return std::move(*outcome.retained); +} + +template +auto MonomialPropagator::evaluate_retained_(const detail::sharded::RetainedEvaluation &retained, + const VecD ¶ms, + bool gradient, + bool *mutation_started) -> std::pair { + // Views of the retained snapshots and `params`, built for this call only. A failure up to and including the + // evaluator's own argument checks happens before its team and mutates nothing. + const auto requests = retained.requests(params); + if (requests.size() == 1 && world_.ranks == 1 && observer_ == nullptr) { + /* + * One shard on one rank: nothing to exchange and no team to coordinate, so the sole shard goes to the + * low-level serial evaluator (MPFunctions.cpp), the loop the sharded phases reduce to at T = 1, without + * their per-step publication, staging and checkpoints. Bitwise the sharded result (sharded_root_tests: + * sharded_root_sole_shard_serial_evaluation_matches_the_phases). A test observer keeps the phases, which it + * observes. + */ + detail::sharded::check_shard_request(requests.front()); + if (mutation_started != nullptr) { + *mutation_started = true; // the evaluator warms the shard's lazy caches and the thread's scratch + } + try { + if (gradient) { + return ev_and_grad(requests.front(), comm_, retained.callbacks.front()); + } + return {ev(requests.front(), comm_, retained.callbacks.front()), VecD{}}; + } + catch (...) { + if (mutation_started != nullptr) { + throw; // the enclosing run_operation_ applies the failure policy once + } + mutation_failed_(std::current_exception()); + } + } + auto outcome = detail::sharded::evaluate_shards(requests, + retained.callbacks, + retained.options, + gradient, + observer_, + world_, + rounds_.get()); + // The team has run: owners warmed their lazy caches and per-thread scratch. + if (mutation_started != nullptr) { + *mutation_started = true; + } + try { + if (outcome.error) { + std::rethrow_exception(outcome.error); + } + detail::sharded::notify_root(observer_, detail::sharded::RootWork::combine, 0); + // Ascending-shard fold, then the physical reduction, then the identity once (as ev_sharded()). + const double local = detail::sharded::combine_contributions(outcome.contributions); + const double expectation_value = mpi::allreduce_sum(local, comm_); + VecD total_gradient; + if (gradient) { + total_gradient = detail::sharded::combine_gradients(outcome.gradients); + if (!params.empty()) { + mpi::allreduce_sum_inplace(total_gradient, comm_); + } + } + return {retained.core_term + expectation_value, std::move(total_gradient)}; + } + catch (...) { + if (mutation_started != nullptr) { + throw; // the enclosing run_operation_ applies the failure policy once + } + mutation_failed_(std::current_exception()); + } +} + +template +auto MonomialPropagator::expectation_value_functional(std::optional pare_threshold) + -> std::function { + return run_operation_(true, [&](bool &mutation_started) -> std::function { + // Prepared once; every call reuses the snapshots. The callbacks borrow the shards' inverted indices, so the + // callable must not outlive this propagator. + auto retained = std::make_shared( + prepare_retained_(pare_threshold, mutation_started)); + const auto expected_epoch = initial_operator_epoch_; + return [owner = this, expected_epoch, retained](const VecD ¶ms) -> double { + owner->enter_operation_(true); + validate_expected_initial_operator(owner->initial_operator_epoch_, expected_epoch); + retained->validate_call(params); + return owner->evaluate_retained_(*retained, params, /*gradient=*/false, nullptr).first; + }; + }); +} + +template +auto MonomialPropagator::expectation_value_and_gradient_functional(std::optional pare_threshold) + -> std::function(const VecD &)> { + using Functional = std::function(const VecD &)>; + return run_operation_(true, [&](bool &mutation_started) -> Functional { + auto retained = std::make_shared( + prepare_retained_(pare_threshold, mutation_started)); + const auto expected_epoch = initial_operator_epoch_; + return [owner = this, expected_epoch, retained](const VecD ¶ms) -> std::pair { + owner->enter_operation_(true); + validate_expected_initial_operator(owner->initial_operator_epoch_, expected_epoch); + retained->validate_call(params); + return owner->evaluate_retained_(*retained, params, /*gradient=*/true, nullptr); + }; + }); +} + +template +auto MonomialPropagator::expectation_value(const VecD ¶meters) -> double { + return run_operation_(true, [&](bool &mutation_started) -> double { + validate_functional_call(parameters, expected_num_params(graph_gate_arrays_().first)); + // A preparation team, then one evaluation team: two operation-scoped regions, independent of depth. + const auto retained = prepare_retained_(std::nullopt, mutation_started); + return evaluate_retained_(retained, parameters, /*gradient=*/false, &mutation_started).first; + }); +} + +template +auto MonomialPropagator::expectation_value_and_gradient(const VecD ¶meters) -> std::pair { + return run_operation_(true, [&](bool &mutation_started) -> std::pair { + validate_functional_call(parameters, expected_num_params(graph_gate_arrays_().first)); + const auto retained = prepare_retained_(std::nullopt, mutation_started); + return evaluate_retained_(retained, parameters, /*gradient=*/true, &mutation_started); + }); +} + +template +auto MonomialPropagator::contract_blocks_(const VecD ¶meters, + bool inplace, + bool &mutation_started, + VecD *concatenated) -> std::vector { + const size_t threads = shards_.size(); + const auto concatenate = [&](const std::vector &blocks) { + if (concatenated == nullptr) { + return; + } + size_t total = 0; + for (const auto &block : blocks) { + total += block.size(); + } + concatenated->clear(); + concatenated->reserve(total); + for (const auto &block : blocks) { + concatenated->insert(concatenated->end(), block.begin(), block.end()); + } + }; + if (parameters.empty()) { + // Nothing to replay: each owner copies its current picture (materializing pending entries, a lazy cache). + mutation_started = true; + std::vector blocks(threads); + const auto error = run_owner_phase_(detail::sharded::RootWork::picture, [&](size_t shard) { + blocks[shard] = shards_[shard]->op.current_picture(schrodinger_); + }); + if (error) { + std::rethrow_exception(error); + } + concatenate(blocks); + return blocks; + } + + const auto [parameter_mapping, gen_coeffs] = graph_gate_arrays_(); + const size_t num_layers = parameter_mapping.size(); + // Heisenberg replays the oldest layers first in reverse optimizer order; Schrödinger replays the reversed window + // of slice_view() at the negated angles, starting from the dense state. + const VecD mapped_params = schrodinger_ ? map_params(parameters, parameter_mapping, gen_coeffs, -1.0) + : map_params(parameters, parameter_mapping, gen_coeffs, 1.0, true); + // From here the pictures and inverted indices (lazy caches) are read, and an in-place contraction consumes the + // graphs. + mutation_started = true; + // An in-place contraction's sliced graphs own the consumed layers; they are bound here, before any view of them, + // and outlive the team. + std::vector> sliced(threads); + std::vector requests; + std::vector callbacks; + requests.reserve(threads); + callbacks.reserve(threads); + for (size_t shard = 0; shard < threads; ++shard) { + auto &state = *shards_[shard]; + const VecD &picture = state.op.current_picture(schrodinger_); + const MPGraphView view = inplace + ? sliced[shard].emplace(state.graph.slice_graph(num_layers, true)).replay_view() + : state.graph.slice_view(num_layers); + requests.push_back(detail::sharded::ReplayRequest{.coeffs = picture, .graph = view}); + callbacks.push_back( + detail::make_cos_callbacks(state.op.inverted_index(), view, basis_, detail::parallel::Options{})); + } + auto outcome = + detail::sharded::replay_shards(requests, mapped_params, callbacks, parallel_, observer_, world_, rounds_.get()); + if (outcome.error) { + std::rethrow_exception(outcome.error); + } + concatenate(outcome.coeffs); + if (inplace) { + // Each block replaces its own shard's picture, which owns the rows the block is indexed by. + for (size_t shard = 0; shard < threads; ++shard) { + auto &op = shards_[shard]->op; + (schrodinger_ ? op.state_coeffs : op.op_coeffs) = std::move(outcome.coeffs[shard]); + } + if (!schrodinger_) { + heisenberg_gates_absorbed_ = true; + } + return {}; + } + return std::move(outcome.coeffs); +} + +template +auto MonomialPropagator::contract_partially(const VecD ¶meters, bool inplace) -> VecD { + return run_operation_(true, [&](bool &mutation_started) -> VecD { + validate_parameters_length(parameters, graph_gate_arrays_().first); + VecD concatenated; + static_cast(contract_blocks_(parameters, inplace, mutation_started, &concatenated)); + return concatenated; + }); +} + +template +auto MonomialPropagator::evolved_operator_terms(const VecD ¶meters, double atol) + -> std::vector>> { + using Term = std::pair>; + return run_operation_(true, [&](bool &mutation_started) -> std::vector { + validate_parameters_length(parameters, graph_gate_arrays_().first); + const auto blocks = contract_blocks_(parameters, false, mutation_started, nullptr); + std::vector terms; + // Each shard's index decodes that shard's own block; shards in ascending order. + for (size_t shard = 0; shard < shards_.size(); ++shard) { + detail::sharded::notify_root(observer_, detail::sharded::RootWork::export_block, shard); + const VecD &evolved = blocks[shard]; + shards_[shard]->op.store->for_each([&](const auto &mono, size_t idx) { + if (idx >= evolved.size()) { + return; + } + const double coeff = evolved[idx]; + if (std::abs(coeff) < atol) { + return; + } + // Round to drop anti-hermitian numerical noise (Majorana un-applies the Hermitian phase). + const auto decoded = algebra_decode_coeff(basis_, coeff, mono); + const std::complex rounded(std::round(decoded.real() * 1e12) / 1e12, + std::round(decoded.imag() * 1e12) / 1e12); + terms.emplace_back(bitset_to_indices(mono), rounded); + }); + } + return terms; + }); +} + +template +auto MonomialPropagator::evolved_operator_coefficients(const VecD ¶meters, const std::vector &terms) + -> std::vector> { + // Checked: these are user-supplied indices. Past the storage width the unchecked encode's bit position + // underflows into an out-of-bounds write; between the logical and storage widths it encodes a monomial outside + // this propagator's system, which reads back 0 instead of raising. + auto keys = terms | std::views::transform([this](const VecZ &term) { + return indices_to_bitset_checked(term, 2 * logical_num_modes_); + }) + | std::ranges::to>>(); + return run_operation_(true, [&](bool &mutation_started) -> std::vector> { + validate_parameters_length(parameters, graph_gate_arrays_().first); + const auto blocks = contract_blocks_(parameters, false, mutation_started, nullptr); + std::vector> out(keys.size()); + std::vector rows(keys.size()); + // Shards hold disjoint rows, so at most one finds a key; a key no shard of this rank holds stays 0, as + // evolved_operator_terms() leaves it out. Rank-local, like evolved_operator_terms(). + for (size_t shard = 0; shard < shards_.size(); ++shard) { + detail::sharded::notify_root(observer_, detail::sharded::RootWork::export_block, shard); + const VecD &evolved = blocks[shard]; + shards_[shard]->op.store->find_batch(keys.data(), keys.size(), rows.data()); + for (size_t q = 0; q < keys.size(); ++q) { + if (rows[q] < evolved.size()) { // kNotFound is size_t max, so this covers a miss too + out[q] = algebra_decode_coeff(basis_, evolved[rows[q]], keys[q]); + } + } + } + // Round off the anti-hermitian numerical noise, as evolved_operator_terms() does. + for (auto &coeff : out) { + coeff = {std::round(coeff.real() * 1e12) / 1e12, std::round(coeff.imag() * 1e12) / 1e12}; + } + // Heisenberg only: the empty monomial is the rank-level core term, never a row of a shard. + if (!schrodinger_) { + const auto core = core_term(); + for (size_t q = 0; q < terms.size(); ++q) { + if (terms[q].empty()) { + out[q] = {core, 0.0}; + } + } + } + return out; + }); +} + +template +auto MonomialPropagator::term_expectation_values(const VecD ¶meters, const std::vector &terms) + -> std::vector> { + return run_operation_(true, [&](bool &mutation_started) -> std::vector> { + // Absorbed gates are no longer in the graph, so the replay would silently leave them out of every value. + if (heisenberg_gates_absorbed_) { + throw GraphStateConflict("term_expectation_values() is unavailable once propagate() or an in-place " + "contract_partially() has absorbed gates into a Heisenberg operator, because " + "the values would miss those gates. Call it after build_graph() and before " + "contracting in place."); + } + // Checked: these are user-supplied indices (see evolved_operator_coefficients()). + const auto keys = terms | std::views::transform([this](const VecZ &term) { + return indices_to_bitset_checked(term, 2 * logical_num_modes_); + }) + | std::ranges::to>>(); + const auto [parameter_mapping, gen_coeffs] = graph_gate_arrays_(); + validate_functional_call(parameters, expected_num_params(parameter_mapping)); + + // Each key lives on at most one shard of one rank. The lookup only reads the frozen stores, and the flag + // reduction is collective over replicated arguments, so a Heisenberg miss is rejected before any mutation. + // A rank-local failure ahead of the reduction would strand the peers in it, hence the guard. + const size_t count = keys.size(); + const size_t threads = shards_.size(); + std::vector> rows; + VecD found; + mpi::guard_distributed(comm_, [&] { + rows.assign(threads, std::vector(count)); + found.assign(count, 0.0); + for (size_t shard = 0; shard < threads; ++shard) { + const auto &op = shards_[shard]->op; + op.store->find_batch(keys.data(), count, rows[shard].data()); + for (size_t q = 0; q < count; ++q) { + if (rows[shard][q] < op.size()) { // kNotFound is size_t max, so this covers a miss too + found[q] = 1.0; + } + } + } + }); + mpi::allreduce_sum_inplace(found, comm_); + if (!schrodinger_) { + for (size_t q = 0; q < count; ++q) { + // The core term is the rank-level identity, never a row. + if (!terms[q].empty() && found[q] == 0.0) { + throw detail::OperatorTermNotFound( + std::format("Operator term [{}] not found in the evolved operator. To evaluate a term " + "outside the initial operator, add it with coefficient 0 before building the " + "graph.", + join_with_separator(terms[q], ", "))); + } + } + } + + /* + * The values are the gradient of expectation_value() with respect to the un-evolved operator coefficients + * (ev_op_gradient() is the one-store form): each shard's state, swept back through its layers newest first at + * the negated angles. The forward kernel at -theta is exactly a layer's transpose, so the sweep never divides + * by cos theta as an inverse would. Scoring the state warms a lazy cache, hence mutation. + */ + mutation_started = true; + std::vector seeds(threads); + const auto seed_error = run_owner_phase_(detail::sharded::RootWork::adjoint_seed, [&](size_t shard) { + auto &op = shards_[shard]->op; + if (schrodinger_) { + seeds[shard] = op.dense_state(); + return; + } + const auto sparse = op.sparse_state(); + EvalState::sparse(op.size(), sparse.rows, sparse.values).scatter_into(seeds[shard]); + }); + if (seed_error) { + std::rethrow_exception(seed_error); + } + std::vector values; + if (parameters.empty()) { + values = std::move(seeds); + } + else { + const VecD mapped_params = map_params(parameters, parameter_mapping, gen_coeffs, -1.0); + std::vector requests; + std::vector callbacks; + requests.reserve(threads); + callbacks.reserve(threads); + for (size_t shard = 0; shard < threads; ++shard) { + const auto &state = *shards_[shard]; + const MPGraphView view = state.graph.reverse_replay_view(); + requests.push_back(detail::sharded::ReplayRequest{.coeffs = seeds[shard], .graph = view}); + callbacks.push_back(detail::make_cos_callbacks(state.op.inverted_index(), + view, + basis_, + detail::parallel::Options{})); + } + auto outcome = detail::sharded::replay_shards(requests, + mapped_params, + callbacks, + parallel_, + observer_, + world_, + rounds_.get()); + if (outcome.error) { + std::rethrow_exception(outcome.error); + } + values = std::move(outcome.coeffs); + } + + VecD gathered(count, 0.0); + for (size_t shard = 0; shard < threads; ++shard) { + detail::sharded::notify_root(observer_, detail::sharded::RootWork::export_block, shard); + for (size_t q = 0; q < count; ++q) { + if (rows[shard][q] < values[shard].size()) { + gathered[q] = values[shard][rows[shard][q]]; + } + } + } + mpi::allreduce_sum_inplace(gathered, comm_); + + std::vector> out(count); + for (size_t q = 0; q < count; ++q) { + if (terms[q].empty()) { // the core term is a fixed identity, which every state reads as 1 + out[q] = 1.0; + continue; + } + if (found[q] == 0.0) { // Schrödinger only: an absent term reads 0 + continue; + } + // The engine stores row M as the Hermitian h·M, with h = 1 or i, so gathered[q] is ⟨h·M⟩. To recover + // ⟨M⟩ we divide by h, which is equivalent to conjugation. + out[q] = std::conj(algebra_decode_coeff(basis_, gathered[q], keys[q])); + } + return out; + }); +} + +} // namespace monoprop diff --git a/cpp/monoprop/detail/mpi/CMakeLists.txt b/cpp/monoprop/detail/mpi/CMakeLists.txt index 9128c8efa..1f9e45bca 100644 --- a/cpp/monoprop/detail/mpi/CMakeLists.txt +++ b/cpp/monoprop/detail/mpi/CMakeLists.txt @@ -6,15 +6,12 @@ target_sources( FILES "CheckedCount.h" "Comm.h" - "CpuRelax.h" "Exchange.h" - "HybridComm.h" "MPICompat.h" "MPIUtils.h" + "OperationFailure.h" "Pairwise.h" - "PartitionBarrier.h" "Routing.h" - "ShmComm.h" ) target_sources(monoprop-objs PRIVATE MPICompat.cpp) diff --git a/cpp/monoprop/detail/mpi/CheckedCount.h b/cpp/monoprop/detail/mpi/CheckedCount.h index 7e71b1cb9..b61366b93 100644 --- a/cpp/monoprop/detail/mpi/CheckedCount.h +++ b/cpp/monoprop/detail/mpi/CheckedCount.h @@ -19,8 +19,7 @@ #include #include -// Below MPICompat.h so the in-process transports (ShmComm, HybridComm), which MPICompat.h includes, -// narrow their counts through the same policy as the MPI collectives. +// The one narrowing policy for every count and displacement handed to MPI; MPICompat.h re-exports it. namespace monoprop::mpi { @@ -45,7 +44,7 @@ inline auto checked_mpi_count(long long value, const char *what = "Aggregate MPI } // A buffer size is not int-bounded to begin with: a per-peer payload is kWords + 1 elements -// per term, so a single wide-mode query round reaches INT_MAX well inside one partition's term space. +// per term, so a single wide-mode query round reaches INT_MAX well inside one shard's term space. // Kept separate from the `long long` overload so a size_t above LLONG_MAX cannot sign-flip on the way // into the check. inline auto checked_mpi_count(size_t value, const char *what = "MPI count") -> int { diff --git a/cpp/monoprop/detail/mpi/Comm.h b/cpp/monoprop/detail/mpi/Comm.h index fb53e7fc3..d9e38d432 100644 --- a/cpp/monoprop/detail/mpi/Comm.h +++ b/cpp/monoprop/detail/mpi/Comm.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -33,40 +34,15 @@ constexpr MPI_Comm MPI_COMM_SELF = 0; namespace monoprop::mpi { -class ShmComm; -class HybridComm; - -// Runtime-tagged communicator handle: the same SPMD code drives real MPI (`Kind::Mpi`) or an in-process -// ShmComm (`Kind::Shm`). Trivially copyable, passed by value. The implicit MPI_Comm constructor is -// deliberate; there is no implicit conversion back (that would silently drop a Shm handle), so read -// `.mpi` explicitly where a raw communicator is required. +// The ordinary communicator handle: a real MPI communicator, or the MPI-off stub (one process). Trivially copyable, +// passed by value. The implicit MPI_Comm constructor is deliberate; read `.mpi` explicitly where a raw communicator is +// required. It carries no shard geometry: a sharded root derives its (P, T) owners from this communicator's size and +// its own captured budget. struct Comm { - // Hybrid = R MPI ranks x S in-process partitions presented as one flat P=R*S SPMD world (size()==P). - enum class Kind : std::uint8_t { Mpi, Shm, Hybrid }; - Kind kind = Kind::Mpi; - MPI_Comm mpi = MPI_COMM_SELF; // valid iff kind == Mpi - ShmComm *shm = nullptr; // non-owning (PartitionGroup owns); valid iff kind == Shm - HybridComm *hyb = nullptr; // non-owning (PartitionGroup owns); valid iff kind == Hybrid - int shm_rank = 0; // this participant's local partition index; valid iff kind == Shm | Hybrid + MPI_Comm mpi = MPI_COMM_SELF; constexpr Comm() = default; constexpr Comm(MPI_Comm c) : mpi(c) {} // NOLINT(google-explicit-constructor): implicit on purpose (see above) - - static auto make_shm(ShmComm *group, int rank) -> Comm { - Comm c; - c.kind = Kind::Shm; - c.shm = group; - c.shm_rank = rank; - return c; - } - - static auto make_hybrid(HybridComm *group, int local_partition) -> Comm { - Comm c; - c.kind = Kind::Hybrid; - c.hyb = group; - c.shm_rank = local_partition; - return c; - } }; // A window-relative index, typed apart from a flat slot: mixing the two stays in bounds but addresses @@ -79,7 +55,7 @@ struct WindowIndex { }; // The contiguous run of flat slots a round can reach. Slots are rank-major, so a sparse plan's one peer -// is S contiguous slots and dense is the count == P case of the same run. See PeerPlan::window. +// is that rank's T contiguous slots and dense is the count == P * T case of the same run. See PeerPlan::window. struct SlotWindow { size_t base = 0; // first reachable flat slot size_t count = 0; // slots in the run @@ -136,12 +112,31 @@ class WindowVec { std::vector v_; }; +/*! + * \brief Read-only span views of every block of `blocks`, over the same window. + * + * Copies no payload: view `wi` aliases `blocks[wi]`, so `blocks` must outlive the views and must not be resized + * while they are read. An empty block yields an empty view, and a window with a nonzero base keeps its base. + * + * \param blocks One owning block per window slot. + * \return One view per window slot, in slot order. + */ +template +[[nodiscard]] auto views_of(const WindowVec> &blocks) -> WindowVec> { + const SlotWindow w = blocks.window(); + WindowVec> views(w); + for (const auto wi : w.indices()) { + views[wi] = std::span(blocks[wi]); + } + return views; +} + // Which destination ranks a round can touch. Dense (the default) is every rank and the collective path; // sparse is the single peer `me ^ shift` that linear routing implies. XOR is an involution, so every // rank derives the same pairing without communicating. // // A wrong `shift` fails two ways: ranks that disagree deadlock, and ranks that agree on the same wrong -// value silently drop blocks outside the peer set. HybridComm::require_plan_covers_traffic_ checks the latter. +// value silently drop blocks outside the peer set. struct PeerPlan { bool sparse = false; int shift = 0; @@ -151,7 +146,8 @@ struct PeerPlan { // `k` indexes the peer set, which is a singleton when sparse. [[nodiscard]] constexpr auto peer(int me, int k) const -> int { return sparse ? (me ^ shift) : k; } [[nodiscard]] constexpr auto contains(int me, int b) const -> bool { return !sparse || b == (me ^ shift); } - // The flat slots reachable from `me_flat` in a `ranks` x `parts` world: the peer rank's, or all. + // The flat slots reachable from `me_flat` in a `ranks` x `parts` world (`parts` owners per rank): the peer rank's, + // or all. [[nodiscard]] constexpr auto window(size_t me_flat, size_t ranks, size_t parts) const -> SlotWindow { const size_t peer_rank = sparse ? ((me_flat / parts) ^ static_cast(shift)) : 0; return SlotWindow{.base = peer_rank * parts, @@ -174,31 +170,10 @@ inline auto require_routable(PeerPlan plan, int ranks) -> void { } } -// Argument bundles for the variable all-to-all verbs, deliberately here rather than in HybridComm.h: -// ShmComm.h takes the resolve bundle and compiles in non-MPI builds, so neither bundle may name an -// MPI-only type. MPI_Datatype therefore stays a separate parameter on the HybridComm verbs that need -// one (MPI-only header, MPI-only argument) instead of becoming a #ifdef-guarded member, which would -// make this installed header's layout depend on monoprop_ENABLE_MPI. -// -// `send`, `send_counts` and `send_displs` are non-owning views into caller memory. The send buffer must -// stay alive and unmodified until the verb's second barrier: peer partitions read it in place rather -// than through a copy, so freeing or mutating it earlier corrupts what they scatter. -// -// Counts and displacements are in ELEMENTS while `send` / `recv` are raw bytes, so every offset is -// scaled by `elem`; the pointers carry no element type to scale by. -struct AlltoallvArgs { - const std::byte *send = nullptr; // read in place by peers until the second barrier - const int *send_counts = nullptr; // [P] elements sent to each destination - const int *send_displs = nullptr; // [P] element offsets into `send` - std::byte *recv = nullptr; // caller-owned, already sized for recv_counts - const int *recv_counts = nullptr; // [P] input: the senders' transpose, same contract as MPI_Alltoallv - const int *recv_displs = nullptr; // [P] element offsets into `recv` - size_t elem = 0; // bytes per element; scales every count/displ above -}; - -// Typed view of the same six spans, for callers that hold `T` buffers: `bytes()` type-erases it for the -// in-process transports, which address payloads as raw bytes, while the MPI path keeps the typed -// pointers it passes alongside a datatype. Same lifetime and element-offset contract as AlltoallvArgs. +// The typed buffers and layout of one variable all-to-all over a communicator (post_flat_alltoallv). `send`, +// `send_counts` and `send_displs` are non-owning views into caller memory; MPI reads the send buffer, and writes the +// receive buffer, until the posted transfer completes, so neither may move, be freed or be modified before then. +// Counts and displacements are in elements of `T`, one per rank. template struct FlatAlltoallvArgs { const T *send = nullptr; @@ -207,31 +182,6 @@ struct FlatAlltoallvArgs { T *recv = nullptr; // caller-owned, already sized for recv_counts const int *recv_counts = nullptr; // [P] input: the senders' transpose, same contract as MPI_Alltoallv const int *recv_displs = nullptr; // [P] element offsets into `recv` - - auto bytes() const -> AlltoallvArgs { - return AlltoallvArgs{.send = reinterpret_cast(send), - .send_counts = send_counts, - .send_displs = send_displs, - .recv = reinterpret_cast(recv), - .recv_counts = recv_counts, - .recv_displs = recv_displs, - .elem = sizeof(T)}; - } -}; - -// The fused resolve verb keeps its own shape because its recv side is an output, not a caller-supplied -// layout: `recv` is a reference the callee resizes, and the two recv arrays are written, not read. Being -// typed, it derives element bytes as sizeof(T) instead of carrying `elem`. Send side: same lifetime and -// element-offset contract as AlltoallvArgs. -template -struct AlltoallvResolveArgs { - const T *send = nullptr; - const int *send_counts = nullptr; // [P] elements sent to each destination - const int *send_displs = nullptr; // [P] element offsets into `send` - std::vector &recv; // output, resized to the resolved total; a reference, so copying the - // bundle still resizes the caller's vector - int *recv_counts = nullptr; // [P] output: resolved per-source counts - int *recv_displs = nullptr; // [P] output: resolved element offsets into `recv` }; } // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/CpuRelax.h b/cpp/monoprop/detail/mpi/CpuRelax.h deleted file mode 100644 index 42ec2fd00..000000000 --- a/cpp/monoprop/detail/mpi/CpuRelax.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) -#include -#endif - -namespace monoprop::mpi::detail { - -// One iteration of a polite busy-wait: a pause-class hint, off the syscall path, while a sibling stores. -inline auto cpu_relax() noexcept -> void { -#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) - _mm_pause(); -#elif defined(__x86_64__) || defined(__i386__) - __builtin_ia32_pause(); -#elif defined(__aarch64__) - __asm__ __volatile__("yield"); -#else - std::this_thread::yield(); -#endif -} - -// cpu_relax() iterations a barrier spinner burns before donating its timeslice. PAUSE is ~140 cycles on -// Sapphire Rapids ⇒ 2048 iters ≈ 0.1 ms, past a balanced exchange's arrival gaps; longer waits yield. -inline constexpr int kSpinPauseIters = 2048; - -} // namespace monoprop::mpi::detail diff --git a/cpp/monoprop/detail/mpi/Exchange.h b/cpp/monoprop/detail/mpi/Exchange.h index a90682b10..28673631f 100644 --- a/cpp/monoprop/detail/mpi/Exchange.h +++ b/cpp/monoprop/detail/mpi/Exchange.h @@ -89,26 +89,7 @@ class [[nodiscard("call wait() on the Ticket to complete the posted transfer")]] template inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, Comm comm, bool pairwise = false) -> Ticket { - // In-process transports take raw bytes, MPI takes typed pointers; offsets are in elements on both. - if (comm.kind == Comm::Kind::Shm) { - // Synchronous: the transfer completes here, so the Ticket's wait() is a no-op. ShmComm needs no - // send counts: its peers pull using the publisher's displacements. - const auto bytes = args.bytes(); - comm.shm->alltoallv(comm.shm_rank, - bytes.send, - bytes.send_displs, - bytes.recv, - bytes.recv_counts, - bytes.recv_displs, - bytes.elem); - return Ticket{}; - } #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - // Narrowed inside the verb: only there is the whole rank's traffic visible. - comm.hyb->alltoallv(comm.shm_rank, args.bytes(), datatype::get(), PeerPlan{}, pairwise); - return Ticket{}; - } if (pairwise) { // Dense plan: walk every rank and post only non-empty legs, so nothing can be dropped. The layout // is symmetric, so both ends skip the same legs. @@ -142,6 +123,8 @@ inline auto post_flat_alltoallv(const FlatAlltoallvArgs &args, int num_ranks, request.data()); return {std::move(request), 1}; #else + (void)comm; + (void)pairwise; for (int i = 0; i < num_ranks; ++i) { const int c = args.recv_counts[i]; for (int j = 0; j < c; ++j) { diff --git a/cpp/monoprop/detail/mpi/HybridComm.h b/cpp/monoprop/detail/mpi/HybridComm.h deleted file mode 100644 index a6d3cef43..000000000 --- a/cpp/monoprop/detail/mpi/HybridComm.h +++ /dev/null @@ -1,861 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "monoprop/detail/mpi/CheckedCount.h" -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/Pairwise.h" -#include "monoprop/detail/mpi/PartitionBarrier.h" -#include "monoprop/detail/mpi/Routing.h" - -// Composes R MPI ranks x S in-process partitions into one flat P=R*S SPMD world. Global id is rank-major -// (g = mpi_rank*S + partition), keeping each rank's partitions contiguous in ascending-global order — the -// contract Resolve.h's positional pairing relies on. Only partition-0 masters call MPI (bracketed by -// intra-rank barriers ⇒ requires MPI_THREAD_SERIALIZED); ascending-order local sums + order-preserving -// MPI ⇒ bit-identical, repeatable results for fixed (R, S). - -namespace monoprop::mpi { - -class MpiThreadLevelUnsupported : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -class HybridComm { // NOSONAR(cpp:S1448): transport phases share one barrier-owned state machine. -public: - // n_local_partitions = S, identical on every rank (the facade ctor checks that before constructing). - HybridComm(MPI_Comm parent, int n_local_partitions) - : parent_(parent), - s_(n_local_partitions), - slots_(static_cast(n_local_partitions)), - row_peer_(static_cast(n_local_partitions), kNoPeer), - row_recv_peer_(static_cast(n_local_partitions), kNoPeer), - barrier_(n_local_partitions) { - MPI_Comm_size(parent_, &r_); - MPI_Comm_rank(parent_, &mpi_rank_); - int provided = MPI_THREAD_SINGLE; - MPI_Query_thread(&provided); - if (provided < MPI_THREAD_SERIALIZED) { - throw MpiThreadLevelUnsupported("HybridComm requires MPI_THREAD_SERIALIZED (partition-0 masters call " - "MPI while peers are parked); provided level is lower. Ensure " - "mpi::init / mpi4py requests SERIALIZED or MULTIPLE."); - } - // Collective, so it runs after the local checks: a rank that throws first must not strand its peers. - pairwise_ = agree_routes_pairwise(parent_, r_, routing::Config::from_env(static_cast(s_))); - // Size all (R,S)-fixed scratch once so per-call paths never allocate; staging grows on demand. - const size_t rss = static_cast(r_) * static_cast(s_) * static_cast(s_); - const size_t p = static_cast(r_) * static_cast(s_); - counts_send_.resize(rss); - counts_recv_.resize(rss); - mpi_send_counts_.resize(static_cast(r_)); - mpi_recv_counts_.resize(static_cast(r_)); - mpi_send_displs_.resize(static_cast(r_)); - mpi_recv_displs_.resize(static_cast(r_)); - pack_off_.resize(static_cast(s_) * p); // same S*R*S elements as before, indexed (u, g) - base_send_.resize(p); - base_recv_.resize(p); - col_sum_.resize(p); - recv_col_.resize(p); - // Rounded up to whole 64-byte lines: each row has its own writer, so a shared line false-shares. - counts_stride_ = round_up_(p, kIntsPerLine); - rows_stride_ = round_up_(static_cast(r_), kLongsPerLine); - // One spare line each: the allocator guarantees alignof(T), not 64, so row 0 must be realigned. - counts_matrix_store_.assign((static_cast(s_) * counts_stride_) + kIntsPerLine, 0); - counts_matrix_ = align_to_line_(counts_matrix_store_.data()); - rows_store_.assign((static_cast(s_) * rows_stride_) + kLongsPerLine, 0LL); - rows_ = align_to_line_(rows_store_.data()); - assert(reinterpret_cast(counts_matrix_) % kLineBytes == 0); - assert(reinterpret_cast(rows_) % kLineBytes == 0); - assert(counts_matrix_ + static_cast(s_) * counts_stride_ - <= counts_matrix_store_.data() + counts_matrix_store_.size()); - assert(rows_ + static_cast(s_) * rows_stride_ <= rows_store_.data() + rows_store_.size()); - } - - HybridComm(const HybridComm &) = delete; - auto operator=(const HybridComm &) -> HybridComm & = delete; - - auto size() const -> int { return r_ * s_; } - auto ranks() const -> int { return r_; } - auto partitions() const -> int { return s_; } - auto global_rank(int local_partition) const -> int { return (mpi_rank_ * s_) + local_partition; } - // Agreed across parent_ at construction, so every rank branches the same way. - auto routes_pairwise() const -> bool { return pairwise_; } - - auto alltoall_counts(int local_partition, - const int *send_counts /*[P]*/, - int *recv_counts /*[P]*/, - PeerPlan plan = {}) -> void { - require_routable(plan, r_); - guard_partition0_(local_partition, "alltoall_counts", [this, local_partition, send_counts, recv_counts, plan] { - alltoall_counts_impl_(local_partition, send_counts, recv_counts, plan); - }); - } - - // See AlltoallvArgs for the send-buffer lifetime and the element-vs-byte convention; `dt` is the MPI - // datatype whose extent is args.elem, and it stays a separate argument because the bundle is shared - // with the non-MPI-capable transport. - // - // `derive_wire_plan` selects point-to-point and has partition 0 read the peer set off the whole rank's - // rows, which no single caller can see. It must be rank-uniform (routes_pairwise()); a symmetric - // layout narrows to one peer, anything else keeps every rank. - auto alltoallv(int local_partition, - const AlltoallvArgs &args, - MPI_Datatype dt, - PeerPlan plan = {}, - bool derive_wire_plan = false) -> void { - require_routable(plan, r_); - guard_partition0_(local_partition, "alltoallv", [this, local_partition, &args, dt, plan, derive_wire_plan] { - alltoallv_impl_(local_partition, args, dt, plan, derive_wire_plan); - }); - } - - // See AlltoallvResolveArgs: the recv side is an output, and args.recv is resized here. - template - auto alltoallv_resolve(int local_partition, - const AlltoallvResolveArgs &args, - MPI_Datatype dt, - PeerPlan plan = {}) -> void { - require_routable(plan, r_); - // `args` by reference, not by value: the impl resizes args.recv and then writes through it. - guard_partition0_(local_partition, "alltoallv_resolve", [this, local_partition, &args, dt, plan] { - alltoallv_resolve_impl_(local_partition, args, dt, plan); - }); - } - - template - auto allreduce_sum(int local_partition, T local_val) -> T { - return guard_partition0_(local_partition, "allreduce_sum", [this, local_partition, local_val] { - return allreduce_sum_impl_(local_partition, local_val); - }); - } - - auto allreduce_sum_inplace(int local_partition, double *values, size_t len) -> void { - guard_partition0_(local_partition, "allreduce_sum_inplace", [this, local_partition, values, len] { - allreduce_sum_inplace_impl_(local_partition, values, len); - }); - } - - auto poison() -> void { barrier_.poison(); } - auto reset() -> void { barrier_.reset(); } - -private: - // Counts travel through counts_matrix_ / rows_, not through here. - struct alignas(64) Slot { - const std::byte *ptr = nullptr; // byte view of this partition's send buffer; null until published - const int *send_displs = nullptr; - const double *vec = nullptr; - double f64 = 0.0; - uint64_t u64 = 0; - }; - - // Once partition 0 -- this rank's only participant on parent_ -- is inside a collective, the peer ranks - // are committed: their partition-0 threads enter theirs and block inside MPI with no timeout. So a - // rank-local failure here must MPI_Abort with the underlying error rather than throw, which would - // hang the job. Single-rank partitioned runs use ShmComm, not HybridComm, and keep their exceptions. - // `body` is invoked synchronously and never stored, so the callers' lambdas may capture by reference. - template - auto guard_partition0_(int local_partition, const char *verb, Body &&body) -> decltype(body()) { - if (local_partition != 0) { - return body(); - } - try { - return body(); - } - catch (const std::exception &e) { - abort_rank_(verb, e.what()); - } - catch (...) { - abort_rank_(verb, "unknown error"); - } - } - - // recv_counts[g] = amount global partition g sends to this partition. 2 barriers + one count round. - auto alltoall_counts_impl_(int local_partition, - const int *send_counts /*[P]*/, - int *recv_counts /*[P]*/, - PeerPlan plan) -> void { - publish_counts_row_(local_partition, send_counts, plan.sparse); - sync(); - if (local_partition == 0) { - require_plan_covers_traffic_(plan); - fill_peers_(plan); - pack_count_matrix_(); - post_count_blocks_(plan); - wait_count_blocks_(); - } - sync(); - // Partition t extracts its row: recv from (rank a, partition su) is contiguous per source rank a. - // Non-peers are zero by definition. - const int t = local_partition; - std::fill(recv_counts, recv_counts + (static_cast(r_) * static_cast(s_)), 0); - for (const int a : peers_) { - for (int su = 0; su < s_; ++su) { - recv_counts[(a * s_) + su] = counts_recv_[counts_idx_(a, t, su)]; - } - } - // No trailing barrier: past the last sync only counts_recv_ is read, and partition 0 cannot rewrite - // it until a future call's second barrier — unreachable until every extractor here has arrived. - // send_counts is consumed before the first sync, so peers may free or reuse it on return. - } - - // Flat variable all-to-all over caller-owned buffers; see AlltoallvArgs for the conventions. - auto alltoallv_impl_(int local_partition, - const AlltoallvArgs &args, - MPI_Datatype dt, - PeerPlan plan, - bool derive_wire_plan = false) -> void { - const size_t u = static_cast(local_partition); - Slot &me = slots_[u]; - me.ptr = args.send; - me.send_displs = args.send_displs; - publish_counts_row_(local_partition, args.send_counts, plan.sparse || derive_wire_plan); - publish_recv_rows_(local_partition, args.recv_counts, plan); - sync(); // B1 - - // B2: partition 0 sizes/reallocates staging; must finish before any partition packs into stage_send_. - if (local_partition == 0) { - // Partition 0 only: written here, read again in B3->B4. - if (derive_wire_plan) { - wire_plan_ = derived_wire_plan_(); - } - else { - wire_plan_ = plan; - require_plan_covers_traffic_(wire_plan_); - } - fill_peers_(wire_plan_); - size_staging_send_(args.elem); - fill_recv_col_([this](int a, int t) { return row_recv_(t)[a]; }); - size_staging_recv_(args.elem); - } - sync(); // B2 - - // B3: each partition packs its own cross-rank blocks into stage_send_ (disjoint writes). - pack_send_(local_partition, args.elem); - sync(); // B3 - - // B4: partition 0 moves the payload while peers park at the barrier. - if (local_partition == 0) { - exchange_payload_(dt, args.elem, wire_plan_, derive_wire_plan || !wire_plan_.dense()); - } - sync(); // B4 - - scatter_recv_(local_partition, args.recv, args.recv_counts, args.recv_displs, args.elem); - // No trailing barrier: base_recv_ is rewritten only in a later verb's B1→B2 window. - } - - // Fused count-resolve + payload alltoallv: folds the standalone count exchange into this verb's - // barriered windows (4 syncs instead of 6). recv_counts / recv_displs and `recv` (resized) are - // outputs. Bit-identical to alltoall_counts + alltoallv. - // - // The sparse count round is posted in B1→B2 and drained in B3→B4, overlapping the packing: nothing - // before fill_recv_col_ reads counts_recv_. The dense arm's MPI_Alltoall blocks in the post. - template - auto alltoallv_resolve_impl_(int local_partition, - const AlltoallvResolveArgs &args, - MPI_Datatype dt, - PeerPlan plan) -> void { - // Typed verb: element bytes are sizeof(T) by construction, so they are derived rather than passed. - constexpr size_t elem = sizeof(T); - const size_t u = static_cast(local_partition); - Slot &me = slots_[u]; - // Typed here but byte-addressed in the slot: pack_send_ copies by (displ, count) in elements and - // never reconstructs T, so the slot stays type-erased for the untyped alltoallv_impl_ above. - me.ptr = reinterpret_cast(args.send); - me.send_displs = args.send_displs; - // Count row only: the recv counts do not exist until the count round is drained in B3→B4. - publish_counts_row_(local_partition, args.send_counts, plan.sparse); - sync(); // B1 - - if (local_partition == 0) { - require_plan_covers_traffic_(plan); - fill_peers_(plan); - pack_count_matrix_(); - post_count_blocks_(plan); - size_staging_send_(elem); - } - sync(); // B2 - - // B3: each partition packs its own blocks while the count round is in flight. - pack_send_(local_partition, elem); - sync(); // B3 - - // B4: the counts land, then the payload moves. - if (local_partition == 0) { - wait_count_blocks_(); - fill_recv_col_([this](int a, int t) { return block_sum_(a, t); }); - size_staging_recv_(elem); - exchange_payload_(dt, elem, plan, !plan.dense()); - } - sync(); // B4 - - // Only past B4 does counts_recv_ exist; partition 0 next rewrites it in a later verb's B1→B2. - const int t = local_partition; - long long total = 0; - const size_t p = static_cast(r_) * static_cast(s_); - std::fill(args.recv_counts, args.recv_counts + p, 0); - std::fill(args.recv_displs, args.recv_displs + p, 0); - for (const int a : peers_) { - for (int su = 0; su < s_; ++su) { - const int g = (a * s_) + su; - const int c = counts_recv_[counts_idx_(a, t, su)]; - args.recv_counts[g] = c; - args.recv_displs[g] = checked_mpi_count(total, "Recv displacement"); - total += c; - } - } - args.recv.resize(static_cast(checked_mpi_count(total, "Total recv count"))); - - scatter_recv_(local_partition, - reinterpret_cast(args.recv.data()), // after the resize: it may reallocate - args.recv_counts, - args.recv_displs, - elem); - // No trailing barrier: same discipline as alltoallv_impl_. - } - - template - auto allreduce_sum_impl_(int local_partition, T local_val) -> T { - Slot &me = slots_[static_cast(local_partition)]; - if constexpr (std::is_floating_point_v) { - me.f64 = static_cast(local_val); - } - else { - me.u64 = static_cast(local_val); - } - sync(); - if (local_partition == 0) { - if constexpr (std::is_floating_point_v) { - double local = 0.0; - for (int s = 0; s < s_; ++s) { - local += slots_[static_cast(s)].f64; - } - MPI_Allreduce(&local, &red_f64_, 1, MPI_DOUBLE, MPI_SUM, parent_); - } - else { - uint64_t local = 0; - for (int s = 0; s < s_; ++s) { - local += slots_[static_cast(s)].u64; - } - MPI_Allreduce(&local, &red_u64_, 1, MPI_UINT64_T, MPI_SUM, parent_); - } - } - sync(); - T out{}; - if constexpr (std::is_floating_point_v) { - out = static_cast(red_f64_); - } - else { - out = static_cast(red_u64_); - } - // No trailing barrier: red_f64_/red_u64_ are rewritten only inside a future verb's barriered window. - return out; - } - - // In-place element-wise allreduce-sum across the flat P-world, slice-partitioned across partitions in - // ascending order (bit-identical to a sequential sum). - auto allreduce_sum_inplace_impl_(int local_partition, double *values, size_t len) -> void { - slots_[static_cast(local_partition)].vec = values; - sync(); // all inputs published - if (local_partition == 0) { - grow_(red_vec_, len); - } - sync(); // red_vec_ sized - constexpr size_t kLine = 64 / sizeof(double); - const size_t lines = (len + kLine - 1) / kLine; - const size_t per = (lines + static_cast(s_) - 1) / static_cast(s_); - const size_t lo = std::min(len, static_cast(local_partition) * per * kLine); - const size_t hi = std::min(len, lo + (per * kLine)); - for (size_t k = lo; k < hi; ++k) { - double acc = 0.0; - for (int s = 0; s < s_; ++s) { - acc += slots_[static_cast(s)].vec[k]; - } - red_vec_[k] = acc; // disjoint line-rounded slices: no two partitions store to one line - } - sync(); // local reduction complete - if (local_partition == 0) { - MPI_Allreduce(MPI_IN_PLACE, red_vec_.data(), static_cast(len), MPI_DOUBLE, MPI_SUM, parent_); - } - sync(); // global result in red_vec_ - std::memcpy(values, red_vec_.data(), len * sizeof(double)); - // No trailing barrier: red_vec_ is rewritten only inside a future verb's barriered phases. - } - - static constexpr size_t kLineBytes = 64; - static constexpr size_t kIntsPerLine = kLineBytes / sizeof(int); - static constexpr size_t kLongsPerLine = kLineBytes / sizeof(long long); - - static auto round_up_(size_t n, size_t m) -> size_t { return ((n + m - 1) / m) * m; } - - template - static auto align_to_line_(T *p) -> T * { - static_assert(kLineBytes % sizeof(T) == 0); - const auto addr = reinterpret_cast(p); - const size_t pad = (kLineBytes - static_cast(addr % kLineBytes)) % kLineBytes; - const auto offset = static_cast(pad / sizeof(T)); - return p + offset; - } - - // (rank, dest partition, source partition) in the R*S*S count matrices: the count message's wire layout. - auto counts_idx_(int b, int t, int su) const -> size_t { - return (((static_cast(b) * static_cast(s_)) + static_cast(t)) * static_cast(s_)) - + static_cast(su); - } - - // The plan's peer ranks, once per verb. Written by partition 0 in B1→B2, so every reader past B2 sees it. - auto fill_peers_(PeerPlan plan) -> void { - const int f = plan.count(r_); - peers_.resize(static_cast(f)); - for (int k = 0; k < f; ++k) { - peers_[static_cast(k)] = plan.peer(mpi_rank_, k); - } - } - - // Zero the peer ranks' slots of a [P] table; every reader walks the same peers, so the rest is dead. - auto zero_peer_slots_(std::vector &table) const -> void { - for (const int b : peers_) { - std::fill_n(table.begin() + (static_cast(b) * s_), s_, 0LL); - } - } - - // Rank a's count block for partition t, summed over source partitions. - auto block_sum_(int a, int t) const -> long long { - const int *blk = counts_recv_.data() + counts_idx_(a, t, 0); - long long sum = 0; - for (int su = 0; su < s_; ++su) { - sum += blk[su]; - } - return sum; - } - - // [S x P] payload offset table: row u is source partition u's staging starts, one per destination g. - auto pack_idx_(int u, int g) const -> size_t { - return (static_cast(u) * static_cast(r_) * static_cast(s_)) + static_cast(g); - } - - auto counts_row_(int u) -> int * { return counts_matrix_ + (static_cast(u) * counts_stride_); } - auto row_recv_(int u) -> long long * { return rows_ + (static_cast(u) * rows_stride_); } - - // Phase P0: partition u writes only its own row, so the pre-barrier write needs no barrier. The one - // cross-partition reader is partition 0 inside B1→B2, and no peer reaches verb k+1's P0 without - // passing verb k's B2, so the rewrite always follows the read. - auto publish_counts_row_(int local_partition, const int *send_counts, bool track_peer) -> void { - const size_t u = static_cast(local_partition); - std::memcpy(counts_row_(local_partition), - send_counts, - static_cast(r_) * static_cast(s_) * sizeof(int)); - row_peer_[u] = kNoPeer; - if (!track_peer) { - return; // a dense round reaches every rank, so no peer can be excluded - } - // Folded per partition so partition 0 checks the plan without an O(R*S^2) serial sweep. - int peer = kNoPeer; - for (int a = 0; a < r_ && peer != kManyPeers; ++a) { - const int *block = send_counts + (static_cast(a) * static_cast(s_)); - if (!std::all_of(block, block + s_, [](int n) { return n == 0; })) { - peer = fold_peer_(peer, a); - } - } - row_peer_[u] = peer; - } - - // Summarises the ranks seen so far as kNoPeer, one rank, or kManyPeers. - static constexpr auto fold_peer_(int acc, int a) -> int { - if (a == kNoPeer || a == acc) { - return acc; - } - return acc == kNoPeer ? a : kManyPeers; - } - - // One row-peer table folded across the S partitions. Partition 0 only. - auto fold_rows_(const std::vector &row_peers) const -> int { - int peer = kNoPeer; - for (const int a : row_peers) { - peer = fold_peer_(peer, a); - } - return peer; - } - - // An agreed plan that excludes a live destination would silently drop those blocks. Checked in B1->B2, - // before anything is posted, so guard_partition0_ can abort with nobody waiting. - auto require_plan_covers_traffic_(PeerPlan plan) const -> void { - if (plan.dense()) { - return; - } - const int peer = fold_rows_(row_peer_); - if (peer == kNoPeer || (peer != kManyPeers && plan.contains(mpi_rank_, peer))) { - return; - } - throw std::runtime_error( - std::format("peer plan shift {} excludes destination ranks this rank is sending to (rank {} of {}); " - "those blocks would be dropped rather than refused", - plan.shift, - mpi_rank_, - r_)); - } - - // long long: it sums S int counts. Only the plan's peers are written, the only entries read. Also - // folds the row's recv peer for derived_wire_plan_, like publish_counts_row_ does for sends. - auto publish_recv_rows_(int local_partition, const int *recv_counts, PeerPlan plan) -> void { - long long *rr = row_recv_(local_partition); - int peer = kNoPeer; - const int f = plan.count(r_); - for (int k = 0; k < f; ++k) { - const int a = plan.peer(mpi_rank_, k); - long long sum = 0; - for (int su = 0; su < s_; ++su) { - sum += recv_counts[(a * s_) + su]; - } - rr[a] = sum; - if (sum != 0) { - peer = fold_peer_(peer, a); - } - } - row_recv_peer_[static_cast(local_partition)] = peer; - } - - // Transpose the published count rows into counts_send_, dest-major then source-minor, for the one - // S*S-int MPI_Alltoall. Partition 0 only, inside a barriered window. Source partition outer, so the - // peer-owned side streams. Every element is written here, so no pre-zeroing. - auto pack_count_matrix_() -> void { - for (int su = 0; su < s_; ++su) { - const int *row = counts_row_(su); - for (const int b : peers_) { - for (int t = 0; t < s_; ++t) { - counts_send_[counts_idx_(b, t, su)] = row[(b * s_) + t]; - } - } - } - } - - // The rank's peer set, read off every partition's published rows. Narrowing only: anything the rows - // do not pin to one peer (several, or send and recv disagreeing) keeps every rank, which drops - // nothing. Nothing occupied narrows to self, whose legs are all empty. - auto derived_wire_plan_() const -> PeerPlan { - const int recv_peer = fold_rows_(row_recv_peer_); - const int send_peer = fold_rows_(row_peer_); - const bool spans_one = recv_peer != kManyPeers && send_peer != kManyPeers; - const bool sides_agree = recv_peer == kNoPeer || send_peer == kNoPeer || recv_peer == send_peer; - if (!spans_one || !sides_agree) { - return PeerPlan{}; - } - const int peer = recv_peer != kNoPeer ? recv_peer : (send_peer != kNoPeer ? send_peer : mpi_rank_); - return PeerPlan{.sparse = true, .shift = mpi_rank_ ^ peer}; - } - - // The count blocks: a blocking S*S-int MPI_Alltoall when dense, else a posted pair per peer that - // wait_count_blocks_ drains; counts_send_/counts_recv_ must stay put until then. Partition 0 only. - auto post_count_blocks_(PeerPlan plan) -> void { - assert(count_posted_ == 0); // an un-drained round would be waited on twice - const int block = s_ * s_; - if (plan.dense()) { - MPI_Alltoall(counts_send_.data(), block, MPI_INT, counts_recv_.data(), block, MPI_INT, parent_); - return; - } - const PeerLayout blocks{.block = block}; - const SparsePairwiseArgs pairwise{ - .plan = plan, - .me = mpi_rank_, - .num_ranks = r_, - .comm = parent_, - .tag = kHybridCountTag, - .datatype = MPI_INT, - .elem = sizeof(int), - .send = reinterpret_cast(counts_send_.data()), - .send_layout = blocks, - .recv = reinterpret_cast(counts_recv_.data()), - .recv_layout = blocks, - }; - count_posted_ = sparse_pairwise(pairwise, count_reqs_); - } - - // Drains post_count_blocks_; a no-op on the dense arm and when the only peer is self. - auto wait_count_blocks_() -> void { - if (count_posted_ != 0) { - MPI_Waitall(count_posted_, count_reqs_.data(), MPI_STATUSES_IGNORE); - count_posted_ = 0; - } - } - - // The staged payload: one MPI_Alltoallv, or point-to-point over `plan`'s peers. `pairwise` is the - // transport and must be rank-uniform; `plan` is only the peer set, which each rank narrows itself. - // `elem` is dt's extent. - auto exchange_payload_(MPI_Datatype dt, size_t elem, PeerPlan plan, bool pairwise) -> void { - if (!pairwise) { - MPI_Alltoallv(stage_send_.data(), - mpi_send_counts_.data(), - mpi_send_displs_.data(), - dt, - stage_recv_.data(), - mpi_recv_counts_.data(), - mpi_recv_displs_.data(), - dt, - parent_); - return; - } - const SparsePairwiseArgs post{ - .plan = plan, - .me = mpi_rank_, - .num_ranks = r_, - .comm = parent_, - .tag = kHybridPayloadTag, - .datatype = dt, - .elem = elem, - .send = stage_send_.data(), - .send_layout = {.counts = mpi_send_counts_.data(), .displs = mpi_send_displs_.data()}, - .recv = stage_recv_.data(), - .recv_layout = {.counts = mpi_recv_counts_.data(), .displs = mpi_recv_displs_.data()}, - }; - const int posted = sparse_pairwise(post, reqs_); - MPI_Waitall(posted, reqs_.data(), MPI_STATUSES_IGNORE); - } - - template - static auto grow_(V &v, size_t need) -> void { - if (v.size() < need) { - v.resize(need); - } - } - - // Partition 0's send-side staging sizing, between B1 and B2, in two sweeps of counts_matrix_. Wire - // block order is destination major, source minor: a per-destination base plus a per-source prefix. - // Serial while S-1 partitions park, so both sweeps walk only the peers. - auto size_staging_send_(size_t elem) -> void { - zero_peer_slots_(col_sum_); - for (int u = 0; u < s_; ++u) { - const int *row = counts_row_(u); - for (const int b : peers_) { - const size_t base = static_cast(b) * static_cast(s_); - for (int t = 0; t < s_; ++t) { - col_sum_[base + static_cast(t)] += row[base + static_cast(t)]; - } - } - } - // Fully zeroed: MPI_Alltoallv reads every entry, and non-peers must contribute 0 to the prefix. - std::ranges::fill(mpi_send_counts_, 0); - std::ranges::fill(mpi_send_displs_, 0); - for (const int b : peers_) { - const long long *col = col_sum_.data() + (static_cast(b) * static_cast(s_)); - long long send_sum = 0; - for (int t = 0; t < s_; ++t) { - send_sum += col[t]; - } - mpi_send_counts_[static_cast(b)] = checked_mpi_count(send_sum, "Per-rank send count"); - } - // peers_ is ascending, so this prefix equals the full 0..R one at every peer. - long long send_running = 0; - for (const int b : peers_) { - mpi_send_displs_[static_cast(b)] = checked_mpi_count(send_running, "Send displacement"); - send_running += mpi_send_counts_[static_cast(b)]; - } - const size_t total_send = static_cast(checked_mpi_count(send_running, "Total send count")); - for (const int b : peers_) { - size_t cur = static_cast(mpi_send_displs_[static_cast(b)]); - for (int t = 0; t < s_; ++t) { - const size_t g = (static_cast(b) * static_cast(s_)) + static_cast(t); - base_send_[g] = cur; - cur += static_cast(col_sum_[g]); - } - } - zero_peer_slots_(col_sum_); - for (int u = 0; u < s_; ++u) { - const int *row = counts_row_(u); - size_t *off = pack_off_.data() + pack_idx_(u, 0); - for (const int b : peers_) { - const size_t base = static_cast(b) * static_cast(s_); - for (int t = 0; t < s_; ++t) { - const size_t g = base + static_cast(t); - off[g] = base_send_[g] + static_cast(col_sum_[g]); - col_sum_[g] += row[g]; - } - } - } - // Grow-only, no zero-fill: pack_send_'s blocks tile [0, total_send) exactly. - grow_(stage_send_, total_send * elem); - } - - // recv_col_[a*S + t] = what partition t receives from rank a, as `value(a, t)` reads it. - template - auto fill_recv_col_(Value &&value) -> void { - zero_peer_slots_(recv_col_); - for (const int a : peers_) { - for (int t = 0; t < s_; ++t) { - recv_col_[(static_cast(a) * static_cast(s_)) + static_cast(t)] = value(a, t); - } - } - } - - // Partition 0's recv-side staging sizing, from recv_col_. Only the per-(rank, partition) base; the - // post-B4 scatter re-derives the per-source offsets as it walks (a, su). - auto size_staging_recv_(size_t elem) -> void { - std::ranges::fill(mpi_recv_counts_, 0); - std::ranges::fill(mpi_recv_displs_, 0); - for (const int a : peers_) { - long long recv_sum = 0; - for (int t = 0; t < s_; ++t) { - recv_sum += recv_col_[(static_cast(a) * static_cast(s_)) + static_cast(t)]; - } - mpi_recv_counts_[static_cast(a)] = checked_mpi_count(recv_sum, "Per-rank recv count"); - } - // Same ascending-peers prefix as the send side. - long long recv_running = 0; - for (const int a : peers_) { - mpi_recv_displs_[static_cast(a)] = checked_mpi_count(recv_running, "Recv displacement"); - recv_running += mpi_recv_counts_[static_cast(a)]; - } - const size_t total_recv = static_cast(checked_mpi_count(recv_running, "Total recv count")); - for (const int a : peers_) { - size_t cur = static_cast(mpi_recv_displs_[static_cast(a)]); - for (int t = 0; t < s_; ++t) { - const size_t g = (static_cast(a) * static_cast(s_)) + static_cast(t); - base_recv_[g] = cur; - cur += static_cast(recv_col_[g]); - } - } - grow_(stage_recv_, total_recv * elem); - } - - auto pack_send_(int local_partition, size_t elem) -> void { - const int u = local_partition; - // Own slot only — no peer's published send buffer is read here, which is what lets every - // partition pack concurrently in the B2→B3 window. - const std::byte *src = slots_[static_cast(u)].ptr; - const int *my_send_counts = counts_row_(u); - const int *my_send_displs = slots_[static_cast(u)].send_displs; - const size_t *off = pack_off_.data() + pack_idx_(u, 0); - for (const int b : peers_) { - const int base = b * s_; - for (int t = 0; t < s_; ++t) { - const int g = base + t; - const int cnt = my_send_counts[g]; - if (cnt != 0) { - std::memcpy(stage_send_.data() + (off[g] * elem), - src + (static_cast(my_send_displs[g]) * elem), - static_cast(cnt) * elem); - } - } - } - } - - // Scatter each source's run out of stage_recv_ (self-rank legs included) to recv_displs[g]. Walks - // (a, su) in accumulation order, so `cur` re-derives block starts from base_recv_. - auto scatter_recv_(int local_partition, std::byte *dst, const int *recv_counts, const int *recv_displs, size_t elem) - -> void { - const int t = local_partition; - for (const int a : peers_) { - size_t cur = base_recv_[(static_cast(a) * static_cast(s_)) + static_cast(t)]; - for (int su = 0; su < s_; ++su) { - const int g = (a * s_) + su; - const int cnt = recv_counts[g]; - if (cnt != 0) { - std::memcpy(dst + (static_cast(recv_displs[g]) * elem), - stage_recv_.data() + (cur * elem), - static_cast(cnt) * elem); - } - cur += static_cast(cnt); - } - } - } - - [[noreturn]] auto abort_rank_(const char *verb, const char *what) -> void { - std::print(stderr, - "monoprop: rank {} cannot complete the collective '{}' ({}). Its peer ranks are " - "blocked inside MPI with no way to be released, so the job is aborted rather than hung.\n", - mpi_rank_, - verb, - what); - std::fflush(stderr); - MPI_Abort(parent_, 1); - std::abort(); // MPI_Abort is not marked [[noreturn]]; unreachable in practice - } - - auto sync() -> void { barrier_.sync(); } - - static constexpr int kNoPeer = -1; - static constexpr int kManyPeers = -2; - - MPI_Comm parent_; - int s_; - int r_ = 1; - int mpi_rank_ = 0; - bool pairwise_ = false; - std::vector slots_; - // Each partition's send / recv peer (kNoPeer, a rank, or kManyPeers): written by its owner in Phase - // P0, folded by partition 0 in B1->B2. - std::vector row_peer_; - std::vector row_recv_peer_; - - // Partition-0-managed, except the two Phase P0 tables at the end, whose rows each partition owns. - // S*S per rank, the counts alltoall. - std::vector counts_send_; - std::vector counts_recv_; - // Per-rank [R] counts/displs for the aggregated payload alltoallv. - std::vector mpi_send_counts_; - std::vector mpi_send_displs_; - std::vector mpi_recv_counts_; - std::vector mpi_recv_displs_; - // [S x P] payload block starts in stage_send_, row u per global destination g. No scatter-side - // table: see size_staging_recv_. - std::vector pack_off_; - // [P] staging bases: base_send_[b*S+t] starts partition (b,t)'s region of the message to rank b, - // base_recv_[a*S+t] starts local partition t's region of the message from rank a. - std::vector base_send_; - std::vector base_recv_; - std::vector col_sum_; - std::vector recv_col_; - // [S x counts_stride_] send-count matrix, row u owned by partition u; padded so no two rows share a - // 64-byte line. Lifetime rule: publish_counts_row_. - std::vector counts_matrix_store_; - int *counts_matrix_ = nullptr; - size_t counts_stride_ = 0; - // [S x rows_stride_] per-partition per-rank recv totals, same ownership and padding rules. - std::vector rows_store_; - long long *rows_ = nullptr; - size_t rows_stride_ = 0; - // Aggregated MPI payload staging, HWM-sized. - std::vector stage_send_; - std::vector stage_recv_; - double red_f64_ = 0.0; - uint64_t red_u64_ = 0; - std::vector red_vec_; - // Partition-0 request scratch for the sparse payload round. - std::vector reqs_; - // The count round's own: it stays live across B2..B3, so sharing reqs_ would let a resize move it. - std::vector count_reqs_; - int count_posted_ = 0; // live entries of count_reqs_; 0 on the dense (blocking) arm - std::vector peers_; // this verb's peer ranks; see fill_peers_ - PeerPlan wire_plan_; // alltoallv's, partition 0 only: written in B1->B2, read in B3->B4 - - PartitionBarrier barrier_; -}; - -} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/MPICompat.cpp b/cpp/monoprop/detail/mpi/MPICompat.cpp index 0ed8cbcb5..8f720d9dd 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.cpp +++ b/cpp/monoprop/detail/mpi/MPICompat.cpp @@ -16,8 +16,9 @@ #include #include +#include +#include #include -#include #include #ifdef monoprop_ENABLE_MPI @@ -31,17 +32,24 @@ namespace monoprop::mpi { auto init(int *argc, char ***argv) -> void { auto initialized = 0; MPI_Initialized(&initialized); + int provided = MPI_THREAD_SINGLE; if (!initialized) { - // serialized (not funneled): under the hybrid the one-at-a-time MPI calls come from each rank's - // partition-0 master, not the main thread. mpi4py already requests >= serialized. - auto required = MPI_THREAD_SERIALIZED; - auto provided = 0; - MPI_Init_thread(argc, argv, required, &provided); - if (provided < required) { - std::print("Sorry, the MPI library does not provide MPI_THREAD_SERIALIZED support, which is required " - "by the partition/MPI hybrid transport.\n"); - MPI_Abort(MPI_COMM_WORLD, 1); - } + // Funneled: only this thread, as the primary of the library's team, ever calls MPI. + MPI_Init_thread(argc, argv, required_thread_level, &provided); + } + else { + // Host-initialized: never reinitialize, and never assume the host's request was granted. + MPI_Query_thread(&provided); + } + if (!thread_level_satisfies(provided, required_thread_level)) { + // Fixed strings only: nothing here may allocate (or throw) before the abort. + std::fputs("monoprop: the MPI library provides ", stderr); + std::fputs(thread_level_name(provided), stderr); + std::fputs(" but monoprop requires ", stderr); + std::fputs(thread_level_name(required_thread_level), stderr); + std::fputs("; aborting.\n", stderr); + std::fflush(stderr); + MPI_Abort(MPI_COMM_WORLD, 1); } } @@ -54,38 +62,73 @@ auto finalize() -> void { } #endif // monoprop_ENABLE_MPI -auto rank(const Comm &comm) -> int { - if (comm.kind == Comm::Kind::Shm) { - return comm.shm_rank; +auto require_thread_support() -> void { +#ifdef monoprop_ENABLE_MPI + int initialized = 0; + MPI_Initialized(&initialized); + if (initialized == 0) { + return; + } + int provided = MPI_THREAD_SINGLE; + MPI_Query_thread(&provided); + if (!thread_level_satisfies(provided, required_thread_level)) { + throw MpiThreadLevelUnsupported( + std::format("monoprop requires {} support from MPI, but the initialized MPI library provides {}. " + "Initialize MPI with MPI_Init_thread requesting {} or higher (mpi4py requests " + "MPI_THREAD_MULTIPLE unless mpi4py.rc.thread_level says otherwise).", + thread_level_name(required_thread_level), + thread_level_name(provided), + thread_level_name(required_thread_level))); } +#endif +} + +auto require_initializing_thread() -> void { #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->global_rank(comm.shm_rank); + // MPI_Initialized, MPI_Finalized and MPI_Is_thread_main may be called from any thread; nothing else is. + int initialized = 0; + int finalized = 0; + MPI_Initialized(&initialized); + MPI_Finalized(&finalized); + if (initialized == 0 || finalized != 0) { + return; } + int is_main = 0; + MPI_Is_thread_main(&is_main); + if (is_main == 0) { + std::fputs("monoprop: an operation that uses MPI was called from a thread other than the thread that " + "initialized MPI. monoprop requires every such call to come from the thread that initialized " + "MPI, whatever thread support MPI provides. This process stops here without communicating; " + "the launcher ends its peers.\n", + stderr); + std::fflush(stderr); + std::abort(); + } +#endif +} + +auto rank(const Comm &comm) -> int { +#ifdef monoprop_ENABLE_MPI int r = 0; if (MPI_Comm_rank(comm.mpi, &r) != MPI_SUCCESS) { throw CollectiveArgumentError("MPI_Comm_rank failed"); } return r; #else + (void)comm; return 0; #endif } auto size(const Comm &comm) -> int { - if (comm.kind == Comm::Kind::Shm) { - return comm.shm->size(); - } #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->size(); - } int s = 0; if (MPI_Comm_size(comm.mpi, &s) != MPI_SUCCESS) { throw CollectiveArgumentError("MPI_Comm_size failed"); } return s; #else + (void)comm; return 1; #endif } @@ -106,71 +149,39 @@ auto routing_keyval() -> int { auto routes_pairwise(const Comm &comm) -> bool { #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->routes_pairwise(); - } - if (comm.kind == Comm::Kind::Mpi) { - const int ranks = size(comm); - if (ranks <= 1) { - return false; // no peer to pair with, and no collective to agree through - } - void *cached = nullptr; - int found = 0; - MPI_Comm_get_attr(comm.mpi, routing_keyval(), &cached, &found); - if (found != 0) { - return reinterpret_cast(cached) != 0; - } - const bool pairwise = agree_routes_pairwise(comm.mpi, ranks, routing::Config::from_env(1)); - MPI_Comm_set_attr(comm.mpi, routing_keyval(), reinterpret_cast(static_cast(pairwise))); - return pairwise; - } -#endif - return false; // Shm is one rank; there is no inter-rank transport to choose -} - -auto geometry(const Comm &comm) -> Geometry { - if (comm.kind == Comm::Kind::Shm) { - return {.ranks = 1, .partitions = comm.shm->size()}; - } -#ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return {.ranks = comm.hyb->ranks(), .partitions = comm.hyb->partitions()}; - } - return {.ranks = size(comm), .partitions = 1}; + const int ranks = size(comm); + if (ranks <= 1) { + return false; // no peer to pair with, and no collective to agree through + } + void *cached = nullptr; + int found = 0; + MPI_Comm_get_attr(comm.mpi, routing_keyval(), &cached, &found); + if (found != 0) { + return reinterpret_cast(cached) != 0; + } + const bool pairwise = agree_routes_pairwise(comm.mpi, ranks, routing::Config::from_env()); + MPI_Comm_set_attr(comm.mpi, routing_keyval(), reinterpret_cast(static_cast(pairwise))); + return pairwise; #else - return {.ranks = 1, .partitions = 1}; + (void)comm; + return false; // one process: there is no inter-rank transport to choose #endif } auto allreduce_sum_inplace(VecD &values, Comm comm) -> void { - if (comm.kind == Comm::Kind::Shm) { - comm.shm->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); - return; - } #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->allreduce_sum_inplace(comm.shm_rank, values.data(), values.size()); - return; - } MPI_Allreduce(MPI_IN_PLACE, values.data(), static_cast(values.size()), MPI_DOUBLE, MPI_SUM, comm.mpi); #else (void)values; // single participant: identity + (void)comm; #endif } auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, PeerPlan plan) -> void { - require_routable(plan, geometry(comm).ranks); - if (comm.kind == Comm::Kind::Shm) { - comm.shm->alltoall_counts(comm.shm_rank, send_counts, recv_counts); - return; - } + require_routable(plan, size(comm)); #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoall_counts(comm.shm_rank, send_counts, recv_counts, plan); - return; - } if (!plan.dense()) { - // S == 1 world: one int with the peer. Non-peers are zero by definition, so clear them. + // One int with the peer. Non-peers are zero by definition, so clear them. int me = 0; MPI_Comm_rank(comm.mpi, &me); std::fill(recv_counts, recv_counts + n, 0); @@ -198,6 +209,7 @@ auto alltoall_counts(const int *send_counts, int *recv_counts, int n, Comm comm, MPI_Alltoall(send_counts, 1, MPI_INT, recv_counts, 1, MPI_INT, comm.mpi); #else (void)plan; // single participant: nothing to narrow + (void)comm; for (int i = 0; i < n; ++i) { recv_counts[i] = send_counts[i]; } diff --git a/cpp/monoprop/detail/mpi/MPICompat.h b/cpp/monoprop/detail/mpi/MPICompat.h index b72a7c1a3..758eb7378 100644 --- a/cpp/monoprop/detail/mpi/MPICompat.h +++ b/cpp/monoprop/detail/mpi/MPICompat.h @@ -27,12 +27,10 @@ #include #include -// Comm.h owns the MPI_Comm typedef (real or non-MPI fallback) and the runtime-tagged mpi::Comm handle. +// Comm.h owns the MPI_Comm typedef (real or non-MPI fallback) and the ordinary mpi::Comm handle. #include "monoprop/detail/mpi/CheckedCount.h" #include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/ShmComm.h" #ifdef monoprop_ENABLE_MPI -#include "monoprop/detail/mpi/HybridComm.h" #include "monoprop/detail/mpi/Pairwise.h" #endif @@ -42,7 +40,44 @@ namespace monoprop::mpi { +/// Initialized MPI provides less thread support than monoprop requires (required_thread_level). +class MpiThreadLevelUnsupported : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + #ifdef monoprop_ENABLE_MPI +/// The MPI thread support monoprop requests and requires. FUNNELED: only the thread that initialized MPI, as the +/// primary of the library's OpenMP team, ever calls MPI. A higher level is accepted and changes nothing: workers still +/// never call MPI, and public calls must still come from the initializing thread. +inline constexpr int required_thread_level = MPI_THREAD_FUNNELED; + +/// Whether \a provided thread support satisfies \a required; MPI orders the levels +/// SINGLE < FUNNELED < SERIALIZED < MULTIPLE. A higher level never relaxes the initializing-thread rule. +constexpr auto thread_level_satisfies(int provided, int required) noexcept -> bool { + return provided >= required; +} + +/// The MPI constant naming a thread support level, for diagnostics. +constexpr auto thread_level_name(int level) noexcept -> const char * { + if (level == MPI_THREAD_SINGLE) { + return "MPI_THREAD_SINGLE"; + } + if (level == MPI_THREAD_FUNNELED) { + return "MPI_THREAD_FUNNELED"; + } + if (level == MPI_THREAD_SERIALIZED) { + return "MPI_THREAD_SERIALIZED"; + } + if (level == MPI_THREAD_MULTIPLE) { + return "MPI_THREAD_MULTIPLE"; + } + return "an unknown MPI thread level"; +} + +/// Initialize MPI requesting required_thread_level unless the host already did, then check the level actually +/// provided in either case: a level below it prints a diagnostic and aborts MPI_COMM_WORLD. Host-initialized MPI is +/// never reinitialized. monoprop_EXPORT auto init(int *argc = nullptr, char ***argv = nullptr) -> void; monoprop_EXPORT auto finalize() -> void; @@ -90,33 +125,33 @@ inline auto init(int * /*argc*/ = nullptr, char *** /*argv*/ = nullptr) -> void inline auto finalize() -> void {} #endif // monoprop_ENABLE_MPI +/// Check that initialized MPI provides at least required_thread_level, whoever initialized it. +/// \throws MpiThreadLevelUnsupported if it does not. A no-op without MPI or before MPI is initialized. +monoprop_EXPORT auto require_thread_support() -> void; + +/// Fail fast unless the caller is the thread that initialized MPI. +/// +/// A call from any other thread breaks the host contract, whatever thread support MPI provides: it prints a +/// diagnostic and calls std::abort without calling MPI_Abort or any other communication, so the launcher +/// must end the peers. A no-op without MPI or while MPI is not initialized. Every MPI-using propagator +/// operation checks it on entry. +monoprop_EXPORT auto require_initializing_thread() -> void; + monoprop_EXPORT auto rank(const Comm &comm) -> int; monoprop_EXPORT auto size(const Comm &comm) -> int; -// size() split into ranks * partitions: routing needs to tell a network hop from a shared-memory copy. -struct Geometry { - int ranks = 1; - int partitions = 1; -}; -monoprop_EXPORT auto geometry(const Comm &comm) -> Geometry; - // Whether this communicator's exchanges go point-to-point: agreed collectively once and cached on it, // because ranks that disagree hang. Throws RoutingDisagreement on every rank together on a mismatch. monoprop_EXPORT auto routes_pairwise(const Comm &comm) -> bool; template inline auto allreduce_sum(T local_val, Comm comm) -> T { - if (comm.kind == Comm::Kind::Shm) { - return comm.shm->allreduce_sum(comm.shm_rank, local_val); - } #ifdef monoprop_ENABLE_MPI - if (comm.kind == Comm::Kind::Hybrid) { - return comm.hyb->allreduce_sum(comm.shm_rank, local_val); - } T global_val{}; MPI_Allreduce(&local_val, &global_val, 1, datatype::get(), MPI_SUM, comm.mpi); return global_val; #else + (void)comm; // single participant: identity return local_val; #endif } @@ -248,8 +283,8 @@ inline auto prepare_recv_layout_(PendingAlltoallv &pending, pending.recv_buffer.resize(static_cast(checked_mpi_count(running, "Total recv count"))); } -// The count exchange runs eagerly (recv_counts known on return); the Kind::Mpi payload is non-blocking -// (wait_into completes it), Shm / single-process transfer here. +// The count exchange runs eagerly (recv_counts known on return); the MPI payload is non-blocking (wait_into +// completes it), and the single-process stub copies here. // skip_self: do not send the self slot (the caller handles self inline) — self send/recv = 0. // known_recv_counts: recv counts already known (e.g. the transpose of the query counts), so skip the // count exchange. The self slot is also zeroed when skip_self is set. @@ -261,13 +296,12 @@ template PeerPlan plan = {}) -> PendingAlltoallv { const int num_ranks = size(comm); const int me = rank(comm); - const auto geom = geometry(comm); - require_routable(plan, geom.ranks); + require_routable(plan, num_ranks); PendingAlltoallv h; h.num_ranks = num_ranks; - // The plan's window is the mask. send_data need only cover it; what lies outside must be empty. - h.window = - plan.window(static_cast(me), static_cast(geom.ranks), static_cast(geom.partitions)); + // The plan's window is the mask (one owner per rank). send_data need only cover it; what lies outside must be + // empty. + h.window = plan.window(static_cast(me), static_cast(num_ranks), 1); const SlotWindow supplied = send_data.window(); if (h.window.stop() > static_cast(num_ranks) || supplied.base > h.window.base || supplied.stop() < h.window.stop()) { @@ -310,88 +344,44 @@ template h.recv_counts.assign(static_cast(num_ranks), 0); - const AlltoallvResolveArgs resolve_args{.send = h.send_buffer.data(), - .send_counts = h.send_counts.data(), - .send_displs = h.send_displs.data(), - .recv = h.recv_buffer, - .recv_counts = h.recv_counts.data(), - .recv_displs = h.recv_displs.data()}; - // Fused fast path (query round, recv layout unknown): resolve recv counts AND move payload in one - // in-process verb, folding away the count exchange's barriers (Shm 4→2, Hybrid 6→4). It fills - // recv_counts/recv_displs and resizes recv_buffer; known-layout and pure-MPI paths fall through. - if (known_recv_counts == nullptr && comm.kind == Comm::Kind::Shm) { - comm.shm->alltoallv_resolve(comm.shm_rank, resolve_args); - return h; - } -#ifdef monoprop_ENABLE_MPI - if (known_recv_counts == nullptr && comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoallv_resolve(comm.shm_rank, resolve_args, datatype::get(), plan); - return h; - } -#endif - prepare_recv_layout_(h, known_recv_counts, self, comm, plan); - // Taken after the resize above: recv_buffer may have reallocated. - const auto flat = FlatAlltoallvArgs{.send = h.send_buffer.data(), - .send_counts = h.send_counts.data(), - .send_displs = h.send_displs.data(), - .recv = h.recv_buffer.data(), - .recv_counts = h.recv_counts.data(), - .recv_displs = h.recv_displs.data()} - .bytes(); - if (comm.kind == Comm::Kind::Shm) { - // ShmComm needs no send counts: its peers pull using the publisher's displacements. - comm.shm->alltoallv(comm.shm_rank, - flat.send, - flat.send_displs, - flat.recv, - flat.recv_counts, - flat.recv_displs, - flat.elem); - } #ifdef monoprop_ENABLE_MPI - else if (comm.kind == Comm::Kind::Hybrid) { - comm.hyb->alltoallv(comm.shm_rank, flat, datatype::get(), plan); + // Taken after the resize above: recv_buffer may have reallocated. + if (plan.dense()) { + h.requests.resize(1); + h.posted = 1; + MPI_Ialltoallv(h.send_buffer.data(), + h.send_counts.data(), + h.send_displs.data(), + datatype::get(), + h.recv_buffer.data(), + h.recv_counts.data(), + h.recv_displs.data(), + datatype::get(), + comm.mpi, + h.requests.data()); } -#endif else { -#ifdef monoprop_ENABLE_MPI - if (plan.dense()) { - h.requests.resize(1); - h.posted = 1; - MPI_Ialltoallv(h.send_buffer.data(), - h.send_counts.data(), - h.send_displs.data(), - datatype::get(), - h.recv_buffer.data(), - h.recv_counts.data(), - h.recv_displs.data(), - datatype::get(), - comm.mpi, - h.requests.data()); - } - else { - // S == 1 world: point-to-point with the peer, left in flight like MPI_Ialltoallv. - const SparsePairwiseArgs pairwise{ - .plan = plan, - .me = me, - .num_ranks = num_ranks, - .comm = comm.mpi, - .tag = kFlatPayloadTag, - .datatype = datatype::get(), - .elem = sizeof(T), - .send = reinterpret_cast(h.send_buffer.data()), - .send_layout = {.counts = h.send_counts.data(), .displs = h.send_displs.data()}, - .recv = reinterpret_cast(h.recv_buffer.data()), - .recv_layout = {.counts = h.recv_counts.data(), .displs = h.recv_displs.data()}, - }; - h.posted = sparse_pairwise(pairwise, h.requests); - } + // Point-to-point with the plan's one peer, left in flight like MPI_Ialltoallv. + const SparsePairwiseArgs pairwise{ + .plan = plan, + .me = me, + .num_ranks = num_ranks, + .comm = comm.mpi, + .tag = kFlatPayloadTag, + .datatype = datatype::get(), + .elem = sizeof(T), + .send = reinterpret_cast(h.send_buffer.data()), + .send_layout = {.counts = h.send_counts.data(), .displs = h.send_displs.data()}, + .recv = reinterpret_cast(h.recv_buffer.data()), + .recv_layout = {.counts = h.recv_counts.data(), .displs = h.recv_displs.data()}, + }; + h.posted = sparse_pairwise(pairwise, h.requests); + } #else - h.recv_buffer = h.send_buffer; // single participant: self round-trip (layouts identical) + h.recv_buffer = h.send_buffer; // single participant: self round-trip (layouts identical) #endif - } return h; } diff --git a/cpp/monoprop/detail/mpi/MPIUtils.h b/cpp/monoprop/detail/mpi/MPIUtils.h index 9e2fb0915..eee29f0c6 100644 --- a/cpp/monoprop/detail/mpi/MPIUtils.h +++ b/cpp/monoprop/detail/mpi/MPIUtils.h @@ -55,11 +55,12 @@ auto find_rank(const Monomial &mono, const routing::Router &router) -> return router.dest(mono); } -// The router for this communicator's geometry and monoprop_ROUTING, bound to this monomial width. +// The router of the low-level one-owner-per-rank engine (detail::build_layer): one flat owner per rank of `comm`, under +// monoprop_ROUTING, bound to this monomial width. A sharded root never uses it: it prepares its own (P, T) router from +// the communicator's size and its captured budget. template inline auto router_for(const mpi::Comm &comm) -> routing::Router { - const auto geom = mpi::geometry(comm); - return routing::make_router(static_cast(geom.ranks), static_cast(geom.partitions)); + return routing::make_router(static_cast(mpi::size(comm)), 1); } // Agree the transport now, so a misconfigured rank throws at construction instead of hanging its peers. diff --git a/cpp/monoprop/detail/mpi/OperationFailure.h b/cpp/monoprop/detail/mpi/OperationFailure.h new file mode 100644 index 000000000..06e7487dc --- /dev/null +++ b/cpp/monoprop/detail/mpi/OperationFailure.h @@ -0,0 +1,118 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/Comm.h" + +namespace monoprop::mpi { + +/*! + * \brief The message carried by \a error, for a diagnostic. + * + * Never throws; an exception that is not a `std::exception` reads as "unknown exception". + */ +inline auto describe_exception(const std::exception_ptr &error) noexcept -> std::string { + try { + std::rethrow_exception(error); + } + catch (const std::exception &e) { + try { + return e.what(); + } + catch (...) { + return {}; + } + } + catch (...) { + return "unknown exception"; + } +} + +/*! + * \brief Handle a failure on the calling rank during a distributed operation, from the controlling thread. + * + * On a communicator with more than one rank, the peers may already be committed to a communication this + * rank will never join, and no exception can release them. So this reports the rank and the underlying + * error on stderr and calls `MPI_Abort` on \a comm; if `MPI_Abort` returns, it calls `std::terminate`. + * Composing the report cannot throw ahead of the abort: if it fails, a fixed diagnostic is printed + * instead. On one rank or without MPI, it rethrows \a error unchanged, preserving the original type. + * + * Callers catch failures where they happen, inside the lifetime of any posted request handle + * (mpi::Ticket, mpi::PendingAlltoallv), so that unwinding never drains requests peers will not complete. + * Never call it from an OpenMP worker or from a thread other than MPI's initializing thread. + * + * \param comm The communicator the operation runs on. + * \param error The failure; must not be null. + */ +[[noreturn]] inline auto operation_failed(const Comm &comm, std::exception_ptr error) -> void { +#ifdef monoprop_ENABLE_MPI + int initialized = 0; + int finalized = 0; + MPI_Initialized(&initialized); + MPI_Finalized(&finalized); + int size = 1; + if (initialized != 0 && finalized == 0 && MPI_Comm_size(comm.mpi, &size) == MPI_SUCCESS && size > 1) { + int rank = 0; + MPI_Comm_rank(comm.mpi, &rank); + try { + const auto line = + std::format("monoprop: rank {} of {}: a distributed operation failed on this rank: {}. Its peers may " + "already be waiting on communication this rank will not complete, so monoprop is " + "aborting the communicator rather than leaving them blocked.\n", + rank, + size, + describe_exception(error)); + std::fputs(line.c_str(), stderr); + } + catch (...) { + std::fputs("monoprop: a distributed operation failed on this rank (its description could not be " + "formatted); aborting the communicator rather than leaving its peers blocked.\n", + stderr); + } + std::fflush(stderr); + MPI_Abort(comm.mpi, 1); + std::terminate(); + } +#else + (void)comm; +#endif + std::rethrow_exception(std::move(error)); +} + +/*! + * \brief Run \a fn and send any exception it throws to operation_failed(). + * + * Wrap each phase that can throw while peers may be committed to communication, and place the wrapper + * inside the lifetime of any request handle the phase overlaps. + * + * \return Whatever \a fn returns. + */ +template +auto guard_distributed(const Comm &comm, Fn &&fn) -> decltype(std::forward(fn)()) { + try { + return std::forward(fn)(); + } + catch (...) { + operation_failed(comm, std::current_exception()); + } +} + +} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/Pairwise.h b/cpp/monoprop/detail/mpi/Pairwise.h index 15f258666..9b40c5d09 100644 --- a/cpp/monoprop/detail/mpi/Pairwise.h +++ b/cpp/monoprop/detail/mpi/Pairwise.h @@ -33,9 +33,7 @@ namespace monoprop::mpi { // in flight matching a payload receive. Engine.h's two rounds may share kFlatPayloadTag: MPI does not // overtake within (src, dst, tag, comm), round 1 completes before round 2 posts, and both ends skip a // zero-count leg on the same (transposed) value. -inline constexpr int kHybridCountTag = 0x6D70; // 'mp' -inline constexpr int kHybridPayloadTag = 0x6D71; -inline constexpr int kFlatPayloadTag = 0x6D72; +inline constexpr int kFlatPayloadTag = 0x6D72; // 'mr' inline constexpr int kFlatCountTag = 0x6D73; // Graph replay payload (Exchange.h), distinct from the build path's rounds on the same communicator. inline constexpr int kFlatReplayTag = 0x6D74; @@ -124,14 +122,9 @@ struct SparsePairwiseArgs { // Allreduce {v, ~v} under MPI_MAX, giving max and min at once: max == min == mine is exact agreement. // Ranks that disagree would hang, so every rank throws together instead. inline auto agree_routes_pairwise(MPI_Comm comm, int ranks, const routing::Config &mine) -> bool { - const std::array probe{mine.linear, - ~mine.linear, - mine.partitions, - ~mine.partitions, - mine.seed, - ~mine.seed}; - std::array agreed{}; - MPI_Allreduce(probe.data(), agreed.data(), 6, MPI_UINT64_T, MPI_MAX, comm); + const std::array probe{mine.linear, ~mine.linear, mine.seed, ~mine.seed}; + std::array agreed{}; + MPI_Allreduce(probe.data(), agreed.data(), static_cast(probe.size()), MPI_UINT64_T, MPI_MAX, comm); if (agreed != probe) { throw routing::RoutingDisagreement( std::format("routing configuration differs across the {} ranks (this one: {}). " diff --git a/cpp/monoprop/detail/mpi/PartitionBarrier.h b/cpp/monoprop/detail/mpi/PartitionBarrier.h deleted file mode 100644 index 1f31a4825..000000000 --- a/cpp/monoprop/detail/mpi/PartitionBarrier.h +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include "monoprop/detail/mpi/CpuRelax.h" - -namespace monoprop::mpi { - -// Turns what would be a permanent hang into an exception on every participant. -class ShmCommPoisoned : public std::runtime_error { -public: - ShmCommPoisoned() : std::runtime_error("ShmComm poisoned: a peer partition threw during a collective") {} -}; - -// Sense-reversing generation barrier for a fixed number of in-process partition threads. Each barrier -// word gets a private cache line: if `gen_` shared `arrived_`'s line, spinners' reloads would miss to -// L3 on every peer arrival — O(S) coherence bounces (measured top hotspot at S=112). -// -// The acquire/release/relaxed orderings below are load-bearing, not an oversight: the release store to -// `gen_` is what publishes the preceding relaxed reset of `arrived_`. Promoting them to seq_cst would -// put a full barrier in the spin loop of that same hotspot, so cpp:S8417 is suppressed for this file -// in sonar-project.properties. Do not "simplify" them to the default ordering. -class PartitionBarrier { -public: - explicit PartitionBarrier(int participants) : participants_(participants) {} - - PartitionBarrier(const PartitionBarrier &) = delete; - auto operator=(const PartitionBarrier &) -> PartitionBarrier & = delete; - - auto sync() -> void { - const unsigned g = gen_.load(std::memory_order_acquire); - if (arrived_.fetch_add(1, std::memory_order_acq_rel) + 1 == participants_) { - arrived_.store(0, std::memory_order_relaxed); - gen_.store(g + 1, std::memory_order_release); - } - else { - // Bounded on-core spin first (pinned partitions ⇒ the release store lands in the pause window, - // no syscall); only long waits (imbalance, oversubscription) fall to yield. - int spins = 0; - while (gen_.load(std::memory_order_acquire) == g) { - if (poisoned_.load(std::memory_order_acquire)) { - throw ShmCommPoisoned(); - } - if (spins < detail::kSpinPauseIters) { - ++spins; - detail::cpu_relax(); - } - else { - std::this_thread::yield(); - } - } - } - if (poisoned_.load(std::memory_order_acquire)) { - throw ShmCommPoisoned(); - } - } - - // Signal that this participant is unwinding (e.g. an engine exception), releasing peers spinning in a - // barrier. Idempotent. - auto poison() -> void { poisoned_.store(true, std::memory_order_release); } - - // Must be called only when every participant is quiescent (between rounds), so a poison-aborted round - // leaves no dirty state. `gen_` deliberately stays monotonic: each participant re-reads it at its next - // barrier. - auto reset() -> void { - poisoned_.store(false, std::memory_order_relaxed); - arrived_.store(0, std::memory_order_relaxed); - } - -private: - int participants_; - alignas(64) std::atomic arrived_{0}; - alignas(64) std::atomic gen_{0}; - alignas(64) std::atomic poisoned_{false}; -}; - -} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/mpi/Routing.h b/cpp/monoprop/detail/mpi/Routing.h index a941f93cb..005dbe159 100644 --- a/cpp/monoprop/detail/mpi/Routing.h +++ b/cpp/monoprop/detail/mpi/Routing.h @@ -32,7 +32,7 @@ // The one owner function: Scan.h emits queries by it and MonomialPropagator seeds by it, so a second // copy would split ownership silently. // -// part = monomial_hash(M) % S splitmix: partition fanout is free, only balance matters +// part = monomial_hash(M) % S splitmix: shard fanout is free, only balance matters // rank = linear_hash(M) & (R - 1) GF(2)-linear: rank(M ^ G) == rank(M) ^ rank(G), so fanout 1 // flat = rank * S + part // @@ -146,9 +146,9 @@ class Router final { // The only way to a linear router. Binds the basis for this width, keeping its static-init guard off // dest(). Throws unless every rank is reachable: R == 2^d with d <= the full-rank column count. template - [[nodiscard]] static auto for_modes(size_t ranks, size_t partitions, bool linear) -> Router { + [[nodiscard]] static auto for_modes(size_t ranks, size_t shards_per_rank, bool linear) -> Router { constexpr size_t max_bits = std::min(kLinearColumns, 2 * NumModes); - Router r{ranks, partitions, linear}; + Router r{ranks, shards_per_rank, linear}; if (r.linear_bits() > max_bits) { throw UnroutableGeometry( std::format("linear routing over {} modes reaches at most 2^{} ranks, got {}. Launch fewer " @@ -166,7 +166,7 @@ class Router final { static constexpr auto splitmix(size_t flat_world) -> Router { return Router{flat_world, 1, false}; } [[nodiscard]] constexpr auto ranks() const noexcept -> size_t { return ranks_; } - [[nodiscard]] constexpr auto partitions() const noexcept -> size_t { return parts_; } + [[nodiscard]] constexpr auto shards_per_rank() const noexcept -> size_t { return shards_per_rank_; } [[nodiscard]] constexpr auto flat_world() const noexcept -> size_t { return flat_; } // False for a splitmix router AND for R == 1, which has no rank bit to take: both route densely. [[nodiscard]] constexpr auto is_linear() const noexcept -> bool { return linear_; } @@ -182,10 +182,10 @@ class Router final { return static_cast(monomial_hash(mono) % flat_); } const auto rank = static_cast(linear_low_(mono)); - if (parts_ == 1) { + if (shards_per_rank_ == 1) { return rank; // S == 1 needs no hash } - return (rank * parts_) + part_of_(monomial_hash(mono)); + return (rank * shards_per_rank_) + part_of_(monomial_hash(mono)); } // dest(M ^ G) for a term M that `my_flat` owns, where `shift` == rank_shift(G). Wrong arguments move @@ -198,10 +198,10 @@ class Router final { return dest(mono); } const size_t rank = rank_of_slot_(my_flat) ^ shift; - if (parts_ == 1) { + if (shards_per_rank_ == 1) { return rank; } - return (rank * parts_) + part_of_(monomial_hash(mono)); + return (rank * shards_per_rank_) + part_of_(monomial_hash(mono)); } // The rank-level shift a generator induces: rank(M^G) == rank(M) ^ shift(G). Zero when not linear. @@ -212,14 +212,14 @@ class Router final { private: // Private: a linear router must go through for_modes, which binds the basis. - constexpr Router(size_t ranks, size_t partitions, bool linear) + constexpr Router(size_t ranks, size_t shards_per_rank, bool linear) : ranks_(ranks == 0 ? 1 : ranks), - parts_(partitions == 0 ? 1 : partitions), - flat_(ranks_ * parts_), + shards_per_rank_(shards_per_rank == 0 ? 1 : shards_per_rank), + flat_(ranks_ * shards_per_rank_), linear_(linear && ranks_ > 1), // R == 1 has no rank bit: dense by construction - parts_pow2_(std::has_single_bit(parts_)), - parts_mask_(parts_ - 1), - parts_log2_(static_cast(std::countr_zero(parts_))) { + shards_pow2_(std::has_single_bit(shards_per_rank_)), + shards_mask_(shards_per_rank_ - 1), + shards_log2_(static_cast(std::countr_zero(shards_per_rank_))) { if (linear && !std::has_single_bit(ranks_)) { throw UnroutableGeometry( std::format("linear routing needs a power-of-two rank count, got {}. Launch 2^k ranks, or set " @@ -228,13 +228,13 @@ class Router final { } } - // parts_ is a runtime value, so strength-reduce the power-of-two case by hand. + // shards_per_rank_ is a runtime value, so strength-reduce the power-of-two case by hand. [[nodiscard]] [[gnu::always_inline]] auto part_of_(uint64_t q) const noexcept -> size_t { - return static_cast(parts_pow2_ ? (q & parts_mask_) : (q % parts_)); + return static_cast(shards_pow2_ ? (q & shards_mask_) : (q % shards_per_rank_)); } [[nodiscard]] [[gnu::always_inline]] auto rank_of_slot_(size_t flat_slot) const noexcept -> size_t { - return parts_pow2_ ? (flat_slot >> parts_log2_) : (flat_slot / parts_); + return shards_pow2_ ? (flat_slot >> shards_log2_) : (flat_slot / shards_per_rank_); } // linear_hash(M) & (R - 1). Runs at seeding and once per generator, never per query. @@ -252,12 +252,12 @@ class Router final { } size_t ranks_; - size_t parts_; + size_t shards_per_rank_; size_t flat_; bool linear_; - bool parts_pow2_; // S is 2^k, so `% S` is a mask and `/ S` a shift - size_t parts_mask_; // S - 1, and parts_log2_ == log2(S); both meaningless unless parts_pow2_ - size_t parts_log2_; + bool shards_pow2_; // S is 2^k, so `% S` is a mask and `/ S` a shift + size_t shards_mask_; // S - 1, and shards_log2_ == log2(S); both meaningless unless shards_pow2_ + size_t shards_log2_; const uint64_t *basis_ = nullptr; // linear_basis(), bound by for_modes size_t basis_bits_ = 0; }; @@ -267,21 +267,17 @@ inline auto linear_requested() -> bool { return config::get().routing_mode.value_or(config::RoutingMode::Linear) == config::RoutingMode::Linear; } -// What must match across ranks for the transports to pair up; raw values, so agreement is exact. +// What must match across ranks for the transports to pair up; raw values, so agreement is exact. The thread count is +// not part of it: the sharded runtime takes a uniform T as a launch precondition, which nothing checks. struct Config { uint64_t linear = 0; - uint64_t partitions = 1; uint64_t seed = 0; - static auto from_env(size_t partitions) -> Config { - return {.linear = static_cast(linear_requested()), - .partitions = static_cast(partitions), - .seed = seed_from_env()}; + static auto from_env() -> Config { + return {.linear = static_cast(linear_requested()), .seed = seed_from_env()}; } - [[nodiscard]] auto describe() const -> std::string { - return std::format("linear={}, partitions={}, seed={}", linear, partitions, seed); - } + [[nodiscard]] auto describe() const -> std::string { return std::format("linear={}, seed={}", linear, seed); } }; // Whether an AGREED config routes point-to-point. Never throws, since the replay path asks from inside a @@ -291,8 +287,8 @@ struct Config { } template -inline auto make_router(size_t ranks, size_t partitions) -> Router { - return Router::for_modes(ranks, partitions, linear_requested()); +inline auto make_router(size_t ranks, size_t shards_per_rank) -> Router { + return Router::for_modes(ranks, shards_per_rank, linear_requested()); } } // namespace monoprop::routing diff --git a/cpp/monoprop/detail/mpi/ShmComm.h b/cpp/monoprop/detail/mpi/ShmComm.h deleted file mode 100644 index b845383da..000000000 --- a/cpp/monoprop/detail/mpi/ShmComm.h +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "monoprop/detail/mpi/CheckedCount.h" -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/PartitionBarrier.h" - -// In-process shared-memory SPMD transport: S partition-master threads each call the same collective -// sequence in program order, every collective two-phase (publish slot → barrier → read peers → -// barrier). Two guarantees the engine relies on: alltoallv delivers each source's block contiguously -// in ascending source-rank order (Resolve.h's positional pairing needs it), and allreduce sums in -// ascending rank order, so results are bit-identical and deterministic per S. - -namespace monoprop::mpi { - -class ShmComm { -public: - explicit ShmComm(int n) : n_(n), slots_(static_cast(n)), barrier_(n) {} - - ShmComm(const ShmComm &) = delete; - auto operator=(const ShmComm &) -> ShmComm & = delete; - - auto size() const -> int { return n_; } - - // recv_counts[s] = what rank s sends to me (the transpose of the send-count matrix). - auto alltoall_counts(int rank, const int *send_counts, int *recv_counts) -> void { - slots_[static_cast(rank)].counts = send_counts; - sync(); - for (int s = 0; s < n_; ++s) { - recv_counts[s] = slots_[static_cast(s)].counts[rank]; - } - sync(); - } - - // Variable all-to-all over caller-owned flat buffers, addressed as raw bytes (counts/displs stay in - // elements, `elem` = element bytes — the pointers carry no element type to scale by). Fills recv per - // source s ascending at recv_displs[s]; recv_counts must already hold the transpose — same contract - // as MPI_Alltoallv. `send` must stay alive and unmodified until the second barrier: peers read it. - auto alltoallv(int rank, - const std::byte *send, - const int *send_displs, - std::byte *recv, - const int *recv_counts, - const int *recv_displs, - size_t elem) -> void { - Slot &me = slots_[static_cast(rank)]; - me.ptr = send; - me.displs = send_displs; - sync(); - auto *dst = recv; - for (int s = 0; s < n_; ++s) { - const Slot &src = slots_[static_cast(s)]; - const auto count = static_cast(recv_counts[s]); - // Bail before offsetting src: a source that sends us nothing need not have published a - // buffer at all, and its displs[rank] may then point past the end of one. - if (count == 0) { - continue; - } - std::memcpy(dst + static_cast(recv_displs[s]) * elem, - src.ptr + static_cast(src.displs[rank]) * elem, - count * elem); - } - sync(); - } - - // Fused count-resolve + payload all-to-all in one round (2 syncs vs 4). Same contiguous - // ascending-source ordering as alltoallv, which the query/response positional pairing depends on. - // See AlltoallvResolveArgs for the lifetime and element-offset contract. - template - auto alltoallv_resolve(int rank, const AlltoallvResolveArgs &args) -> void { - Slot &me = slots_[static_cast(rank)]; - // Typed here but byte-addressed in the slot: peers scatter by (displ, count) in elements and - // never reconstruct T, so the slot stays type-erased for the untyped alltoallv above. - me.ptr = reinterpret_cast(args.send); - me.displs = args.send_displs; - me.counts = args.send_counts; - sync(); // B1: send buffers published - long long total = 0; - for (int s = 0; s < n_; ++s) { - const int c = slots_[static_cast(s)].counts[rank]; // what s sends to me - args.recv_counts[s] = c; - args.recv_displs[s] = checked_mpi_count(total, "Recv displacement"); - total += c; - } - args.recv.resize(static_cast(checked_mpi_count(total, "Total recv count"))); - auto *dst = reinterpret_cast(args.recv.data()); // after the resize: it may reallocate - for (int s = 0; s < n_; ++s) { - const Slot &src = slots_[static_cast(s)]; - const auto count = static_cast(args.recv_counts[s]); - // See alltoallv: an unpublished source must not be offset by its displacement. - if (count == 0) { - continue; - } - std::memcpy(dst + static_cast(args.recv_displs[s]) * sizeof(T), - src.ptr + static_cast(src.displs[rank]) * sizeof(T), - count * sizeof(T)); - } - sync(); // B2: peers finished reading our send buffer before the caller may reuse it - } - - template - auto allreduce_sum(int rank, T local_val) -> T { - Slot &me = slots_[static_cast(rank)]; - if constexpr (std::is_floating_point_v) { - me.f64 = static_cast(local_val); - } - else { - me.u64 = static_cast(local_val); - } - sync(); - T acc{}; - for (int s = 0; s < n_; ++s) { - const Slot &src = slots_[static_cast(s)]; - if constexpr (std::is_floating_point_v) { - acc += static_cast(src.f64); - } - else { - acc += static_cast(src.u64); - } - } - sync(); - return acc; - } - - // Safe in place: each element is read then overwritten by its single slice owner, and slices are - // cache-line-rounded. - auto allreduce_sum_inplace(int rank, double *values, size_t len) -> void { - slots_[static_cast(rank)].vec = values; - sync(); - constexpr size_t kLine = 64 / sizeof(double); - const size_t lines = (len + kLine - 1) / kLine; - const size_t per = (lines + static_cast(n_) - 1) / static_cast(n_); - const size_t lo = std::min(len, static_cast(rank) * per * kLine); - const size_t hi = std::min(len, lo + per * kLine); - for (size_t k = lo; k < hi; ++k) { - double acc = 0.0; - for (int s = 0; s < n_; ++s) { // ascending rank order - acc += slots_[static_cast(s)].vec[k]; - } - for (int s = 0; s < n_; ++s) { // publish the same bits into every rank's buffer - slots_[static_cast(s)].vec[k] = acc; - } - } - sync(); // peers write into our buffer (and read from it) until here - } - - // See PartitionBarrier::poison / ::reset for when each is legal to call. - auto poison() -> void { barrier_.poison(); } - - auto reset() -> void { barrier_.reset(); } - -private: - // One cache-line-isolated publish slot per rank. A rank writes only its own slot, and reads peers' - // slots only between the two barriers of a collective. - struct alignas(64) Slot { - const std::byte *ptr = nullptr; // byte view of the publisher's send buffer; null until published - const int *counts = nullptr; - const int *displs = nullptr; - double *vec = nullptr; // mutable: allreduce_sum_inplace writes results back through peers' slots - double f64 = 0.0; - uint64_t u64 = 0; - }; - - auto sync() -> void { barrier_.sync(); } - - int n_; - std::vector slots_; - PartitionBarrier barrier_; -}; - -} // namespace monoprop::mpi diff --git a/cpp/monoprop/detail/operator/InvertedIndex.h b/cpp/monoprop/detail/operator/InvertedIndex.h index 3268e7d6d..16f0fc3cc 100644 --- a/cpp/monoprop/detail/operator/InvertedIndex.h +++ b/cpp/monoprop/detail/operator/InvertedIndex.h @@ -219,15 +219,6 @@ inline constexpr size_t kColumnBlockWords = 1024; // 8 KB block ≈ L1-resident static_assert(kColumnBlockWords % 64 == 0, "the zero-block summary holds one bit per block word"); -// Per-thread fold block, overwritten on every use. -inline auto column_block_scratch() -> std::vector & { - static thread_local std::vector blk; - if (blk.size() < kColumnBlockWords) { - blk.assign(kColumnBlockWords, 0); - } - return blk; -} - // Block that is all-zero between uses, with one summary bit per touched word. Users must re-zero what // they touch. struct ZeroBlockScratch { @@ -236,11 +227,40 @@ struct ZeroBlockScratch { std::array summary{}; }; -// Per-thread zero blocks for the fold and the sparse pivot, which are expanded together. +/* + * The thread's fold block. A fold runs one kernel per block, so the word sweep (which overwrites the words) and the + * posting fold (which needs them all-zero and leaves them so) share it: a second block per thread was about 0.8 MiB + * at T = 96, where every owner folds. `zero` records whether the words are all-zero now; the summary always is between + * uses. Neither accessor's result may be held across a fold on the same thread. + */ +struct SharedFoldBlock { + ZeroBlockScratch block; + bool zero = true; +}; + +inline auto shared_fold_block() -> SharedFoldBlock & { + static thread_local SharedFoldBlock shared; + return shared; +} + +// The fold block for a word sweep, which overwrites it. +inline auto column_block_scratch() -> std::vector & { + SharedFoldBlock &shared = shared_fold_block(); + shared.zero = false; + return shared.block.words; +} + +// The fold block for the posting fold: all-zero, re-zeroed first if a word sweep last wrote it. inline auto fold_zero_scratch() -> ZeroBlockScratch & { - static thread_local ZeroBlockScratch scratch; - return scratch; + SharedFoldBlock &shared = shared_fold_block(); + if (!shared.zero) { + std::ranges::fill(shared.block.words, 0); + shared.zero = true; + } + return shared.block; } + +// The sparse pivot's zero block, expanded while the fold's block is in use. inline auto pivot_zero_scratch() -> ZeroBlockScratch & { static thread_local ZeroBlockScratch scratch; return scratch; @@ -449,8 +469,7 @@ inline auto for_each_fold_word(const InvertedIndex &sc, } } - uint64_t *const blk = column_block_scratch().data(); - ZeroBlockScratch &zs = fold_zero_scratch(); + // The block is fetched per kernel, never held across both: they share it (SharedFoldBlock). for (size_t bb = wlo; bb < whi; bb += kColumnBlockWords) { const size_t be = std::min(bb + kColumnBlockWords, whi); size_t postings = 0; @@ -466,7 +485,8 @@ inline auto for_each_fold_word(const InvertedIndex &sc, postings_eligible && static_cast(postings) <= kSparseFoldPostingsPerWord * static_cast(be - bb); if (by_postings) { - fold_block_by_postings(sc, cols, begin, end, zs, bb, be, last_word, last_word_mask, on_word); + fold_block_by_postings< + NumModes>(sc, cols, begin, end, fold_zero_scratch(), bb, be, last_word, last_word_mask, on_word); } else { fold_block_by_words(sc, @@ -474,7 +494,7 @@ inline auto for_each_fold_word(const InvertedIndex &sc, dense_seed, begin, end, - blk, + column_block_scratch().data(), bb, be, last_word, diff --git a/cpp/monoprop/detail/operator/MPOperator.h b/cpp/monoprop/detail/operator/MPOperator.h index 14fedde1a..6ae4cdffb 100644 --- a/cpp/monoprop/detail/operator/MPOperator.h +++ b/cpp/monoprop/detail/operator/MPOperator.h @@ -15,16 +15,24 @@ #pragma once #include +#include +#include #include #include #include #include +#include #include #include #include #include +#ifdef __linux__ +#include +#include +#endif + #include "monoprop/TypeAliases.h" #include "monoprop/Utilities.h" #include "monoprop/core/Monomial.h" @@ -58,6 +66,54 @@ class OperatorTermNotFound : public std::runtime_error { using std::runtime_error::runtime_error; }; +//! Vectors at least this large give their spare capacity back in place; smaller ones are copied down to their size. +inline constexpr size_t spare_release_bytes = size_t{256} << 10; + +//! Whether release_spare_capacity() keeps a block of `capacity_bytes` in place (Linux only) rather than copying it. +[[nodiscard]] constexpr auto releases_in_place(size_t capacity_bytes) noexcept -> bool { +#ifdef __linux__ + return capacity_bytes >= spare_release_bytes; +#else + static_cast(capacity_bytes); + return false; +#endif +} + +/*! + * \brief Give `v`'s spare capacity back to the system. + * + * A block for which releases_in_place() holds stays where it is, with its capacity, and the pages wholly inside its + * spare capacity are discarded (`MADV_DONTNEED`): no copy, so no transient second block, and no freed block left in + * the calling thread's malloc arena. Spare capacity holds no objects, and a later growth into it reads fresh zero + * pages. Any other vector is copied down to its size (`shrink_to_fit`). + */ +template +auto release_spare_capacity(std::vector &v) -> void { + static_assert(std::is_trivially_copyable_v, "discarded spare pages must hold no objects"); +#ifdef __linux__ + if (releases_in_place(v.capacity() * sizeof(T))) { + const auto page = static_cast(sysconf(_SC_PAGESIZE)); + const auto first = (reinterpret_cast(v.data() + v.size()) + page - 1) & ~(page - 1); + const auto last = reinterpret_cast(v.data() + v.capacity()) & ~(page - 1); + if (first < last) { + // Advisory: a refusal leaves the pages resident, which is the unreleased state, not an error. + (void)madvise(reinterpret_cast(first), last - first, MADV_DONTNEED); + } + return; + } +#endif + v.shrink_to_fit(); +} + +/*! + * \brief The bytes `v` is accounted for: its entries when its block is released in place (its spare capacity is then + * not resident at rest; growth writes only the entries it adds), its capacity otherwise, which at rest is its size. + */ +template +[[nodiscard]] auto accounted_bytes(const std::vector &v) noexcept -> size_t { + return (releases_in_place(v.capacity() * sizeof(T)) ? v.size() : v.capacity()) * sizeof(T); +} + template struct MPOperator { // The store is non-copyable/non-movable, so it is heap-owned by unique_ptr (keeping MPOperator @@ -79,7 +135,8 @@ struct MPOperator { Basis basis{Basis::Majorana}; mutable std::optional> inverted_index_{std::nullopt}; - MPOperator() noexcept = default; + // Not noexcept: the store's member initializer allocates, and an allocation failure must reach the caller. + MPOperator() = default; MPOperator(MPOperator &&) noexcept = default; MPOperator &operator=(MPOperator &&) noexcept = default; @@ -177,18 +234,92 @@ struct MPOperator { return state_coeffs; } + // The picture's live coefficients: the operator with its pending initial entries placed (Heisenberg), or the + // dense state with its new rows scored (Schrödinger). Both extend to the store size. + auto current_picture(bool schrodinger) -> const VecD & { return schrodinger ? dense_state() : get_operator(); } + + // Grow `coeffs` to the store size after a layer's inserts: the current picture's values for rows it lacks, then + // zeros. When `coeffs` is the picture's own live vector, bringing the picture up to date is the whole extension + // (a fresh Heisenberg row picks up a pending initial entry; a fresh Schrödinger row is scored). + auto extend_from_current_picture(VecD &coeffs, bool schrodinger) -> void { + if (coeffs.size() >= size()) { + return; + } + const auto ¤t = current_picture(schrodinger); + if (&coeffs == ¤t) { + return; + } + if (coeffs.size() < current.size()) { + coeffs.insert(coeffs.end(), current.begin() + static_cast(coeffs.size()), current.end()); + } + coeffs.resize(size(), 0.0); + } + auto shrink_state_to_fit() -> void { - state_rows_.shrink_to_fit(); - state_vals_.shrink_to_fit(); - state_coeffs.shrink_to_fit(); + release_spare_capacity(state_rows_); + release_spare_capacity(state_vals_); + release_spare_capacity(state_coeffs); + } + + // Materialize the caches every operation expects after construction or a gate loop: pending initial entries + // placed onto their rows, the picture's state, then the inverted index. Heisenberg warms the sparse state only; + // densifying it here would defeat it. Schrödinger's dense vector IS the live evolved vector. + auto initialize_caches(bool schrodinger) -> void { + warm_caches(schrodinger); + release_slack(); } - // Each term lands on its existing evolved-operator row, or in the pending map if not yet materialized. - // Heisenberg rejects a term absent from both (new monomials may have no graph paths); Schrödinger - // admits them freely (the state was already evolved). Returns the supplied terms with their encoded - // coefficients, in order. + // The first half of initialize_caches(): every cache materialized, no capacity released. + auto warm_caches(bool schrodinger) -> void { + (void)get_operator(); + if (schrodinger) { + (void)dense_state(); + } + else { + (void)sparse_state(); + } + (void)inverted_index(); + } + + // The second half: the coefficient and state vectors' spare capacity given back (release_spare_capacity()). Large + // vectors keep their blocks; a small one is copied down, holding its old and new block at once. + auto release_slack() -> void { + release_spare_capacity(op_coeffs); + shrink_state_to_fit(); + } + + // The bytes release_slack() copies: every vector with spare capacity not released in place is copied whole. + [[nodiscard]] auto slack_copy_bytes() const noexcept -> size_t { + const auto copied = [](const auto &v) { + const bool copies = v.capacity() > v.size() && !releases_in_place(v.capacity() * sizeof(v[0])); + return copies ? v.size() * sizeof(v[0]) : 0uz; + }; + return copied(op_coeffs) + copied(state_rows_) + copied(state_vals_) + copied(state_coeffs); + } + + // Throws OperatorTermNotFound for the first term update_initial_operator() would reject, changing nothing: + // Heisenberg rejects a term that is neither pending nor an evolved-operator row (new monomials may have no graph + // paths); Schrödinger admits every term (the state was already evolved). + auto require_updatable_terms(const OperatorDict &op_dict, bool schrodinger) const -> void { + if (schrodinger) { + return; + } + for (const auto &k : op_dict | std::views::keys) { + // Unchecked by design: the only caller bounds-checks against its logical_num_modes_. + const auto mono = indices_to_bitset(k); + if (!init_op_map.contains(mono) && !store->find(mono)) { + const auto term_repr = std::format("[{}]", join_with_separator(k, ", ")); + throw OperatorTermNotFound(std::format("Operator term {} not found in the operator.", term_repr)); + } + } + } + + // Each term lands on its existing evolved-operator row, or in the pending map if not yet materialized; the terms + // are checked with require_updatable_terms() first. Returns the supplied terms with their encoded coefficients, in + // order. auto update_initial_operator(const OperatorDict &op_dict, bool schrodinger) -> std::pair, VecD> { + require_updatable_terms(op_dict, schrodinger); MonomialMap new_op_map; std::pair, VecD> new_grad_op; VecD new_op_coeffs(size(), 0.0); @@ -204,12 +335,8 @@ struct MPOperator { if (rank_init_op != init_op_map.end()) { new_op_map[mono] = coeff; } - else if (rank_evolved_op) { - new_op_coeffs[*rank_evolved_op] = coeff; - } else { - const auto term_repr = std::format("[{}]", join_with_separator(k, ", ")); - throw OperatorTermNotFound(std::format("Operator term {} not found in the operator.", term_repr)); + new_op_coeffs[*rank_evolved_op] = coeff; } } else { @@ -327,11 +454,10 @@ template inline auto estimate_memory_usage(const MPOperator &op) -> MPOperatorMemoryBreakdown { MPOperatorMemoryBreakdown breakdown; breakdown.operator_terms_bytes = op.store->memory_bytes(); - breakdown.op_coeffs_bytes = op.op_coeffs.capacity() * sizeof(double); + breakdown.op_coeffs_bytes = accounted_bytes(op.op_coeffs); // Every representation of the state at once: the sparse scored set plus the dense vector. - breakdown.state_coeffs_bytes = op.state_coeffs.capacity() * sizeof(double) - + op.state_rows_.capacity() * sizeof(TermIndex) - + op.state_vals_.capacity() * sizeof(double); + breakdown.state_coeffs_bytes = + accounted_bytes(op.state_coeffs) + accounted_bytes(op.state_rows_) + accounted_bytes(op.state_vals_); breakdown.indexing_bytes = op.store->index_estimated_memory_bytes(); breakdown.init_operator_bytes = unordered_flat_map_storage_bytes(op.init_op_map); breakdown.init_operator_entries = op.init_op_map.size(); diff --git a/cpp/monoprop/detail/operator/OperatorIndex.h b/cpp/monoprop/detail/operator/OperatorIndex.h index e96afd27d..30ba6d16c 100644 --- a/cpp/monoprop/detail/operator/OperatorIndex.h +++ b/cpp/monoprop/detail/operator/OperatorIndex.h @@ -32,6 +32,7 @@ #include "monoprop/TypeAliases.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/parallel/Options.h" namespace monoprop::detail { @@ -44,7 +45,8 @@ class TermIndexCeilingReached : public std::runtime_error { // those rows. Row layout: slot 0 = popcount c (or kOverflowMarker if c > inline_width_), slots 1..c = // ascending set-bit positions; stride_ is fixed for the container's life so row offsets stay stable. // inline_width_ is a free parameter -- any width is correct, over-long rows spill losslessly to overflow. -// Single-writer: one partition, one thread; parallelism is cross-partition. +// Single-writer: every mutation (growth, set, overflow, table insertion) runs on one thread. The const lookups +// (find, find_batch, find_batch_positions) may run concurrently only while nothing mutates the store. template class OperatorIndex { public: @@ -364,8 +366,14 @@ class OperatorIndex { // bulk_insert with the hashes already in hand: same precondition (n distinct rows, already written, // at consecutive indices) and the same slot assignment. `hashes[k]` must be fold_hash of the key of // row base+k -- a wrong one leaves the row unfindable, which surfaces later as a duplicate insert. + // Neither this nor bulk_insert deduplicates: two rows with one key are both indexed and find() returns + // the first (pinned by operator_index_tests). `options` is the publication seam an alternative index may + // use to publish in parallel; this packed table ignores it and always inserts serially, in order. template - auto bulk_insert_hashed(size_t n, mapped_type base, HashFn &&hash_at) -> void { + auto bulk_insert_hashed(size_t n, + mapped_type base, + HashFn &&hash_at, + [[maybe_unused]] parallel::Options options = {}) -> void { if (n == 0) { return; } @@ -548,8 +556,8 @@ class OperatorIndex { static auto check_append_fits(size_t base, size_t n) -> void { if (n > kIndexCeiling - base) { throw TermIndexCeilingReached( - std::format("OperatorIndex: appending {} terms to a partition holding {} would pass the " - "2^32 TermIndex ceiling; raise the partition or rank count to split it further.", + std::format("OperatorIndex: appending {} terms to a store holding {} would pass the " + "2^32 TermIndex ceiling; raise the rank or thread (shard) count to split it further.", n, base)); } @@ -557,8 +565,8 @@ class OperatorIndex { static auto check_index_fits(size_t value) -> void { if (value >= kIndexCeiling) { - throw TermIndexCeilingReached("OperatorIndex: this partition's term count reached the 2^32 TermIndex " - "ceiling; raise the partition or rank count to split it further."); + throw TermIndexCeilingReached("OperatorIndex: this store's term count reached the 2^32 TermIndex " + "ceiling; raise the rank or thread (shard) count to split it further."); } } diff --git a/cpp/monoprop/detail/parallel/CMakeLists.txt b/cpp/monoprop/detail/parallel/CMakeLists.txt new file mode 100644 index 000000000..0bf966039 --- /dev/null +++ b/cpp/monoprop/detail/parallel/CMakeLists.txt @@ -0,0 +1,12 @@ +target_sources( + monoprop + PUBLIC + FILE_SET headers + TYPE HEADERS + FILES + "Options.h" + "ThreadBudget.h" + "Workshare.h" +) + +target_sources(monoprop-objs PRIVATE ThreadBudget.cpp) diff --git a/cpp/monoprop/detail/parallel/Options.h b/cpp/monoprop/detail/parallel/Options.h new file mode 100644 index 000000000..b603392ba --- /dev/null +++ b/cpp/monoprop/detail/parallel/Options.h @@ -0,0 +1,33 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// Deliberately free of : signatures can carry Options without exposing OpenMP. + +namespace monoprop::detail::parallel { + +/*! + * \brief A requested thread budget, never a global OpenMP runtime setting. + * + * The value is what a region requests, not a promise of what the runtime supplies. Whether a consumer + * tolerates fewer workers is that consumer's contract: for_blocks() stays correct with any smaller team, + * including its serial and nested fallbacks, while sharded::run_team() assumes the launch supplies exactly + * `threads` workers and has no reduced-team path. + */ +struct Options { + int threads = 1; //!< Requested positive per-object budget; not a global runtime setting. +}; + +} // namespace monoprop::detail::parallel diff --git a/cpp/monoprop/detail/parallel/ThreadBudget.cpp b/cpp/monoprop/detail/parallel/ThreadBudget.cpp new file mode 100644 index 000000000..6dc5e396f --- /dev/null +++ b/cpp/monoprop/detail/parallel/ThreadBudget.cpp @@ -0,0 +1,76 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/parallel/ThreadBudget.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace monoprop::detail::parallel { +namespace { + +constexpr const char *budget_variable = "monoprop_NUM_THREADS"; + +[[noreturn]] auto reject(std::string_view text) -> void { + throw std::invalid_argument(std::format("{}=\"{}\" is not a thread count: it must be a decimal integer from 1 " + "to {} with no sign or spaces. Set it to the threads per process, or " + "unset it to use the OpenMP runtime default.", + budget_variable, + text, + INT_MAX)); +} + +} // namespace + +auto resolve_thread_budget(std::optional configured, int runtime_default) -> Options { + if (!configured.has_value()) { + if (runtime_default < 1) { + throw std::invalid_argument(std::format("{} is unset and the OpenMP runtime default thread count is {}, " + "not positive; set {} explicitly.", + budget_variable, + runtime_default, + budget_variable)); + } + return {.threads = runtime_default}; + } + const std::string_view text = *configured; + // from_chars alone would accept a partial parse; requiring digits throughout also rejects signs, + // whitespace and lists before any narrowing. + if (text.empty() || !std::ranges::all_of(text, [](char c) { return c >= '0' && c <= '9'; })) { + reject(text); + } + std::uint64_t value = 0; + const auto [end, ec] = std::from_chars(text.data(), text.data() + text.size(), value, 10); + if (ec != std::errc{} || end != text.data() + text.size() || value < 1 + || value > static_cast(INT_MAX)) { + reject(text); + } + return {.threads = static_cast(value)}; +} + +auto capture_thread_budget() -> Options { + const char *configured = std::getenv(budget_variable); + return resolve_thread_budget(configured != nullptr ? std::optional(configured) : std::nullopt, + omp_get_max_threads()); +} + +} // namespace monoprop::detail::parallel diff --git a/cpp/monoprop/detail/parallel/ThreadBudget.h b/cpp/monoprop/detail/parallel/ThreadBudget.h new file mode 100644 index 000000000..5236459bf --- /dev/null +++ b/cpp/monoprop/detail/parallel/ThreadBudget.h @@ -0,0 +1,54 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/monopropExport.h" + +namespace monoprop::detail::parallel { + +/*! + * \brief Resolve a per-object thread budget from a configured value and the OpenMP runtime default. + * + * Pure: no environment read, hardware query or runtime mutation. The value is never clamped to the + * hardware or the allocation. + * + * \param configured The `monoprop_NUM_THREADS` text, or `std::nullopt` when the variable is unset. A + * present value must be nonempty ASCII decimal digits whose value lies in + * `[1, INT_MAX]`; leading zeros are accepted. A valid value wins over any fallback. + * \param runtime_default The OpenMP default (`omp_get_max_threads()`), used only when `configured` is + * absent; it must then be positive. + * \return The resolved budget. + * \throws std::invalid_argument for an empty, signed, padded, list-valued, non-digit, zero or overflowing + * configured value, or for an absent value with a non-positive runtime default. + */ +monoprop_EXPORT auto resolve_thread_budget(std::optional configured, int runtime_default) -> Options; + +/*! + * \brief Read `monoprop_NUM_THREADS` once and resolve it against `omp_get_max_threads()`. + * + * Called while constructing a propagator, never during an operation; each call rereads the + * environment and nothing is cached. An unset variable selects the OpenMP runtime default, while a + * present but empty one is an error. + * + * \return The resolved budget. + * \throws std::invalid_argument as resolve_thread_budget() does. + */ +monoprop_EXPORT auto capture_thread_budget() -> Options; + +} // namespace monoprop::detail::parallel diff --git a/cpp/monoprop/detail/parallel/Workshare.h b/cpp/monoprop/detail/parallel/Workshare.h new file mode 100644 index 000000000..d477b6906 --- /dev/null +++ b/cpp/monoprop/detail/parallel/Workshare.h @@ -0,0 +1,99 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/Options.h" + +namespace monoprop::detail::parallel { + +/*! + * \brief Invoke `body(block)` once for every logical block in `[0, count)`, then return. + * + * Runs serially, in ascending block order, when the budget or `count` is one, or when called from inside + * any OpenMP region (active or not). Otherwise opens one region requesting `min(options.threads, count)` + * workers with a static schedule; the runtime may provide fewer, and every block still runs exactly once. + * + * Worker exceptions never leave the region. Each worker keeps its first exception and stops taking + * blocks; after all workers join, the exception of the lowest-numbered failing worker is rethrown on + * the caller. The serial path lets an exception propagate from the failing block directly. After a + * failure, which other blocks ran is unspecified, and nothing is rolled back. + * + * The helper owns no threads and changes no OpenMP runtime setting. + * + * \param count Number of logical blocks; must not exceed `PTRDIFF_MAX`. + * \param options Thread budget; `options.threads` must be positive. + * \param body Callable taking a `std::size_t` block ID. Distinct blocks may run concurrently, so it + * must only write state owned by its block and must not call MPI. + * \throws std::invalid_argument if `options.threads` is not positive, before any block runs. + * \throws std::length_error if `count` exceeds `PTRDIFF_MAX`, before any block runs. + */ +template +auto for_blocks(std::size_t count, Options options, Fn &&body) -> void { + if (options.threads < 1) { + throw std::invalid_argument( + std::format("parallel::for_blocks: the thread budget must be positive, got {}", options.threads)); + } + // The region iterates a signed index, as the most portable OpenMP loop form requires. + constexpr auto max_count = static_cast(std::numeric_limits::max()); + if (count > max_count) { + throw std::length_error( + std::format("parallel::for_blocks: {} blocks exceed the loop bound {}", count, max_count)); + } + + const auto team = std::min(static_cast(options.threads), count); + if (team <= 1 || omp_get_level() > 0) { + for (std::size_t block = 0; block < count; ++block) { + body(block); + } + return; + } + + // Allocated before the region, so allocation failure propagates normally. The actual team never + // exceeds the num_threads request, so every worker ID indexes a slot. + std::vector errors(team); + const auto blocks = static_cast(count); + +#pragma omp parallel for schedule(static) num_threads(static_cast(team)) + for (std::ptrdiff_t block = 0; block < blocks; ++block) { + auto &error = errors[static_cast(omp_get_thread_num())]; + if (error) { + continue; + } + try { + body(static_cast(block)); + } + catch (...) { + error = std::current_exception(); + } + } + + for (const auto &error : errors) { + if (error) { + std::rethrow_exception(error); + } + } +} + +} // namespace monoprop::detail::parallel diff --git a/cpp/monoprop/detail/pare/PareGraph.cpp b/cpp/monoprop/detail/pare/PareGraph.cpp index b490a0a92..a16f6b6f0 100644 --- a/cpp/monoprop/detail/pare/PareGraph.cpp +++ b/cpp/monoprop/detail/pare/PareGraph.cpp @@ -40,10 +40,12 @@ inline auto keep_mask_for_block(const std::vector &keep, size_t base, uint return mask; } -// Filter the cosine set to the kept nodes; {{}, true} means nothing was pruned (replay folds the full set). -auto filter_layer_cosine_data(const CosMask &cos, const std::vector &nodes_to_keep) -> std::pair { +// Filter the cosine set to the kept nodes into `filtered` (contents replaced, capacity reused); true means nothing was +// pruned (replay folds the full set) and leaves `filtered` unspecified. +auto filter_layer_cosine_data(const CosMask &cos, const std::vector &nodes_to_keep, CosMask &filtered) -> bool { const size_t n = cos.blocks.size(); - CosMask filtered; + filtered.blocks.clear(); + filtered.total_count = 0; bool preserves = true; for (size_t k = 0; k < n; ++k) { const auto [base, bits] = cos.blocks[k]; @@ -56,10 +58,7 @@ auto filter_layer_cosine_data(const CosMask &cos, const std::vector &nodes filtered.total_count += static_cast(std::popcount(kept)); } } - if (preserves) { - return {{}, true}; - } - return {std::move(filtered), false}; + return preserves; } // The cos pass (not the D-apply) scales every D target, since cos holds all anticommuting indices, so no @@ -87,14 +86,18 @@ auto mark_cross_rank_endpoints_kept(const LayerTraversal &layer, size_t my_rank, } // namespace -auto pare_graph(const MPGraph &graph, - const VecZ &nonzero_inds, - size_t local_index_count, - bool schrodinger, - mpi::Comm comm, - const std::function &full_cos_of_layer) -> MPGraph { +} // namespace monoprop + +namespace monoprop::detail { + +auto pare_graph_owner(const MPGraph &graph, + const VecZ &nonzero_inds, + size_t local_index_count, + bool schrodinger, + size_t flat_owner, + const std::function &full_cos_of_layer) -> MPGraph { const size_t num_layers = graph.layers(); - const auto my_rank = static_cast(mpi::rank(comm)); + const size_t my_rank = flat_owner; std::vector nodes_to_keep(local_index_count, 0); for (auto idx : nonzero_inds) { @@ -104,6 +107,13 @@ auto pare_graph(const MPGraph &graph, } std::vector layers(num_layers); + /* + * One buffer for every layer's full set and one for its filtered set: growing a fresh vector per layer, interleaved + * with the stored sets, extends the owner's heap one reallocation at a time, and at T owners those heap-growth + * calls serialize on the process's memory map. Only the stored sets are allocated, each at its exact size. + */ + CosMask full; + CosMask filtered; // Single backward sweep, entirely rank-local: every cross-rank endpoint is force-kept (see // mark_cross_rank_endpoints_kept), so nodes_to_keep stays consistent across ranks with no exchange. @@ -116,13 +126,36 @@ auto pare_graph(const MPGraph &graph, // Order is load-bearing for bit-exact pruning: endpoints kept before the cosine filter. mark_cross_rank_endpoints_kept(lt, my_rank, nodes_to_keep); - const CosMask full = full_cos_of_layer(layer_idx); - auto [filtered, preserves] = filter_layer_cosine_data(full, nodes_to_keep); - - layers[layer_idx] = preserves ? Layer(layer.shared_core()) : Layer(layer.shared_core(), std::move(filtered)); + full_cos_of_layer(layer_idx, full); + if (filter_layer_cosine_data(full, nodes_to_keep, filtered)) { + layers[layer_idx] = Layer(layer.shared_core()); + } + else { + layers[layer_idx] = Layer(layer.shared_core(), + CosMask{.blocks = {filtered.blocks.begin(), filtered.blocks.end()}, + .total_count = filtered.total_count}); + } } return MPGraph(graph.is_schrodinger(), std::move(layers)); } +} // namespace monoprop::detail + +namespace monoprop { + +auto pare_graph(const MPGraph &graph, + const VecZ &nonzero_inds, + size_t local_index_count, + bool schrodinger, + mpi::Comm comm, + const std::function &full_cos_of_layer) -> MPGraph { + return detail::pare_graph_owner(graph, + nonzero_inds, + local_index_count, + schrodinger, + static_cast(mpi::rank(comm)), + [&full_cos_of_layer](size_t i, CosMask &out) { out = full_cos_of_layer(i); }); +} + } // namespace monoprop diff --git a/cpp/monoprop/detail/pare/PareGraph.h b/cpp/monoprop/detail/pare/PareGraph.h index 59bc3e7f0..96f6cb2c8 100644 --- a/cpp/monoprop/detail/pare/PareGraph.h +++ b/cpp/monoprop/detail/pare/PareGraph.h @@ -14,8 +14,44 @@ #pragma once +#include +#include + // pare_graph is declared in MPFunctions.h. #include "monoprop/MPFunctions.h" #include "monoprop/MPGraph.h" +#include "monoprop/MPGraphEncoding.h" #include "monoprop/TypeAliases.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/monopropExport.h" + +namespace monoprop::detail { + +/*! + * \brief Prune one owner's graph to the subgraph reaching `nonzero_inds`: the communicator-free core of pare_graph(). + * + * A single backward sweep over the layers in the picture's order (last layer first in Heisenberg, first layer first + * in Schrödinger). At each layer every partner-owner endpoint is kept before the cosine set is filtered: a + * sin_recv target always, and a sin_send source only for a slot other than `flat_owner` (another shard of the same + * process is a partner owner like a remote one). Cross-owner lists are never pruned. A layer whose filtered cosine + * set is unchanged keeps recomputing; any other layer stores its filtered set, and an empty stored set replays + * nothing. Cores stay shared with `graph`; the returned layer sequence is independently owned. + * + * \param graph The owner's graph. + * \param nonzero_inds Rows to keep; indices at or past `local_index_count` are ignored. + * \param local_index_count The owner's row count. + * \param schrodinger The picture, which fixes the sweep order. + * \param flat_owner This owner's flat routing slot, `rank * T + shard`; never derived from a communicator. + * \param full_cos_of_layer Writes layer i's full cosine set into its second argument, replacing the contents. The + * sweep passes the same buffer for every layer, so a writer that reuses its capacity + * allocates once per owner instead of once per layer. + * \return The pared graph. Each stored set is allocated at its exact size. + */ +monoprop_EXPORT auto pare_graph_owner(const MPGraph &graph, + const VecZ &nonzero_inds, + size_t local_index_count, + bool schrodinger, + size_t flat_owner, + const std::function &full_cos_of_layer) -> MPGraph; + +} // namespace monoprop::detail diff --git a/cpp/monoprop/detail/partition/CMakeLists.txt b/cpp/monoprop/detail/partition/CMakeLists.txt deleted file mode 100644 index 19d4b69bc..000000000 --- a/cpp/monoprop/detail/partition/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -target_sources( - monoprop - PUBLIC - FILE_SET headers - TYPE HEADERS - FILES - "CpuTopology.h" - "PartitionGroup.h" -) - -target_sources(monoprop-objs PRIVATE CpuTopology.cpp) diff --git a/cpp/monoprop/detail/partition/CpuTopology.cpp b/cpp/monoprop/detail/partition/CpuTopology.cpp deleted file mode 100644 index 81f0be4db..000000000 --- a/cpp/monoprop/detail/partition/CpuTopology.cpp +++ /dev/null @@ -1,410 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "monoprop/detail/partition/CpuTopology.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace monoprop::detail::partition { - -namespace { - -/* ── Process-lifetime hwloc topology ──────────────────────────────────────── */ - -// hwloc_topology_t is safe for concurrent read-only access after hwloc_topology_load(). -struct TopologyHolder { - hwloc_topology_t topo = nullptr; - - TopologyHolder() noexcept { - if (hwloc_topology_init(&topo) < 0) { - topo = nullptr; - return; - } - /* Keep all L3 cache objects so shared-L3 domains can always be identified, even on - * topologies where the L3 appears private and would otherwise be suppressed by the - * default HWLOC_TYPE_FILTER_KEEP_STRUCTURE filter. */ - hwloc_topology_set_type_filter(topo, HWLOC_OBJ_L3CACHE, HWLOC_TYPE_FILTER_KEEP_ALL); - if (hwloc_topology_load(topo) < 0) { - hwloc_topology_destroy(topo); - topo = nullptr; - } - } - - ~TopologyHolder() { - if (topo) { - hwloc_topology_destroy(topo); - } - } - - TopologyHolder(const TopologyHolder &) = delete; - auto operator=(const TopologyHolder &) -> TopologyHolder & = delete; -}; - -// Returns the loaded topology, or nullptr when initialization failed. -// The static local is initialized once; subsequent calls return the cached handle. -auto get_topology() -> hwloc_topology_t { - static TopologyHolder holder; - return holder.topo; -} - -/* ── Effective allowed cpuset for the calling thread ──────────────────────── */ - -// Queries the current thread's affinity to respect any launcher-imposed restriction (cgroup, MPI -// process binding) narrower than the topology's own allowed cpuset. Falls back to the topology -// allowed cpuset when the cpubind query is unsupported on this platform. Caller must free the bitmap. -auto effective_allowed_cpuset(hwloc_topology_t topo) -> hwloc_cpuset_t { - hwloc_cpuset_t set = hwloc_bitmap_alloc(); - if (!set) { - return nullptr; - } - if (hwloc_get_cpubind(topo, set, HWLOC_CPUBIND_THREAD) == 0) { - return set; - } - /* cpubind query not supported (e.g. macOS without OS X binding): fall back. */ - hwloc_bitmap_free(set); - return hwloc_bitmap_dup(hwloc_topology_get_allowed_cpuset(topo)); -} - -} // anonymous namespace - -/* ── topo_detail::placement_order ─────────────────────────────────────────── */ - -namespace topo_detail { - -auto placement_order(const std::vector &cores, size_t n, size_t group_index, size_t group_count) - -> std::vector { - if (cores.empty() || group_count == 0 || group_count * n > cores.size()) { - return {}; - } - - int max_domain = 0; - for (const auto &c : cores) { - max_domain = std::max(max_domain, c.l3_domain); - } - - /* Bucket representative PU indices by L3 domain id. */ - std::vector> by_domain(static_cast(max_domain) + 1); - for (const auto &c : cores) { - by_domain[static_cast(c.l3_domain)].push_back(c.cpu); - } - - /* Interleave buckets depth-first: b0[0], b1[0], …, b0[1], b1[1], … */ - const auto interleave = [](const std::vector> &buckets) { - std::vector out; - for (size_t depth = 0;; ++depth) { - bool any = false; - for (const auto &bucket : buckets) { - if (depth < bucket.size()) { - out.push_back(bucket[depth]); - any = true; - } - } - if (!any) { - return out; - } - } - }; - - std::vector order; - size_t offset = 0; - if (group_count <= by_domain.size()) { - /* Interleave arm: deal domains round-robin across ranks. */ - std::vector> mine; - for (size_t d = group_index; d < by_domain.size(); d += group_count) { - mine.push_back(by_domain[d]); - } - order = interleave(mine); - } - else { - /* More co-located ranks than L3 domains: flat domain-major order, one contiguous slice each. */ - for (const auto &bucket : by_domain) { - order.insert(order.end(), bucket.begin(), bucket.end()); - } - offset = group_index * n; - } - - if (offset + n > order.size()) { - return {}; - } - return {order.begin() + static_cast(offset), - order.begin() + static_cast(offset + n)}; -} - -} // namespace topo_detail - -/* ── enumerate_physical_cores ──────────────────────────────────────────────── */ - -auto enumerate_physical_cores() -> std::vector { - auto *const topo = get_topology(); - if (!topo) { - return {}; - } - - const hwloc_cpuset_t allowed = effective_allowed_cpuset(topo); - if (!allowed) { - return {}; - } - - const int core_depth = hwloc_get_type_depth(topo, HWLOC_OBJ_CORE); - if (core_depth == HWLOC_TYPE_DEPTH_UNKNOWN || core_depth == HWLOC_TYPE_DEPTH_MULTIPLE) { - hwloc_bitmap_free(allowed); - return {}; - } - - std::vector cores; - std::map l3_domain_map; // l3->logical_index → domain id - int next_domain_id = 0; - - const unsigned num_cores = hwloc_get_nbobjs_by_depth(topo, core_depth); - for (unsigned i = 0; i < num_cores; ++i) { - auto *const core = hwloc_get_obj_by_depth(topo, core_depth, i); - if (!core || !core->cpuset) { - continue; - } - - /* Skip cores that have no PU in the calling thread's effective allowed mask. */ - if (!hwloc_bitmap_intersects(core->cpuset, allowed)) { - continue; - } - - /* Lowest allowed PU on this core is the representative OS index. */ - hwloc_cpuset_t core_allowed = hwloc_bitmap_alloc(); - if (!core_allowed) { - continue; - } - hwloc_bitmap_and(core_allowed, core->cpuset, allowed); - const int rep = hwloc_bitmap_first(core_allowed); - hwloc_bitmap_free(core_allowed); - if (rep < 0) { - continue; - } - - /* Find the L3 cache ancestor and assign a stable domain id. Cores sharing an L3 object - * (same logical_index) receive the same domain id. Cores without an L3 ancestor each - * receive their own singleton domain so the placement algorithm can still spread across - * whatever structure the topology does have. */ - int domain; - auto *const l3 = hwloc_get_ancestor_obj_by_type(topo, HWLOC_OBJ_L3CACHE, core); - if (l3) { - const auto [it, inserted] = l3_domain_map.emplace(l3->logical_index, next_domain_id); - if (inserted) { - ++next_domain_id; - } - domain = it->second; - } - else { - domain = next_domain_id++; - } - - cores.push_back(PhysicalCore{.cpu = rep, .l3_domain = domain}); - } - - hwloc_bitmap_free(allowed); - return cores; -} - -/* ── affinity_mask_words ───────────────────────────────────────────────────── */ - -auto affinity_mask_words(uint64_t *out, size_t nwords) -> bool { - if (out == nullptr || nwords == 0) { - return false; - } - std::fill_n(out, nwords, uint64_t{0}); - auto *const topo = get_topology(); - if (!topo) { - return false; - } - const hwloc_cpuset_t allowed = effective_allowed_cpuset(topo); - if (!allowed) { - return false; - } - // Refused rather than truncated: a truncated mask could compare disjoint against a peer it overlaps. - const int last = hwloc_bitmap_last(allowed); - const bool representable = last >= 0 && static_cast(last) < nwords * 64; - if (representable) { - for (int pu = hwloc_bitmap_first(allowed); pu >= 0; pu = hwloc_bitmap_next(allowed, pu)) { - out[static_cast(pu) / 64] |= uint64_t{1} << (static_cast(pu) % 64); - } - } - hwloc_bitmap_free(allowed); - return representable; -} - -/* ── masks_are_pairwise_disjoint ───────────────────────────────────────────── */ - -auto masks_are_pairwise_disjoint(const uint64_t *masks, size_t n, size_t words) -> bool { - if (masks == nullptr || words == 0 || n < 2) { - return false; - } - // An all-zero mask is disjoint from everything, so empty is rejected before the pairwise test. - for (size_t r = 0; r < n; ++r) { - bool any = false; - for (size_t w = 0; w < words && !any; ++w) { - any = masks[(r * words) + w] != 0; - } - if (!any) { - return false; - } - } - for (size_t a = 0; a < n; ++a) { - for (size_t b = a + 1; b < n; ++b) { - for (size_t w = 0; w < words; ++w) { - if ((masks[(a * words) + w] & masks[(b * words) + w]) != 0) { - return false; // two peers share a CPU: not private - } - } - } - } - return true; -} - -/* ── summarize_masks ──────────────────────────────────────────────────────── */ - -// hwloc indexes a bitmap in unsigned long units, so a 64-bit word must be one of them. -static_assert(sizeof(unsigned long) == sizeof(uint64_t), "the mask word is not an hwloc bitmap unit"); - -auto summarize_masks(const uint64_t *masks, size_t n, size_t words, size_t self) -> std::optional { - if (masks == nullptr || n == 0 || words == 0 || self >= n) { - return std::nullopt; - } - auto *const row = hwloc_bitmap_alloc(); - auto *const all = hwloc_bitmap_alloc(); - if (row == nullptr || all == nullptr) { - hwloc_bitmap_free(row); - hwloc_bitmap_free(all); - return std::nullopt; - } - MaskSummary out; - bool ok = true; - for (size_t r = 0; r < n; ++r) { - hwloc_bitmap_zero(row); - for (size_t w = 0; w < words; ++w) { - hwloc_bitmap_set_ith_ulong(row, static_cast(w), masks[(r * words) + w]); - } - const int weight = hwloc_bitmap_weight(row); - if (weight <= 0) { // an all-zero row is a mask that did not fit the exchange window - ok = false; - break; - } - if (r == self) { - out.cpus = static_cast(weight); - } - hwloc_bitmap_or(all, all, row); - } - if (ok) { - out.node_cpus = static_cast(hwloc_bitmap_weight(all)); - std::array text{}; - const int need = hwloc_bitmap_list_snprintf(text.data(), text.size(), all); - out.cpu_list = text.data(); - // Truncation is stated, never silent: a cut list read as complete is a smaller machine. Cut - // back to the last whole range first, so the marker never follows a half-written CPU id. - if (std::cmp_greater_equal(need, text.size())) { - const size_t last = out.cpu_list.rfind(','); - out.cpu_list.resize(last == std::string::npos ? 0 : last + 1); - out.cpu_list += "+"; - } - } - hwloc_bitmap_free(row); - hwloc_bitmap_free(all); - return ok ? std::optional{out} : std::nullopt; -} - -auto format_place_line(int mpi_rank, int node_rank, int node_size, const char *verdict, const MaskSummary &summary) - -> std::string { - return std::format("COMMPLACE rank={} node_rank={} node_size={} masks={} cpus={} node_cpus={} cpu_list={}\n", - mpi_rank, - node_rank, - node_size, - verdict, - summary.cpus, - summary.node_cpus, - summary.cpu_list); -} - -auto place_line_is_new(std::string_view line) -> bool { - static std::mutex mu; - static std::string last; - const std::lock_guard lock(mu); - if (line == last) { - return false; - } - last = line; - return true; -} - -/* ── partition_cpusets ─────────────────────────────────────────────────────── */ - -auto partition_cpusets(size_t n, size_t group_index, size_t group_count, NodeMask mask) -> std::vector { - const auto cores = enumerate_physical_cores(); - - // A private mask IS this rank's share: the launcher already separated co-located ranks. - if (mask == NodeMask::PerRank) { - group_index = 0; - group_count = 1; - } - const auto order = topo_detail::placement_order(cores, n, group_index, group_count); - - if (order.empty()) { - static std::once_flag warned; - std::call_once(warned, [&] { - std::print(stderr, - "monoprop: partition pinning requested but not possible " - "({} cores visible, {} groups x {} partitions); threads run unpinned.\n", - cores.size(), - group_count, - n); - std::fflush(stderr); - }); - } - - std::vector sets(order.size()); - for (size_t i = 0; i < order.size(); ++i) { - sets[i] = CpuSet{order[i]}; - } - return sets; -} - -/* ── pin_this_thread ───────────────────────────────────────────────────────── */ - -auto pin_this_thread(const CpuSet &set) -> void { - if (set.pu < 0) { - return; - } - auto *const topo = get_topology(); - if (!topo) { - return; - } - hwloc_cpuset_t cpuset = hwloc_bitmap_alloc(); - if (!cpuset) { - return; - } - hwloc_bitmap_only(cpuset, static_cast(set.pu)); - /* Errors are intentionally ignored: pinning is performance-only, not a correctness requirement. */ - hwloc_set_cpubind(topo, cpuset, HWLOC_CPUBIND_THREAD | HWLOC_CPUBIND_STRICT); - hwloc_bitmap_free(cpuset); -} - -} // namespace monoprop::detail::partition diff --git a/cpp/monoprop/detail/partition/CpuTopology.h b/cpp/monoprop/detail/partition/CpuTopology.h deleted file mode 100644 index 3355d1dad..000000000 --- a/cpp/monoprop/detail/partition/CpuTopology.h +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/*! - * @file CpuTopology.h - * @brief CPU-topology helpers for partition placement. - * - * Uses hwloc for cross-platform topology discovery and thread binding. - * Policy: one partition per physical core, spread across L3/CCX domains, each worker thread pinned - * to its representative PU. Falls back to unpinned execution when hwloc cannot load the topology or - * when binding is unsupported — pinning is a performance optimisation, not a correctness requirement. - * Pinning has no runtime knob: leaving placement to the launcher measured propagate[hubbard] 2.90x slower. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "monoprop/monopropExport.h" - -namespace monoprop::detail::partition { - -/*! - * @brief One physical CPU core the process may use, tagged with its L3 cache domain. - * - * Produced by enumerate_physical_cores(). The representative PU is the lowest OS index in the - * calling thread's allowed affinity mask for the core, so a cgroup or launcher-imposed restriction - * is always respected. - */ -struct PhysicalCore { - int cpu = 0; //!< OS index of the representative PU (lowest allowed SMT sibling of the core). - int l3_domain = 0; //!< Sequential L3-cache domain id assigned by enumerate_physical_cores(). -}; - -/*! - * @brief Lightweight placement token: identifies the single PU a partition worker thread is pinned to. - */ -struct CpuSet { - int pu = -1; //!< OS PU index. -1 ⇒ invalid / not placed. -}; - -namespace topo_detail { -/*! - * @brief Apply the L3-domain interleaving placement policy to a synthetic core list. - * - * Factored out of partition_cpusets() so the scheduling logic can be exercised without depending - * on hwloc or live hardware. Cores are bucketed by their l3_domain, then interleaved depth-first - * across the domains dealt to this rank. When there are more co-located ranks than L3 domains the - * algorithm falls back to flat domain-major order with one contiguous slice per rank. - * - * @param cores Physical cores available to allocate from, with L3 domain tags. - * @param n Number of partitions (PUs) requested for this rank. - * @param group_index This rank's 0-based index among the co-located ranks on the host. - * @param group_count Total number of co-located ranks on the host. - * @returns Ordered PU OS indices of length @p n for the slice assigned to this rank, - * or empty when the request cannot be filled (empty core list, oversubscription, - * or computed offset out of range). - */ -monoprop_EXPORT auto placement_order(const std::vector &cores, - size_t n, - size_t group_index, - size_t group_count) -> std::vector; -} // namespace topo_detail - -/*! - * @brief Enumerate physical cores (one per SMT sibling group) the process may use. - * - * Queries the calling thread's CPU affinity via hwloc to respect any cgroup or launcher-imposed - * restriction. For each core whose cpuset intersects that affinity mask, the lowest matching OS - * index is recorded as the representative PU and the core is labelled with the sequential id of - * its nearest L3 cache ancestor (or a unique singleton id when no L3 object is present). - * - * @returns Vector of PhysicalCore in hwloc logical-core order, or empty when hwloc cannot load - * the topology or when no core passes the affinity filter. - */ -monoprop_EXPORT auto enumerate_physical_cores() -> std::vector; - -//! Affinity-mask exchange width, in 64-bit words. A mask needing more is "cannot classify", never private. -inline constexpr size_t kAffinityMaskWords = 64; - -static_assert(kAffinityMaskWords > 0 && kAffinityMaskWords <= static_cast(INT_MAX), - "the affinity-mask width is an MPI_Allgather element count, which is an int"); - -//! This process's allowed cpuset as @p nwords 64-bit words; false with @p out zeroed when it does not fit. -monoprop_EXPORT auto affinity_mask_words(uint64_t *out, size_t nwords) -> bool; - -/*! @brief Whether the @p n masks of @p words words laid end to end in @p masks are pairwise disjoint. - * False for @p n < 2 and for any empty mask: all-zero is disjoint from everything, and shared is the safe error. - */ -[[nodiscard]] monoprop_EXPORT auto masks_are_pairwise_disjoint(const uint64_t *masks, size_t n, size_t words) -> bool; - -//! What one host's exchanged affinity masks add up to. The union says what the JOB got, not what one rank got. -struct MaskSummary { - size_t cpus = 0; //!< CPUs in our own mask - size_t node_cpus = 0; //!< CPUs in the union over the host - std::string cpu_list = "none"; //!< that union as ascending ranges, "0-15,64-79" -}; - -/*! @brief Summarize the @p n masks of @p words words laid end to end in @p masks, row @p self being ours. - * @returns nullopt for a null/empty argument or any all-zero row: a mask that did not fit the exchange - * window cannot be summed with the others. Diagnostic only; nothing branches on the result. - */ -[[nodiscard]] monoprop_EXPORT auto summarize_masks(const uint64_t *masks, size_t n, size_t words, size_t self) - -> std::optional; - -/* COMMPLACE: a rank seeing 16 of a host's 128 CPUs is equally "my own 16" and "eight of us share - * these 16", and only the co-located ranks' masks separate them, so the exchange PartitionGroup - * already runs to place also reports. Report-only, unconditional; nothing branches on the line. */ - -/*! @brief One newline-terminated COMMPLACE line naming what the launcher handed this rank. - * @param verdict how the co-located masks relate: "private" (pairwise disjoint), "shared" (two ranks - * can land on one CPU), "alone" (the only rank on its host, which is NOT "private"), or - * "unknown" (a mask did not fit the exchanged window, so no verdict is sound). - * Returned, not written, so the formatting is testable without a live rank. - */ -[[nodiscard]] monoprop_EXPORT auto format_place_line(int mpi_rank, - int node_rank, - int node_size, - const char *verdict, - const MaskSummary &summary) -> std::string; - -/*! @brief True the first time @p line is seen, and whenever it differs from the previous call. - * One process has one placement, so a propagator built 437 times reports it 437 times identically; - * the whole line is compared rather than a once-flag set, because a process that builds an `alone` - * propagator before an MPI one has two placements and both are worth saying. Process-wide, locked. - */ -[[nodiscard]] monoprop_EXPORT auto place_line_is_new(std::string_view line) -> bool; - -//! Whether the launcher has already handed this rank a private slice of the node, or the node's CPUs are shared. -enum class NodeMask { Shared, PerRank }; - -/*! - * @brief Build placement tokens for one MPI rank's partitions. - * - * Calls enumerate_physical_cores() and applies topo_detail::placement_order() to select @p n - * distinct physical cores for this rank. Two co-located ranks must never share a core: one rank's - * busy-polling collectives would starve the other's barrier spins. - * - * @param n Number of partitions to place. - * @param group_index This rank's 0-based index among the co-located ranks on the host. - * @param group_count Total number of co-located ranks on the host. - * @param mask NodeMask::PerRank only when the co-located ranks' affinity masks have been measured - * pairwise DISJOINT (PartitionGroup::classify_node_masks_), so this mask is our share. - * @returns Vector of @p n CpuSet tokens, or empty when hwloc is unavailable or fewer than - * @p group_count x @p n cores are visible (@p n under PerRank). - * - * @note Under NodeMask::PerRank the group split is skipped: our share is already this rank's alone. - */ -monoprop_EXPORT auto partition_cpusets(size_t n, - size_t group_index = 0, - size_t group_count = 1, - NodeMask mask = NodeMask::Shared) -> std::vector; - -/*! - * @brief Bind the calling thread to the PU identified by @p set. - * - * Allocates a temporary hwloc bitmap, sets the single bit for @c set.pu, and calls - * @c hwloc_set_cpubind with @c HWLOC_CPUBIND_THREAD | @c HWLOC_CPUBIND_STRICT. The call is - * best-effort: hwloc errors are silently ignored because only performance, not correctness, - * depends on successful pinning. - * - * @param set Placement token as returned by partition_cpusets(). A token with @c pu == -1 - * is a no-op. - */ -monoprop_EXPORT auto pin_this_thread(const CpuSet &set) -> void; -} // namespace monoprop::detail::partition diff --git a/cpp/monoprop/detail/partition/PartitionGroup.h b/cpp/monoprop/detail/partition/PartitionGroup.h deleted file mode 100644 index 724769c60..000000000 --- a/cpp/monoprop/detail/partition/PartitionGroup.h +++ /dev/null @@ -1,371 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/MPICompat.h" // mpi::size for the transport choice -#include "monoprop/detail/mpi/ShmComm.h" -#ifdef monoprop_ENABLE_MPI -#include "monoprop/detail/mpi/HybridComm.h" -#endif -#include "monoprop/detail/partition/CpuTopology.h" - -// Intra-process partition runtime: S master threads, each pinned to a core and running an independent -// MonomialPropagator over one hash partition, with an in-process comm standing in for the network. -// run_on_all must fan out to all masters: the collectives are barrier-synced. - -namespace monoprop { - -template -class MonomialPropagator; // completed before any PartitionGroup member body is instantiated (Impl.h) - -namespace detail::partition { - -template -class PartitionGroup { -public: - // Builds each partition's propagator via `factory(partition_comm)` ON its master thread, so heap allocations - // are first-touched on the owning core. `factory` must build a partitions=1 propagator. - using Factory = std::function>(mpi::Comm)>; - - // `parent` is the enclosing communicator (size R): R == 1 ⇒ an in-process ShmComm; R > 1 ⇒ a - // HybridComm folding R ranks x S partitions into one flat P=R*S world. - PartitionGroup(int n_partitions, const Factory &factory, mpi::Comm parent) - : n_(n_partitions), - parent_(parent), - partitions_(static_cast(n_partitions)), - errs_(static_cast(n_partitions)) { - make_transport_(); - discover_node_peers_(); - cpusets_ = topo_partition_cpusets(n_, node_rank_, node_size_, node_mask_); - start_masters_(); - // The masters are already running, so a ctor throw must not escape: ~PartitionGroup would never run, - // and destroying joinable threads during unwinding calls std::terminate. - try { - run_on_all([&](int r) { partitions_[static_cast(r)] = factory(comm_for_(r)); }); - } - catch (...) { - stop_and_join_(); - throw; - } - } - - // Fresh transport and threads over the same parent; each partition is deep-copied on its new master, then - // its comm rebound (the copy inherited src's handle). - PartitionGroup(const PartitionGroup &src) - : n_(src.n_), - parent_(src.parent_), - node_rank_(src.node_rank_), - node_size_(src.node_size_), - node_mask_(src.node_mask_), - partitions_(static_cast(src.n_)), - errs_(static_cast(src.n_)) { - make_transport_(); - cpusets_ = topo_partition_cpusets(n_, node_rank_, node_size_, node_mask_); - start_masters_(); - try { // see the primary ctor: a throw past live masters would std::terminate - run_on_all([&](int r) { - auto p = src.partitions_[static_cast(r)]->clone_(); // virtual: keeps the derived type - p->comm_ = comm_for_(r); // PartitionGroup is a friend of MonomialPropagator - partitions_[static_cast(r)] = std::move(p); - }); - } - catch (...) { - stop_and_join_(); - throw; - } - } - auto operator=(const PartitionGroup &) -> PartitionGroup & = delete; - - ~PartitionGroup() { stop_and_join_(); } - - auto partition_count() const -> int { return n_; } - auto partition(int s) -> MonomialPropagator & { return *partitions_[static_cast(s)]; } - auto partition(int s) const -> const MonomialPropagator & { return *partitions_[static_cast(s)]; } - - // Run `body(partition_rank)` on all masters, block until every one finishes, then rethrow the exception - // that caused the failure: the lowest-ranked one that is not ShmCommPoisoned. A throw on one master never - // hangs the rest, since peers are released via poison. - // ShmCommPoisoned only ever follows another master's throw, so it is rethrown only when nothing else was - // raised. It can even reach a master whose collective already completed, if a peer throws between the - // barrier's release and that master's post-barrier check. - auto run_on_all(const std::function &body) -> void { - { - std::lock_guard lk(m_); - transport_reset_(); // clear any poison/arrival state left by a previously aborted round - for (auto &e : errs_) { - e = nullptr; - } - job_ = &body; - done_count_ = 0; - ++job_gen_; - } - cv_start_.notify_all(); - { - std::unique_lock lk(m_); - cv_done_.wait(lk, [&] { return done_count_ == n_; }); - } - std::exception_ptr poisoned; - for (auto &e : errs_) { - if (!e) { - continue; - } - try { - std::rethrow_exception(e); - } - catch (const mpi::ShmCommPoisoned &) { - if (!poisoned) { - poisoned = e; - } - } - } - if (poisoned) { - std::rethrow_exception(poisoned); - } - } - -private: - // Poison first so a master parked in a barrier is released rather than joined-on forever. - auto stop_and_join_() noexcept -> void { - transport_poison_(); - { - std::lock_guard lk(m_); - stop_ = true; - } - cv_start_.notify_all(); - for (auto &t : masters_) { - if (t.joinable()) { - t.join(); - } - } - } - - // Free-function wrapper so the header compiles on non-Linux (where partition_cpusets returns {}). - static auto topo_partition_cpusets(int n, int group_index, int group_count, NodeMask mask) - -> std::vector { - return monoprop::detail::partition::partition_cpusets(static_cast(n), - static_cast(group_index), - static_cast(group_count), - mask); - } - - // Under an MPI parent, find how many ranks share this host and which we are, so each co-located rank - // pins to a disjoint core block (see partition_cpusets). Collective over `parent`; clones copy the result. - auto discover_node_peers_() -> void { -#ifdef monoprop_ENABLE_MPI - if (parent_.kind == mpi::Comm::Kind::Mpi && mpi::size(parent_) > 1) { - MPI_Comm node = MPI_COMM_NULL; - MPI_Comm_split_type(parent_.mpi, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &node); - MPI_Comm_rank(node, &node_rank_); - MPI_Comm_size(node, &node_size_); - classify_node_masks_(node); - MPI_Comm_free(&node); - return; - } -#endif - report_placement_(nullptr, 0, "alone"); - } - -#ifdef monoprop_ENABLE_MPI - // A rank seeing 16 of 128 CPUs is equally "my own 16" and "eight of us share these 16": only the masks tell. - auto classify_node_masks_(MPI_Comm node) -> void { - node_mask_ = NodeMask::Shared; - if (node_size_ <= 1) { - report_placement_(nullptr, 0, "alone"); - return; // nobody to collide with; the normal split already handles group_count == 1 - } - constexpr size_t kMaskWords = monoprop::detail::partition::kAffinityMaskWords; - std::array mine{}; - // Refusal zeroes `mine` and an all-zero row is never private, so no reduction of the verdict is needed. - monoprop::detail::partition::affinity_mask_words(mine.data(), kMaskWords); - std::vector all(kMaskWords * static_cast(node_size_), 0); - MPI_Allgather(mine.data(), kMaskWords, MPI_UINT64_T, all.data(), kMaskWords, MPI_UINT64_T, node); - const bool disjoint = monoprop::detail::partition::masks_are_pairwise_disjoint(all.data(), - static_cast(node_size_), - kMaskWords); - node_mask_ = disjoint ? NodeMask::PerRank : NodeMask::Shared; - report_placement_(all.data(), static_cast(node_size_), disjoint ? "private" : "shared"); - } -#endif - - /* COMMPLACE only, over the array MPI_Allgather already filled: no extra collective, and no - * reduction either, since every rank reads the same rows and so reaches the same verdict. `masks` - * nullptr means no peers, so measure our own mask; the verdict is then "alone", which is NOT - * evidence that a multi-rank launcher did the right thing. Reached only from the primary ctor, so - * a clone does not re-emit -- the mask belongs to the process, not the object. */ - auto report_placement_(const uint64_t *masks, size_t peers, const char *verdict) -> void { - constexpr size_t kWords = monoprop::detail::partition::kAffinityMaskWords; - std::array own{}; - if (masks == nullptr && affinity_mask_words(own.data(), kWords)) { - masks = own.data(); - peers = 1; - } - // No summary is "unknown" rather than a plausible zero: some mask did not fit the window. - const auto sum = summarize_masks(masks, peers, kWords, static_cast(node_rank_)); - const std::string line = format_place_line(mpi::rank(parent_), - node_rank_, - node_size_, - sum ? verdict : "unknown", - sum.value_or(MaskSummary{})); - if (place_line_is_new(line)) { - std::fputs(line.c_str(), stderr); - std::fflush(stderr); - } - } - - auto make_transport_() -> void { -#ifdef monoprop_ENABLE_MPI - if (parent_.kind == mpi::Comm::Kind::Mpi && mpi::size(parent_) > 1) { - hyb_ = std::make_unique(parent_.mpi, n_); - return; - } -#endif - shm_ = std::make_unique(n_); - } - auto comm_for_(int r) -> mpi::Comm { -#ifdef monoprop_ENABLE_MPI - if (hyb_) { - return mpi::Comm::make_hybrid(hyb_.get(), r); - } -#endif - return mpi::Comm::make_shm(shm_.get(), r); - } - auto transport_poison_() -> void { -#ifdef monoprop_ENABLE_MPI - if (hyb_) { - hyb_->poison(); - return; - } -#endif - shm_->poison(); - } - auto transport_reset_() -> void { -#ifdef monoprop_ENABLE_MPI - if (hyb_) { - hyb_->reset(); - return; - } -#endif - shm_->reset(); - } - - // Spawning is itself a failure point: one std::thread per partition is O(100) threads, and a mid-loop - // std::system_error must not escape with live masters parked in masters_ (see the primary ctor). - auto start_masters_() -> void { - masters_.reserve(static_cast(n_)); - try { - for (int r = 0; r < n_; ++r) { - masters_.emplace_back([this, r] { master_loop_(r); }); - } - } - catch (...) { - stop_and_join_(); - throw; - } - } - - auto master_loop_(int rank) -> void { - if (!cpusets_.empty()) { - pin_this_thread(cpusets_[static_cast(rank)]); - } - unsigned seen = 0; - for (;;) { - const std::function *job = nullptr; - { - std::unique_lock lk(m_); - cv_start_.wait(lk, [&] { return stop_ || job_gen_ != seen; }); - if (stop_) { - return; - } - seen = job_gen_; - job = job_; - } - try { - (*job)(rank); - } - catch (...) { - errs_[static_cast(rank)] = std::current_exception(); - transport_poison_(); - } - { - std::lock_guard lk(m_); - ++done_count_; - } - cv_done_.notify_one(); - } - } - - int n_; - mpi::Comm parent_; // enclosing communicator (size R) — decides the transport - int node_rank_ = 0; // this rank's index among the ranks sharing the host - int node_size_ = 1; // how many parent ranks share the host (1 unless MPI R>1) - NodeMask node_mask_ = NodeMask::Shared; // set by classify_node_masks_; copied, never re-derived, by the copy ctor - std::unique_ptr shm_; // set iff R == 1 -#ifdef monoprop_ENABLE_MPI - std::unique_ptr hyb_; // set iff R > 1 -#endif - std::vector>> partitions_; - std::vector errs_; - std::vector cpusets_; - std::vector masters_; - - // Job dispatch: the facade thread publishes one job and waits for all masters to complete it. - std::mutex m_; - std::condition_variable cv_start_, cv_done_; - const std::function *job_ = nullptr; - unsigned job_gen_ = 0; - int done_count_ = 0; - bool stop_ = false; -}; - -// One result per partition, indexed by partition rank. The slots are written from the owning master, so -// `body` must not touch the vector itself. Staged into a non-bit-packed `Slot` type: std::vector is -// the bit-packed specialization, so concurrent partition-master writes to different logical elements can -// tear the same underlying word (a data race) even though their indices are disjoint. -template > -auto collect_on_all(PartitionGroup &group, Body body) -> std::vector { - using Slot = std::conditional_t, std::uint8_t, R>; - std::vector staging(static_cast(group.partition_count())); - group.run_on_all([&](int r) { staging[static_cast(r)] = static_cast(body(r)); }); - if constexpr (std::is_same_v) { - return std::vector(staging.begin(), staging.end()); - } - else { - return staging; - } -} - -// collect_on_all over the partition propagators themselves: `body(partition)` on each partition's master. -template &>> -auto map_partitions(PartitionGroup &group, Body body) -> std::vector { - return collect_on_all(group, [&](int r) -> R { return body(group.partition(r)); }); -} - -} // namespace detail::partition -} // namespace monoprop diff --git a/cpp/monoprop/detail/sharded/CMakeLists.txt b/cpp/monoprop/detail/sharded/CMakeLists.txt new file mode 100644 index 000000000..697a79d6b --- /dev/null +++ b/cpp/monoprop/detail/sharded/CMakeLists.txt @@ -0,0 +1,20 @@ +target_sources( + monoprop + PUBLIC + FILE_SET headers + TYPE HEADERS + FILES + "Construction.h" + "Evaluation.h" + "Exchange.h" + "RootObserver.h" + "State.h" + "Team.h" +) + +target_sources( + monoprop-objs + PRIVATE + Evaluation.cpp + Exchange.cpp +) diff --git a/cpp/monoprop/detail/sharded/Construction.h b/cpp/monoprop/detail/sharded/Construction.h new file mode 100644 index 000000000..111a725d4 --- /dev/null +++ b/cpp/monoprop/detail/sharded/Construction.h @@ -0,0 +1,1269 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/MPGraph.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/Validation.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/Engine.h" +#include "monoprop/detail/evolution/layer_build/FusedApply.h" +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/OperationFailure.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/sharded/Evaluation.h" +#include "monoprop/detail/sharded/Exchange.h" +#include "monoprop/detail/sharded/State.h" +#include "monoprop/detail/sharded/Team.h" + +/* + * Graph construction and graph-free propagation over the T shards of one process, at geometry (P, T). + * + * One run_team() spans the whole gate loop. Worker t is the owner of shard t, whose flat routing slot is + * rank * T + t; every other shard is a cross owner, in this process or another. Owners of this process exchange + * partner queries through published buffers instead of communicators; blocks for and from other processes go through + * one physical round per handoff (Exchange.h), whose MPI calls the primary makes. Each phase below ends in a + * collective checkpoint (phase()); a phase body mutates only its owner's shard and gate buffers, and reads other + * owners' buffers only after the checkpoint that published them. Within-shard kernels run serially: the parallelism + * is across owners. + * + * Per gate, in this order (P = publication checkpoint): + * + * traverse + prepare leaders scan the shard, resolve same-shard leaders, publish leader payloads (P1) + * resolve leaders each destination resolves every source's block, in ascending source + * order, inserts its misses and publishes its answers (P2) + * consume leaders + prepare fold the answers into the sink; then filter followers against the now + * followers complete leader marks, resolve same-shard followers, publish payloads (P3) + * resolve followers as for leaders (P4) + * consume followers + finish fold the answers; insert the deferred same-shard leader then follower + * misses; finalize the layer, or extend and apply the fused records (P5) + * + * Coefficient-informed graph construction (build_graph_informed) runs the same gate phases with the owner's evolving + * coefficients and build angle, after a seed phase that copies the current picture and, when a graph already exists, + * replays it through replay_forward_in_team() in the same team. Its P5 then also extends the coefficients and begins + * the replay of the new layer over its transient cosine set (snapshot, publish, cosine pass), and one more checkpoint + * (P6) finishes it from the partners' published snapshots before the next gate's traversal reads the coefficients. + * + * The identity generator skips the exchange phases; with one flat owner there is no cross owner, so only the resolve + * and consume work is skipped, and same-shard resolution still runs. Row insertion order is therefore cross-owner + * leaders, cross-owner followers, deferred same-shard leaders, deferred same-shard followers, as the removed partition + * runtime inserted them at the same (P, T) geometry. + * + * Other processes. A gate's window reaches another rank exactly when its peer plan does: the single peer + * rank ^ shift(generator) under linear routing (none when the shift is zero), every other rank under splitmix. That + * depends on the replicated generator and router only, so every rank takes the same branch. Each pass then adds: + * + * P1 / P3 the owner also writes its query send row: the size of its block for every remote destination + * Q1 primary: lay out the query round, post its count round + * Q2 owners pack their remote query blocks into the round's send staging + * Q3 primary: complete the counts, lay out the receive side, post and complete the queries + * P2 / P4 destinations resolve local views and received blocks together, in ascending flat-slot order, then + * write their answer rows: one answer per query resolved for each remote source, and one per own query + * to each remote destination (answer counts are query counts, not the streams' word counts) + * R1 primary: lay out the answer round + * R2 owners pack the answers remote sources wait for + * R3 primary: post and complete the answers + * P3 / P5 sources consume local and received answers, in window order + * + * The query staging is read in P2 (P4) and rewritten only by the next pass's Q3; the answer staging is read in P3 + * (P5) and rewritten only by the next pass's R3. Construction's peer plan picks the transport: pairwise for the single + * linear peer, one collective MPI_Ialltoallv under splitmix. Informed construction's new-layer replay goes through + * a replay round as evaluation does (Evaluation.h), with the communicator-agreed transport: P5 publishes and writes + * the replay rows, a primary phase lays the round out, owners pack, and a phase posts, runs every owner's cosine + * pass while the transfer is in flight and completes it before P6 reads the received blocks. + * + * Buffers (owner t writes only its own; no buffer is resized, moved or freed while another owner may read it): + * + * | Buffer | Writer | Published | Readers | Last read | Reset | + * | --------------------------------------- | ------- | --------- | ---------------------- | --------- | ----------- | + * | leader payload (engine queries_r or | owner t | P1 | every destination | P2 | prepare of | + * | fused scratch), query inbox row t | | | (resolve leaders) | | followers | + * | leader answers, answer inbox row t | owner t | P2 | every source (consume) | P3 | resolve of | + * | | | | | | followers | + * | follower payload, query inbox row t | owner t | P3 | every destination | P4 | next gate | + * | follower answers, answer inbox row t | owner t | P4 | every source (consume) | P5 | next gate | + * | own sources, plain queries, values | owner t | (private) | owner t (consume) | P3 / P5 | next pass | + * | same-shard stage, deferred misses, | owner t | (private) | owner t | P5 | next gate | + * | decoded incoming positions, scratch | | | | | | + * | fused records, cosine set (propagate) | owner t | (private) | owner t (apply) | P5 | next gate | + * | informed: new-layer replay publication, | owner t | P5 | every partner (P6) | P6 | next P5 | + * | board replay_pairs[t] | | | | | | + * | informed: evolving coefficients, self | owner t | (private) | owner t | caches | frame freed | + * | sources, transient cosine set | | | | | | + * + * "Next gate" means the owner's first phase of the following gate, which starts after P5 has passed, or the final + * cache phase, which releases the gate buffers. In that phase at most max(4, T / 8) owners at a time copy their + * vectors down to size when the copy is 256 KiB or more (SectionLimit), so the copies' transients do not coincide. The + * two T x T inboxes of views (row t written by owner t, column d read by owner d), the replay board, the frames' + * pointer vector and the physical rounds are allocated on the caller and outlive the team, so every buffer and request + * stays alive through the join on failure. + * + * Opaque callbacks: a cutoff predicate that is not a typed built-in (CutoffEvaluator::parallel_safe()), including a + * basis-change closure, is never called from two threads at once. The traversal then runs on the primary, shard by + * shard, in a phase every worker enters; the other owners wait at its checkpoint and resume with their own + * resolution. This is an exclusive phase, not a smaller team: the shards and their owners are unchanged. + * + * Failure: every call that can throw runs inside a phase. A failure suppresses every later phase body on every + * worker, the team joins, and the original exception of the lowest-numbered failing worker is returned (not + * rethrown) together with whether mutation had started. Nothing is rolled back: once mutation started, the shards + * are partly updated and the rank-level owner must be treated as invalid. Independent copies are unaffected. On a + * multi-rank communicator the seam hands every failure, before or after mutation, to mpi::operation_failed() itself, + * before any request owner is destroyed, which aborts the communicator: the peers would otherwise wait for this rank in + * the operation's rounds or later collectives. + */ + +namespace monoprop::detail::sharded { + +//! The owner work a construction reports to its observer, in phase order. +enum class ConstructionWork : std::uint8_t { + frame, //!< Allocate the owner's operation frame; mutates no shard. + picture, //!< Propagation only: bring the live picture coefficients up to date. + traverse, //!< Scan the shard for this gate's partner queries. + prepare_leaders, //!< Resolve same-shard leaders and expose the leader payload. + resolve_leaders, //!< Resolve the leader queries published to this shard; insert and index misses. + consume_leaders, //!< Fold the answers to this shard's leader queries. + prepare_followers, //!< Filter followers, resolve same-shard followers and expose the follower payload. + resolve_followers, //!< Resolve the follower queries published to this shard; insert and index misses. + consume_followers, //!< Fold the answers to this shard's follower queries. + finalize, //!< Insert deferred same-shard misses; finalize the layer and append it to the graph. + apply, //!< Propagation only: extend the coefficients and apply the fused records. + caches, //!< After the last gate: release gate buffers, warm the shard's caches, shrink its vectors. + seed, //!< Informed construction: copy the picture, prepare and run the seed replay. + replay, //!< Informed construction: begin (P5) or finish (P6) the replay of the new layer. + exchange, //!< Multi-rank: the primary lays out, posts or completes a physical round (shard 0 only). + pack, //!< Multi-rank: copy this owner's remote blocks into a round's send staging. +}; + +/*! + * \brief The observer production code uses: it does nothing and compiles away. + * + * `visit(work, step, shard)` runs inside the protected phase, on the thread that performs `work` for `shard`, before + * that work; `step` is the gate-loop step or no_step. `kernels(shard)` supplies the range observer the shard's + * kernels and exchange phases report to (see NoRangeObserver and LayerBuildEngine); it is called once, by the owner, + * in its frame phase. Tests substitute observers that record the executing worker, capture streams or throw; an + * exception from either method is that phase's failure. A test observer may also define + * `replayed(shard, step, const VecD &coeffs)`, detected at compile time, which informed construction calls after each + * new layer's replay with the owner's evolving coefficients, `queries_packed(step, shard, PhysicalExchange &)`, + * called after the owner packed its remote query blocks, which may rewrite them to inject malformed traffic, and + * `fused_records(step, shard, const FusedGateView &)`, which propagation calls with the owner's real fused records + * just before it applies them. + */ +struct NoConstructionObserver { + auto visit(ConstructionWork /*work*/, size_t /*step*/, size_t /*shard*/) const noexcept -> void {} + [[nodiscard]] auto kernels(size_t /*shard*/) const noexcept -> NoRangeObserver { return {}; } +}; + +/*! + * \brief Test-only view of one owner's fused records for one propagation gate, just before they are applied. + * + * Everything apply_fused_contract() receives, read-only: an observer may copy it and replay the apply itself. + */ +struct FusedGateView { + const FusedContract &records; //!< The gate's hit, insert and cross-rank-half records for this shard. + const CosMask &cos; //!< The gate's cosine set on this shard. + std::span coeffs; //!< The picture coefficients the records apply to, already extended. + double apply_angle; //!< The angle apply_fused_contract() receives. + bool schrodinger; //!< Picture. + bool fused_scale; //!< Whether the traversal already applied the cosine (fused sweep). +}; + +/*! + * \brief Rank-level configuration of a construction, read concurrently and never mutated by the owners. + */ +template +struct ConstructionContext { + const CutoffFn &cutoff_fn; //!< Structural cutoff on partners; must outlive the call. + routing::Router router; //!< Flat-owner routing at geometry (P, T), as the shards were seeded. + std::optional lower_atol; //!< Lower sine cutoff; applies only where coefficients are supplied. + std::optional upper_atol; //!< Upper rescue threshold; applies only where coefficients are supplied. + Basis basis = Basis::Majorana; //!< Rotation signs, and Schrödinger fresh-insert scoring. + bool schrodinger = false; //!< Picture: forward gate order and state coefficients when true. + //! This process among the router's ranks; read on the caller (PhysicalWorld::of()) before the call. + PhysicalWorld world{}; + //! The owner's physical rounds, reused across its calls; null allocates rounds for this call only. + PhysicalRounds *rounds = nullptr; +}; + +/*! + * \brief A validated circuit for graph construction, in circuit order, with every array one entry per gate. + * + * Views the caller's arrays, which must outlive the call. Generators are already converted and bounds-checked. + */ +template +struct GraphCircuit { + std::span> generators; //!< Gate generators, in circuit order. + std::span parameter_mapping; //!< Parameter index of each gate's layer. + std::span gen_coeffs; //!< Generator coefficient of each gate's layer. + std::span gate_indices; //!< Absolute gate index of each layer (offset already applied). + std::optional only_rotate_len_k; //!< Rotate only sources of at most this many slots. +}; + +/*! + * \brief A validated circuit for graph-free propagation, in circuit order, with one angle per gate. + * + * Views the caller's arrays, which must outlive the call. + */ +template +struct PropagationCircuit { + std::span> generators; //!< Gate generators, in circuit order. + std::span mapped_params; //!< Build angle of each gate: map_params(..., 1.0). + std::optional only_rotate_len_k; //!< Rotate only sources of at most this many slots. +}; + +/*! + * \brief A validated circuit for coefficient-informed graph construction. + * + * Views the caller's arrays, which must outlive the call. + */ +template +struct InformedCircuit { + GraphCircuit graph; //!< Generators and layer metadata, as for build_graph(). + std::span mapped_params; //!< Build angle of each gate: map_params(parameters, ..., 1.0). + //! Seed replay angle of each existing layer, in replay order (see replay_shards()); required exactly when the + //! shards already hold layers, which are then replayed to form the seed. Without it the seed is the picture. + std::optional> seed_params; +}; + +/*! + * \brief What a construction reports after its team has joined. + * + * The seam a rank-level caller applies its validity policy at: with an error and `mutation_started`, the shards are + * partly updated and the owner must be invalidated before the error is rethrown; with an error and no mutation, the + * shards are unchanged and the owner stays usable. A multi-rank failure never gets here: the seam aborts it. + */ +struct ConstructionOutcome { + std::exception_ptr error; //!< The lowest-numbered failing worker's original exception, or empty. + bool mutation_started = false; //!< Whether any phase that may change a shard began. +}; + +/*! + * \brief Read-only views of the blocks the owners of one process published for `owner`, one per window slot. + * + * Slot `s` of `window` is the flat owner whose local index is `s - first_local`. Its view is that owner's block for + * `owner` when it published a buffer whose window reaches `owner`; a slot outside this process's owners, an owner + * that published nothing, or a buffer whose window does not reach `owner` yields an empty view. Views alias the + * published buffers, which must stay unchanged while the views are read; nothing is copied. + * + * \param window The reading owner's window over flat slots. + * \param owner The reading owner's flat slot. + * \param first_local The flat slot of this process's first shard, `rank * T`. + * \param published Per local shard, its published buffer or null. + * \return One view per slot of `window`, in slot order. + */ +template +[[nodiscard]] auto gather_published(mpi::SlotWindow window, + size_t owner, + size_t first_local, + std::span> *const> published) + -> mpi::WindowVec> { + mpi::WindowVec> views(window); + for (const auto wi : window.indices()) { + const size_t slot = window.slot(wi); + if (slot < first_local || slot - first_local >= published.size()) { + continue; + } + const auto *const buffer = published[slot - first_local]; + if (buffer != nullptr && buffer->window().contains(owner)) { + views[wi] = std::span(buffer->at_slot(owner)); + } + } + return views; +} + +namespace construction_detail { + +/* + * Post one source's published blocks into its row of the `threads` x `threads` inbox of views: entry + * source * threads + d views the block for local destination d, or is empty when the buffer's window does not reach + * it. Every entry of the row is rewritten, so no view from an earlier pass survives, and a row has one writer, so posts + * by different sources share no cache line except at row boundaries. The views alias `buffer`, which must stay + * unchanged until its readers are done. Read from the destination's side, this equals gather_published(). + */ +template +auto post_blocks(const mpi::WindowVec> &buffer, + size_t source, + size_t first_local, + size_t threads, + std::span> inbox) -> void { + const mpi::SlotWindow window = buffer.window(); + const auto row = inbox.subspan(source * threads, threads); + for (size_t d = 0; d < threads; ++d) { + const size_t slot = first_local + d; + row[d] = window.contains(slot) ? std::span(buffer.at_slot(slot)) : std::span{}; + } +} + +/* + * Destination `dest`'s views over `window` from its inbox column: slot s of the window gets the entry local source + * s - first_local posted for `dest` when s is a local shard, and an empty view otherwise. The column's entries are + * independent loads, one per source row. + */ +template +[[nodiscard]] auto collect_blocks(mpi::SlotWindow window, + size_t first_local, + size_t dest, + size_t threads, + std::span> inbox) -> mpi::WindowVec> { + mpi::WindowVec> views(window); + for (const auto wi : window.indices()) { + const size_t slot = window.slot(wi); + if (slot >= first_local && slot - first_local < threads) { + views[wi] = inbox[((slot - first_local) * threads) + dest]; + } + } + return views; +} + +//! Within-shard kernels run serially: the team's parallelism is across owners. +inline constexpr parallel::Options kSerial{}; + +/* + * The peer ranks a gate's handoffs reach: the single linear peer rank ^ shift (none when that is this rank), or every + * other rank under a dense plan. A function of the replicated generator and router only, so every rank computes the + * same set for the same gate. + */ +inline auto gate_peers(mpi::PeerPlan plan, const PhysicalWorld &world) -> std::vector { + if (plan.sparse) { + const size_t peer = world.rank ^ static_cast(plan.shift); + return peer == world.rank ? std::vector{} : std::vector{peer}; + } + return other_ranks(world); +} + +// The flat slot of shard `shard` of rank `rank`, checked against the window it must index. +inline auto window_slot(mpi::SlotWindow window, size_t rank, size_t shard, size_t threads) -> size_t { + const size_t slot = (rank * threads) + shard; + if (!window.contains(slot)) { + throw std::logic_error( + std::format("sharded construction: remote slot {} lies outside the gate's window [{}, {})", + slot, + window.base, + window.stop())); + } + return slot; +} + +// Owner `own`'s send row: the size of its block for every shard of every peer. +template +auto write_send_rows(PhysicalExchange &round, + const mpi::WindowVec> &blocks, + std::span peers, + size_t threads, + size_t own) -> void { + round.reset_rows(own); + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t t = 0; t < threads; ++t) { + round.set_send_count(own, k, t, blocks.at_slot(window_slot(blocks.window(), peers[k], t, threads)).size()); + } + } +} + +/* + * Owner `own`'s rows of an answer round: it sends one answer per query it resolved for each remote source, and receives + * one per query of its own (`sources`, one entry per query) addressed to each remote destination. Answer counts are + * query counts, not the query streams' word counts, so they are known only after resolution. + */ +template +auto write_answer_rows(PhysicalExchange &round, + const mpi::WindowVec> &answers, + const mpi::WindowVec> &sources, + std::span peers, + size_t threads, + size_t own) -> void { + round.reset_rows(own); + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t t = 0; t < threads; ++t) { + round.set_send_count(own, + k, + t, + answers.at_slot(window_slot(answers.window(), peers[k], t, threads)).size()); + round.set_recv_count(own, + k, + t, + sources.at_slot(window_slot(sources.window(), peers[k], t, threads)).size()); + } + } +} + +// Owner `own` copies its block for every remote shard into its slices of the round's send staging. +template +auto pack_blocks(PhysicalExchange &round, + const mpi::WindowVec> &blocks, + std::span peers, + size_t threads, + size_t own) -> void { + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t t = 0; t < threads; ++t) { + const auto &block = blocks.at_slot(window_slot(blocks.window(), peers[k], t, threads)); + const auto slice = round.send_block(own, k, t); + if (slice.size() != block.size()) { + throw std::logic_error(std::format("sharded construction: a {}-element block meets a {}-element slice", + block.size(), + slice.size())); + } + std::ranges::copy(block, slice.begin()); + } + } +} + +// Views of what owner `own` received from every remote shard, placed at their slots of `views`. +template +auto add_received(mpi::WindowVec> &views, + const PhysicalExchange &round, + std::span peers, + size_t threads, + size_t own) -> void { + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t su = 0; su < threads; ++su) { + views.at_slot(window_slot(views.window(), peers[k], su, threads)) = round.recv_block(own, k, su); + } + } +} + +// One owner's buffers for the current gate. The fused records precede the engine, which refers to them. +template +struct GateWork { + using Response = typename Sink::Response; + GateScan scan; + FusedContract fc; + CosMask cos; + std::optional> engine; + const mpi::WindowVec *payload = nullptr; // the engine's payload for the current pass (multi-rank packing) + mpi::WindowVec> answers; // this owner's answers for the current pass + std::shared_ptr core; // informed: the new layer, replayed at P5/P6 +}; + +// One owner's operation frame, allocated by the owner in the frame phase. +template +struct OwnerFrame { + explicit OwnerFrame(KernelObserver kernels_) : kernels(std::move(kernels_)) {} + KernelObserver kernels; + VecD *coeffs = nullptr; // propagation: the live picture coefficients; informed: `informed` below + std::optional> work; + VecD informed; // informed: the evolving coefficients (seed, then gate by gate) + std::optional seed_view; // informed: the existing graph's replay window + CosCallbacks seed_callbacks; // informed: the seed replay's callbacks + ForwardScratch replay_scratch; // informed: self sources of the new layer's replay + PublishedPair replay_published; // informed: read by partners at P6 +}; + +// Construction's work reported through an evaluation observer: the seed replay's steps. +template +class SeedObserver final : public EvaluationObserver { +public: + explicit SeedObserver(const Observer &observer) : observer_(&observer) {} + auto visit(EvaluationWork /*work*/, size_t step, size_t shard) const -> void override { + observer_->visit(ConstructionWork::seed, step, shard); + } + +private: + const Observer *observer_; +}; + +//! Which gate loop run_gates() executes. +enum class GateMode : std::uint8_t { graph, informed, propagate }; + +template +auto check_arguments(const char *what, + parallel::Options options, + const Shards &shards, + const ConstructionContext &ctx, + size_t gates, + std::initializer_list lengths, + std::optional only_rotate_len_k) -> void { + if (options.threads < 1 || shards.size() != static_cast(options.threads) + || std::ranges::any_of(shards, [](const auto &state) { return !state; })) { + throw std::invalid_argument( + std::format("sharded::{}: expected {} non-null shards, got {}", what, options.threads, shards.size())); + } + // The router's geometry must be this process's: its ranks, and T shards per rank. + if (ctx.router.ranks() != ctx.world.ranks || ctx.world.rank >= ctx.world.ranks + || ctx.router.shards_per_rank() != shards.size()) { + throw std::invalid_argument(std::format("sharded::{}: routing geometry ({} ranks, {} shards per rank) does not " + "match rank {} of {} with {} shards", + what, + ctx.router.ranks(), + ctx.router.shards_per_rank(), + ctx.world.rank, + ctx.world.ranks, + shards.size())); + } + for (const size_t length : lengths) { + if (length != gates) { + throw std::invalid_argument( + std::format("sharded::{}: a per-gate array has {} entries for {} gates", what, length, gates)); + } + } + validate_only_rotate_len_k_(only_rotate_len_k, 2 * NumModes); +} + +/* + * The gate loop shared by graph construction (GraphSink), coefficient-informed graph construction (GraphSink with + * coefficients) and propagation (ContractSink). `angle_of(idx)` is the build angle of gate idx (informed and + * propagation); `append(state, storage, idx)` appends a finished layer (both graph modes); `seed_params` are the + * informed seed replay's angles, or null for the picture seed. + */ +template +auto run_gates(parallel::Options options, + Shards &shards, + const ConstructionContext &ctx, + std::span> generators, + std::optional only_rotate_len_k, + const Observer &observer, + AngleOf &&angle_of, + Append &&append, + const std::span *seed_params = nullptr) -> ConstructionOutcome { + constexpr bool Fused = Mode == GateMode::propagate; + constexpr bool Informed = Mode == GateMode::informed; + using Sink = std::conditional_t, GraphSink>; + using Response = typename Sink::Response; + using KernelObserver = std::remove_cvref_t().kernels(size_t{}))>; + using Frame = OwnerFrame; + using W = ConstructionWork; + + const size_t threads = shards.size(); + const size_t gates = generators.size(); + const size_t flat_world = ctx.router.flat_world(); + const size_t first_local = ctx.world.rank * threads; + const bool schrodinger = ctx.schrodinger; + // Opaque predicates keep the exclusive traversal; see the header comment. + const bool owner_traversal = CutoffEvaluator(ctx.cutoff_fn).parallel_safe(); + + // Caller-side metadata: T frame pointers. Entry t has one writer, owner t; readers dereference it only after the + // checkpoint that published it. + std::vector> frames(threads); + // Handoff tables (T x T): entry s * T + d views source s's block for destination d. Source s writes its row in the + // phase that publishes the blocks; destination d reads its column only after that phase's checkpoint, with one + // independent load per source instead of chasing every source's buffer. + std::vector> query_inbox(threads * threads); + std::vector> answer_inbox(threads * threads); + // Informed only: the new layers' replay publications (entry t written by owner t in its frame phase) and the + // seed replay job (entry t filled by owner t in the seed phase, read by partners after its checkpoint). + std::vector replay_pairs(threads, nullptr); + ForwardReplayJob seed_job; + if constexpr (Informed) { + if (seed_params != nullptr) { + seed_job.params = *seed_params; + seed_job.first_local = first_local; + seed_job.owners.resize(threads); + } + } + const SeedObserver seed_observer(observer); + bool mutation_started = false; // written by the primary only, read after the join + + // Physical rounds, only when there are other processes; allocated on the caller and alive through the join. + const PhysicalWorld &world = ctx.world; + const bool multirank = world.ranks > 1; + // The owner's rounds when it supplies them (ctx.rounds), so their staging persists across calls; otherwise this + // call's own. + PhysicalRounds own_rounds; + PhysicalExchange *queries = nullptr; + PhysicalExchange *answers = nullptr; + PhysicalExchange *replay = nullptr; + if (multirank) { + PhysicalRounds &rounds = ctx.rounds != nullptr ? *ctx.rounds : own_rounds; + using Kind = PhysicalRounds::Kind; + queries = &rounds.get(Kind::queries, world, threads); + answers = &rounds.get(Fused ? Kind::fused_answers : Kind::graph_answers, world, threads); + if constexpr (Informed) { + replay = &rounds.get(Kind::replay, world, threads); + } + } + const routing::Router &router = ctx.router; + // Construction's transport follows its peer plan: pairwise for the linear peer, collective under splitmix. + const auto transport = router.is_linear() ? ExchangeTransport::pairwise : ExchangeTransport::collective; + const ReplayRound replay_round{ + .round = replay, + .transport = world.replay_pairwise ? ExchangeTransport::pairwise : ExchangeTransport::collective}; + if constexpr (Informed) { + seed_job.remote = replay_round; + } + + /* + * In the last phase every owner copies its coefficient and state vectors down to size, each copy holding the old + * and the new block at once. With every owner copying together, those transients add up to the operation's peak + * (about 5 % of the operator at T = 96); T / 8 concurrent copies, never fewer than 4, keep the memory bandwidth + * busy without that pile-up. A copy below 256 KiB skips the limit: such copies cannot pile up to much, and the + * shared counter would cost a small operation more than the copy. + */ + auto slack_copies = SectionLimit(std::max(4, threads / 8)); + constexpr size_t small_slack_copy_bytes = size_t{256} << 10; + const auto error = run_team(options, [&](size_t t, TeamFailure &failure) noexcept { + const size_t flat = first_local + t; + ShardState &own = *shards[t]; + + if (!phase(failure, t, [&] { + observer.visit(W::frame, no_step, t); + frames[t] = std::make_unique(observer.kernels(t)); + replay_pairs[t] = &frames[t]->replay_published; + })) { + return; + } + if (t == 0) { + mutation_started = true; + } + Frame &frame = *frames[t]; + if constexpr (Informed) { + // The seed: the current picture, or the existing graph replayed from it at the seed angles. + if (!phase(failure, t, [&] { + observer.visit(W::seed, no_step, t); + frame.informed = own.op.current_picture(schrodinger); + frame.coeffs = &frame.informed; + if (seed_params != nullptr) { + frame.seed_view.emplace(own.graph.slice_view(own.graph.layers())); + frame.seed_callbacks = + make_cos_callbacks(own.op.inverted_index(), *frame.seed_view, ctx.basis, kSerial); + auto &entry = seed_job.owners[t]; + entry.coeffs = &frame.informed; + entry.graph = &*frame.seed_view; + entry.callbacks = &frame.seed_callbacks; + } + })) { + return; + } + if (seed_params != nullptr && !replay_forward_in_team(failure, t, seed_job, &seed_observer)) { + return; + } + } + if constexpr (Fused) { + if (!phase(failure, t, [&] { + observer.visit(W::picture, no_step, t); + // Materializes pending initial entries (Heisenberg) or scores the state (Schrödinger). + (void)own.op.current_picture(schrodinger); + frame.coeffs = schrodinger ? &own.op.state_coeffs : &own.op.op_coeffs; + })) { + return; + } + } + + for (size_t step = 0; step < gates; ++step) { + // Heisenberg applies the circuit in reverse; Schrödinger forward, with the apply angle negated. + const size_t idx = schrodinger ? step : gates - 1 - step; + const Monomial &gen = generators[idx]; + const bool passes = gen.any(); // identity: nothing anticommutes, nothing is exchanged + const bool cross = passes && flat_world > 1; + // The other processes this gate's handoffs reach; the same set on every rank (see the header comment). + const auto peers = + multirank && passes + ? gate_peers(mpi::PeerPlan{.sparse = router.is_linear(), + .shift = static_cast(router.rank_shift(gen))}, + world) + : std::vector{}; + const bool remote = !peers.empty(); + + // Scans shard s into a fresh gate buffer; the previous gate's buffers were last read before P5. + const auto traverse = [&](size_t s) { + observer.visit(W::traverse, step, s); + Frame &f = *frames[s]; + ShardState &state = *shards[s]; + f.work.reset(); + auto &work = f.work.emplace(); + if constexpr (Fused) { + work.scan = scan_gate(state.op, + gen, + ctx.cutoff_fn, + ctx.lower_atol, + std::cref(*f.coeffs), + ctx.upper_atol, + angle_of(idx), + only_rotate_len_k, + ctx.router, + first_local + s, + /*use_fused=*/true, + f.coeffs, + ctx.basis, + kSerial, + f.kernels); + } + else if constexpr (Informed) { + // A coefficient-informed build: coefficients and build angle, no fused sweep. + work.scan = scan_gate(state.op, + gen, + ctx.cutoff_fn, + ctx.lower_atol, + std::cref(*f.coeffs), + ctx.upper_atol, + angle_of(idx), + only_rotate_len_k, + ctx.router, + first_local + s, + /*use_fused=*/false, + nullptr, + ctx.basis, + kSerial, + f.kernels); + } + else { + work.scan = scan_gate(state.op, + gen, + ctx.cutoff_fn, + ctx.lower_atol, + std::nullopt, + ctx.upper_atol, + std::nullopt, + only_rotate_len_k, + ctx.router, + first_local + s, + /*use_fused=*/false, + nullptr, + ctx.basis, + kSerial, + f.kernels); + } + }; + // Owner-local: builds the engine (which starts the matched epoch) and runs the leader prepare. + const auto prepare_leaders = [&] { + observer.visit(W::prepare_leaders, step, t); + auto &work = *frame.work; + auto &engine = [&]() -> auto & { + if constexpr (Fused) { + return work.engine.emplace(own.op, + flat_world, + flat, + own.matched, + work.scan.combined_size, + Sink{.R = flat_world, + .my_rank = flat, + .fc = work.fc, + .op_coeffs = *frame.coeffs, + .fused_scale = work.scan.fused_scale, + .inv_cos = work.scan.inv_cos, + .schrodinger = schrodinger, + .basis = ctx.basis}, + work.scan.window, + kSerial, + frame.kernels); + } + else { + return work.engine.emplace(own.op, + flat_world, + flat, + own.matched, + work.scan.combined_size, + Sink(flat_world, flat), + work.scan.window, + kSerial, + frame.kernels); + } + }(); + if (passes) { + auto &streams = work.scan.streams; + work.payload = &engine.prepare_exchange(true, + std::move(streams.leader_queries), + std::move(streams.leader_src), + std::move(streams.leader_val), + std::move(streams.leader_self)); + post_blocks(*work.payload, + t, + first_local, + threads, + std::span>(query_inbox)); + if (remote) { + write_send_rows(*queries, *work.payload, peers, threads, t); + } + } + }; + const auto resolve = [&](bool leaders) { + observer.visit(leaders ? W::resolve_leaders : W::resolve_followers, step, t); + auto &work = *frame.work; + auto incoming = collect_blocks(work.engine->window, + first_local, + t, + threads, + std::span>(query_inbox)); + if (remote) { + add_received(incoming, *queries, peers, threads, t); + } + work.answers = work.engine->resolve_published(incoming, leaders); + post_blocks(work.answers, + t, + first_local, + threads, + std::span>(answer_inbox)); + if (remote) { + write_answer_rows(*answers, work.answers, work.engine->src_idx_r, peers, threads, t); + } + }; + const auto consume = [&](bool leaders) { + observer.visit(leaders ? W::consume_leaders : W::consume_followers, step, t); + auto &work = *frame.work; + auto received = collect_blocks(work.engine->window, + first_local, + t, + threads, + std::span>(answer_inbox)); + if (remote) { + add_received(received, *answers, peers, threads, t); + } + work.engine->consume_published(received); + }; + // Q1-Q3 of a pass whose queries reach other processes; see the header comment. + const auto query_round = [&]() -> bool { + if (!remote) { + return true; + } + return phase(failure, + t, + [&] { + if (t == 0) { + observer.visit(W::exchange, step, t); + queries->plan_send(peers, transport); + queries->post_counts(); + } + }) + && phase(failure, + t, + [&] { + observer.visit(W::pack, step, t); + pack_blocks(*queries, *frame.work->payload, peers, threads, t); + // Test-only: an observer that asks may rewrite this owner's slices. + if constexpr (requires { observer.queries_packed(step, t, *queries); }) { + observer.queries_packed(step, t, *queries); + } + }) + && phase(failure, t, [&] { + if (t == 0) { + observer.visit(W::exchange, step, t); + queries->wait_counts(); + queries->plan_recv(); + queries->post(); + queries->wait(); + } + }); + }; + // R1-R3: the answers remote sources wait for, laid out from the rows the resolvers wrote. + const auto answer_round = [&]() -> bool { + if (!remote) { + return true; + } + return phase(failure, + t, + [&] { + if (t == 0) { + observer.visit(W::exchange, step, t); + answers->plan_send(peers, transport); + answers->plan_recv(); + } + }) + && phase(failure, + t, + [&] { + observer.visit(W::pack, step, t); + pack_blocks(*answers, frame.work->answers, peers, threads, t); + }) + && phase(failure, t, [&] { + if (t == 0) { + observer.visit(W::exchange, step, t); + answers->post(); + answers->wait(); + } + }); + }; + + // P1: traversal and leader preparation. + bool ok = false; + if (owner_traversal) { + ok = phase(failure, t, [&] { + traverse(t); + prepare_leaders(); + }); + } + else { + ok = phase(failure, + t, + [&] { + if (t == 0) { + for (size_t s = 0; s < threads; ++s) { + traverse(s); + } + } + }) + && phase(failure, t, prepare_leaders); + } + if (!ok || !query_round()) { + return; + } + // P2: every destination resolves the leader payloads published at P1. + if (cross && !phase(failure, t, [&] { resolve(true); })) { + return; + } + if (!answer_round()) { + return; + } + // P3: leader answers consumed; the follower filter below reads this owner's completed leader marks. + if (passes && !phase(failure, t, [&] { + if (cross) { + consume(true); + } + observer.visit(W::prepare_followers, step, t); + auto &work = *frame.work; + auto &streams = work.scan.streams; + work.payload = &work.engine->prepare_exchange(false, + std::move(streams.follower_queries), + std::move(streams.follower_src), + std::move(streams.follower_val), + std::move(streams.follower_self)); + post_blocks(*work.payload, + t, + first_local, + threads, + std::span>(query_inbox)); + if (remote) { + write_send_rows(*queries, *work.payload, peers, threads, t); + } + })) { + return; + } + if (!query_round()) { + return; + } + // P4: every destination resolves the follower payloads published at P3. + if (cross && !phase(failure, t, [&] { resolve(false); })) { + return; + } + if (!answer_round()) { + return; + } + // P5: follower answers consumed, then the owner-local finish. + if (!phase(failure, t, [&] { + if (cross) { + consume(false); + } + observer.visit(W::finalize, step, t); + auto &work = *frame.work; + if constexpr (Fused) { + (void)work.engine->finish(std::move(work.scan.cos_all), &work.cos); + observer.visit(W::apply, step, t); + // After the inserts, before the insert-target gather of the apply. + own.op.extend_from_current_picture(*frame.coeffs, schrodinger); + const double build_angle = angle_of(idx); + if constexpr (requires(const FusedGateView &view) { observer.fused_records(step, t, view); }) { + observer.fused_records( + step, + t, + FusedGateView{.records = work.fc, + .cos = work.cos, + .coeffs = *frame.coeffs, + .apply_angle = schrodinger ? -build_angle : build_angle, + .schrodinger = schrodinger, + .fused_scale = work.scan.fused_scale}); + } + apply_fused_contract(work.fc, + *frame.coeffs, + work.cos, + schrodinger ? -build_angle : build_angle, + schrodinger, + work.scan.fused_scale, + kSerial, + frame.kernels); + } + else if constexpr (Informed) { + // The cosine set is not persisted on the layer; it drives only this gate's replay. + work.core = work.engine->finish(std::move(work.scan.cos_all), &work.cos); + stamp_layer_metadata(*work.core, gen, own.op); + append(own, work.core, idx); + observer.visit(W::replay, step, t); + // After the inserts, before the new layer's endpoints are read. + own.op.extend_from_current_picture(*frame.coeffs, schrodinger); + const double build_angle = angle_of(idx); + const double apply_angle = schrodinger ? -build_angle : build_angle; + forward_publish(*frame.coeffs, + LayerTraversal(*work.core), + flat, + frame.replay_scratch, + frame.replay_published[0]); + if (multirank) { + // The cosine pass waits for the replay round's post, so the transfer overlaps it. + write_replay_rows(*replay, frame.replay_published[0], t); + } + else { + forward_scale_mask(*frame.coeffs, work.cos, apply_angle); + } + } + else { + auto storage = work.engine->finish(std::move(work.scan.cos_all), nullptr); + stamp_layer_metadata(*storage, gen, own.op); + append(own, std::move(storage), idx); + } + })) { + return; + } + // P6 (informed): finish the new layer's replay from the partners' snapshots published at P5. With other + // processes, the replay round runs first, and the owners' cosine passes run while it is in flight. + if constexpr (Informed) { + if (multirank + && !replay_round.run( + failure, + t, + [&] { + observer.visit(W::pack, step, t); + pack_replay_column(*replay, replay_pairs, 0, t); + }, + [&] { + const double build_angle = angle_of(idx); + forward_scale_mask(*frame.coeffs, + frame.work->cos, + schrodinger ? -build_angle : build_angle); + })) { + return; + } + if (!phase(failure, t, [&] { + observer.visit(W::replay, step, t); + const double build_angle = angle_of(idx); + const EndpointBoard board{.owners = replay_pairs, + .buffer = 0, + .first_local = first_local, + .remote = replay}; + forward_finish(*frame.coeffs, + LayerTraversal(*frame.work->core), + schrodinger ? -build_angle : build_angle, + flat, + frame.replay_scratch, + board); + // Test-only: an observer that asks sees the evolving coefficients; production has no hook. + if constexpr (requires { observer.replayed(t, step, std::as_const(*frame.coeffs)); }) { + observer.replayed(t, step, std::as_const(*frame.coeffs)); + } + })) { + return; + } + } + } + + // The last phase: its decision ends the sequence either way. + static_cast(phase(failure, t, [&] { + observer.visit(W::caches, no_step, t); + frame.work.reset(); + own.op.warm_caches(schrodinger); + if (own.op.slack_copy_bytes() < small_slack_copy_bytes) { + own.op.release_slack(); + } + else { + slack_copies.run([&] { own.op.release_slack(); }); + } + })); + }); + /* + * The peers run this same operation, so a failure here, before mutation (a frame allocation) or after it, leaves + * them waiting in its rounds or in a later collective, and a request still live belongs to a round some peer may + * never complete. On a multi-rank communicator the failure goes to the distributed policy now, before any request + * owner is destroyed (see the header comment). + */ + if (error && multirank) { + mpi::operation_failed(world.comm, error); + } + // The owner's construction rounds keep no staging between operations: no evaluation reads it, so holding it would + // add the largest construction round to every later operation's memory. The next construction maps it again. The + // replay round keeps its staging, which every evaluation reuses. A round with a live request keeps its staging. + if (multirank && ctx.rounds != nullptr) { + ctx.rounds->release_staging(PhysicalRounds::Kind::queries); + ctx.rounds->release_staging(Fused ? PhysicalRounds::Kind::fused_answers : PhysicalRounds::Kind::graph_answers); + } + return ConstructionOutcome{.error = error, .mutation_started = mutation_started}; +} + +} // namespace construction_detail + +/*! + * \brief Append one layer per gate to every shard's graph, building the partner rows the layers need. + * + * The graph-construction gate loop at geometry (1, T), with one team for the whole circuit. Gates run in reverse + * circuit order (Heisenberg) or in circuit order (Schrödinger); layer i carries `parameter_mapping[i]`, + * `gen_coeffs[i]` and `gate_indices[i]`, its generator words and its post-insertion `scaled_count`, and is appended + * with the picture's graph semantics. After the last gate every shard's caches are warmed. An empty circuit returns + * at once, without a team. + * + * \param options The shards' captured budget; its thread count T is the team size and the shard count. + * \param shards The process's T shards, seeded or copied before the call; mutated in place, each by its owner. + * \param ctx Rank-level configuration; see ConstructionContext. + * \param circuit The validated circuit; see GraphCircuit. + * \param observer Test-only seam; see NoConstructionObserver. + * \return The outcome after the team has joined; see ConstructionOutcome. Phase failures are returned, not thrown. + * \throws std::invalid_argument before the team, with nothing mutated, if the shards, geometry or per-gate arrays do + * not fit; as validate_only_rotate_len_k_() for an out-of-range `only_rotate_len_k`; std::bad_alloc if the + * caller-side metadata cannot be allocated. + */ +template +[[nodiscard]] auto build_graph(parallel::Options options, + Shards &shards, + const ConstructionContext &ctx, + const GraphCircuit &circuit, + const Observer &observer = {}) -> ConstructionOutcome { + const size_t gates = circuit.generators.size(); + construction_detail::check_arguments( + "build_graph", + options, + shards, + ctx, + gates, + {circuit.parameter_mapping.size(), circuit.gen_coeffs.size(), circuit.gate_indices.size()}, + circuit.only_rotate_len_k); + if (gates == 0) { + return {}; + } + return construction_detail::run_gates( + options, + shards, + ctx, + circuit.generators, + circuit.only_rotate_len_k, + observer, + [](size_t) { return 0.0; }, + [&](ShardState &state, std::shared_ptr storage, size_t idx) { + state.graph.append(std::move(storage), + circuit.parameter_mapping[idx], + circuit.gen_coeffs[idx], + circuit.gate_indices[idx]); + }); +} + +/*! + * \brief Coefficient-informed graph construction: build_graph() with the lower/upper atol cutoffs applied to the + * evolving coefficients. + * + * Coefficient-informed graph construction at geometry (1, T), in one team for the seed and the whole circuit. + * Each owner's seed is a copy of its current picture; when the shards already hold layers, every shard replays its + * existing graph from that copy at `seed_params` (replay_forward_in_team(), no nested team), as contract_partially() + * would. Each gate then traverses with the coefficients and its build angle `mapped_params[i]`, appends the layer + * with unchanged metadata (as build_graph()), extends the coefficients to the new rows, and replays the new layer over + * its transient cosine set at the picture's apply angle (negated for Schrödinger), so the next gate's cutoff + * decisions see the evolved values. The coefficients are then discarded; after the last gate every shard's caches + * are warmed. An empty circuit returns at once, without a team. + * + * \param options The shards' captured budget; its thread count T is the team size and the shard count. + * \param shards The process's T shards; mutated in place, each by its owner. + * \param ctx Rank-level configuration; its atols apply. + * \param circuit The validated circuit and angles; see InformedCircuit. + * \param observer Test-only seam; see NoConstructionObserver. Seed replay steps report ConstructionWork::seed. + * \return The outcome after the team has joined; see ConstructionOutcome. Phase failures are returned, not thrown. + * \throws As build_graph(), before the team, with nothing mutated; std::invalid_argument if `mapped_params` does not + * have one angle per gate, or if `seed_params` is absent while the shards hold layers or does not have one + * angle per existing layer of every shard. + */ +template +[[nodiscard]] auto build_graph_informed(parallel::Options options, + Shards &shards, + const ConstructionContext &ctx, + const InformedCircuit &circuit, + const Observer &observer = {}) -> ConstructionOutcome { + const auto &graph = circuit.graph; + const size_t gates = graph.generators.size(); + construction_detail::check_arguments("build_graph_informed", + options, + shards, + ctx, + gates, + {graph.parameter_mapping.size(), + graph.gen_coeffs.size(), + graph.gate_indices.size(), + circuit.mapped_params.size()}, + graph.only_rotate_len_k); + if (gates == 0) { + return {}; + } + const bool has_layers = std::ranges::any_of(shards, [](const auto &state) { return state->graph.layers() > 0; }); + if (has_layers != circuit.seed_params.has_value() + || (circuit.seed_params && std::ranges::any_of(shards, [&](const auto &state) { + return state->graph.layers() != circuit.seed_params->size(); + }))) { + throw std::invalid_argument( + "sharded::build_graph_informed: the seed angles must be given exactly when the shards " + "hold layers, one per existing layer of every shard"); + } + const std::span *seed = circuit.seed_params ? &*circuit.seed_params : nullptr; + return construction_detail::run_gates( + options, + shards, + ctx, + graph.generators, + graph.only_rotate_len_k, + observer, + [&](size_t idx) { return circuit.mapped_params[idx]; }, + [&](ShardState &state, std::shared_ptr storage, size_t idx) { + state.graph.append(std::move(storage), + graph.parameter_mapping[idx], + graph.gen_coeffs[idx], + graph.gate_indices[idx]); + }, + seed); +} + +/*! + * \brief Propagate every shard's picture coefficients through the circuit, contracting each gate immediately. + * + * The graph-free gate loop at geometry (1, T), with one team for the whole circuit: Heisenberg evolves the operator + * coefficients in reverse circuit order at angle `mapped_params[i]`; Schrödinger evolves the state coefficients in + * circuit order at the negated angle. Coefficients are captured before the fused cosine sweep overwrites them, the + * two-pass fallback runs where the sweep cannot (a length cap or a zero cosine), and fresh rows are extended from + * the picture before the apply. No graph layer is recorded. After the last gate every shard's caches are warmed. + * An empty circuit returns at once, without a team. + * + * \param options The shards' captured budget; its thread count T is the team size and the shard count. + * \param shards The process's T shards; mutated in place, each by its owner. + * \param ctx Rank-level configuration; see ConstructionContext. + * \param circuit The validated circuit; see PropagationCircuit. + * \param observer Test-only seam; see NoConstructionObserver. + * \return The outcome after the team has joined; see ConstructionOutcome. Phase failures are returned, not thrown. + * \throws As build_graph(), before the team, with nothing mutated. + */ +template +[[nodiscard]] auto propagate(parallel::Options options, + Shards &shards, + const ConstructionContext &ctx, + const PropagationCircuit &circuit, + const Observer &observer = {}) -> ConstructionOutcome { + const size_t gates = circuit.generators.size(); + construction_detail::check_arguments("propagate", + options, + shards, + ctx, + gates, + {circuit.mapped_params.size()}, + circuit.only_rotate_len_k); + if (gates == 0) { + return {}; + } + return construction_detail::run_gates( + options, + shards, + ctx, + circuit.generators, + circuit.only_rotate_len_k, + observer, + [&](size_t idx) { return circuit.mapped_params[idx]; }, + [](ShardState &, std::shared_ptr, size_t) {}); +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/Evaluation.cpp b/cpp/monoprop/detail/sharded/Evaluation.cpp new file mode 100644 index 000000000..7c72aa4b9 --- /dev/null +++ b/cpp/monoprop/detail/sharded/Evaluation.cpp @@ -0,0 +1,807 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/sharded/Evaluation.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/evolution/LayerReplay.h" +#include "monoprop/detail/mpi/OperationFailure.h" + +namespace monoprop::detail::sharded { + +namespace { + +// derive_exchange_layout() for one owner's layer, skipping its slot walk when the layer pairs no row of this owner with +// another owner (always so with one owner): every count and displacement is then zero, as derive_exchange_layout() +// computes them. Each replay step publishes one, so at small sizes the walk is a visible fixed cost per step. +auto publication_layout(const PackedCrossRankStorage &cross_rank, + size_t flat_owner, + int scale, + LayerExchangeLayout &out, + const char *what) -> void { + const auto &occupied = cross_rank.occupied; + if (occupied.empty() || (occupied.size() == 1 && static_cast(occupied.front().slot) == flat_owner)) { + const size_t ranks = cross_rank.rank_count(); + out.counts.assign(ranks, 0); + out.displs.assign(ranks, 0); + out.total_count = 0; + return; + } + derive_exchange_layout(cross_rank, flat_owner, scale, out, what); +} + +// Within-shard kernels run serially: the team's parallelism is across owners. +constexpr parallel::Options kSerial{}; + +auto notify(const EvaluationObserver *observer, EvaluationWork work, size_t step, size_t shard) -> void { + if (observer != nullptr) { + observer->visit(work, step, shard); + } +} + +/* + * The blocks partners published for this owner, `Scale` values per endpoint of its slot for them, read through the + * owner's `staging`. + * + * Every partner's block was written by another core and is read once. Copying a run of consecutive blocks back to back + * keeps many of those transfers in flight, where reading them inside the apply loop exposes each one, and the apply + * then reads the run from cache. The first run is staged before the apply starts; the apply visits the partners in + * their canonical order, so each later request either hits the current run or starts the next one. A block larger + * than a run is not staged: it forms a run of its own, read where the partner published it. The block checks, the + * values and the order the apply reads them in are unchanged. The staging is the owner's, reached through a plain + * reference: a thread-local object here costs a TLS lookup at every use in the copy loop. + */ +template +auto board_reader(const EndpointBoard &board, const LayerTraversal &layer, size_t flat_owner, PartnerStaging &staging) { + staging.position.assign(layer.cross_rank_rank_count(), 0); + staging.blocks.clear(); + size_t total = 0; + layer.for_each_occupied_slot([&](size_t rank, const CrossRankSlotView &slot) { + if (rank != flat_owner) { + staging.position[rank] = staging.blocks.size(); + staging.blocks.push_back({.rank = rank, .count = Scale * slot.sin_send_count}); + total += Scale * slot.sin_send_count; + } + }); + grow_to(staging.values, staging_capacity(staging.run_values, total)); + // Stage one run of consecutive blocks starting at `first`; returns the index after the run. + const auto stage = [&board, &staging, flat_owner](size_t first) { + auto &blocks = staging.blocks; + const size_t room = staging.values.size(); + if (first < blocks.size() && blocks[first].count > room) { + // Larger than a run: a run of its own, read in place. + blocks[first].data = board.block(blocks[first].rank, flat_owner, blocks[first].count); + return first + 1; + } + size_t filled = 0; + size_t next = first; + // The first block fits: a larger one took the branch above. + while (next < blocks.size() && blocks[next].count <= room - filled) { + double *const dst = staging.values.data() + filled; + std::copy_n(board.block(blocks[next].rank, flat_owner, blocks[next].count), blocks[next].count, dst); + blocks[next].data = dst; + filled += blocks[next].count; + ++next; + } + return next; + }; + const size_t staged = stage(0); + return [&staging, stage, run = size_t{0}, next = staged](size_t rank) mutable -> const double * { + const size_t first = staging.position[rank]; + if (first < run) { + // A later run has overwritten this block's staged values. + throw std::logic_error(std::format("sharded replay: partner {} was read out of order", rank)); + } + if (first >= next) { + run = first; + next = stage(first); + } + return staging.blocks[first].data; + }; +} + +// One owner's working frame. Thread-local to the executing worker thread and retained across calls, as the low-level +// evaluator's per-thread eval scratch is (MPFunctions.cpp); every call rebinds it in the owner's first phase and resets +// what it reads. +struct OwnerFrame { + VecD op; // the working operator, evolved in place + VecD state; // gradient only: the dense working state + ForwardScratch forward; // forward self sources + replay::DerivativeSnapshotScratch snap; // reverse pre-cosine snapshots + replay::CosRecords records; // gradient only: selective pre-layer records + PublishedPair published; // read by partners + double accumulated = 0.0; // reverse: the raw accumulation (before ×sec) between begin and finish + bool self_pre = false; // reverse: whether the self pass reads the snapshotted entry op +}; + +auto owner_frame() -> OwnerFrame & { + static thread_local OwnerFrame frame; + return frame; +} + +// Pure argument checks, before the team. +auto same_bits(double a, double b) -> bool { + return std::bit_cast(a) == std::bit_cast(b); +} + +auto check_team(const char *what, parallel::Options options, size_t entries, size_t callbacks) -> void { + if (options.threads < 1 || entries != static_cast(options.threads) + || callbacks != static_cast(options.threads)) { + throw std::invalid_argument(std::format("sharded::{}: expected {} requests and callback sets, got {} and {}", + what, + options.threads, + entries, + callbacks)); + } +} + +// What every shard of one evaluation shares, decided on the caller from the replicated inputs. +struct EvaluationPlan { + bool empty = false; // no parameters: contract the un-evolved operator, no replay + size_t layers = 0; // forward steps L + size_t steps = 0; // L, or 2L with the reverse pass + bool opaque = false; // some callback set is opaque: callback work runs on the primary + VecD mapped; // replay angle per forward step (layer order) + std::vector wanted; // gradient: per layer record flags + // Gradient: reverse step j's factors, built once per call instead of twice per step on every owner. + std::vector reverse; +}; + +auto plan_evaluation(std::span requests, + std::span callbacks, + parallel::Options options, + bool gradient) -> EvaluationPlan { + check_team("evaluate", options, requests.size(), callbacks.size()); + const EvalRequest &first = requests.front(); + EvaluationPlan plan; + plan.empty = first.params.empty(); + for (const EvalRequest &request : requests) { + // The team runs one phase sequence, so everything that shapes it must agree on every shard. + if (!same_bits(request.e_core, first.e_core) || request.params.size() != first.params.size() + || !std::ranges::equal(request.params, first.params, same_bits) + || request.parameter_mapping != first.parameter_mapping + || !std::ranges::equal(request.gen_coeffs, first.gen_coeffs, same_bits)) { + throw std::invalid_argument( + "sharded::evaluate: the replicated identity, parameters, mapping and generator coefficients must " + "agree on every shard"); + } + check_shard_request(request); + } + if (plan.empty) { + return plan; + } + // The low-level evaluator's checks (MPFunctions.cpp), in its order: accumulate (gradient), then scale, then + // indices. + for (const CosCallbacks &cos : callbacks) { + if (gradient && !cos.accumulate) { + throw MissingLayerCallback("ev_and_grad requires a cos_acc (reverse) callback."); + } + } + for (const CosCallbacks &cos : callbacks) { + if (!cos.scale) { + throw MissingLayerCallback("Evaluating at non-empty parameters requires a cos_scale (forward) callback."); + } + } + plan.layers = first.parameter_mapping.size(); + plan.steps = gradient ? 2 * plan.layers : plan.layers; + plan.mapped = map_params(first.params, first.parameter_mapping, first.gen_coeffs, 1.0, true); + if (gradient) { + plan.reverse.reserve(plan.layers); + for (size_t j = 0; j < plan.layers; ++j) { + plan.reverse.emplace_back(first.params[first.parameter_mapping[j]], first.gen_coeffs[j]); + } + } + if (gradient && replay::plan_cos_records(plan.mapped, plan.wanted)) { + for (const CosCallbacks &cos : callbacks) { + if (!cos.indices) { + throw MissingLayerCallback("A layer whose cosine vanishes requires a cos_indices (reverse) callback."); + } + } + } + plan.opaque = std::ranges::any_of(callbacks, [](const CosCallbacks &cos) { return !cos.owner_parallel; }); + return plan; +} + +// The evaluation's shared, caller-owned state for one team. +struct EvaluationRun { + std::span requests; + std::span callbacks; + const EvaluationPlan &plan; + bool gradient; + const EvaluationObserver *observer; + size_t first_local = 0; + ReplayRound remote{}; // the steps' physical round; inert when P = 1 + std::vector frames; // entry t written by owner t in its frame phase + std::vector pairs; // entry t written by owner t in its frame phase + std::vector *contributions; // entry t written by owner t + std::vector *gradients; // entry t written by owner t + + [[nodiscard]] auto board(size_t step) const -> EndpointBoard { + return {.owners = pairs, .buffer = step % 2, .first_local = first_local, .remote = remote.round}; + } + + // Reverse step j = step - L replays layer L - 1 - j at parameter mapping[j]. + [[nodiscard]] auto reverse_layer(size_t step) const -> size_t { return (2 * plan.layers) - 1 - step; } + [[nodiscard]] auto reverse_trig(size_t step) const -> const replay::TrigValues & { + return plan.reverse[step - plan.layers]; + } + + auto frame(size_t t) -> void { + notify(observer, EvaluationWork::frame, no_step, t); + OwnerFrame &f = owner_frame(); + const EvalRequest &request = requests[t]; + if (!plan.empty) { + f.op = request.op; + if (gradient) { + request.state.scatter_into(f.state); + replay::reset_records(f.records, plan.layers); + // An opaque set's callbacks run only on the primary, so its records keep growing on demand. + replay::reserve_records(f.records, + plan.wanted, + request.graph, + plan.opaque ? LayerCosCount{} : callbacks[t].count); + (*gradients)[t].assign(request.params.size(), 0.0); + } + } + f.forward.partners.run_values = observer != nullptr ? observer->staging_run() : staging_run_values; + f.accumulated = 0.0; + f.self_pre = false; + frames[t] = &f; + pairs[t] = &f.published; + } + + // The callback part of a begin: on the owner, or on the primary for an opaque set. + auto callback_part(size_t s, size_t step) -> void { + OwnerFrame &f = *frames[s]; + const EvalRequest &request = requests[s]; + const CosCallbacks &cos = callbacks[s]; + if (step < plan.layers) { + if (gradient) { + notify(observer, EvaluationWork::record, step, s); + replay::record_pre_layer(f.records, plan.wanted, request.graph, cos.indices, step, f.op); + if (step + 1 == plan.layers) { + replay::close_records(f.records, plan.layers); + } + } + notify(observer, EvaluationWork::cosine, step, s); + forward_scale(f.op, step, plan.mapped[step], cos.scale); + return; + } + notify(observer, EvaluationWork::reverse_accumulate, step, s); + const size_t idx = reverse_layer(step); + f.accumulated = replay::accumulate_layer_raw(f.state, + f.op, + idx, + reverse_trig(step), + replay::layer_record_in(f.records, idx), + cos.accumulate); + } + + auto begin(size_t t, size_t step) -> void { + OwnerFrame &f = *frames[t]; + const EvalRequest &request = requests[t]; + const size_t flat = first_local + t; + PublishedEndpoints &out = f.published[step % 2]; + if (step < plan.layers) { + notify(observer, EvaluationWork::publish, step, t); + forward_publish(f.op, request.graph.get_layer_traversal(step), flat, f.forward, out); + } + else { + notify(observer, EvaluationWork::reverse_publish, step, t); + const size_t idx = reverse_layer(step); + const auto layer = request.graph.get_layer_traversal(idx); + const auto record = replay::layer_record_in(f.records, idx); + replay::snapshot_remote_endpoints(f.state, f.op, layer, flat, f.snap); + // The self slot reads its entry op off the post-cos slots, which a record overwrites first. + f.self_pre = record.count > 0 && flat < layer.cross_rank_rank_count(); + if (f.self_pre) { + replay::snapshot_self_recv_op(f.op, layer, f.snap); + } + // Scale 2: each rotation endpoint carries both the state and the op value. + publication_layout(layer.cross_rank(), flat, 2, out.layout, "Layer derivative exchange"); + grow_to(out.values, out.layout.total_count); + replay::pack_derivative_payload(f.snap, layer, flat, [&out](size_t rank) { + return out.values.data() + out.layout.displs[rank]; + }); + } + if (remote.round != nullptr) { + // The callback part runs while the step's round is in flight; see run(). + write_replay_rows(*remote.round, out, t); + } + else if (!plan.opaque) { + callback_part(t, step); + } + } + + auto finish(size_t t, size_t step) -> void { + OwnerFrame &f = *frames[t]; + const EvalRequest &request = requests[t]; + const size_t flat = first_local + t; + const EndpointBoard published = board(step); + if (step < plan.layers) { + notify(observer, EvaluationWork::finish, step, t); + forward_finish(f.op, + request.graph.get_layer_traversal(step), + plan.mapped[step], + flat, + f.forward, + published); + return; + } + notify(observer, EvaluationWork::reverse_finish, step, t); + const size_t idx = reverse_layer(step); + const auto layer = request.graph.get_layer_traversal(idx); + const replay::TrigValues &trig = reverse_trig(step); + replay::EndpointContrib ep; + if (flat < layer.cross_rank_rank_count()) { + ep = replay::apply_self_slot_derivative_paired(f.state, + f.op, + layer, + trig, + f.self_pre ? f.snap.self_recv_op.data() : nullptr); + } + const auto remote = + replay::apply_derivative_payload(f.state, + f.op, + layer, + f.snap, + trig, + flat, + board_reader<2>(published, layer, flat, f.forward.partners)); + ep = replay::combine_endpoint_contrib(ep, remote); + const size_t param_ind = request.parameter_mapping[step - plan.layers]; + (*gradients)[t][param_ind] += replay::layer_derivative_from_raw(trig, f.accumulated, ep); + } + + auto contribution(size_t t) -> void { + notify(observer, EvaluationWork::contribution, no_step, t); + const EvalRequest &request = requests[t]; + if (plan.empty) { + (*contributions)[t] = request.state.dot(request.op); + return; + } + const OwnerFrame &f = *frames[t]; + (*contributions)[t] = gradient ? inner_product(f.state, f.op) : request.state.dot(f.op); + } + + // The whole sequence of one worker: frame, then phase p = finish(p - 1), contribution at p = L, begin(p). + auto run(size_t t, TeamFailure &failure) noexcept -> void { + if (!phase(failure, t, [&] { frame(t); })) { + return; + } + for (size_t p = 0; p <= plan.steps; ++p) { + const bool ok = phase(failure, t, [&] { + if (p > 0) { + finish(t, p - 1); + } + if (p == plan.layers) { + contribution(t); + } + if (p < plan.steps) { + begin(t, p); + } + }); + if (!ok) { + return; + } + // Opaque callbacks: the primary runs every shard's callback part while the other owners wait. + const auto callbacks_of_step = [&] { + if (!plan.opaque) { + callback_part(t, p); + } + else if (t == 0) { + for (size_t s = 0; s < frames.size(); ++s) { + callback_part(s, p); + } + } + }; + if (remote.round != nullptr && p < plan.steps) { + if (!remote.run( + failure, + t, + [&] { pack_replay_column(*remote.round, pairs, p % 2, t); }, + callbacks_of_step)) { + return; + } + } + else if (plan.opaque && p < plan.steps && !phase(failure, t, callbacks_of_step)) { + return; + } + } + } +}; + +// The forward replay of replay_forward_in_team() and replay_shards(): phase p = finish(p - 1), begin(p). +auto forward_steps(TeamFailure &failure, + size_t t, + ForwardReplayJob &job, + std::span pairs, + const EvaluationObserver *observer) noexcept -> bool { + const size_t steps = job.params.size(); + const size_t flat = job.first_local + t; + PhysicalExchange *const round = job.remote.round; + job.owners[t].scratch.partners.run_values = observer != nullptr ? observer->staging_run() : staging_run_values; + const auto cosine = [&](size_t s, size_t step) { + ForwardReplayOwner &owner = job.owners[s]; + notify(observer, EvaluationWork::cosine, step, s); + forward_scale(*owner.coeffs, step, job.params[step], owner.callbacks->scale); + }; + for (size_t p = 0; p <= steps; ++p) { + const bool ok = phase(failure, t, [&] { + ForwardReplayOwner &own = job.owners[t]; + if (p > 0) { + notify(observer, EvaluationWork::finish, p - 1, t); + const EndpointBoard board{.owners = pairs, + .buffer = (p - 1) % 2, + .first_local = job.first_local, + .remote = round}; + forward_finish(*own.coeffs, + own.graph->get_layer_traversal(p - 1), + job.params[p - 1], + flat, + own.scratch, + board); + } + if (p < steps) { + notify(observer, EvaluationWork::publish, p, t); + forward_publish(*own.coeffs, + own.graph->get_layer_traversal(p), + flat, + own.scratch, + own.published[p % 2]); + if (round != nullptr) { + // The cosine pass runs while the step's round is in flight, below. + write_replay_rows(*round, own.published[p % 2], t); + } + else if (job.owner_parallel) { + cosine(t, p); + } + } + }); + if (!ok) { + return false; + } + if (p == steps) { + continue; + } + const auto cosines_of_step = [&] { + if (job.owner_parallel) { + cosine(t, p); + } + else if (t == 0) { + for (size_t s = 0; s < job.owners.size(); ++s) { + cosine(s, p); + } + } + }; + if (round != nullptr) { + if (!job.remote.run(failure, t, [&] { pack_replay_column(*round, pairs, p % 2, t); }, cosines_of_step)) { + return false; + } + } + else if (!job.owner_parallel && !phase(failure, t, cosines_of_step)) { + return false; + } + } + return true; +} + +auto check_replay_job(const ForwardReplayJob &job) -> void { + for (const ForwardReplayOwner &owner : job.owners) { + if (owner.coeffs == nullptr || owner.graph == nullptr || owner.callbacks == nullptr) { + throw std::logic_error("sharded replay: an owner entry was not filled"); + } + if (owner.graph->layers() != job.params.size()) { + throw std::invalid_argument(std::format("sharded replay: a window has {} layers for {} angles", + owner.graph->layers(), + job.params.size())); + } + if (!job.params.empty() && !owner.callbacks->scale) { + throw MissingLayerCallback("Evaluating at non-empty parameters requires a cos_scale (forward) callback."); + } + } +} + +// The replay round of an evaluation or a partial contraction in `world`, or none at P = 1: the owner's own (`rounds`), +// which persists across its calls, or else one for this call in `local`. +auto replay_round_for(const PhysicalWorld &world, size_t threads, PhysicalRounds *rounds, PhysicalRounds &local) + -> PhysicalExchange * { + if (world.ranks <= 1) { + return nullptr; + } + return &(rounds != nullptr ? *rounds : local).get(PhysicalRounds::Kind::replay, world, threads); +} + +auto transport_for(const PhysicalWorld &world) -> ExchangeTransport { + return world.replay_pairwise ? ExchangeTransport::pairwise : ExchangeTransport::collective; +} + +// After a failed join with requests still live, the distributed policy runs before the round is destroyed. +auto hand_off_live_failure(const std::exception_ptr &error, const PhysicalExchange *round, const PhysicalWorld &world) + -> void { + if (error && round != nullptr && round->live() != 0) { + mpi::operation_failed(world.comm, error); + } +} + +} // namespace + +auto check_shard_request(const EvalRequest &request) -> void { + if (request.state.length() > request.op.size()) { + throw EvalStateArgumentError("EvalState::dot: the operator is shorter than the state."); + } + if (request.params.empty()) { + return; + } + if (request.graph.layers() != request.parameter_mapping.size()) { + throw std::invalid_argument(std::format("sharded::evaluate: a graph has {} layers for {} mapped parameters", + request.graph.layers(), + request.parameter_mapping.size())); + } + if (request.gen_coeffs.size() != request.parameter_mapping.size()) { + throw std::invalid_argument("sharded::evaluate: parameter_mapping and gen_coeffs differ in length"); + } + for (const size_t index : request.parameter_mapping) { + if (index >= request.params.size()) { + throw std::invalid_argument( + std::format("sharded::evaluate: parameter index {} is out of range for {} parameters", + index, + request.params.size())); + } + } +} + +auto write_replay_rows(PhysicalExchange &round, const PublishedEndpoints &out, size_t shard) -> void { + round.write_symmetric_rows(shard, out.layout.counts); +} + +auto pack_replay_blocks(PhysicalExchange &round, const PublishedEndpoints &out, size_t shard) -> void { + const size_t threads = round.threads(); + const auto peers = round.peers(); + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t t = 0; t < threads; ++t) { + const auto slice = round.send_block(shard, k, t); + if (slice.empty()) { + continue; + } + const size_t slot = (peers[k] * threads) + t; + const auto *const first = out.values.data() + out.layout.displs[slot]; + std::copy_n(first, slice.size(), slice.begin()); + } + } +} + +auto pack_replay_column(PhysicalExchange &round, + std::span owners, + size_t buffer, + size_t shard) -> void { + const size_t threads = round.threads(); + const auto peers = round.peers(); + for (size_t k = 0; k < peers.size(); ++k) { + const size_t slot = (peers[k] * threads) + shard; + const auto column = round.send_column(shard, k); + for (size_t u = 0; u < threads; ++u) { + const auto slice = column.block(u); + if (slice.empty()) { + continue; + } + const PublishedEndpoints &out = (*owners[u])[buffer]; + std::copy_n(out.values.data() + out.layout.displs[slot], slice.size(), slice.begin()); + } + } +} + +auto forward_publish(const VecD &coeffs, + const LayerTraversal &layer, + size_t flat_owner, + ForwardScratch &scratch, + PublishedEndpoints &out) -> void { + replay::snapshot_self_sources(coeffs, layer, scratch.self_sources); + publication_layout(layer.cross_rank(), flat_owner, 1, out.layout, "Layer exchange"); + grow_to(out.values, out.layout.total_count); + replay::pack_evolution_payload(coeffs, layer, flat_owner, [&out](size_t rank) { + return out.values.data() + out.layout.displs[rank]; + }); +} + +auto forward_scale(VecD &coeffs, size_t layer_idx, double param, const LayerCosScale &scale) -> void { + const double cos_val = std::cos(2 * param); + scale(layer_idx, coeffs.data(), cos_val); +} + +auto forward_scale_mask(VecD &coeffs, const CosMask &cos, double param) -> void { + const double cos_val = std::cos(2 * param); + scale_cos_mask(coeffs.data(), cos, cos_val, kSerial); +} + +auto forward_finish(VecD &coeffs, + const LayerTraversal &layer, + double param, + size_t flat_owner, + ForwardScratch &scratch, + const EndpointBoard &board) -> void { + const double sin_val = std::sin(2 * param); + replay::apply_evolution_payload(coeffs, + layer, + sin_val, + flat_owner, + board_reader<1>(board, layer, flat_owner, scratch.partners)); + replay::apply_self_sources(coeffs, layer, sin_val, scratch.self_sources); +} + +auto replay_forward_in_team(TeamFailure &failure, + size_t shard, + ForwardReplayJob &job, + const EvaluationObserver *observer) noexcept -> bool { + // The pair pointers are derived, not stored: every worker builds the same span over the caller's entries. + std::vector pairs; + bool ok = phase(failure, shard, [&] { + if (shard == 0) { + check_replay_job(job); + } + pairs.reserve(job.owners.size()); + for (const ForwardReplayOwner &owner : job.owners) { + pairs.push_back(&owner.published); + } + }); + return ok && forward_steps(failure, shard, job, pairs, observer); +} + +auto evaluate_shards(std::span requests, + std::span callbacks, + parallel::Options options, + bool gradient, + const EvaluationObserver *observer, + const PhysicalWorld &world, + PhysicalRounds *rounds) -> EvaluationOutcome { + const EvaluationPlan plan = plan_evaluation(requests, callbacks, options, gradient); + const size_t threads = requests.size(); + EvaluationOutcome outcome{.error = {}, .contributions = std::vector(threads, 0.0), .gradients = {}}; + if (gradient) { + outcome.gradients.resize(threads); + } + // Caller-owned, so its requests and staging outlive the team. + PhysicalRounds local; + PhysicalExchange *const round = replay_round_for(world, threads, rounds, local); + EvaluationRun run{.requests = requests, + .callbacks = callbacks, + .plan = plan, + .gradient = gradient, + .observer = observer, + .first_local = world.rank * threads, + .remote = {.round = round, .transport = transport_for(world)}, + .frames = std::vector(threads, nullptr), + .pairs = std::vector(threads, nullptr), + .contributions = &outcome.contributions, + .gradients = &outcome.gradients}; + // Empty parameters leave every gradient empty, as the low-level evaluator returns. + outcome.error = run_team(options, [&run](size_t t, TeamFailure &failure) noexcept { run.run(t, failure); }); + hand_off_live_failure(outcome.error, round, world); + return outcome; +} + +auto combine_contributions(std::span contributions) -> double { + auto total = 0.0; + for (const double contribution : contributions) { + total += contribution; + } + return total; +} + +auto combine_gradients(std::span gradients) -> VecD { + if (gradients.empty()) { + return {}; + } + const size_t length = gradients.front().size(); + for (const VecD &g : gradients) { + if (g.size() != length) { + throw std::invalid_argument("sharded::combine_gradients: the shards' gradients differ in length"); + } + } + VecD total(length, 0.0); + for (size_t k = 0; k < length; ++k) { + auto acc = 0.0; + for (const VecD &g : gradients) { + acc += g[k]; + } + total[k] = acc; + } + return total; +} + +auto ev_sharded(std::span requests, + std::span callbacks, + parallel::Options options, + mpi::Comm comm, + const EvaluationObserver *observer) -> double { + const auto world = PhysicalWorld::of(comm); + auto outcome = evaluate_shards(requests, callbacks, options, /*gradient=*/false, observer, world); + if (outcome.error) { + mpi::operation_failed(comm, outcome.error); + } + const double local = combine_contributions(outcome.contributions); + return requests.front().e_core + mpi::allreduce_sum(local, comm); +} + +auto ev_and_grad_sharded(std::span requests, + std::span callbacks, + parallel::Options options, + mpi::Comm comm, + const EvaluationObserver *observer) -> std::pair { + const auto world = PhysicalWorld::of(comm); + auto outcome = evaluate_shards(requests, callbacks, options, /*gradient=*/true, observer, world); + if (outcome.error) { + mpi::operation_failed(comm, outcome.error); + } + const double local = combine_contributions(outcome.contributions); + const double expectation_value = mpi::allreduce_sum(local, comm); + VecD gradient = combine_gradients(outcome.gradients); + if (!requests.front().params.empty()) { + mpi::allreduce_sum_inplace(gradient, comm); + } + return {requests.front().e_core + expectation_value, std::move(gradient)}; +} + +auto replay_shards(std::span requests, + std::span mapped_params, + std::span callbacks, + parallel::Options options, + const EvaluationObserver *observer, + const PhysicalWorld &world, + PhysicalRounds *rounds) -> ReplayOutcome { + check_team("replay_shards", options, requests.size(), callbacks.size()); + const size_t threads = requests.size(); + PhysicalRounds local; + PhysicalExchange *const round = replay_round_for(world, threads, rounds, local); + ForwardReplayJob job{ + .params = mapped_params, + .first_local = world.rank * threads, + .owner_parallel = std::ranges::all_of(callbacks, [](const CosCallbacks &cos) { return cos.owner_parallel; }), + .owners = std::vector(threads), + .remote = {.round = round, .transport = transport_for(world)}}; + for (size_t t = 0; t < threads; ++t) { + job.owners[t].graph = &requests[t].graph; + job.owners[t].callbacks = &callbacks[t]; + } + // Slot t is filled by owner t in its frame phase, so its storage is first touched there. + ReplayOutcome outcome{.error = {}, .coeffs = std::vector(threads)}; + for (size_t t = 0; t < threads; ++t) { + job.owners[t].coeffs = &outcome.coeffs[t]; + } + check_replay_job(job); + outcome.error = run_team(options, [&](size_t t, TeamFailure &failure) noexcept { + if (!phase(failure, t, [&] { + notify(observer, EvaluationWork::frame, no_step, t); + outcome.coeffs[t] = requests[t].coeffs; // allocated on the owner + })) { + return; + } + static_cast(replay_forward_in_team(failure, t, job, observer)); + }); + hand_off_live_failure(outcome.error, round, world); + return outcome; +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/Evaluation.h b/cpp/monoprop/detail/sharded/Evaluation.h new file mode 100644 index 000000000..4cf92fafc --- /dev/null +++ b/cpp/monoprop/detail/sharded/Evaluation.h @@ -0,0 +1,755 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/MPFunctions.h" +#include "monoprop/MPGraph.h" +#include "monoprop/MPGraphEncoding.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/Validation.h" +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/evolution/CosineRecomputeCallbacks.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/pare/PareGraph.h" +#include "monoprop/detail/sharded/Exchange.h" +#include "monoprop/detail/sharded/State.h" +#include "monoprop/detail/sharded/Team.h" +#include "monoprop/monopropExport.h" + +/* + * Snapshot-safe replay, energy, gradients and retained functionals over the T shards of one process, at geometry + * (P, T). + * + * One run_team() spans a whole evaluation, forward and reverse loops included. Worker t owns shard t, whose flat + * routing slot is rank * T + t; every other shard is a cross owner, in this process or another. A replay step of layer + * l is split at one checkpoint: its begin (snapshot and publish the endpoint values partners read, then the cosine + * pass) and its finish (read the partners' published snapshots, add the sine terms). Finishes read published + * snapshots, never a partner's live coefficients, which that partner's cosine pass has already overwritten. + * + * Other processes (P > 1). Every step then also moves the blocks of partners on other ranks through one physical + * round (Exchange.h, ReplayRound), with the communicator-agreed transport: the begin publishes and writes the owner's + * replay rows (send and receive counts are equal: a layer's partner layout is symmetric); a primary phase lays the + * round out over every other rank; owners pack their remote blocks from the published snapshots; then the primary + * posts, every owner runs its callback part (record, cosine pass or reverse accumulation) while the transfer is in + * flight, and the primary completes it. The finish then reads remote partners' blocks from the round's receive + * staging, which the next step's round rewrites only after that finish. Each step thus has four checkpoints instead + * of one; P = 1 keeps the one-checkpoint sequence. + * + * Phase p (one checkpoint) = finish of step p-1, then, at p = L, the owner's local contribution, then the begin of + * step p. Steps 0 .. L-1 are the forward replay; a gradient continues with the reverse steps L .. 2L-1 (reverse layer + * L-1 first). Step p publishes into buffer p % 2 of its owner's pair: + * + * | Buffer | Writer | Published | Readers | Last read | Earliest reuse | + * | ------------------------------ | ------- | -------------- | ----------------- | ----------- | ------------------ | + * | published[p % 2] (step p) | owner t | end of phase p | partners' finish | phase p + 1 | begin, phase p + 2 | + * | forward self sources | owner t | (private) | owner t (finish) | phase p + 1 | begin, phase p + 1 | + * | derivative snapshots, A | owner t | (private) | owner t (finish) | phase p + 1 | begin, phase p + 1 | + * | partner staging (finish) | owner t | (private) | owner t (finish) | phase p + 1 | next finish | + * | records (gradient) | owner t | (private) | owner t (reverse) | last phase | next call | + * | working op / state | owner t | (private) | owner t | last phase | next call | + * | contributions[t], gradients[t] | owner t | join | caller | after join | caller's | + * + * Working storage is thread-local to the executing worker thread, retained across calls and rebound and reset in the + * owner's first phase of every call: an OpenMP worker index is not a permanent OS-thread identity. Frames, published + * pairs and results stay alive through the join on failure. Hot paths reach the frame through the pointer the frame + * phase stores, never through the thread-local itself. A finish copies its partners' published blocks, in runs of + * consecutive partners, into the owner's partner staging before applying them (PartnerStaging), and a gradient's frame + * phase reserves the records exactly (replay::reserve_records()), so recording never reallocates. + * + * Callbacks: a set marked CosCallbacks::owner_parallel (built by detail::make_cos_callbacks) runs on its owner, in + * parallel with the other owners. If any shard's set is opaque, every callback call (records, cosine pass, reverse + * accumulation) runs on the primary, shard by shard, in an exclusive phase every worker enters after each begin; + * snapshots, publication, finishes and contributions stay owner-parallel. Within-shard kernels are serial. + * + * Failure: arguments are checked before the team, with nothing mutated. Every call that can throw runs inside a + * phase; a failure suppresses every later phase body, the team joins, and the lowest failing worker's original + * exception is returned by evaluate_shards() / replay_shards(), and delivered by ev_sharded() / + * ev_and_grad_sharded() through mpi::operation_failed() before any result reduction. + */ + +namespace monoprop::detail::sharded { + +//! The step reported for work outside a step loop. +inline constexpr size_t no_step = std::numeric_limits::max(); + +//! The owner work an evaluation reports to its observer. +enum class EvaluationWork : std::uint8_t { + frame, //!< Rebind and reset the owner's working frame; copy its operator (and state). + record, //!< Forward step: take the selective pre-layer record. + publish, //!< Forward step: snapshot own sources, publish partner endpoints. + cosine, //!< Forward step: the cosine pass (callback). + finish, //!< Forward step: add partners' published and own snapshotted sine terms. + contribution, //!< The owner's local energy term. + reverse_publish, //!< Reverse step: snapshot endpoints, publish partner pairs. + reverse_accumulate, //!< Reverse step: predivide, cosine accumulation (callback), restore. + reverse_finish, //!< Reverse step: self pairs, partner endpoints, the local parameter contribution. + retain, //!< Retained preparation: snapshot, pare and build callbacks. +}; + +//! The values one run of staged partner blocks holds at least: small enough to stay cache-resident until applied. +inline constexpr size_t staging_run_values = 4096; + +/*! + * \brief Test-only observation seam; production passes none. + * + * `visit(work, step, shard)` runs inside the protected phase, on the thread that performs `work` for `shard` (its + * owner, or the primary for an opaque callback), before that work. `step` is the replay step or no_step. An + * exception it throws is that phase's failure. + */ +class EvaluationObserver { +public: + EvaluationObserver() = default; //!< Stateless base. + EvaluationObserver(const EvaluationObserver &) = default; //!< Copyable. + auto operator=(const EvaluationObserver &) -> EvaluationObserver & = default; //!< Copyable. + EvaluationObserver(EvaluationObserver &&) = default; //!< Movable. + auto operator=(EvaluationObserver &&) -> EvaluationObserver & = default; //!< Movable. + virtual ~EvaluationObserver() = default; //!< Polymorphic base. + + //! Called before `work` of `step` for `shard`; may throw. + virtual auto visit(EvaluationWork work, size_t step, size_t shard) const -> void = 0; + + //! Test-only: the minimum staging run of every owner's finishes, so small layers can cover multi-run staging. + [[nodiscard]] virtual auto staging_run() const noexcept -> size_t { return staging_run_values; } +}; + +// --- Owner-local replay phases ---------------------------------------------------------------------------------- + +/*! + * \brief One owner's endpoint values for one replay step, published to its partners at a checkpoint. + */ +struct PublishedEndpoints { + VecD values; //!< Per partner slot, its block of pre-cosine values (pairs for a reverse step). + LayerExchangeLayout layout; //!< Blocks over flat slots: counts and offsets; the owner's own slot is empty. +}; + +//! The double buffer one owner publishes through: step p uses entry p % 2. +using PublishedPair = std::array; + +/*! + * \brief Read-only view of the publications for one replay step: the local owners' pairs, and the step's physical + * round for partners on other ranks. + */ +struct EndpointBoard { + std::span owners; //!< Per local shard, its publication pair. + size_t buffer = 0; //!< The entry of each pair that holds this step. + size_t first_local = 0; //!< Flat slot of local shard 0, `rank * T`. + const PhysicalExchange *remote = nullptr; //!< The step's completed round, or null when P = 1. + + /*! + * \brief The block `source` published for `reader` at this step. + * + * \param source The partner's flat slot. + * \param reader The reading owner's flat slot; a shard of this process. + * \param count The number of values the reader's layer expects from `source`. + * \throws std::logic_error if `source` is neither a local shard nor a shard of a peer of `remote`, or its block + * does not hold exactly `count` values. + */ + [[nodiscard]] auto block(size_t source, size_t reader, size_t count) const -> const double * { + if (source < first_local || source - first_local >= owners.size()) { + return remote_block_(source, reader, count); + } + const auto &published = (*owners[source - first_local])[buffer]; + if (reader >= published.layout.counts.size() || static_cast(published.layout.counts[reader]) != count) { + throw std::logic_error( + std::format("sharded replay: partner {} did not publish {} values for slot {}", source, count, reader)); + } + return published.values.data() + published.layout.displs[reader]; + } + +private: + [[nodiscard]] auto remote_block_(size_t source, size_t reader, size_t count) const -> const double * { + if (remote == nullptr) { + throw std::logic_error( + std::format("sharded replay: partner slot {} is not a shard of this process", source)); + } + const size_t threads = remote->threads(); + const auto peers = remote->peers(); + const auto found = std::ranges::lower_bound(peers, source / threads); + if (found == peers.end() || *found != source / threads) { + throw std::logic_error(std::format("sharded replay: partner slot {} is on no peer of the round", source)); + } + const auto received = remote->recv_block(reader - first_local, + static_cast(found - peers.begin()), + source % threads); + if (received.size() != count) { + throw std::logic_error(std::format("sharded replay: partner {} sent {} values for slot {}, expected {}", + source, + received.size(), + reader, + count)); + } + return received.data(); + } +}; + +/*! + * \brief Owner `shard`'s rows of a replay round from its publication: what it sends to and receives from every shard + * of every other rank, then its row totals (PhysicalExchange::close_rows()). A layer's partner layout is + * symmetric, so both counts are the published block size. + */ +monoprop_EXPORT auto write_replay_rows(PhysicalExchange &round, const PublishedEndpoints &out, size_t shard) -> void; + +/*! + * \brief evaluate_shards()' argument checks of one shard's request, in its order: the state against the operator, then, + * with parameters, the graph's layer count, the generator coefficients' length and every parameter index. + * + * Mutates nothing. A root that evaluates its sole shard with the low-level serial evaluator runs these first, so a + * bad call fails as evaluate_shards() would, before anything is mutated. + * + * \throws EvalStateArgumentError; std::invalid_argument, with evaluate_shards()' messages. + */ +monoprop_EXPORT auto check_shard_request(const EvalRequest &request) -> void; + +/*! + * \brief Owner `shard` copies its published blocks for every shard of every other rank into the round's send slices. + * + * Requires the round's offsets for this owner's row, which plan_send() lays out; ReplayRound plans owner-parallel + * and packs by column instead (pack_replay_column()). + */ +monoprop_EXPORT auto pack_replay_blocks(PhysicalExchange &round, const PublishedEndpoints &out, size_t shard) -> void; + +/*! + * \brief Owner `shard` copies every local owner's published block for remote shard `shard` of every other rank into + * the round's send slices: column `shard`, whose offsets it placed itself (PhysicalExchange::place_column()). + * + * Reads other owners' publications, which are immutable from their publication checkpoint until consumed. + * + * \param round The step's round, planned with plan_totals(). + * \param owners Per local shard, its publication pair; every entry non-null. + * \param buffer The entry of each pair that holds this step. + * \param shard This owner, the remote shard index of the column it fills. + */ +monoprop_EXPORT auto pack_replay_column(PhysicalExchange &round, + std::span owners, + size_t buffer, + size_t shard) -> void; + +/*! + * \brief A replay step's physical round with the communicator-agreed transport; inert (null) when P = 1. + */ +struct ReplayRound { + PhysicalExchange *round = nullptr; //!< Caller-owned; outlives the team. + ExchangeTransport transport = ExchangeTransport::pairwise; //!< mpi::routes_pairwise() of the communicator. + + /*! + * \brief The round's three phases, after the phase in which every owner wrote its rows (write_replay_rows()). + * + * The round is planned owner-parallel over every other rank: the primary sizes it from the owners' row totals + * while every owner sums its column; every owner then places its column's offsets and runs `pack()`, which fills + * that column (pack_replay_column()); finally the primary posts, every owner runs `overlap()` while the transfer + * is in flight, and the primary completes it. Every worker must call this at the same point of its sequence. + * + * \return The last checkpoint decision, identical on every worker; on false the caller ends its sequence. + */ + template + auto run(TeamFailure &failure, size_t shard, Pack &&pack, Overlap &&overlap) const noexcept -> bool { + return phase(failure, + shard, + [&] { + if (shard == 0) { + round->plan_totals(transport); + } + round->plan_column(shard); + }) + && phase(failure, + shard, + [&] { + round->place_column(shard); + pack(); + }) + && phase(failure, shard, [&] { + if (shard == 0) { + round->post(); + } + overlap(); + if (shard == 0) { + round->wait(); + } + }); + } +}; + +/*! + * \brief One owner's staging of its partners' blocks while it finishes a step. + * + * Private to the owner and reused across steps. `values` holds one run of consecutive partner blocks, sized by + * staging_capacity(). + */ +struct PartnerStaging { + //! One partner of the step. + struct Block { + size_t rank = 0; //!< The partner's flat slot. + size_t count = 0; //!< The values it published for this owner. + const double *data = nullptr; //!< Where the apply reads them: staged, or the partner's block if over a run. + }; + std::vector position; //!< Per flat slot: its index in `blocks`. + std::vector blocks; //!< The step's partners, in the order the apply visits them. + VecD values; //!< The current run. + size_t run_values = staging_run_values; //!< The minimum run; EvaluationObserver::staging_run() in tests. +}; + +/*! + * \brief The size of a step's staging run: the step's values when they fit one run of `run_values`, else one run (and + * never less than one value). + * + * A step whose values fit one run stages all of them in one run either way, so sizing by need changes no copy and no + * read, only how much an owner reserves: a whole default run per owner was most of a small evaluation's first-call + * page faults and, at T = 96, 3 MiB of resident memory. A block larger than the run is read where its partner + * published it rather than staged: widening every owner's staging to the step's largest block held about 1.3 MiB more + * at T = 96 on Hubbard models, and a block that large is a sequential read the apply streams on its own. + */ +[[nodiscard]] constexpr auto staging_capacity(size_t run_values, size_t step_values) -> size_t { + return std::max(std::min(run_values, step_values), size_t{1}); +} + +/*! + * \brief Resize `v` to `n` elements, reallocating only past its capacity and then to at least twice that capacity. + * + * std::vector::resize() grows from the current size: an owner's per-step buffer whose size falls and rises from step to + * step would reallocate by small increments each time it passes its old capacity, thousands of times in a T = 96 + * evaluation, and every reallocation copies the block and leaves the old one free in the owner's malloc arena. Values + * below `n` are kept, as resize() keeps them. + */ +template +auto grow_to(std::vector &v, size_t n) -> void { + if (n > v.capacity()) { + v.reserve(std::max(n, 2 * v.capacity())); + } + v.resize(n); +} + +//! One owner's private forward-step scratch: its self slot's pre-cosine sources and its partner staging. +struct ForwardScratch { + VecD self_sources; //!< The self slot's sin_send values before the cosine pass. + PartnerStaging partners; //!< Partner blocks staged by the finish; the reverse finish reuses it. +}; + +/*! + * \brief Begin of a forward step, before the cosine pass: snapshot the self slot and publish every partner's block. + * + * Owner-local; reads `coeffs`, writes `scratch` and `out` only. + * + * \param coeffs The owner's coefficients, pre-layer. + * \param layer The owner's layer. + * \param flat_owner The owner's flat slot. + * \param scratch The owner's scratch; its self snapshot is overwritten. + * \param out The publication for this step; overwritten, then read by partners until their finish. + */ +monoprop_EXPORT auto forward_publish(const VecD &coeffs, + const LayerTraversal &layer, + size_t flat_owner, + ForwardScratch &scratch, + PublishedEndpoints &out) -> void; + +/*! + * \brief The cosine pass of a forward step: `scale(layer_idx, coeffs, cos(2 * param))`. + */ +monoprop_EXPORT auto forward_scale(VecD &coeffs, size_t layer_idx, double param, const LayerCosScale &scale) -> void; + +/*! + * \brief The cosine pass of a forward step over a transient cosine mask, with serial kernels. + */ +monoprop_EXPORT auto forward_scale_mask(VecD &coeffs, const CosMask &cos, double param) -> void; + +/*! + * \brief Finish of a forward step, after every partner published and after the cosine pass. + * + * Adds `sin(2 * param)` times each partner's published pre-cosine value into this owner's targets, partners in + * ascending slot order, then the self slot's snapshotted sources. + * + * \param coeffs The owner's coefficients, already cos-scaled. + * \param layer The same layer as the begin. + * \param param The replay angle. + * \param flat_owner The owner's flat slot. + * \param scratch The owner's scratch from the begin; its partner staging is overwritten. + * \param board The step's publications. + */ +monoprop_EXPORT auto forward_finish(VecD &coeffs, + const LayerTraversal &layer, + double param, + size_t flat_owner, + ForwardScratch &scratch, + const EndpointBoard &board) -> void; + +/*! + * \brief One owner's entry of a multi-step forward replay run inside an existing team. + */ +struct ForwardReplayOwner { + VecD *coeffs = nullptr; //!< The owner's coefficients, evolved in place. + const MPGraphView *graph = nullptr; //!< The replay window; must outlive the replay. + const CosCallbacks *callbacks = nullptr; //!< Its `scale` is the cosine pass. + ForwardScratch scratch; //!< Private. + PublishedPair published; //!< Read by partners. +}; + +/*! + * \brief A forward replay of every shard of a running team, sized and owned by the caller. + */ +struct ForwardReplayJob { + std::span params; //!< Replay angle of each step, shared by every shard. + size_t first_local = 0; //!< Flat slot of local shard 0. + bool owner_parallel = true; //!< False: every cosine pass runs on the primary. + std::vector owners; //!< One per shard; entry t is filled by owner t. + ReplayRound remote{}; //!< The physical round for partners on other ranks, or inert at P = 1. +}; + +/*! + * \brief Replay `job.params.size()` forward steps of every shard inside the calling team; opens no team. + * + * Every worker of the team must call it with its own shard, after the checkpoint that published every entry of + * `job.owners`. Uses `params.size() + 1` checkpoints (twice as many cosine phases with opaque callbacks), each + * through phase(); its return value is the last checkpoint decision, and on false the caller must end its sequence. + * + * \param failure The team's failure state. + * \param shard The calling worker's shard. + * \param job The replay; the caller keeps it alive through the join. + * \param observer Test-only seam, or null. + * \return Whether every checkpoint passed, identically on every worker. + */ +monoprop_EXPORT auto replay_forward_in_team(TeamFailure &failure, + size_t shard, + ForwardReplayJob &job, + const EvaluationObserver *observer = nullptr) noexcept -> bool; + +// --- Rank-level evaluation -------------------------------------------------------------------------------------- + +/*! + * \brief What evaluate_shards() reports after its team has joined. + */ +struct EvaluationOutcome { + std::exception_ptr error; //!< The lowest failing worker's original exception, or empty. + std::vector contributions; //!< Per shard: its local energy term (identity excluded). + std::vector gradients; //!< Per shard: its local gradient; empty for an energy-only evaluation. +}; + +/*! + * \brief Evaluate T shards in one team and return their local contributions, without combining them. + * + * The S4 seam beneath ev_sharded() and ev_and_grad_sharded(). Request t must view shard t's retained state, + * operator and graph; parameters, mapping, generator coefficients and identity are replicated. Energy-only + * evaluation contracts with EvalState::dot (sparse stays sparse); a gradient scatters the state densely and + * contracts with inner_product(). Empty parameters use EvalState::dot and need no callbacks. + * + * \param requests One request per shard, in shard order. + * \param callbacks One callback set per shard, built for that shard's index and graph. + * \param options The captured budget; its thread count T is the team size. + * \param gradient Whether to run the reverse pass. + * \param observer Test-only seam, or null. + * \param world This process among the ranks the shards' graphs were built over (PhysicalWorld::of() on the + * caller); the default is one process. Every rank of a multi-rank world must call this together. + * \return The outcome; a phase failure is returned, never thrown, except that a failure leaving a physical round's + * requests live goes to mpi::operation_failed() before the round is destroyed. + * \throws std::invalid_argument before the team, with nothing mutated, for mismatched counts or shapes; + * MissingLayerCallback for a missing callback the path needs; EvalStateArgumentError for a state longer + * than its operator. + */ +monoprop_EXPORT auto evaluate_shards(std::span requests, + std::span callbacks, + parallel::Options options, + bool gradient, + const EvaluationObserver *observer = nullptr, + const PhysicalWorld &world = {}, + PhysicalRounds *rounds = nullptr) -> EvaluationOutcome; + +/*! + * \brief The reference fold of per-shard scalars: `0.0 + c[0] + c[1] + ...`, in ascending shard order. + */ +monoprop_EXPORT auto combine_contributions(std::span contributions) -> double; + +/*! + * \brief Per component, the reference fold of per-shard gradients in ascending shard order. + * \throws std::invalid_argument if the shards' gradients differ in length. + */ +monoprop_EXPORT auto combine_gradients(std::span gradients) -> VecD; + +/*! + * \brief Expectation value over T shards: the replicated identity plus the ascending-shard fold of their terms, + * reduced over `comm`. + * + * One team spans the whole replay. Outside the team the communicator is used only on the caller: its rank, size and + * agreed replay transport are read before the team (PhysicalWorld::of()), and the result reduction runs after the + * join. Inside it, only the primary posts and completes the replay rounds. A phase failure goes to + * mpi::operation_failed(comm, error) before the reduction, which at one rank rethrows the original exception. + * + * \param requests One request per shard; see evaluate_shards(). + * \param callbacks One callback set per shard. + * \param options The captured budget; its thread count T is the team size. + * \param comm The ordinary physical communicator of P ranks the shards' graphs were built over. + * \param observer Test-only seam, or null. + * \throws As evaluate_shards() before the team, and std::invalid_argument if `comm` is not an ordinary communicator. + */ +monoprop_EXPORT auto ev_sharded(std::span requests, + std::span callbacks, + parallel::Options options, + mpi::Comm comm, + const EvaluationObserver *observer = nullptr) -> double; + +/*! + * \brief As ev_sharded(), plus the full gradient: per component, the ascending-shard fold, reduced over `comm`. + */ +monoprop_EXPORT auto ev_and_grad_sharded(std::span requests, + std::span callbacks, + parallel::Options options, + mpi::Comm comm, + const EvaluationObserver *observer = nullptr) -> std::pair; + +// --- Partial contraction ---------------------------------------------------------------------------------------- + +/*! + * \brief One shard's input to replay_shards(): the coefficients to evolve and the replay window. + */ +struct ReplayRequest { + const VecD &coeffs; //!< Un-evolved coefficients; copied, never modified. + MPGraphView graph; //!< Replay window over layers the caller keeps alive. +}; + +/*! + * \brief What replay_shards() reports after its team has joined. + */ +struct ReplayOutcome { + std::exception_ptr error; //!< The lowest failing worker's original exception, or empty. + std::vector coeffs; //!< Per shard: its evolved coefficients, allocated by its owner. +}; + +/*! + * \brief Evolve every shard's coefficients through its replay window in one team: the partial-contraction seam. + * + * `mapped_params` gives each replay step's angle. For a Heisenberg partial contraction they are + * `map_params(parameters, mapping, gen_coeffs, 1.0, true)` over `graph.slice_view(k)`; for Schrödinger + * `map_params(parameters, mapping, gen_coeffs, -1.0)` over the reversed `slice_view(k)`, starting from the dense + * state. Shard coefficient blocks concatenated in shard order form the rank-local result. + * + * `world` is this process among the ranks the graphs were built over, as for evaluate_shards(); every rank of a + * multi-rank world must call this together, and a failure leaving the round's requests live goes to + * mpi::operation_failed() before the round is destroyed. + * + * \throws std::invalid_argument before the team for mismatched counts or window lengths, MissingLayerCallback for + * a missing `scale` when there is a step to replay. + */ +monoprop_EXPORT auto replay_shards(std::span requests, + std::span mapped_params, + std::span callbacks, + parallel::Options options, + const EvaluationObserver *observer = nullptr, + const PhysicalWorld &world = {}, + PhysicalRounds *rounds = nullptr) -> ReplayOutcome; + +// --- Retained functionals --------------------------------------------------------------------------------------- + +/*! + * \brief Rank-level inputs of a retained functional, replicated over the shards. + */ +struct RetainedContext { + double core_term = 0.0; //!< The replicated identity coefficient, added once per evaluation. + VecZ parameter_mapping; //!< Optimizer order, as MonomialPropagator::graph_gate_arrays_(). + VecD gen_coeffs; //!< Optimizer order, parallel to parameter_mapping. + std::optional pare_threshold; //!< Pare each shard's graph against this threshold, if set. + Basis basis = Basis::Majorana; //!< Fold encoding of the callbacks. + bool schrodinger = false; //!< Picture: which vector is thresholded, and the state's form. + size_t rank = 0; //!< This process's rank; flat owner of shard t is rank * T + t. +}; + +/*! + * \brief One shard's retained evaluation inputs. + * + * `graph` is either a pared graph owned here or an aliasing handle to the shard's live graph. `callbacks` borrow the + * shard's inverted index and `graph`'s stored masks: they must not outlive the shards, and `graph` keeps the masks + * alive. + */ +struct RetainedShard { + EvalState state; //!< Owned snapshot: sparse Heisenberg scores, or the dense Schrödinger state. + VecD op; //!< Owned snapshot of the un-evolved operator coefficients. + std::shared_ptr graph; //!< The replayed graph. +}; + +/*! + * \brief A retained evaluation over T shards: what a functional captures, with views built only at call time. + */ +struct RetainedEvaluation { + double core_term = 0.0; //!< Replicated identity. + VecZ parameter_mapping; //!< Optimizer order. + VecD gen_coeffs; //!< Optimizer order. + size_t num_params = 0; //!< Required parameter count. + size_t expected_layers = 0; //!< Graph layers at preparation (identical on every shard). + parallel::Options options; //!< The captured budget. + std::vector shards; //!< One per shard. + std::vector callbacks; //!< One per shard, parallel to `shards`. + + /*! + * \brief The functional's own argument checks: parameter count and unchanged graph layer counts. + * \throws As validate_functional_call() and validate_expected_graph_layers(). + */ + auto validate_call(const VecD ¶ms) const -> void { + validate_functional_call(params, num_params); + for (const auto &shard : shards) { + validate_expected_graph_layers(shard.graph->layers(), expected_layers); + } + } + + /*! + * \brief One request per shard viewing this object's snapshots and `params`; valid while both live unchanged. + */ + [[nodiscard]] auto requests(const VecD ¶ms) const -> std::vector { + std::vector out; + out.reserve(shards.size()); + for (const auto &shard : shards) { + out.push_back(EvalRequest{.e_core = core_term, + .state = shard.state, + .op = shard.op, + .parameter_mapping = parameter_mapping, + .gen_coeffs = gen_coeffs, + .graph = shard.graph->replay_view(), + .params = params, + .parallel = parallel::Options{}}); + } + return out; + } +}; + +/*! + * \brief What prepare_retained() reports after its team has joined. + */ +struct RetainedOutcome { + std::exception_ptr error; //!< The lowest failing worker's original exception, or empty. + bool mutation_started = false; //!< Whether any owner began warming caches or paring. + std::optional retained; //!< Set on success. +}; + +/*! + * \brief Snapshot, optionally pare, and build callbacks for every shard, each on its owner, in one team. + * + * Per shard, as MonomialPropagator::make_functional_() does for one store: the state (sparse Heisenberg scores, or + * the dense Schrödinger state) with its length snapshotted, a copy of the operator coefficients, then -- with a + * threshold -- the keep set of the picture's driving vector and pare_graph_owner() at the shard's flat slot, and the + * callbacks, built with serial kernels. Preparation warms the shard's lazy caches, so a failure after the first + * phase started leaves `mutation_started` set. + * + * \param options The shards' captured budget. + * \param shards The process's T shards; their caches are warmed in place. + * \param context Rank-level inputs. + * \param observer Test-only seam, or null. + * \throws std::invalid_argument before the team if the shards do not fit `options` or disagree on layer counts. + */ +template +auto prepare_retained(parallel::Options options, + Shards &shards, + const RetainedContext &context, + const EvaluationObserver *observer = nullptr) -> RetainedOutcome { + if (options.threads < 1 || shards.size() != static_cast(options.threads) + || std::ranges::any_of(shards, [](const auto &state) { return !state; })) { + throw std::invalid_argument(std::format("sharded::prepare_retained: expected {} non-null shards, got {}", + options.threads, + shards.size())); + } + const size_t layers = shards.front()->graph.layers(); + for (const auto &state : shards) { + if (state->graph.layers() != layers) { + throw std::invalid_argument("sharded::prepare_retained: the shards' graphs disagree on the layer count"); + } + } + if (context.parameter_mapping.size() != layers || context.gen_coeffs.size() != layers) { + throw std::invalid_argument("sharded::prepare_retained: the gate arrays do not match the graph layers"); + } + const size_t threads = shards.size(); + // Caller-owned slots: entry t is written by owner t only, and read after the join. + std::vector> slots(threads); + std::vector callbacks(threads); + const parallel::Options serial{}; // within-shard kernels are serial + bool mutation_started = false; + const auto error = run_team(options, [&](size_t t, TeamFailure &failure) noexcept { + if (t == 0) { + mutation_started = true; + } + static_cast(phase(failure, t, [&] { + if (observer != nullptr) { + observer->visit(EvaluationWork::retain, no_step, t); + } + auto &op = shards[t]->op; + const auto num_terms = op.size(); + auto state = [&] { + if (context.schrodinger) { + return EvalState::dense(op.dense_state()); + } + const auto sparse = op.sparse_state(); + return EvalState::sparse(num_terms, sparse.rows, sparse.values); + }(); + VecD coeffs = op.get_operator(); + const auto &inverted_index = op.inverted_index(); + std::shared_ptr graph; + const MPGraph &live = shards[t]->graph; + if (context.pare_threshold.has_value()) { + const auto full_cos_of_layer = [&](size_t i, CosMask &out) { + full_cos_mask_into(inverted_index, live.get_layer_traversal(i), context.basis, out); + }; + const auto keep = context.schrodinger ? indices_above(coeffs, *context.pare_threshold) + : state.indices_above(*context.pare_threshold); + const auto count = context.schrodinger ? coeffs.size() : state.length(); + graph = std::make_shared(pare_graph_owner(live, + keep, + count, + context.schrodinger, + (context.rank * threads) + t, + full_cos_of_layer)); + } + else { + graph = std::shared_ptr(std::shared_ptr{}, &live); + } + callbacks[t] = make_cos_callbacks(inverted_index, graph->replay_view(), context.basis, serial); + slots[t].emplace( + RetainedShard{.state = std::move(state), .op = std::move(coeffs), .graph = std::move(graph)}); + })); + }); + RetainedOutcome outcome{.error = error, .mutation_started = mutation_started, .retained = std::nullopt}; + if (error) { + return outcome; + } + // Assembled on the caller after the join; a failure here still follows the post-mutation outcome contract. + try { + RetainedEvaluation retained{.core_term = context.core_term, + .parameter_mapping = context.parameter_mapping, + .gen_coeffs = context.gen_coeffs, + .num_params = expected_num_params(context.parameter_mapping), + .expected_layers = layers, + .options = options, + .shards = {}, + .callbacks = std::move(callbacks)}; + retained.shards.reserve(threads); + for (auto &slot : slots) { + retained.shards.push_back(std::move(*slot)); + } + outcome.retained.emplace(std::move(retained)); + } + catch (...) { + outcome.error = std::current_exception(); + } + return outcome; +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/Exchange.cpp b/cpp/monoprop/detail/sharded/Exchange.cpp new file mode 100644 index 000000000..5073e44f3 --- /dev/null +++ b/cpp/monoprop/detail/sharded/Exchange.cpp @@ -0,0 +1,978 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "monoprop/detail/sharded/Exchange.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/MPICompat.h" + +namespace monoprop::detail::sharded { + +namespace { + +auto element_bytes(ExchangeElement element) -> size_t { + switch (element) { + case ExchangeElement::u32: + return 4; + case ExchangeElement::u64: + case ExchangeElement::f64: + return 8; + } + return 8; +} + +#ifdef monoprop_ENABLE_MPI +auto element_datatype(ExchangeElement element) -> MPI_Datatype { + switch (element) { + case ExchangeElement::u32: + return MPI_UINT32_T; + case ExchangeElement::u64: + return MPI_UINT64_T; + case ExchangeElement::f64: + return MPI_DOUBLE; + } + return MPI_DOUBLE; +} +#endif + +/* + * Staging memory straight from the kernel. A round's staging is large and grows with the operator; returned to + * malloc, every outgrown block would raise glibc's dynamic mmap threshold, after which the owners' per-gate transients + * stay resident in fragmented thread arenas instead of being unmapped on free. Mapping it directly leaves malloc's + * heuristics alone. Whole pages, zero-filled by the kernel. + */ +template +struct MappedAllocator { + using value_type = T; + + MappedAllocator() noexcept = default; + template + MappedAllocator(const MappedAllocator & /*other*/) noexcept {} + + [[nodiscard]] static auto bytes_for(size_t n) -> size_t { + static const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + if (n > (std::numeric_limits::max() - page) / sizeof(T)) { + throw std::bad_alloc(); + } + return ((n * sizeof(T)) + page - 1) / page * page; + } + + [[nodiscard]] auto allocate(size_t n) -> T * { + if (n == 0) { + return nullptr; + } + void *p = ::mmap(nullptr, bytes_for(n), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + throw std::bad_alloc(); + } + return static_cast(p); + } + + auto deallocate(T *p, size_t n) noexcept -> void { + if (p != nullptr) { + ::munmap(p, bytes_for(n)); + } + } + + template + auto operator==(const MappedAllocator & /*other*/) const noexcept -> bool { + return true; + } +}; + +// AddressSanitizer neither poisons nor tracks memory mapped outside its allocator (no redzones, and its pointer-pair +// checks reject end - begin of such a block), so a sanitized build stages through malloc, where ASan checks every +// access; the mapped staging is a release-build memory-behaviour choice, not a correctness one. +#if defined(__SANITIZE_ADDRESS__) +constexpr bool sanitized_addresses = true; +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +constexpr bool sanitized_addresses = true; +#else +constexpr bool sanitized_addresses = false; +#endif +#else +constexpr bool sanitized_addresses = false; +#endif +using Staging = + std::vector, MappedAllocator>>; + +// A wide running total narrowed to an MPI int, or a length_error naming what overflowed. +auto checked_int(long long value, const char *what) -> int { + if (value < 0 || value > std::numeric_limits::max()) { + throw std::length_error(std::format("sharded exchange: {} {} does not fit an MPI int", what, value)); + } + return static_cast(value); +} + +#ifdef monoprop_ENABLE_MPI +// Write `counts` (non-negative) to `wire` in the narrowest of 1, 2 or 4 unsigned bytes per count that holds the +// largest, little-endian as the host stores it; return that width. +auto encode_counts(std::span counts, std::byte *wire) -> size_t { + int largest = 0; + for (const int count : counts) { + largest = std::max(largest, count); + } + const auto store = [&](U /*tag*/) { + for (size_t i = 0; i < counts.size(); ++i) { + const auto value = static_cast(counts[i]); + std::memcpy(wire + (i * sizeof(U)), &value, sizeof(U)); + } + return sizeof(U); + }; + if (largest <= std::numeric_limits::max()) { + return store(std::uint8_t{}); + } + if (largest <= std::numeric_limits::max()) { + return store(std::uint16_t{}); + } + return store(std::uint32_t{}); +} + +// Count `i` of a block written by encode_counts() with `width` bytes per count. +auto decode_count(const std::byte *wire, size_t i, size_t width) -> int { + if (width == 1) { + return static_cast(std::to_integer(wire[i])); + } + if (width == 2) { + std::uint16_t value = 0; + std::memcpy(&value, wire + (i * 2), 2); + return static_cast(value); + } + std::uint32_t value = 0; + std::memcpy(&value, wire + (i * 4), 4); + return static_cast(value); +} + +#endif + +// MPI ownership: the primary of the team, which is the thread that entered it and initialized MPI. +auto require_primary(const char *what) -> void { + if (omp_get_thread_num() != 0) { + throw std::logic_error(std::format("sharded exchange: {} called on OpenMP worker {}, not the primary", + what, + omp_get_thread_num())); + } +#ifdef monoprop_ENABLE_MPI + int initialized = 0; + MPI_Initialized(&initialized); + if (initialized != 0) { + int is_main = 0; + MPI_Is_thread_main(&is_main); + if (is_main == 0) { + throw std::logic_error( + std::format("sharded exchange: {} called on a thread other than the one that initialized MPI", what)); + } + } +#endif +} + +} // namespace + +struct PhysicalExchange::State { + mpi::Comm comm; + size_t rank = 0; + size_t ranks = 1; + size_t threads = 1; + ExchangeElement element = ExchangeElement::f64; + size_t elem = 8; + int tag = 0; + size_t max_peers = 0; // ranks - 1 + size_t stride = 0; // max_peers * threads: one row's length + + // Rows: entry [shard * stride + peer * threads + other] is the count between local `shard` and shard `other` of + // peer `peer`. Row `shard` has one writer, its owner, except where the primary fills receive rows. + std::vector send_rows; + std::vector recv_rows; + // Element offsets into the staging; derived by the primary's planning, or owner-parallel. recv_off has the rows' + // shape. The send side is stored by column, entry [dest * stride + peer * threads + source], with a copy of the + // column's counts in send_col_rows: owner-parallel planning has owner t place and pack column t, so its entries are + // contiguous and private to it. Stored by row, every line of a column was shared by eight owners, each writing + // its own entry (place_column()), and each packing read missed on lines the others had just written (about a + // quarter of a small 2 x 48 evaluation's call time). + std::vector send_off; + std::vector send_col_rows; + std::vector recv_off; + // Owner-parallel planning: per-owner row totals [shard * max_peers + peer] (one writer each, close_rows()), and + // per-column send sums [peer * threads + column] (one writer per column, plan_column()). + std::vector send_totals; + std::vector recv_totals; + std::vector column_sums; + std::vector column_start; // lay_out() scratch: one entry per remote shard + + std::vector peers; + ExchangeTransport transport = ExchangeTransport::pairwise; + bool send_planned = false; + bool recv_planned = false; + // Per communicator rank, zero outside the peers: what MPI_Ialltoallv reads, and the pairwise legs. + std::vector mpi_send_counts; + std::vector mpi_send_displs; + std::vector mpi_recv_counts; + std::vector mpi_recv_displs; + // High-water-mark staging, never resized while a request may point into it. + Staging send_stage; + Staging recv_stage; + // Count round: [peer][dest shard][source shard] counts. count_send holds them as ints; on the wire each peer's + // block travels in the narrowest unsigned width (1, 2 or 4 bytes) that holds its largest count, so the small + // blocks of a light round stay below the transport's eager limit instead of paying a rendezvous; the receiver + // reads the width from the received size. + std::vector count_send; + std::vector count_wire_send; + std::vector count_wire_recv; +#ifdef monoprop_ENABLE_MPI + std::vector requests; + std::vector count_requests; + std::vector count_statuses; +#endif + int posted = 0; + int count_posted = 0; + + auto drain() noexcept -> void { +#ifdef monoprop_ENABLE_MPI + if (count_posted != 0) { + MPI_Waitall(count_posted, count_requests.data(), MPI_STATUSES_IGNORE); + count_posted = 0; + } + if (posted != 0) { + MPI_Waitall(posted, requests.data(), MPI_STATUSES_IGNORE); + posted = 0; + } +#endif + } + + [[nodiscard]] auto at(size_t shard, size_t peer, size_t other) const -> size_t { + if (shard >= threads || peer >= max_peers || other >= threads) { + throw std::out_of_range(std::format("sharded exchange: (shard {}, peer {}, shard {}) is outside a table of " + "{} shards and {} peers", + shard, + peer, + other, + threads, + max_peers)); + } + return (shard * stride) + (peer * threads) + other; + } + + auto require_idle(const char *what) const -> void { + if (posted != 0 || count_posted != 0) { + throw std::logic_error(std::format("sharded exchange: {} while a posted round is still in flight", what)); + } + } + + auto set_peers(std::span next, ExchangeTransport how) -> void { + if (next.size() > max_peers) { + throw std::invalid_argument( + std::format("sharded exchange: {} peers in a world of {} ranks", next.size(), ranks)); + } + for (size_t k = 0; k < next.size(); ++k) { + if (next[k] >= ranks || next[k] == rank || (k > 0 && next[k] <= next[k - 1])) { + throw std::invalid_argument( + std::format("sharded exchange: peer {} is not another rank in strictly ascending order (rank {} " + "of {})", + next[k], + rank, + ranks)); + } + } + peers.assign(next.begin(), next.end()); + transport = how; + } + + // Offsets of one side from its rows: the message of peer k is destination-shard major, source-shard minor. On + // the send side the row owner is the source (minor), on the receive side the destination (major). Both passes read + // rows contiguously through local pointers (the planning runs serially on the primary, once per round side); the + // send side writes its offsets and counts by column (see send_off). Only each peer's end is checked, which bounds + // every partial sum below it. + auto lay_out(const std::vector &rows, + std::vector &off, + std::vector &mpi_counts, + std::vector &mpi_displs, + bool send_side, + const char *what) -> size_t { + std::ranges::fill(mpi_counts, 0); + std::ranges::fill(mpi_displs, 0); + const size_t n = threads; + const size_t row_stride = stride; + const int *const in = rows.data(); + long long *const out = off.data(); + long long *const start = column_start.data(); + // The rows were just written by T different owners and the offsets were last read by them: request every line + // of the round's segments at once, so those transfers overlap instead of arriving one row at a time. + constexpr size_t line_counts = 64 / sizeof(int); + constexpr size_t line_offsets = 64 / sizeof(long long); + const size_t used = peers.size() * n; + for (size_t row = 0; row < n; ++row) { + for (size_t i = 0; i < used; i += line_counts) { + __builtin_prefetch(in + (row * row_stride) + i, 0, 3); + } + for (size_t i = 0; i < used; i += line_offsets) { + __builtin_prefetch(out + (row * row_stride) + i, 1, 3); + } + } + long long running = 0; + for (size_t k = 0; k < peers.size(); ++k) { + const long long base = running; + const size_t segment = k * n; + if (send_side) { + // Column t (remote destination t) starts after the earlier columns; local sources ascend within it. + std::fill_n(start, n, 0LL); + for (size_t u = 0; u < n; ++u) { + const int *row = in + (u * row_stride) + segment; + for (size_t t = 0; t < n; ++t) { + start[t] += row[t]; + } + } + for (size_t t = 0; t < n; ++t) { + const long long width = start[t]; + start[t] = running; + running += width; + } + int *const col_counts = send_col_rows.data(); + for (size_t u = 0; u < n; ++u) { + const int *row = in + (u * row_stride) + segment; + for (size_t t = 0; t < n; ++t) { + out[(t * row_stride) + segment + u] = start[t]; + col_counts[(t * row_stride) + segment + u] = row[t]; + start[t] += row[t]; + } + } + } + else { + // Row t (local destination t) holds remote sources in ascending order: already contiguous. + for (size_t t = 0; t < n; ++t) { + const int *row = in + (t * row_stride) + segment; + long long *dst = out + (t * row_stride) + segment; + for (size_t su = 0; su < n; ++su) { + dst[su] = running; + running += row[su]; + } + } + } + const size_t r = peers[k]; + mpi_displs[r] = checked_int(base, what); + mpi_counts[r] = checked_int(running - base, what); + } + return static_cast(checked_int(running, what)); + } + + // Owner-parallel planning's primary part, once the peers are set: per-peer MPI counts and displacements from the + // owners' row totals, and both staging sizes. + auto plan_from_totals() -> void; + + auto size_stage(Staging &stage, size_t elements) const -> void { + // At least one element, so MPI never sees a null buffer. + const size_t need = std::max(elements, 1) * elem; + if (stage.capacity() < need) { + // A round is sized while idle, before anything is packed, so the staging holds nothing yet: the old block + // is unmapped before the larger one is mapped, rather than copied into it with both resident. Capacity + // still grows geometrically, as resize() would; only the `need` bytes written become resident. + const size_t capacity = std::max(need, 2 * stage.size()); + Staging{}.swap(stage); + stage.reserve(capacity); + } + if (stage.size() < need) { + stage.resize(need); + } + } +}; + +auto PhysicalWorld::of(const mpi::Comm &comm) -> PhysicalWorld { + PhysicalWorld world; + world.comm = comm; + world.rank = static_cast(mpi::rank(comm)); + world.ranks = static_cast(mpi::size(comm)); + world.replay_pairwise = mpi::routes_pairwise(comm); + return world; +} + +PhysicalExchange::PhysicalExchange() noexcept = default; + +PhysicalExchange::PhysicalExchange(const PhysicalWorld &world, size_t threads, ExchangeElement element, int tag) + : state_(std::make_unique()) { + if (threads == 0 || world.ranks == 0 || world.rank >= world.ranks) { + throw std::invalid_argument( + std::format("sharded exchange: {} threads at rank {} of {}", threads, world.rank, world.ranks)); + } + State &s = *state_; + s.comm = world.comm; + s.rank = world.rank; + s.ranks = world.ranks; + s.threads = threads; + s.element = element; + s.elem = element_bytes(element); + s.tag = tag; + s.max_peers = world.ranks - 1; + s.stride = s.max_peers * threads; + const size_t table = threads * s.stride; + s.send_rows.assign(table, 0); + s.recv_rows.assign(table, 0); + s.send_off.assign(table, 0); + s.send_col_rows.assign(table, 0); + s.recv_off.assign(table, 0); + s.send_totals.assign(threads * s.max_peers, 0); + s.recv_totals.assign(threads * s.max_peers, 0); + s.column_sums.assign(s.max_peers * threads, 0); + s.column_start.assign(threads, 0); + s.mpi_send_counts.assign(world.ranks, 0); + s.mpi_send_displs.assign(world.ranks, 0); + s.mpi_recv_counts.assign(world.ranks, 0); + s.mpi_recv_displs.assign(world.ranks, 0); + s.count_send.assign(s.max_peers * threads * threads, 0); + s.count_wire_send.assign(s.max_peers * threads * threads * sizeof(std::uint32_t), std::byte{0}); + s.count_wire_recv.assign(s.max_peers * threads * threads * sizeof(std::uint32_t), std::byte{0}); +#ifdef monoprop_ENABLE_MPI + // Sized once: MPI holds pointers into these while requests are live. + s.requests.assign(2 * std::max(s.max_peers, 1), MPI_REQUEST_NULL); + s.count_requests.assign(2 * std::max(s.max_peers, 1), MPI_REQUEST_NULL); + s.count_statuses.resize(s.count_requests.size()); +#endif +} + +PhysicalExchange::PhysicalExchange(PhysicalExchange &&other) noexcept = default; + +auto PhysicalExchange::operator=(PhysicalExchange &&other) noexcept -> PhysicalExchange & { + if (this != &other) { + if (state_) { + state_->drain(); // never drop a request this exchange already owns + } + state_ = std::move(other.state_); + } + return *this; +} + +PhysicalExchange::~PhysicalExchange() { + if (state_) { + state_->drain(); + } +} + +namespace { + +auto require_state(const void *state) -> void { + if (state == nullptr) { + throw std::logic_error("sharded exchange: an inert (default-constructed or moved-from) exchange was used"); + } +} + +} // namespace + +auto PhysicalExchange::threads() const noexcept -> size_t { + return state_ ? state_->threads : 0; +} + +auto PhysicalExchange::rank() const noexcept -> size_t { + return state_ ? state_->rank : 0; +} + +auto PhysicalExchange::ranks() const noexcept -> size_t { + return state_ ? state_->ranks : 1; +} + +auto PhysicalExchange::peers() const noexcept -> std::span { + return state_ ? std::span(state_->peers) : std::span{}; +} + +auto PhysicalExchange::live() const noexcept -> int { + return state_ ? state_->posted + state_->count_posted : 0; +} + +auto PhysicalExchange::staging_bytes() const noexcept -> size_t { + return state_ ? state_->send_stage.size() + state_->recv_stage.size() : 0; +} + +auto PhysicalExchange::release_staging() noexcept -> void { + if (!state_ || live() != 0) { + return; + } + State &s = *state_; + Staging{}.swap(s.send_stage); + Staging{}.swap(s.recv_stage); + // Staging is sized by the plans: the next round must plan again before it packs. + s.send_planned = false; + s.recv_planned = false; +} + +auto PhysicalExchange::reset_rows(size_t shard) -> void { + require_state(state_.get()); + State &s = *state_; + if (shard >= s.threads) { + throw std::out_of_range(std::format("sharded exchange: shard {} of {}", shard, s.threads)); + } + std::fill_n(s.send_rows.begin() + static_cast(shard * s.stride), s.stride, 0); + std::fill_n(s.recv_rows.begin() + static_cast(shard * s.stride), s.stride, 0); +} + +auto PhysicalExchange::set_send_count(size_t shard, size_t peer, size_t dest, size_t count) -> void { + require_state(state_.get()); + const size_t i = state_->at(shard, peer, dest); + state_->send_rows[i] = checked_int(static_cast(std::min(count, size_t{1} << 62)), "send count"); +} + +auto PhysicalExchange::set_recv_count(size_t shard, size_t peer, size_t source, size_t count) -> void { + require_state(state_.get()); + const size_t i = state_->at(shard, peer, source); + state_->recv_rows[i] = + checked_int(static_cast(std::min(count, size_t{1} << 62)), "receive count"); +} + +auto PhysicalExchange::send_count(size_t shard, size_t peer, size_t dest) const -> size_t { + require_state(state_.get()); + return static_cast(state_->send_rows[state_->at(shard, peer, dest)]); +} + +auto PhysicalExchange::recv_count(size_t shard, size_t peer, size_t source) const -> size_t { + require_state(state_.get()); + return static_cast(state_->recv_rows[state_->at(shard, peer, source)]); +} + +auto PhysicalExchange::plan_send(std::span peers, ExchangeTransport transport) -> void { + require_state(state_.get()); + State &s = *state_; + s.require_idle("plan_send"); + s.send_planned = false; + s.recv_planned = false; + s.set_peers(peers, transport); + const size_t total = + s.lay_out(s.send_rows, s.send_off, s.mpi_send_counts, s.mpi_send_displs, /*send_side=*/true, "send total"); + s.size_stage(s.send_stage, total); + s.send_planned = true; +} + +auto PhysicalExchange::close_rows(size_t shard) -> void { + require_state(state_.get()); + State &s = *state_; + if (shard >= s.threads) { + throw std::out_of_range(std::format("sharded exchange: shard {} of {}", shard, s.threads)); + } + for (size_t k = 0; k < s.max_peers; ++k) { + long long sent = 0; + long long received = 0; + for (size_t t = 0; t < s.threads; ++t) { + const size_t i = (shard * s.stride) + (k * s.threads) + t; + sent += s.send_rows[i]; + received += s.recv_rows[i]; + } + s.send_totals[(shard * s.max_peers) + k] = sent; + s.recv_totals[(shard * s.max_peers) + k] = received; + } +} + +auto PhysicalExchange::write_symmetric_rows(size_t shard, std::span slot_counts) -> void { + require_state(state_.get()); + State &s = *state_; + if (shard >= s.threads) { + throw std::out_of_range(std::format("sharded exchange: shard {} of {}", shard, s.threads)); + } + int *const send = s.send_rows.data() + (shard * s.stride); + int *const recv = s.recv_rows.data() + (shard * s.stride); + size_t k = 0; + for (size_t b = 0; b < s.ranks; ++b) { + if (b == s.rank) { + continue; + } + long long total = 0; + for (size_t u = 0; u < s.threads; ++u) { + const size_t slot = (b * s.threads) + u; + const int count = slot < slot_counts.size() ? slot_counts[slot] : 0; + if (count < 0) { + throw std::length_error(std::format("sharded exchange: negative count {} for slot {}", count, slot)); + } + send[(k * s.threads) + u] = count; + recv[(k * s.threads) + u] = count; + total += count; + } + s.send_totals[(shard * s.max_peers) + k] = total; + s.recv_totals[(shard * s.max_peers) + k] = total; + ++k; + } +} + +auto PhysicalExchange::plan_totals(std::span peers, ExchangeTransport transport) -> void { + require_state(state_.get()); + State &s = *state_; + s.require_idle("plan_totals"); + s.send_planned = false; + s.recv_planned = false; + s.set_peers(peers, transport); + s.plan_from_totals(); +} + +auto PhysicalExchange::plan_totals(ExchangeTransport transport) -> void { + require_state(state_.get()); + State &s = *state_; + s.require_idle("plan_totals"); + s.send_planned = false; + s.recv_planned = false; + s.peers.clear(); // keeps its capacity: no allocation after the first round + for (size_t b = 0; b < s.ranks; ++b) { + if (b != s.rank) { + s.peers.push_back(b); + } + } + s.transport = transport; + s.plan_from_totals(); +} + +auto PhysicalExchange::State::plan_from_totals() -> void { + State &s = *this; + // As lay_out(): peer k's message occupies [base_k, base_k + total_k), the bases a running sum over the peers. + const auto place = [&](const std::vector &totals, + std::vector &mpi_counts, + std::vector &mpi_displs, + const char *what) -> size_t { + std::ranges::fill(mpi_counts, 0); + std::ranges::fill(mpi_displs, 0); + long long running = 0; + for (size_t k = 0; k < s.peers.size(); ++k) { + long long total = 0; + for (size_t shard = 0; shard < s.threads; ++shard) { + total += totals[(shard * s.max_peers) + k]; + } + const size_t r = s.peers[k]; + mpi_displs[r] = checked_int(running, what); + mpi_counts[r] = checked_int(total, what); + running += total; + } + return static_cast(checked_int(running, what)); + }; + const size_t sent = place(s.send_totals, s.mpi_send_counts, s.mpi_send_displs, "send total"); + const size_t received = place(s.recv_totals, s.mpi_recv_counts, s.mpi_recv_displs, "receive total"); + s.size_stage(s.send_stage, sent); + s.size_stage(s.recv_stage, received); + s.send_planned = true; + s.recv_planned = true; +} + +auto PhysicalExchange::plan_column(size_t shard) -> void { + require_state(state_.get()); + State &s = *state_; + if (shard >= s.threads) { + throw std::out_of_range(std::format("sharded exchange: shard {} of {}", shard, s.threads)); + } + for (size_t k = 0; k < s.max_peers; ++k) { + long long sum = 0; + for (size_t u = 0; u < s.threads; ++u) { + sum += s.send_rows[(u * s.stride) + (k * s.threads) + shard]; + } + s.column_sums[(k * s.threads) + shard] = sum; + } +} + +auto PhysicalExchange::place_column(size_t shard) -> void { + require_state(state_.get()); + State &s = *state_; + if (!s.send_planned || !s.recv_planned) { + throw std::logic_error("sharded exchange: place_column() before plan_totals()"); + } + if (shard >= s.threads) { + throw std::out_of_range(std::format("sharded exchange: shard {} of {}", shard, s.threads)); + } + // Every partial sum is bounded by its peer's checked total. + for (size_t k = 0; k < s.peers.size(); ++k) { + const size_t r = s.peers[k]; + // Send: destination-shard major, so column `shard` starts after the earlier columns; sources ascend within it. + long long running = s.mpi_send_displs[r]; + for (size_t t = 0; t < shard; ++t) { + running += s.column_sums[(k * s.threads) + t]; + } + // Column `shard` of the row tables, read once; written to the owner's own contiguous column entries. + const size_t column = (shard * s.stride) + (k * s.threads); + for (size_t u = 0; u < s.threads; ++u) { + const int count = s.send_rows[(u * s.stride) + (k * s.threads) + shard]; + s.send_off[column + u] = running; + s.send_col_rows[column + u] = count; + running += count; + } + // Receive: local-destination major, so row `shard` starts after the earlier rows; sources ascend within it. + running = s.mpi_recv_displs[r]; + for (size_t t = 0; t < shard; ++t) { + running += s.recv_totals[(t * s.max_peers) + k]; + } + for (size_t su = 0; su < s.threads; ++su) { + const size_t i = (shard * s.stride) + (k * s.threads) + su; + s.recv_off[i] = running; + running += s.recv_rows[i]; + } + } +} + +auto PhysicalExchange::post_counts() -> void { + require_primary("post_counts"); + require_state(state_.get()); + State &s = *state_; + if (!s.send_planned) { + throw std::logic_error("sharded exchange: post_counts() before plan_send()"); + } + s.require_idle("post_counts"); + const size_t shards = s.threads; + const size_t t2 = shards * shards; + for (size_t k = 0; k < s.peers.size(); ++k) { + int *const block = s.count_send.data() + (k * t2); + for (size_t u = 0; u < shards; ++u) { + const int *row = s.send_rows.data() + (u * s.stride) + (k * shards); + for (size_t t = 0; t < shards; ++t) { + block[(t * shards) + u] = row[t]; + } + } + } + if (s.peers.empty()) { + return; + } +#ifdef monoprop_ENABLE_MPI + const size_t capacity = t2 * sizeof(std::uint32_t); + const int receivable = checked_int(static_cast(capacity), "count block"); + int n = 0; + for (size_t k = 0; k < s.peers.size(); ++k) { + const int peer = static_cast(s.peers[k]); + const int *const counts = s.count_send.data() + (k * t2); + std::byte *const wire = s.count_wire_send.data() + (k * capacity); + const size_t width = encode_counts(std::span(counts, t2), wire); + MPI_Irecv(s.count_wire_recv.data() + (k * capacity), + receivable, + MPI_BYTE, + peer, + sharded_count_tag, + s.comm.mpi, + &s.count_requests[static_cast(n++)]); + MPI_Isend(wire, + checked_int(static_cast(t2 * width), "count block"), + MPI_BYTE, + peer, + sharded_count_tag, + s.comm.mpi, + &s.count_requests[static_cast(n++)]); + } + s.count_posted = n; +#else + throw std::logic_error("sharded exchange: an MPI-off build has no peer ranks"); +#endif +} + +auto PhysicalExchange::wait_counts() -> void { + require_primary("wait_counts"); + require_state(state_.get()); + State &s = *state_; + const size_t n = s.threads; + const size_t t2 = n * n; +#ifdef monoprop_ENABLE_MPI + const bool waited = s.count_posted != 0; + if (waited) { + MPI_Waitall(s.count_posted, s.count_requests.data(), s.count_statuses.data()); + s.count_posted = 0; + } + // Peer k's block holds [my destination t][its source su]: row t of the receive table. Its receive request is + // number 2k (post_counts() posts the receive before the send, peer by peer). + for (size_t k = 0; waited && k < s.peers.size(); ++k) { + int bytes = 0; + MPI_Get_count(&s.count_statuses[2 * k], MPI_BYTE, &bytes); + const std::byte *const wire = s.count_wire_recv.data() + (k * t2 * sizeof(std::uint32_t)); + const size_t width = static_cast(bytes) / t2; + if (bytes < 0 || width * t2 != static_cast(bytes) || (width != 1 && width != 2 && width != 4)) { + throw std::length_error(std::format("sharded exchange: rank {} sent a {}-byte count block for {} counts", + s.peers[k], + bytes, + t2)); + } + int smallest = 0; + for (size_t t = 0; t < n; ++t) { + int *row = s.recv_rows.data() + (t * s.stride) + (k * n); + for (size_t su = 0; su < n; ++su) { + const int count = decode_count(wire, (t * n) + su, width); + smallest = std::min(smallest, count); + row[su] = count; + } + } + if (smallest < 0) { + throw std::length_error( + std::format("sharded exchange: rank {} announced a negative count {}", s.peers[k], smallest)); + } + } +#else + static_cast(t2); +#endif +} + +auto PhysicalExchange::plan_recv() -> void { + require_state(state_.get()); + State &s = *state_; + if (!s.send_planned) { + throw std::logic_error("sharded exchange: plan_recv() before plan_send()"); + } + s.require_idle("plan_recv"); + s.recv_planned = false; + const size_t total = s.lay_out(s.recv_rows, + s.recv_off, + s.mpi_recv_counts, + s.mpi_recv_displs, + /*send_side=*/false, + "receive total"); + s.size_stage(s.recv_stage, total); + s.recv_planned = true; +} + +auto PhysicalExchange::post() -> void { + require_primary("post"); + require_state(state_.get()); + State &s = *state_; + if (!s.send_planned || !s.recv_planned) { + throw std::logic_error("sharded exchange: post() before the round's layout is planned"); + } + s.require_idle("post"); +#ifdef monoprop_ENABLE_MPI + const MPI_Datatype dt = element_datatype(s.element); + if (s.transport == ExchangeTransport::collective) { + // Every rank joins, even with nothing to send: a rank that skipped would strand the others. + MPI_Ialltoallv(s.send_stage.data(), + s.mpi_send_counts.data(), + s.mpi_send_displs.data(), + dt, + s.recv_stage.data(), + s.mpi_recv_counts.data(), + s.mpi_recv_displs.data(), + dt, + s.comm.mpi, + s.requests.data()); + s.posted = 1; + return; + } + int n = 0; + for (const size_t peer : s.peers) { + const int recv = s.mpi_recv_counts[peer]; + const int send = s.mpi_send_counts[peer]; + if (recv != 0) { + MPI_Irecv(s.recv_stage.data() + (static_cast(s.mpi_recv_displs[peer]) * s.elem), + recv, + dt, + static_cast(peer), + s.tag, + s.comm.mpi, + &s.requests[static_cast(n++)]); + } + if (send != 0) { + MPI_Isend(s.send_stage.data() + (static_cast(s.mpi_send_displs[peer]) * s.elem), + send, + dt, + static_cast(peer), + s.tag, + s.comm.mpi, + &s.requests[static_cast(n++)]); + } + } + s.posted = n; +#else + if (!s.peers.empty()) { + throw std::logic_error("sharded exchange: an MPI-off build has no peer ranks"); + } +#endif +} + +auto PhysicalExchange::wait() -> void { + require_primary("wait"); + require_state(state_.get()); +#ifdef monoprop_ENABLE_MPI + State &s = *state_; + if (s.posted != 0) { + MPI_Waitall(s.posted, s.requests.data(), MPI_STATUSES_IGNORE); + s.posted = 0; + } +#endif +} + +auto PhysicalRounds::get(Kind kind, const PhysicalWorld &world, size_t threads) -> PhysicalExchange & { + PhysicalExchange &round = rounds_.at(static_cast(kind)); + if (round.threads() != threads || round.ranks() != world.ranks || round.rank() != world.rank) { + switch (kind) { + case Kind::queries: + round = PhysicalExchange(world, threads, ExchangeElement::u64, sharded_query_tag); + break; + case Kind::graph_answers: + round = PhysicalExchange(world, threads, ExchangeElement::u32, sharded_answer_tag); + break; + case Kind::fused_answers: + round = PhysicalExchange(world, threads, ExchangeElement::f64, sharded_answer_tag); + break; + case Kind::replay: + round = PhysicalExchange(world, threads, ExchangeElement::f64, sharded_replay_tag); + break; + } + } + return round; +} + +auto PhysicalExchange::send_bytes_(size_t shard, size_t peer, size_t dest, ExchangeElement element) + -> std::span { + require_state(state_.get()); + State &s = *state_; + if (element != s.element || !s.send_planned) { + throw std::logic_error("sharded exchange: send_block() with the wrong element type or before plan_send()"); + } + const size_t i = s.at(shard, peer, dest); + if (peer >= s.peers.size()) { + throw std::out_of_range(std::format("sharded exchange: peer {} of a {}-peer round", peer, s.peers.size())); + } + const size_t column_entry = (dest * s.stride) + (peer * s.threads) + shard; + return {s.send_stage.data() + (static_cast(s.send_off[column_entry]) * s.elem), + static_cast(s.send_rows[i]) * s.elem}; +} + +auto PhysicalExchange::send_column_bytes_(size_t shard, size_t peer, ExchangeElement element) -> ColumnBytes { + require_state(state_.get()); + State &s = *state_; + if (element != s.element || !s.send_planned) { + throw std::logic_error("sharded exchange: send_column() with the wrong element type or before planning"); + } + if (shard >= s.threads || peer >= s.peers.size()) { + throw std::out_of_range( + std::format("sharded exchange: column {} of peer {} in a round of {} shards and {} peers", + shard, + peer, + s.threads, + s.peers.size())); + } + const size_t column = (shard * s.stride) + (peer * s.threads); + return {.stage = s.send_stage.data(), + .offsets = std::span(s.send_off.data() + column, s.threads), + .counts = std::span(s.send_col_rows.data() + column, s.threads)}; +} + +auto PhysicalExchange::recv_bytes_(size_t shard, size_t peer, size_t source, ExchangeElement element) const + -> std::span { + require_state(state_.get()); + const State &s = *state_; + if (element != s.element || !s.recv_planned) { + throw std::logic_error("sharded exchange: recv_block() with the wrong element type or before plan_recv()"); + } + const size_t i = s.at(shard, peer, source); + if (peer >= s.peers.size()) { + throw std::out_of_range(std::format("sharded exchange: peer {} of a {}-peer round", peer, s.peers.size())); + } + return {s.recv_stage.data() + (static_cast(s.recv_off[i]) * s.elem), + static_cast(s.recv_rows[i]) * s.elem}; +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/Exchange.h b/cpp/monoprop/detail/sharded/Exchange.h new file mode 100644 index 000000000..a927550e9 --- /dev/null +++ b/cpp/monoprop/detail/sharded/Exchange.h @@ -0,0 +1,391 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/monopropExport.h" + +/* + * One physical-MPI round of the sharded runtime: the inter-process part of a handoff between the T owners of this + * process and the T owners of each peer process. It is not a collective facade: it exposes physical post and wait + * operations to the primary, and disjoint payload slices and read-only received blocks to the owners. Shards of the + * same process never go through it; they read each other's published buffers directly. + * + * A round, in the phases of one team (O: every owner, inside its protected phase; P: the primary only): + * + * O reset_rows(t), then set_send_count(t, ...) for every remote destination, and set_recv_count(t, ...) when the + * receive counts are known (replay: the layer is symmetric; answers: one per query, known on both sides) + * P plan_send(peers, transport); post_counts() when the receive counts are unknown + * O pack send_block(t, peer, dest) + * P wait_counts() if counted, plan_recv(), post(), and later wait() + * O read recv_block(t, peer, source) + * + * with a team checkpoint between consecutive lines. Planning and MPI calls are serial on the primary; packing and + * reading are owner-parallel over disjoint memory. + * + * A round whose receive counts its owners know (a replay step) can be planned owner-parallel instead, with the same + * number of checkpoints and the same offsets as plan_send() + plan_recv(), but O(P x T) work on the primary instead of + * O(P x T^2): + * + * O reset_rows(t), set_send_count / set_recv_count for every index of every other rank, then close_rows(t); or, for + * a symmetric round over every other rank, write_symmetric_rows(t, counts) alone + * P plan_totals(peers, transport) O plan_column(t) + * O place_column(t), then pack column t, the block of every local source u for remote shard t: send_column(t, + * peer) gives the whole column at once, send_block(u, peer, t) one block + * P post(), and later wait() + * O read recv_block(t, peer, source) + * + * Column t's offsets and counts are owner t's own contiguous entries, written by place_column(t) and read by its + * packing: no line of them is shared with another owner's column. + * + * Wire layout: the message to peer rank b holds, for each of b's shards t in + * ascending order, each local source shard's block for (b, t) in ascending source order. Received messages have the + * same layout, so a local destination's part of a message is contiguous and its blocks ascend by source shard. + * Every count, per-peer total and displacement is a checked int; a violation throws before anything is posted. + * + * Transports: pairwise posts an Irecv and an Isend per non-empty leg of each peer (both ends skip the same legs, + * since receive counts are exchanged or known to match the sender's); collective posts one MPI_Ialltoallv over the + * whole communicator, which every rank joins even with nothing to send. The count round is always pairwise over the + * peer set: T x T counts per peer, each peer's block in the narrowest of 1, 2 or 4 bytes per count that holds its + * largest, so light rounds stay below the transport's eager limit; the receiver reads the width from the received size. + * + * MPI ownership: post_counts(), wait_counts(), post() and wait() throw std::logic_error unless they run on OpenMP + * thread 0 and, in MPI builds, on the thread that initialized MPI. Only the primary of a team, which is the thread + * that entered it, ever posts or waits. + * + * Lifetime: the state lives on the heap, so a move keeps every buffer and request MPI points into. The move + * constructor adopts; move assignment drains the destination's live requests first; the destructor drains. A caller + * that may fail while requests are live must keep the exchange alive until it has handed the failure to + * mpi::operation_failed(), which aborts a multi-rank communicator: draining a request a failed peer never completes + * would hang. + */ + +namespace monoprop::detail::sharded { + +//! Element type of a round's payload. +enum class ExchangeElement : std::uint8_t { + u32, //!< 32-bit unsigned (graph answers: TermIndex). + u64, //!< 64-bit unsigned (queries: size_t words). + f64, //!< double (replay endpoints, fused answers). +}; + +//! How a round's payload is posted. +enum class ExchangeTransport : std::uint8_t { + pairwise, //!< Irecv and Isend per non-empty leg of each peer. + collective, //!< One MPI_Ialltoallv over every rank of the communicator. +}; + +//! Tags of the sharded rounds, distinct from each other and from the low-level engine's (Pairwise.h). +inline constexpr int sharded_query_tag = 0x6D75; +inline constexpr int sharded_answer_tag = 0x6D76; +inline constexpr int sharded_replay_tag = 0x6D77; +inline constexpr int sharded_count_tag = 0x6D78; //!< Every count round. + +//! The element kind of a payload type. +template +inline constexpr ExchangeElement exchange_element_of = [] { + if constexpr (std::is_same_v) { + return ExchangeElement::u32; + } + else if constexpr (std::is_same_v + || (std::is_same_v && sizeof(std::size_t) == sizeof(std::uint64_t))) { + return ExchangeElement::u64; + } + else { + static_assert(std::is_same_v, "a sharded round carries u32, u64 or double payloads"); + return ExchangeElement::f64; + } +}(); + +/*! + * \brief This process's place among the physical ranks of an ordinary communicator, read once on the caller. + */ +struct PhysicalWorld { + mpi::Comm comm{}; //!< The ordinary communicator; must outlive every exchange built from this. + size_t rank = 0; //!< This process's rank. + size_t ranks = 1; //!< The communicator's size P. + bool replay_pairwise = false; //!< The communicator-agreed replay transport (mpi::routes_pairwise()). + + /*! + * \brief Rank, size and agreed replay transport of `comm`. + * + * Call on MPI's initializing thread, outside any team: the first call on a multi-rank communicator agrees the + * routing configuration collectively and caches the answer on the communicator. + * + * \throws routing::RoutingDisagreement on every rank together if the ranks resolved different routing + * configurations. + */ + monoprop_EXPORT static auto of(const mpi::Comm &comm) -> PhysicalWorld; +}; + +//! Every rank of `world` except this one, ascending: the peer set of a dense round. +[[nodiscard]] inline auto other_ranks(const PhysicalWorld &world) -> std::vector { + std::vector peers; + peers.reserve(world.ranks - 1); + for (size_t b = 0; b < world.ranks; ++b) { + if (b != world.rank) { + peers.push_back(b); + } + } + return peers; +} + +/*! + * \brief One owner's column of a round planned owner-parallel: its slice for every local source shard, for one remote + * shard of one peer (PhysicalExchange::send_column()). + */ +template +struct SendColumn { + std::byte *stage = nullptr; //!< The round's send staging. + std::span offsets{}; //!< Per local source: its block's element offset into the staging. + std::span counts{}; //!< Per local source: its block's element count. + + //! The slice of local source `source`; unchecked, `source` must be below the team size. + [[nodiscard]] auto block(size_t source) const -> std::span { + return {reinterpret_cast(stage) + offsets[source], static_cast(counts[source])}; + } +}; + +/*! + * \brief A reusable physical round between this process's T owners and those of a set of peer ranks. + */ +class PhysicalExchange { +public: + monoprop_EXPORT + PhysicalExchange() noexcept; //!< An inert exchange: every operation except destruction and assignment throws. + + /*! + * \brief An exchange for `threads` owners per process, over the ranks of `world`. + * + * Allocates the count tables for every possible peer, P - 1 of them. + * + * \param world The physical world; its communicator must outlive the exchange. + * \param threads The team size T, identical on every rank. + * \param element The payload type of every round. + * \param tag The payload tag; rounds that may be in flight together need different tags. + * \throws std::invalid_argument if `threads` is zero or `world` has no ranks; std::bad_alloc. + */ + monoprop_EXPORT PhysicalExchange(const PhysicalWorld &world, size_t threads, ExchangeElement element, int tag); + + PhysicalExchange(const PhysicalExchange &) = delete; //!< Owns requests: move-only. + auto operator=(const PhysicalExchange &) -> PhysicalExchange & = delete; //!< Owns requests: move-only. + monoprop_EXPORT PhysicalExchange(PhysicalExchange &&other) noexcept; //!< Adopts, live requests included. + //! Drains this exchange's live requests, then adopts `other`'s. + monoprop_EXPORT auto operator=(PhysicalExchange &&other) noexcept -> PhysicalExchange &; + monoprop_EXPORT ~PhysicalExchange(); //!< Drains live requests; see the header comment. + + [[nodiscard]] monoprop_EXPORT auto threads() const noexcept -> size_t; //!< T. + [[nodiscard]] monoprop_EXPORT auto rank() const noexcept -> size_t; //!< This process's rank. + [[nodiscard]] monoprop_EXPORT auto ranks() const noexcept -> size_t; //!< The communicator's size P. + //! The peer ranks of the current round, ascending, as plan_send() received them. + [[nodiscard]] monoprop_EXPORT auto peers() const noexcept -> std::span; + //! Requests posted and not yet waited, payload and counts together. + [[nodiscard]] monoprop_EXPORT auto live() const noexcept -> int; + //! Bytes of send and receive staging held now: the largest round planned since the last release, 0 if none. + [[nodiscard]] monoprop_EXPORT auto staging_bytes() const noexcept -> size_t; + /*! + * \brief Unmap the send and receive staging; the next plan maps what it then needs. + * + * The count tables and peers are kept; the plans are dropped, so the next round plans before it packs, as every + * round does. Does nothing while a request is live, since MPI may still read or write the staging, and nothing on + * an inert exchange. + */ + monoprop_EXPORT auto release_staging() noexcept -> void; + + // --- Owner side: count rows (row `shard` has one writer, owner `shard`) ------------------------------------------ + + //! Zero both of owner `shard`'s count rows, for every possible peer. + monoprop_EXPORT auto reset_rows(size_t shard) -> void; + /*! + * \brief Set how many elements local `shard` sends to shard `dest` of peer number `peer`. + * \throws std::out_of_range for an index outside the table; std::length_error for a count above INT_MAX. + */ + monoprop_EXPORT auto set_send_count(size_t shard, size_t peer, size_t dest, size_t count) -> void; + /*! + * \brief Set how many elements local `shard` receives from shard `source` of peer number `peer`. + * \throws As set_send_count(). + */ + monoprop_EXPORT auto set_recv_count(size_t shard, size_t peer, size_t source, size_t count) -> void; + //! The send count set for (shard, peer, dest). + [[nodiscard]] monoprop_EXPORT auto send_count(size_t shard, size_t peer, size_t dest) const -> size_t; + //! The receive count for (shard, peer, source): set by its owner or by wait_counts(). + [[nodiscard]] monoprop_EXPORT auto recv_count(size_t shard, size_t peer, size_t source) const -> size_t; + + // --- Owner-parallel planning (rounds whose rows cover every other rank, such as replay steps) ------------------ + + //! Owner `shard`, after writing both of its rows: record its per-peer row totals for plan_totals(). + monoprop_EXPORT auto close_rows(size_t shard) -> void; + /*! + * \brief Owner `shard` writes both of its rows for a symmetric round over every other rank, and closes them. + * + * Equivalent to reset_rows(), set_send_count() and set_recv_count() with the same count for every shard of every + * other rank, then close_rows(). `slot_counts[b * T + u]` is the count with shard u of rank b; slots past the end + * count zero, and this rank's own slots are not read. + * + * \throws std::out_of_range for a shard outside the table; std::length_error for a negative count (the row is then + * left partly written and must be rewritten before planning). + */ + monoprop_EXPORT auto write_symmetric_rows(size_t shard, std::span slot_counts) -> void; + /*! + * \brief Start a round over `peers` from the owners' row totals (close_rows()): per-peer MPI counts and + * displacements and both staging sizes, in O(P x T) on the primary. + * + * The owners' offsets follow from plan_column() (same phase) and place_column() (next phase); they equal what + * plan_send() and plan_recv() lay out. Row index k must be the k-th peer of `peers` for every possible peer index. + * + * \throws As plan_send(): std::invalid_argument for an invalid peer set, std::length_error if a total does not fit + * an int, std::logic_error while requests are live. Nothing is posted. + */ + monoprop_EXPORT auto plan_totals(std::span peers, ExchangeTransport transport) -> void; + //! plan_totals() over every other rank, ascending, without allocating once the round has been planned before. + monoprop_EXPORT auto plan_totals(ExchangeTransport transport) -> void; + //! Owner `shard`, in plan_totals()' phase: the send-side sum of column `shard` (local sources to remote shard + //! `shard`) for every possible peer index. + monoprop_EXPORT auto plan_column(size_t shard) -> void; + /*! + * \brief Owner `shard`, one checkpoint after plan_totals() and every plan_column(): the send offsets of column + * `shard` (every local source's block for remote shard `shard`) and the receive offsets of row `shard`. + * \throws std::logic_error before plan_totals(); std::out_of_range for a shard outside the table. + */ + monoprop_EXPORT auto place_column(size_t shard) -> void; + + // --- Primary side: planning and MPI ----------------------------------------------------------------------------- + + /*! + * \brief Start a round over `peers`: lay out and size the send staging from every owner's send row. + * + * \param peers Peer ranks, strictly ascending, none of them this rank. + * \param transport How post() moves the payload; must be the same on every rank of the round. + * \throws std::invalid_argument for an invalid peer set; std::length_error if a per-peer total or offset does not + * fit an int; std::logic_error while requests are live. Nothing is posted. + */ + monoprop_EXPORT auto plan_send(std::span peers, ExchangeTransport transport) -> void; + //! Post the count round over the peers. Primary only; requires plan_send(). + monoprop_EXPORT auto post_counts() -> void; + //! Complete the count round and fill every owner's receive row from it. Primary only. + monoprop_EXPORT auto wait_counts() -> void; + /*! + * \brief Lay out and size the receive staging from every owner's receive row. + * \throws std::length_error as plan_send(); std::logic_error without plan_send() or while requests are live. + */ + monoprop_EXPORT auto plan_recv() -> void; + //! Post the payload. Primary only; requires plan_recv(). + monoprop_EXPORT auto post() -> void; + //! Complete the payload. Primary only. + monoprop_EXPORT auto wait() -> void; + + // --- Owner side: payload ---------------------------------------------------------------------------------------- + + /*! + * \brief Owner `shard`'s slice for shard `dest` of peer number `peer`, valid until the next plan_send(). + * \throws std::logic_error if `T` is not the round's element type or no round is planned; std::out_of_range. + */ + template + [[nodiscard]] auto send_block(size_t shard, size_t peer, size_t dest) -> std::span { + const auto bytes = send_bytes_(shard, peer, dest, exchange_element_of); + return {reinterpret_cast(bytes.data()), bytes.size() / sizeof(T)}; + } + + /*! + * \brief Column `shard` of peer number `peer`: the slice of every local source for remote shard `shard`, placed by + * place_column(shard) and valid until the next plan. + * \throws std::logic_error if `T` is not the round's element type or no round is planned; std::out_of_range. + */ + template + [[nodiscard]] auto send_column(size_t shard, size_t peer) -> SendColumn { + const ColumnBytes column = send_column_bytes_(shard, peer, exchange_element_of); + return {.stage = column.stage, .offsets = column.offsets, .counts = column.counts}; + } + + /*! + * \brief What local `shard` received from shard `source` of peer number `peer`, valid until the next plan_recv(). + * \throws As send_block(); the round's receive side must have been planned with plan_recv(). + */ + template + [[nodiscard]] auto recv_block(size_t shard, size_t peer, size_t source) const -> std::span { + const auto bytes = recv_bytes_(shard, peer, source, exchange_element_of); + return {reinterpret_cast(bytes.data()), bytes.size() / sizeof(T)}; + } + +private: + struct State; + //! A column's staging and its offsets and counts in elements, before the element type is applied. + struct ColumnBytes { + std::byte *stage = nullptr; + std::span offsets; + std::span counts; + }; + + monoprop_EXPORT auto send_bytes_(size_t shard, size_t peer, size_t dest, ExchangeElement element) + -> std::span; + monoprop_EXPORT auto send_column_bytes_(size_t shard, size_t peer, ExchangeElement element) -> ColumnBytes; + [[nodiscard]] monoprop_EXPORT auto recv_bytes_(size_t shard, + size_t peer, + size_t source, + ExchangeElement element) const -> std::span; + + std::unique_ptr state_; //!< Heap-stable: MPI holds pointers into it once a round is posted. +}; + +/*! + * \brief The physical rounds of one rank-level owner, kept across its operations. + * + * Every construction, evaluation and replay call of an owner reuses these rounds instead of allocating its own. The + * replay round's staging grows to a high-water mark once and stays, since every evaluation reuses it; the construction + * rounds unmap theirs when a construction ends (release_staging()), since no evaluation reads it. Staging is mapped + * directly, so neither its growth nor its release moves glibc's dynamic mmap threshold: rounds allocated per call + * through malloc raised it at every free of a large block, after which the per-gate transient allocations stayed + * resident in fragmented thread arenas. + * + * One owner uses its rounds from one operation at a time, on its calling thread; copies of an owner get their own empty + * set, never a shared one. A round is created on first use, and recreated if the geometry it was created for changed. + */ +class PhysicalRounds { +public: + //! The round kinds; rounds of different kinds may be in flight in the same operation. + enum class Kind : std::uint8_t { + queries, //!< Construction queries (u64 words). + graph_answers, //!< Graph construction answers (u32 term indices). + fused_answers, //!< Propagation answers (double coefficients). + replay, //!< Replay endpoints of evaluation, partial contraction and informed construction (double). + }; + + /*! + * \brief The round of `kind` for `threads` owners per process over the ranks of `world`. + * \throws As PhysicalExchange's constructor, when the round has to be created. + */ + monoprop_EXPORT auto get(Kind kind, const PhysicalWorld &world, size_t threads) -> PhysicalExchange &; + + //! The round of `kind` as it is, inert (threads() == 0) if never used; for inspection, never creates one. + [[nodiscard]] auto peek(Kind kind) const -> const PhysicalExchange & { + return rounds_.at(static_cast(kind)); + } + + //! Unmap the staging of the round of `kind` (PhysicalExchange::release_staging()); the round itself is kept. + auto release_staging(Kind kind) noexcept -> void { rounds_[static_cast(kind)].release_staging(); } + +private: + std::array rounds_; //!< Indexed by Kind; inert until first use. +}; + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/RootObserver.h b/cpp/monoprop/detail/sharded/RootObserver.h new file mode 100644 index 000000000..30322aa92 --- /dev/null +++ b/cpp/monoprop/detail/sharded/RootObserver.h @@ -0,0 +1,132 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/sharded/Construction.h" +#include "monoprop/detail/sharded/Evaluation.h" +#include "monoprop/detail/sharded/State.h" + +/* + * Test-only observation of the sharded root. + * + * The rank-level MonomialPropagator holds a pointer to a RootObserver that is null in production and can be set only + * through the test suite's PropagatorTestAccess. Every seam the root calls -- seeding, copying, construction, + * evaluation, replay, retained preparation and the root's own owner teams -- reports to it through the adapters + * below, which compile to a null check when no observer is set. An observer method runs inside the protected phase + * of the work it reports, on the thread performing that work, so recording the executing worker observes actual + * participation, and throwing injects a real failure at that point. Observations are evidence collected by tests, + * never a production geometry check. + */ + +namespace monoprop::detail::sharded { + +//! Work the root performs in its own owner teams or on the caller, as reported to RootObserver::root(). +enum class RootWork : std::uint8_t { + initial_operator, //!< Owner phase: apply this shard's share of an initial-operator update. + remap, //!< Owner phase: relabel this shard's layers for a new parameter mapping. + picture, //!< Owner phase: copy this shard's current picture (contraction at empty parameters). + export_block, //!< Caller: decode this shard's evolved block (evolved_operator_terms() and friends). + combine, //!< Caller, after the evaluation team joined: fold and reduce the shards' results. + adjoint_seed, //!< Owner phase: scatter this shard's state for term_expectation_values()' adjoint sweep. +}; + +/*! + * \brief Test-only observer of the root; every method defaults to doing nothing. + * + * Inherits the evaluation seam's visit(), so one object observes evaluation, replay and retained preparation too. + */ +class RootObserver : public EvaluationObserver { +public: + //! Evaluation, replay and retained-preparation work (see EvaluationObserver); may throw. + auto visit(EvaluationWork /*work*/, size_t /*step*/, size_t /*shard*/) const -> void override {} + + //! Seed or copy work of `shard` (see NoShardObserver): `end` is false before, true after; may throw. + virtual auto shard_work(ShardWork /*work*/, bool /*end*/, size_t /*shard*/) const -> void {} + + //! Construction work (see NoConstructionObserver); may throw. + virtual auto construction(ConstructionWork /*work*/, size_t /*step*/, size_t /*shard*/) const -> void {} + + //! The root's own work; `shard` is the shard concerned. May throw. + virtual auto root(RootWork /*work*/, size_t /*shard*/) const -> void {} + + //! Multi-rank construction, after `shard` packed its remote query blocks into `round`; may rewrite its slices. + virtual auto queries_packed(size_t /*step*/, size_t /*shard*/, PhysicalExchange & /*round*/) const -> void {} + + //! Propagation, just before `shard` applies its fused records for gate `step`; may throw. + virtual auto fused_records(size_t /*step*/, size_t /*shard*/, const FusedGateView & /*gate*/) const -> void {} +}; + +//! The seed_shards() / copy_shards() observer the root passes: forwards to a RootObserver when one is set. +struct RootShardObserver { + const RootObserver *observer = nullptr; //!< Null in production. + + //! Before the owner's substantial allocation. + auto begin(ShardWork work, size_t shard) const -> void { + if (observer != nullptr) { + observer->shard_work(work, false, shard); + } + } + + //! After the owner's state is complete, before it is published. + auto end(ShardWork work, size_t shard) const -> void { + if (observer != nullptr) { + observer->shard_work(work, true, shard); + } + } +}; + +//! The construction observer the root passes: forwards visits; kernels report to no range observer. +struct RootConstructionObserver { + const RootObserver *observer = nullptr; //!< Null in production. + + //! Before `work` of `step` for `shard`. + auto visit(ConstructionWork work, size_t step, size_t shard) const -> void { + if (observer != nullptr) { + observer->construction(work, step, shard); + } + } + + //! Within-shard kernels and exchange phases report to the production default. + [[nodiscard]] auto kernels(size_t /*shard*/) const noexcept -> NoRangeObserver { return {}; } + + //! After `shard` packed its remote query blocks. + auto queries_packed(size_t step, size_t shard, PhysicalExchange &round) const -> void { + if (observer != nullptr) { + observer->queries_packed(step, shard, round); + } + } + + //! Before `shard` applies its fused records for gate `step`. + auto fused_records(size_t step, size_t shard, const FusedGateView &gate) const -> void { + if (observer != nullptr) { + observer->fused_records(step, shard, gate); + } + } +}; + +/*! + * \brief Report the root's own work to `observer` when one is set. + */ +inline auto notify_root(const RootObserver *observer, RootWork work, size_t shard) -> void { + if (observer != nullptr) { + observer->root(work, shard); + } +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/State.h b/cpp/monoprop/detail/sharded/State.h new file mode 100644 index 000000000..c9ddc3dbe --- /dev/null +++ b/cpp/monoprop/detail/sharded/State.h @@ -0,0 +1,457 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/MPGraph.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/sharded/Team.h" + +/* + * Owner-initialized shard state for sharded execution. + * + * A rank-level propagator owns T shard states, one per worker of its fixed team. Shard t is the flat routing owner + * rank * T + t at geometry (P, T). Its state is mathematical only: an operator store with its coefficients, state and + * caches, a primary graph whose endpoints are rows of that store, and the layer build's matched marks. It is not a + * propagator, communicator or executor. + * + * Memory ownership: + * + * - The caller allocates only the Shards pointer vector (T pointers) and the team's error slots. + * - Each owner allocates its whole ShardState inside its protected phase: the state object, the packed rows, hash + * index and overflow map, coefficient/state vectors, pending initial-operator entries, inverted index, graph layer + * list and matched marks. The pointees never move, so their addresses are stable for the state's lifetime, + * including when the vector itself is moved. + * - A copy deep-copies every mutable member; only the immutable graph LayerCore objects stay shared between the + * source's and the copy's layer lists. + * - On failure, shards other owners already published are destroyed by the caller after the team has joined. + * + * Seeding is split into caller-side preparation, which may validate, throw configuration errors and query MPI, and + * an owner-local seed that only reads the prepared inputs. + */ + +namespace monoprop::detail::sharded { + +/*! + * \brief Size of the Schrödinger paired initial basis and its per-owner share. + */ +struct PairedBasisBounds { + size_t max_pairs = 0; //!< Pair bound: the effective Schrödinger cutoff halved, rounded up. + size_t global_terms = 0; //!< Paired basis size; `SIZE_MAX` when too large to count. + size_t share = 0; //!< `global_terms` divided by the number of flat owners. + + //! Whether the basis size was countable at all. + [[nodiscard]] auto countable() const noexcept -> bool { return global_terms != std::numeric_limits::max(); } + + //! Row reservation for each owner's store; never zero. + [[nodiscard]] auto reserve() const noexcept -> size_t { return std::max(1, share); } +}; + +/*! + * \brief Bound the streamed paired basis for a Schrödinger cutoff. + * + * Pure. The cutoff is first capped at `2 * logical_num_modes`, then rounded up to whole pairs, so an odd cutoff + * admits one more pair than it covers. Rejecting an unaddressable share is the caller's configuration policy. + * + * \param schrodinger_cutoff The configured Schrödinger cutoff. + * \param logical_num_modes Active logical modes. + * \param flat_owners Flat owners P * T sharing the basis; 0 is treated as 1. + * \return The pair bound, global size and per-owner share. + */ +inline auto paired_basis_bounds(unsigned int schrodinger_cutoff, size_t logical_num_modes, size_t flat_owners) + -> PairedBasisBounds { + const auto effective = std::min(schrodinger_cutoff, static_cast(2 * logical_num_modes)); + auto bounds = PairedBasisBounds{}; + bounds.max_pairs = (effective / 2) + (effective % 2); + bounds.global_terms = paired_op_size(bounds.max_pairs, logical_num_modes); + bounds.share = bounds.global_terms / std::max(1, flat_owners); + return bounds; +} + +/*! + * \brief Packed-row inline width for a new store. + * + * A performance hint, never a correctness constraint: longer rows spill losslessly to the overflow map. + * Heisenberg uses the cutoff's structural position bound, capped at the store's maximum; Schrödinger and cutoffs + * without a bound use the store's default. + * + * \param schrodinger Whether the propagator evolves the state. + * \param cutoff_fn The propagator's cutoff predicate. + * \return The width to construct each owner's OperatorIndex with. + */ +template +auto packed_inline_width(bool schrodinger, const CutoffFn &cutoff_fn) -> size_t { + constexpr auto max_width = OperatorIndex::kMaxInlinePositions; + constexpr auto default_width = OperatorIndex::kDefaultInlinePositions; + if (schrodinger) { + return default_width; + } + // Already in physical slots (CutoffEvaluator::max_slot_bound), so nothing to scale. + const auto bound = CutoffEvaluator(cutoff_fn).max_slot_bound(); + if (!bound) { + return default_width; + } + return std::min(*bound, max_width); +} + +/*! + * \brief Caller-side check of an initial operator; returns its encoded identity coefficient. + * + * Converts every term within the logical-mode bound and encodes its coefficient, in the dictionary's order, so the + * first invalid term throws before any owner starts. The identity coefficient is rank-level metadata: it is seeded + * into no shard. + * + * \param initial_operator The operator dictionary. + * \param basis The coefficient encoding. + * \param logical_num_modes Active logical modes; indices must lie below `2 * logical_num_modes`. + * \return The encoded identity coefficient, or 0 when the operator has no identity term. + * \throws As indices_to_bitset_checked() for an out-of-range index, and as algebra_encode_coeff() for a coefficient + * the basis cannot encode. + */ +template +auto validate_initial_operator(const OperatorDict &initial_operator, Basis basis, size_t logical_num_modes) -> double { + auto core_term = 0.0; + for (const auto &[indices, coefficient] : initial_operator) { + const auto mono = indices_to_bitset_checked(indices, 2 * logical_num_modes); + const auto encoded = algebra_encode_coeff(basis, coefficient, mono); + if (indices.empty()) { + core_term = encoded; + } + } + return core_term; +} + +/*! + * \brief Read-only, caller-prepared inputs for seeding every owner of one propagator. + * + * Built once on the caller after validate_initial_operator() and any MPI-dependent configuration (router geometry, + * routing agreement). Owners read it concurrently; nothing here is mutated during seeding. + */ +template +struct OperatorSeed { + const OperatorDict &initial_operator; //!< Validated operator; must outlive seeding. + const VecZ &initial_state; //!< Copied into every owner's store. + routing::Router router; //!< Flat-owner routing at geometry (P, T). + Basis basis; //!< Coefficient encoding and state scoring. + size_t logical_num_modes; //!< Active logical modes. + std::optional paired; //!< Schrödinger's streamed paired basis; empty for Heisenberg. + size_t inline_width; //!< From packed_inline_width(). +}; + +/*! + * \brief Seed one owner's operator: its routed share of the initial rows, pending entries and warm caches. + * + * Owner-local: reads `seed` and allocates only into `op` plus transient owner-local lists, so it may run concurrently + * for different owners. Rows are this owner's keys in input order (Heisenberg) or in the paired basis's emission order + * (Schrödinger), numbered 0, 1, 2, ... as they are kept, so fixed geometry reproduces the same rows. The paired basis + * is streamed, never materialized. Heisenberg owners reserve their own term count; Schrödinger owners reserve the + * basis's per-owner share. The identity term is skipped here: its coefficient is rank-level metadata, while + * Schrödinger's identity basis row is an ordinary routed row with coefficient 0. + * + * The operator is filled where it lives rather than returned: an MPOperator is about 112 NumModes bytes (its inverted + * index holds its columns inline), and a copy in a worker's frame would stay resident on every worker's stack. + * + * \param seed Prepared inputs; the operator must have passed validate_initial_operator(). + * \param flat_owner This owner's flat routing slot, `rank * T + shard`. + * \param op A default-constructed operator; on return it is seeded, with the sparse state warmed for + * Heisenberg and the dense live state for Schrödinger, then the inverted index. + */ +template +auto seed_operator(const OperatorSeed &seed, size_t flat_owner, MPOperator &op) -> void { + op.basis = seed.basis; + const auto owns = [&](const Monomial &mono) { + return seed.router.template dest(mono) == flat_owner; + }; + + MonomialList heisenberg_rows; + for (const auto &[indices, coefficient] : seed.initial_operator) { + if (indices.empty()) { + continue; + } + const auto mono = indices_to_bitset_checked(indices, 2 * seed.logical_num_modes); + if (!owns(mono)) { + continue; + } + op.init_op_map[mono] = algebra_encode_coeff(seed.basis, coefficient, mono); + if (!seed.paired) { + heisenberg_rows.push_back(mono); + } + } + + const auto reserve = seed.paired ? seed.paired->reserve() : std::max(1, heisenberg_rows.size()); + op.store = std::make_unique>(seed.inline_width); + op.store->reserve(reserve); + // Store replaced: drop any lazy inverted index so it rebuilds against the new store. + op.inverted_index_.reset(); + + // The kept monomials are distinct, so emplace (insert-if-absent) is an assigning insert. A row index is a + // position in the kept subsequence, so the enumeration order is load-bearing. + size_t row = 0; + const auto keep_if_owned = [&](const Monomial &mono) { + if (owns(mono)) { + op.append_term(mono); + op.store->emplace(mono, row++); + } + }; + if (seed.paired) { + // Streamed: every owner walks the basis concurrently, so a materialized list would hold P * T copies. + for_each_paired_monomial(seed.paired->max_pairs, seed.logical_num_modes, keep_if_owned); + } + else { + for (const auto &mono : heisenberg_rows) { + keep_if_owned(mono); + } + } + + op.initial_state = seed.initial_state; + op.initialize_caches(seed.paired.has_value()); +} + +/*! + * \brief One owner's mutable mathematical state. + * + * Owned through Shards at a stable address and mutated only by its owner during ordinary phases. Copying + * deep-copies the store (OperatorIndex::clone()), coefficients, pending entries, sparse/dense state, inverted index, + * layer list and matched marks; the layers' immutable LayerCore objects stay shared. + */ +template +struct ShardState { + MPOperator op; //!< Store, coefficients, state and lazy caches of this shard. + MPGraph graph; //!< Primary graph; its endpoints are rows of `op`. + MatchedEpochSet matched; //!< Layer-build follower marks, indexed by rows of `op`. + + /*! + * \brief Adopt an operator with an empty graph. + * \param op The operator, moved in. + * \param schrodinger Whether the graph records Schrödinger layers. + */ + ShardState(MPOperator op, bool schrodinger) : op(std::move(op)), graph(schrodinger) {} + + /*! + * \brief An empty operator built in place, with an empty graph; seed_operator() then fills it where it lives. + * \param schrodinger Whether the graph records Schrödinger layers. + */ + explicit ShardState(bool schrodinger) : graph(schrodinger) {} + + ShardState(const ShardState &) = default; //!< Deep copy; graph cores stay shared. + auto operator=(const ShardState &) -> ShardState & = delete; //!< States are replaced, not assigned. + ShardState(ShardState &&) = delete; //!< Held at a stable address. + auto operator=(ShardState &&) -> ShardState & = delete; //!< Held at a stable address. + ~ShardState() = default; //!< Releases every owned allocation. + + //! Rows in this shard's store. + [[nodiscard]] auto size() const -> size_t { return op.size(); } + + //! This shard's operator accounting, including its matched marks. + [[nodiscard]] auto operator_memory_usage() const -> MPOperatorMemoryBreakdown { + auto breakdown = estimate_memory_usage(op); + breakdown.matched_scratch_bytes = matched.memory_bytes(); + return breakdown; + } + + //! This shard's graph accounting; shared cores are counted by every graph that holds them. + [[nodiscard]] auto graph_memory_usage() const -> GraphMemoryBreakdown { return graph.storage_memory_usage(); } +}; + +//! The shard states of one rank-level propagator, indexed by shard; entries are never null once built. +template +using Shards = std::vector>>; + +//! The substantial work an owner performs in seed_shards() or copy_shards(), as reported to an observer. +enum class ShardWork : std::uint8_t { + seed, //!< seed_operator() and the new state. + copy, //!< Copy construction from the source shard. +}; + +/*! + * \brief The observer production code uses: it does nothing and compiles away. + * + * seed_shards() and copy_shards() call `begin(work, shard)` on the owner inside its protected phase before any of + * the shard's substantial allocation, and `end(work, shard)` after the state is complete and before it is + * published. Tests substitute an observer that records the executing worker or arms a failure; no production caller + * passes one. An exception thrown by an observer is that owner's failure. + */ +struct NoShardObserver { + auto begin(ShardWork /*work*/, size_t /*shard*/) const noexcept -> void {} + auto end(ShardWork /*work*/, size_t /*shard*/) const noexcept -> void {} +}; + +/*! + * \brief Build T shard states, each allocated and initialized on its owner worker. + * + * Allocates the pointer vector on the caller, then runs one team of `options.threads` workers in which worker t + * calls `initialize(t)` inside a protected phase and publishes the result in slot t. The initializer is invoked + * concurrently from every worker on the same object; it must perform only owner-local initialization (no MPI, no + * synchronization, no mutation of shared state). Configuration, routing and any distributed failure guard belong to + * the caller, and this helper owns no MPI requests. + * + * \param options The already-captured budget; its thread count is T. The environment is not read. + * \param initialize Callable as `initialize(size_t shard)`, returning a non-null + * `std::unique_ptr>`. + * \return T non-null states, in shard order. + * \throws std::invalid_argument if `options.threads` is not positive, before anything is allocated. + * \throws std::logic_error if an initializer returns a null state. + * \throws The original exception of the lowest-numbered failing owner, after the team has joined. States other owners + * had published are then destroyed on the caller; a failing initializer's own locals unwind inside it. + */ +template + requires std::is_same_v, std::unique_ptr>> +auto make_shards(parallel::Options options, Initialize &&initialize) -> Shards { + // Checked before the negative value could become a container size. + if (options.threads < 1) { + throw std::invalid_argument( + std::format("sharded::make_shards: the team size must be positive, got {}", options.threads)); + } + auto shards = Shards(static_cast(options.threads)); + const auto error = run_team(options, [&](size_t shard, TeamFailure &failure) noexcept { + // The only phase: its decision ends the sequence either way. + static_cast(phase(failure, shard, [&] { + auto state = std::invoke(initialize, shard); + if (!state) { + throw std::logic_error( + std::format("sharded::make_shards: the initializer for shard {} returned no state", shard)); + } + shards[shard] = std::move(state); + })); + }); + if (error) { + std::rethrow_exception(error); + } + return shards; +} + +/*! + * \brief Seed one rank's T shards from caller-prepared inputs, each on its owner. + * + * Shard t seeds flat owner `rank * T + t` through seed_operator(). An empty share still yields a valid empty state, + * and its owner still participates in the team. + * + * \param options The already-captured budget; its thread count T must equal `seed.router.shards_per_rank()`. + * \param seed Prepared inputs; see OperatorSeed. + * \param rank This process's rank, below `seed.router.ranks()`. + * \param observer Test-only seam; see NoShardObserver. + * \return This rank's T seeded states. + * \throws std::invalid_argument if T is not positive, disagrees with the router's shard count, or `rank` is out of + * range, before the team starts. + * \throws As make_shards() for owner failures. + */ +template +auto seed_shards(parallel::Options options, + const OperatorSeed &seed, + size_t rank, + const Observer &observer = {}) -> Shards { + if (options.threads < 1 || seed.router.shards_per_rank() != static_cast(options.threads) + || rank >= seed.router.ranks()) { + throw std::invalid_argument(std::format("sharded::seed_shards: {} shards on rank {} do not fit routing " + "geometry ({} ranks, {} shards per rank)", + options.threads, + rank, + seed.router.ranks(), + seed.router.shards_per_rank())); + } + const auto threads = static_cast(options.threads); + return make_shards(options, [&](size_t shard) { + observer.begin(ShardWork::seed, shard); + // Built on the heap and seeded in place: no operator passes through this worker's frame. + auto state = std::make_unique>(seed.paired.has_value()); + seed_operator(seed, (rank * threads) + shard, state->op); + observer.end(ShardWork::seed, shard); + return state; + }); +} + +/*! + * \brief Copy T shards, each on the owner of the destination shard. + * + * Owner t deep-copies `source[t]` only. The source must not be mutated during the call; validity of the object that + * owns it is the caller's check, made before calling. The copy has the source's shard count, so a copy keeps the + * source's captured budget rather than resolving a new one. + * + * \param options The source owner's captured budget; its thread count must equal `source.size()`. + * \param source The states to copy; every entry non-null. + * \param observer Test-only seam; see NoShardObserver. + * \return T independent states. + * \throws std::invalid_argument if the shard count does not match or an entry is null, before the team starts. + * \throws As make_shards() for owner failures; the source is left unchanged. + */ +template +auto copy_shards(parallel::Options options, const Shards &source, const Observer &observer = {}) + -> Shards { + if (options.threads < 1 || source.size() != static_cast(options.threads) + || std::ranges::any_of(source, [](const auto &state) { return !state; })) { + throw std::invalid_argument(std::format("sharded::copy_shards: expected {} non-null source shards, got {}", + options.threads, + source.size())); + } + return make_shards(options, [&](size_t shard) { + observer.begin(ShardWork::copy, shard); + auto state = std::make_unique>(*source[shard]); + observer.end(ShardWork::copy, shard); + return state; + }); +} + +//! Rows over all shards of this rank, the rank-local operator size. +template +auto total_size(const Shards &shards) -> size_t { + size_t total = 0; + for (const auto &state : shards) { + total += state->size(); + } + return total; +} + +//! Operator accounting summed over this rank's shards, in shard order; the fields are additive. +template +auto operator_memory_usage(const Shards &shards) -> MPOperatorMemoryBreakdown { + auto total = MPOperatorMemoryBreakdown{}; + for (const auto &state : shards) { + total += state->operator_memory_usage(); + } + return total; +} + +//! Graph accounting summed over this rank's shards, in shard order. +template +auto graph_memory_usage(const Shards &shards) -> GraphMemoryBreakdown { + auto total = GraphMemoryBreakdown{}; + for (const auto &state : shards) { + total += state->graph_memory_usage(); + } + return total; +} + +} // namespace monoprop::detail::sharded diff --git a/cpp/monoprop/detail/sharded/Team.h b/cpp/monoprop/detail/sharded/Team.h new file mode 100644 index 000000000..f8a294fbe --- /dev/null +++ b/cpp/monoprop/detail/sharded/Team.h @@ -0,0 +1,294 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/Options.h" + +/* + * Operation-scoped team and phase protocol for sharded execution. + * + * One OpenMP team of exactly T workers spans a whole operation. Worker t owns shard t for the duration of one + * run_team() call; this is an OpenMP team index, not a persistent OS-thread identity across calls. The + * orchestration body runs a common sequence of phases on every worker, each ending in a collective checkpoint. + * + * Failure protocol, for a TeamFailure shared by the team. A checkpoint is one barrier; its generation is the number + * of checkpoints the calling worker has passed before it, identical on every worker because all of them run the same + * phase sequence. + * + * - Error slots. Slot t is written only by worker t, through record() inside its own phase, before that + * worker arrives at the checkpoint's barrier. Nobody else writes it during the region. + * - Stamping. record() also lowers the shared first-failed generation to the recording worker's current generation, + * before that worker arrives at the barrier. The value only ever decreases. + * - Decision. After the barrier, each worker reads the first-failed generation and proceeds past checkpoint N if and + * only if it is greater than N. The barrier orders every stamp of phase N before every read for N. + * - Generations. A fast worker that leaves checkpoint N and fails in phase N + 1 stamps N + 1, which is still greater + * than N, so a slower worker still reading for N sees the same decision. A failure in phase N + 1 is seen by + * checkpoint N + 1, which every worker reaches because it proceeds past N. Once a checkpoint fails, every later + * generation is greater than the stamp, so no later checkpoint can proceed. + * - Join. Errors are delivered only after the implicit barrier ending the parallel region, when every worker + * has returned from the orchestration body. No worker can still be inside a phase, touching buffers or + * waiting at a barrier, when the caller inspects or acts on the error. + * + * run_team() returns the error instead of rethrowing it: a future caller may still own posted MPI requests + * and the buffers they reference, and must decide on invalidation, abort or rethrow before any destructor + * can wait on a peer. This header performs no MPI, owns no threads, queues or persistent state, and never + * changes an OpenMP runtime setting. + */ + +namespace monoprop::detail::sharded { + +//! The default checkpoint observer: does nothing. +struct NoCheckpointObserver { + //! Ignore the checkpoint generation. + auto operator()(std::size_t /*generation*/) const noexcept -> void {} +}; + +/*! + * \brief Operation-local failure state for one team: one error slot per worker and the first failed generation. + * + * Constructed by run_team() before its parallel region; not copyable or movable, so the team always shares + * one instance. The ownership, publication and generation rules are described at the top of this header. + */ +class TeamFailure { +public: + /*! + * \brief Allocate one empty error slot per worker. + * \param threads The team size T. + * \throws std::bad_alloc if the slots cannot be allocated. + */ + explicit TeamFailure(std::size_t threads) : slots_(threads) {} + + TeamFailure(const TeamFailure &) = delete; //!< The team shares one instance. + auto operator=(const TeamFailure &) -> TeamFailure & = delete; //!< The team shares one instance. + TeamFailure(TeamFailure &&) = delete; //!< The team shares one instance. + auto operator=(TeamFailure &&) -> TeamFailure & = delete; //!< The team shares one instance. + ~TeamFailure() = default; //!< Releases the captured errors. + + /*! + * \brief Record a worker's exception in its own slot; a slot keeps its first nonempty error. + * + * Inside a team, only worker `shard` may call this, and only before it reaches the next checkpoint. Lowers the + * first failed generation to that worker's current one, so the next checkpoint fails on every worker. An empty + * `error` records nothing. + * + * \param shard The calling worker's index; must be less than the team size. + * \param error The captured exception. + */ + auto record(std::size_t shard, std::exception_ptr error) noexcept -> void { + if (!error) { + return; + } + auto &slot = slots_[shard]; + if (!slot.error) { + slot.error = std::move(error); + } + auto first = first_failed_.load(std::memory_order_relaxed); + while (slot.generation < first + && !first_failed_.compare_exchange_weak(first, slot.generation, std::memory_order_release)) { + } + } + + /*! + * \brief Collective checkpoint: every worker of the team must call it, in the same phase sequence. + * + * Must be called directly from the orchestration body (or code it calls), never inside a worksharing, + * `single`, `masked` or `critical` construct, and never from a nested region. Outside any parallel + * region it binds to a one-thread team, whose worker is 0. + * + * \param after_barrier Test-only observer, called as `after_barrier(generation)` on each worker between the + * barrier and its read of the decision, so tests can delay that read deterministically. Must not throw. + * \return True, identically on every worker, if no error was recorded at or before this checkpoint's + * generation. + */ + template + auto checkpoint(Observer &&after_barrier = {}) noexcept -> bool { + // The barrier orders every record() of this phase before any worker's read below. A one-worker team has nothing + // to order and runs without a region (run_team()), so it skips the barrier's runtime call. + if (slots_.size() > 1) { +#pragma omp barrier + } + auto &slot = slots_[static_cast(omp_get_thread_num())]; + const auto generation = slot.generation++; + std::invoke(after_barrier, generation); + return first_failed_.load(std::memory_order_acquire) > generation; + } + + /*! + * \brief The error of the lowest-numbered failing worker, or an empty pointer. + * + * Call only after the team has joined, or outside any parallel region. + */ + [[nodiscard]] auto first_error() const noexcept -> std::exception_ptr { + const auto found = std::ranges::find_if(slots_, [](const Slot &slot) { return static_cast(slot.error); }); + return found == slots_.end() ? std::exception_ptr{} : found->error; + } + +private: + // One cache line per slot: each worker advances its own generation at every checkpoint. + static constexpr std::size_t slot_alignment = 64; + static constexpr std::size_t no_failure = std::numeric_limits::max(); + + //! One worker's failure state. + struct alignas(slot_alignment) Slot { + std::exception_ptr error; //!< The worker's first recorded error. + std::size_t generation = 0; //!< Checkpoints this worker has passed. + }; + + std::vector slots_; //!< Slot t: written by worker t only; see the header comment. + std::atomic first_failed_{no_failure}; //!< Lowest generation with a recorded error. +}; + +/*! + * \brief An orchestration body for run_team(): noexcept, callable as `body(shard, failure)`. + * + * Anything that may throw belongs inside phase(); an exception escaping the body terminates the program. + */ +template +concept TeamBody = std::is_nothrow_invocable_v; + +/*! + * \brief Run `body(shard, failure)` once on every worker of one OpenMP team of `options.threads` workers. + * + * Allocates the failure state, then opens one `omp parallel num_threads(options.threads)` region. Each worker + * calls the body once with its OpenMP worker index, which stays fixed for that call, and the shared + * TeamFailure. Worker 0 is the primary: the thread that called run_team(). The body is invoked concurrently + * on the same object, so it must not mutate its own state without synchronization. + * + * A one-worker team opens no region: the caller runs the body directly as worker 0, at its own OpenMP level, and + * checkpoints skip their barrier. The outcome is the same as a one-thread region's; only the runtime's per-region and + * per-barrier cost, a fixed cost on every phase of a small operation, is saved. + * + * Preconditions, not checked: the caller is outside any OpenMP region, and the launch supplies exactly + * `options.threads` workers (`OMP_DYNAMIC=FALSE`, no thread limit below the budget). There is no reduced-team + * fallback; unlike parallel::for_blocks(), correctness requires the full team. + * + * \param options Requested team size; `options.threads` must be positive. + * \param body The noexcept orchestration body, a TeamBody. + * \return After every worker has returned from the body and the team has joined: an empty pointer on + * success, otherwise the original exception of the lowest-numbered failing worker. Phase exceptions + * are never rethrown here. + * \throws std::invalid_argument if `options.threads` is not positive, before the region. + * \throws std::bad_alloc if the failure state cannot be allocated, before the region. + */ +template + requires TeamBody +auto run_team(parallel::Options options, Fn &&body) -> std::exception_ptr { + if (options.threads < 1) { + throw std::invalid_argument( + std::format("sharded::run_team: the team size must be positive, got {}", options.threads)); + } + auto failure = TeamFailure(static_cast(options.threads)); + if (options.threads == 1) { + std::invoke(body, std::size_t{0}, failure); + return failure.first_error(); + } +#pragma omp parallel num_threads(options.threads) + { + std::invoke(body, static_cast(omp_get_thread_num()), failure); + } + return failure.first_error(); +} + +/*! + * \brief Run one worker's local part of a phase, then the collective checkpoint. + * + * Invokes `body()` and catches every exception, including values not derived from `std::exception`, + * recording it in the calling worker's slot. The worker then reaches the checkpoint whether its body had + * work, had none or failed. + * + * Every worker must call phase() for every phase of the common sequence and check the result: false ends the + * sequence on every worker, and no later phase body may run. The body must not contain synchronization + * (barriers, locks another worker needs, MPI waits) that an exception could bypass. + * + * \param failure The team's failure state, as passed to the orchestration body. + * \param shard The calling worker's index. + * \param body The local phase work, callable with no arguments. + * \return The checkpoint decision, identical on every worker. + */ +template + requires std::invocable +auto phase(TeamFailure &failure, std::size_t shard, Fn &&body) noexcept -> bool { + try { + std::invoke(body); + } + catch (...) { + failure.record(shard, std::current_exception()); + } + return failure.checkpoint(); +} + +/*! + * \brief Admits at most a fixed number of a team's workers into a section at once. + * + * For memory, not correctness: a section whose transient footprint grows with the number of workers inside it (each + * copying a large block, say) is entered by at most `limit` workers, while the others yield until a slot frees. The + * order of entry is unspecified, and every worker that calls run() enters exactly once. + * + * Shared by the team for one section; not copyable or movable. The section must not synchronize with other workers + * (no barrier, no wait on another worker's progress), or the workers queued outside it could never be admitted. + */ +class SectionLimit { +public: + //! A limit of zero admits one worker at a time. + explicit SectionLimit(std::size_t limit) noexcept + : free_(static_cast(std::max(limit, 1))) {} + SectionLimit(const SectionLimit &) = delete; + auto operator=(const SectionLimit &) -> SectionLimit & = delete; + + /*! + * \brief Wait for a slot, run `section` and release the slot, also when `section` throws. + * \return What `section` returns. + */ + template + requires std::invocable + auto run(Fn &§ion) -> decltype(auto) { + for (auto free = free_.load(std::memory_order_relaxed);;) { + if (free > 0) { + if (free_.compare_exchange_weak(free, free - 1, std::memory_order_acquire, std::memory_order_relaxed)) { + break; + } + continue; + } + std::this_thread::yield(); + free = free_.load(std::memory_order_relaxed); + } + struct Release { + std::atomic &free; + ~Release() { free.fetch_add(1, std::memory_order_release); } + } release{free_}; + return std::invoke(section); + } + +private: + std::atomic free_; // slots not held +}; + +} // namespace monoprop::detail::sharded diff --git a/cpp/tests/AllocationProbe.cpp b/cpp/tests/AllocationProbe.cpp new file mode 100644 index 000000000..5309767b5 --- /dev/null +++ b/cpp/tests/AllocationProbe.cpp @@ -0,0 +1,190 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "AllocationProbe.h" + +#if defined(__APPLE__) +#include +#else +#include +#endif + +#include +#include +#include +#include + +/* + * Replacement of every replaceable global allocation and deallocation function, so allocation and + * deallocation stay paired on malloc/free under ASan/TSan, whose own operator new/delete this replaces. + * The thread-local counters are constant-initialized PODs, so reading them inside operator new needs no + * dynamic TLS initialization. No new_handler is consulted: a test build has none installed. + */ + +namespace { + +auto usable_size(void *block) noexcept -> std::size_t { +#if defined(__APPLE__) + return malloc_size(block); +#else + return malloc_usable_size(block); +#endif +} + +thread_local std::size_t t_bytes = 0; +thread_local std::size_t t_fail_after = 0; // 0: disarmed; n: the n-th next allocation fails + +std::atomic g_tracking{false}; +std::atomic g_live{0}; + +auto should_fail() noexcept -> bool { + return t_fail_after != 0 && --t_fail_after == 0; +} + +auto track_allocation(void *block) noexcept -> void { + if (block != nullptr && g_tracking.load(std::memory_order_relaxed)) { + g_live.fetch_add(static_cast(usable_size(block)), std::memory_order_relaxed); + } +} + +auto release(void *block) noexcept -> void { + if (block != nullptr && g_tracking.load(std::memory_order_relaxed)) { + g_live.fetch_sub(static_cast(usable_size(block)), std::memory_order_relaxed); + } + std::free(block); +} + +auto allocate(std::size_t size, std::size_t alignment) noexcept -> void * { + if (should_fail()) { + return nullptr; + } + t_bytes += size; + size = size == 0 ? 1 : size; + void *block = nullptr; + if (alignment <= alignof(std::max_align_t)) { + block = std::malloc(size); + } + else { + block = std::aligned_alloc(alignment, (size + alignment - 1) / alignment * alignment); + } + track_allocation(block); + return block; +} + +auto allocate_or_throw(std::size_t size, std::size_t alignment) -> void * { + void *block = allocate(size, alignment); + if (block == nullptr) { + throw std::bad_alloc(); + } + return block; +} + +} // namespace + +namespace test_utils::allocation { + +auto thread_bytes() noexcept -> std::size_t { + return t_bytes; +} + +auto arm_failure(std::size_t nth) noexcept -> void { + t_fail_after = nth; +} + +auto disarm_failure() noexcept -> bool { + const bool pending = t_fail_after != 0; + t_fail_after = 0; + return pending; +} + +LiveBytes::LiveBytes() noexcept { + g_live.store(0); + g_tracking.store(true); +} + +LiveBytes::~LiveBytes() { + g_tracking.store(false); +} + +auto LiveBytes::net() const noexcept -> long long { + return g_live.load(); +} + +} // namespace test_utils::allocation + +#if MONOPROP_TEST_ALLOCATION_PROBE +// NOLINTBEGIN(misc-new-delete-overloads) +auto operator new(std::size_t size) -> void * { + return allocate_or_throw(size, alignof(std::max_align_t)); +} +auto operator new[](std::size_t size) -> void * { + return allocate_or_throw(size, alignof(std::max_align_t)); +} +auto operator new(std::size_t size, std::align_val_t alignment) -> void * { + return allocate_or_throw(size, static_cast(alignment)); +} +auto operator new[](std::size_t size, std::align_val_t alignment) -> void * { + return allocate_or_throw(size, static_cast(alignment)); +} +auto operator new(std::size_t size, const std::nothrow_t &) noexcept -> void * { + return allocate(size, alignof(std::max_align_t)); +} +auto operator new[](std::size_t size, const std::nothrow_t &) noexcept -> void * { + return allocate(size, alignof(std::max_align_t)); +} +auto operator new(std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept -> void * { + return allocate(size, static_cast(alignment)); +} +auto operator new[](std::size_t size, std::align_val_t alignment, const std::nothrow_t &) noexcept -> void * { + return allocate(size, static_cast(alignment)); +} + +auto operator delete(void *block) noexcept -> void { + release(block); +} +auto operator delete[](void *block) noexcept -> void { + release(block); +} +auto operator delete(void *block, std::size_t) noexcept -> void { + release(block); +} +auto operator delete[](void *block, std::size_t) noexcept -> void { + release(block); +} +auto operator delete(void *block, std::align_val_t) noexcept -> void { + release(block); +} +auto operator delete[](void *block, std::align_val_t) noexcept -> void { + release(block); +} +auto operator delete(void *block, std::size_t, std::align_val_t) noexcept -> void { + release(block); +} +auto operator delete[](void *block, std::size_t, std::align_val_t) noexcept -> void { + release(block); +} +auto operator delete(void *block, const std::nothrow_t &) noexcept -> void { + release(block); +} +auto operator delete[](void *block, const std::nothrow_t &) noexcept -> void { + release(block); +} +auto operator delete(void *block, std::align_val_t, const std::nothrow_t &) noexcept -> void { + release(block); +} +auto operator delete[](void *block, std::align_val_t, const std::nothrow_t &) noexcept -> void { + release(block); +} +// NOLINTEND(misc-new-delete-overloads) +#endif diff --git a/cpp/tests/AllocationProbe.h b/cpp/tests/AllocationProbe.h new file mode 100644 index 000000000..a25a80744 --- /dev/null +++ b/cpp/tests/AllocationProbe.h @@ -0,0 +1,79 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +/* + * Allocation observation and controlled allocation failure for the unit-test executable. + * + * AllocationProbe.cpp replaces the global operator new/delete family for the whole of + * monoprop_unit_tests.x (and only it; the library and installed consumers keep the default). While + * nothing is armed the replacement is a plain malloc/free pass-through that also counts, per thread, + * the bytes requested through operator new. A test can make the n-th following operator new on the + * calling thread throw std::bad_alloc (the nothrow forms return nullptr), which exercises the real + * allocation sites of a constructor instead of a callback that merely throws. + * + * Live-byte tracking is process-wide and only meaningful while no unrelated thread allocates. + */ + +#include + +/* + * Sanitizer runtimes linked statically (Clang's ASan/TSan, GCC's TSan) define the operator new/delete family + * themselves, so the replacement is compiled out there: available() is false, thread_bytes() stays 0 and nothing + * can be armed. Cases that need the probe skip through a precondition. + */ +#if defined(__SANITIZE_THREAD__) +#define MONOPROP_TEST_ALLOCATION_PROBE 0 +#elif defined(__clang__) && defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) || __has_feature(memory_sanitizer) +#define MONOPROP_TEST_ALLOCATION_PROBE 0 +#endif +#endif +#ifndef MONOPROP_TEST_ALLOCATION_PROBE +#define MONOPROP_TEST_ALLOCATION_PROBE 1 +#endif + +namespace test_utils::allocation { + +//! Whether this build replaces operator new/delete, so the functions below observe and inject anything. +constexpr auto available() noexcept -> bool { + return MONOPROP_TEST_ALLOCATION_PROBE != 0; +} + +//! Bytes requested through operator new on the calling thread since it started. +auto thread_bytes() noexcept -> std::size_t; + +//! Make the `nth` (>= 1) next operator new on the calling thread fail; 0 disarms. +auto arm_failure(std::size_t nth) noexcept -> void; + +//! Disarm the calling thread; returns true if an armed failure had not fired yet. +auto disarm_failure() noexcept -> bool; + +/*! + * \brief Process-wide live-byte accounting over a scope: blocks allocated minus blocks freed, in usable + * bytes, through operator new/delete while the scope is active. Not nestable. + */ +class LiveBytes { +public: + LiveBytes() noexcept; //!< Starts tracking from zero. + ~LiveBytes(); //!< Stops tracking. + LiveBytes(const LiveBytes &) = delete; //!< One active scope. + auto operator=(const LiveBytes &) -> LiveBytes & = delete; //!< One active scope. + + //! Net usable bytes allocated and not yet freed since construction. + [[nodiscard]] auto net() const noexcept -> long long; +}; + +} // namespace test_utils::allocation diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 50d224e7c..423f715a6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -23,6 +23,8 @@ file( CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/*.cpp" ) +# Its own main() ends every scenario by aborting the job, so it stays out of the unit runner. +list(FILTER _tests_cpps EXCLUDE REGEX "/mpi_failure_driver\\.cpp$") add_executable(monoprop_unit_tests.x ${_tests_cpps}) @@ -47,7 +49,6 @@ target_link_libraries( monoprop-objs Boost::unit_test_framework msgpack-cxx - PkgConfig::HWLOC ) include(${CMAKE_CURRENT_LIST_DIR}/boost-test.cmake) @@ -72,6 +73,16 @@ if(monoprop_TEST_EXCLUDE_MPI_FABRIC AND monoprop_ENABLE_MPI) ) endif() +# The launch-time thread policy of every discovered case, set by the registration because the runner supplies no +# budget: ordinary (raw-layout) cases run at exactly T = 1, per case and in the whole-suite MPI variants alike; teams of +# T > 1 are dedicated fresh-process launches below, and kernel cases request their own teams explicitly. +set( + _monoprop_unit_environment + "monoprop_NUM_THREADS=1" + "OMP_NUM_THREADS=1" + "OMP_DYNAMIC=FALSE" +) + # Automatic discovery of unit tests. # Default CTest run includes per-case serial tests plus suite-level MPI variants # for monoprop_MPI_TEST_PROCS. @@ -80,8 +91,473 @@ discover_tests( PROPERTIES LABELS "unit" + ENVIRONMENT + ${_monoprop_unit_environment} SERIAL_ENVIRONMENT ${_monoprop_serial_env_entries} ) +# Fresh processes for the construction-time thread budget: each entry reruns one unit case under its own +# launch environment, since an object captures monoprop_NUM_THREADS once, when it is constructed. +function(monoprop_add_launch_env_case NAME CASE) + cmake_parse_arguments("" "" "" "ENVIRONMENT;ENVIRONMENT_MODIFICATION" ${ARGN}) + add_test( + NAME ${NAME} + COMMAND + monoprop_unit_tests.x --run_test=${CASE} + ) + set( + _environment + ${_monoprop_serial_env_entries} + ${_ENVIRONMENT} + ) + set_tests_properties( + ${NAME} + PROPERTIES + LABELS + "unit;openmp_env" + ENVIRONMENT + "${_environment}" + ) + if(_ENVIRONMENT_MODIFICATION) + set_tests_properties( + ${NAME} + PROPERTIES + ENVIRONMENT_MODIFICATION + "${_ENVIRONMENT_MODIFICATION}" + ) + endif() +endfunction() + +# Budget capture at the launch: every root captures it, and its T is its shard count, so each launch names an exact +# team. The unset launch removes monoprop_NUM_THREADS whatever the caller's environment holds, so the OpenMP runtime +# default decides; neither CTest nor the runner may supply it. +monoprop_add_launch_env_case( + openmp_env_root_budget_1 + openmp_root_budget_matches_the_launch_environment + ENVIRONMENT "OMP_DYNAMIC=FALSE;monoprop_NUM_THREADS=1;monoprop_TEST_EXPECT_BUDGET=1" +) +monoprop_add_launch_env_case( + openmp_env_root_budget_3 + openmp_root_budget_matches_the_launch_environment + ENVIRONMENT "OMP_DYNAMIC=FALSE;monoprop_NUM_THREADS=3;monoprop_TEST_EXPECT_BUDGET=3" +) +monoprop_add_launch_env_case( + openmp_env_root_budget_openmp_default_3 + openmp_root_budget_matches_the_launch_environment + ENVIRONMENT "OMP_DYNAMIC=FALSE;OMP_NUM_THREADS=3;monoprop_TEST_EXPECT_BUDGET=3;monoprop_TEST_EXPECT_UNSET_BUDGET=3" + ENVIRONMENT_MODIFICATION "monoprop_NUM_THREADS=unset:" +) +# The runtime contract at an exact team of four: settings and affinity untouched, the budget read once, and the +# invalid-owner rule, all inside real multi-owner operations. +monoprop_add_launch_env_case( + openmp_env_root_t4 + "openmp_root_budget_matches_the_launch_environment,openmp_root_captures_once_and_copies_preserve_it,openmp_propagator_work_leaves_openmp_settings_unchanged,openmp_failed_build_invalidates_the_owner,openmp_validation_errors_leave_the_owner_usable,openmp_invalid_owner_can_still_be_destroyed" + ENVIRONMENT "OMP_DYNAMIC=FALSE;monoprop_NUM_THREADS=4;OMP_NUM_THREADS=4;monoprop_TEST_EXPECT_BUDGET=4" +) +monoprop_add_launch_env_case( + openmp_env_no_affinity_change + openmp_propagator_work_does_not_change_affinity + ENVIRONMENT "OMP_PROC_BIND=false;OMP_DYNAMIC=FALSE;monoprop_NUM_THREADS=2;OMP_NUM_THREADS=2" +) + +# The threaded cosine, fused-apply, scan and resolve kernels under runtime-limited and dynamic teams: exact results and +# participation must not depend on receiving the full budget. +set( + _monoprop_kernel_cases + "openmp_cos_mask_exact,openmp_cos_mask_workers_participate,openmp_cos_lazy_workers_participate,openmp_fused_apply_exact_on_disjoint_records,openmp_fused_apply_workers_participate,openmp_kernel_worker_failures_join_before_rethrow,combined_threaded_lazy_fold_matches_oracle,openmp_scan_ranges_match_serial_majorana,openmp_scan_workers_participate,openmp_scan_worker_failure_joins_before_rethrow,openmp_scan_prepares_cold_caches_on_the_caller,openmp_scan_pauli_ranges_match_serial,openmp_probe_frozen_positions_matches_one_batch,openmp_probe_frozen_positions_workers_participate,openmp_incoming_probe_threaded_matches_serial,openmp_incoming_decode_and_probe_workers_participate,openmp_incoming_decode_worker_failure_joins_before_probing,openmp_resolve_scatter_threaded_matches_serial,openmp_resolve_scatter_workers_participate,openmp_self_windows_match_serial,openmp_mixed_self_and_remote_publication_order" +) +# These exercise the kernels' own explicit budgets and for_blocks' reduced-team contract, which is independent of the +# root: no fixed-team root operation runs under these limited or dynamic teams. +monoprop_add_launch_env_case( + openmp_env_kernels_thread_limit_2 + "${_monoprop_kernel_cases}" + ENVIRONMENT "OMP_THREAD_LIMIT=2" +) +monoprop_add_launch_env_case( + openmp_env_kernels_dynamic_teams + "${_monoprop_kernel_cases}" + ENVIRONMENT "OMP_DYNAMIC=TRUE" +) + +# The fixed-team phase primitive (detail/sharded/Team.h) assumes exactly T workers, so its cases run only under +# exact-team launches, never under the limited or dynamic teams above, which test for_blocks' reduced-team contract. +# Each T is a fresh process; cases that need a nonprimary worker are left out of T = 1. The passive-wait launch parks +# waiting workers, which widens the windows the checkpoint-generation cases probe. +set( + _monoprop_sharded_team_single_cases + "sharded_team_visits_each_owner,sharded_team_observes_worker_identity,sharded_team_keeps_owner_across_phases,sharded_team_primary_only_phase_runs_on_the_caller,sharded_team_empty_owners_reach_the_checkpoint,sharded_team_publishes_writes_to_the_next_phase,sharded_team_single_worker_failure_is_returned_after_join,sharded_team_failure_is_returned_not_rethrown,sharded_team_failure_selects_the_lowest_recorded_slot,sharded_team_preserves_the_original_exception,sharded_team_fresh_invocation_has_no_stale_failure,sharded_team_rejects_a_nonpositive_budget_before_the_region,sharded_team_leaves_runtime_settings_unchanged,sharded_team_launch_supplies_the_expected_team,sharded_team_section_limit_bounds_concurrent_holders,sharded_team_section_limit_releases_on_exception,sharded_team_section_limit_admits_at_least_one" +) +foreach(_threads IN ITEMS 1 2 4) + if(_threads EQUAL 1) + set(_cases "${_monoprop_sharded_team_single_cases}") + else() + set(_cases "sharded_team_*") + endif() + set( + _team_environment + "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE;monoprop_TEST_EXPECT_TEAM=${_threads}" + ) + monoprop_add_launch_env_case( + sharded_team_env_t${_threads} + "${_cases}" + ENVIRONMENT "${_team_environment}" + ) + set_tests_properties( + sharded_team_env_t${_threads} + PROPERTIES + TIMEOUT + 120 + ) +endforeach() +monoprop_add_launch_env_case( + sharded_team_env_t4_passive + "sharded_team_*" + ENVIRONMENT "monoprop_NUM_THREADS=4;OMP_NUM_THREADS=4;OMP_DYNAMIC=FALSE;OMP_WAIT_POLICY=PASSIVE;monoprop_TEST_EXPECT_TEAM=4" +) +set_tests_properties( + sharded_team_env_t4_passive + PROPERTIES + TIMEOUT + 120 +) + +# Owner-initialized shard state (detail/sharded/State.h) under the same exact-team launches: one fresh process per T, +# never a limited or dynamic team. Cases that need a nonprimary worker are left out of T = 1. +set( + _monoprop_sharded_state_single_cases + "sharded_state_make_shards_initializes_each_owner_once,sharded_state_seed_matches_an_independent_reference,sharded_state_routed_rows_belong_to_their_flat_owner,sharded_state_keeps_sparse_heisenberg_and_dense_schrodinger_caches,sharded_state_counts_and_accounting_sum_over_shards,sharded_state_rejects_invalid_arguments_before_the_team,sharded_state_copy_is_exact_and_owner_allocated,sharded_state_copies_mutate_independently,sharded_state_graph_cores_are_shared_and_lists_independent,sharded_state_copy_keeps_the_captured_budget,sharded_state_initializer_failure_on_the_primary_is_returned,sharded_state_seed_allocation_failures_are_catchable,sharded_state_copy_allocation_failures_leave_sources_intact" +) +foreach(_threads IN ITEMS 1 2 4) + if(_threads EQUAL 1) + set(_cases "${_monoprop_sharded_state_single_cases}") + else() + set(_cases "sharded_state_*") + endif() + monoprop_add_launch_env_case( + sharded_state_env_t${_threads} + "${_cases}" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) + set_tests_properties( + sharded_state_env_t${_threads} + PROPERTIES + TIMEOUT + 300 + ) +endforeach() + +# evolved_operator_coefficients() probes every shard of the rank: its suite reruns at T = 2 and 4, where the keys spread +# over several shards, against the shard-transparent evolved_operator_terms() oracle at the same T. +foreach(_threads IN ITEMS 2 4) + monoprop_add_launch_env_case( + evolved_coefficients_env_t${_threads} + "evolved_coefficients_*" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) +endforeach() + +# term_expectation_values() sweeps every shard's state back through its layers and gathers each key from the one shard +# holding it: its suite reruns at T = 2 and 4 against the shard-transparent expectation_value() oracle at the same T. +foreach(_threads IN ITEMS 2 4) + monoprop_add_launch_env_case( + term_expectation_values_env_t${_threads} + "term_expectation_values_*" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) +endforeach() + +# Direct-buffer construction and propagation (detail/sharded/Construction.h) under the same exact-team launches: one +# fresh process per T. Cases that need a nonprimary worker are left out of T = 1. A timeout is a hang, not a pass. +set( + _monoprop_sharded_construction_single_cases + "sharded_construction_rows_follow_the_ordering_contract,sharded_construction_propagation_matches_an_independent_reference,sharded_construction_owners_do_their_own_work,sharded_construction_opaque_cutoff_traverses_on_the_primary,sharded_construction_empty_operators_circuits_and_identity,sharded_construction_published_views_cover_the_window,sharded_construction_published_resolution_matches_owning_blocks,sharded_construction_large_self_streams_match_the_root,sharded_construction_wide_terms_match_the_root,sharded_construction_failure_in_each_phase_suppresses_later_work,sharded_construction_allocation_failures_are_contained,sharded_construction_rejects_invalid_arguments_before_the_team" +) +foreach(_threads IN ITEMS 1 2 4) + if(_threads EQUAL 1) + set(_cases "${_monoprop_sharded_construction_single_cases}") + else() + set(_cases "sharded_construction_*") + endif() + monoprop_add_launch_env_case( + sharded_construction_env_t${_threads} + "${_cases}" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) + set_tests_properties( + sharded_construction_env_t${_threads} + PROPERTIES + TIMEOUT + 900 + ) +endforeach() + +# Snapshot-safe replay, energy, gradients, retained/pared functionals and coefficient-informed construction +# (detail/sharded/Evaluation.h, Construction.h) under the same exact-team launches: one fresh process per T. Cases that +# need a nonprimary worker are left out of T = 1. A timeout is a hang, not a pass. +set( + _monoprop_sharded_evaluation_single_cases + "sharded_evaluation_identity_once_and_empty_parameters,sharded_evaluation_rejects_invalid_arguments_before_the_team,sharded_evaluation_matches_independent_references,sharded_evaluation_deep_circuit_gradients,sharded_evaluation_repeated_parameters_and_duplicate_records,sharded_evaluation_pared_functionals_match_the_root,sharded_evaluation_cosine_counts_reserve_records_exactly,sharded_evaluation_empty_stored_mask_applies_nothing,sharded_evaluation_repeated_instances_copies_and_index_growth,sharded_evaluation_owners_do_their_own_work,sharded_evaluation_opaque_callbacks_run_on_the_primary,sharded_evaluation_combination_folds_in_shard_order,sharded_evaluation_failure_in_each_phase_suppresses_later_work,sharded_evaluation_callback_paring_and_construction_failures,sharded_evaluation_allocation_failures_are_contained" +) +foreach(_threads IN ITEMS 1 2 4) + if(_threads EQUAL 1) + set(_cases "${_monoprop_sharded_evaluation_single_cases}") + else() + set(_cases "sharded_evaluation_*") + endif() + monoprop_add_launch_env_case( + sharded_evaluation_env_t${_threads} + "${_cases}" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) + set_tests_properties( + sharded_evaluation_env_t${_threads} + PROPERTIES + TIMEOUT + 900 + ) +endforeach() + +# The root (MonomialPropagator over detail/sharded/*) through its public operations, with private shard inspection, +# phase observation and fault injection (PropagatorTestAccess.h). One fresh process per T, exact teams only; a timeout +# is a hang, not a pass. Cases needing a nonprimary worker skip themselves at T = 1. +foreach(_threads IN ITEMS 1 2 4) + monoprop_add_launch_env_case( + sharded_root_env_t${_threads} + "sharded_root_*" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE;monoprop_TEST_EXPECT_TEAM=${_threads}" + ) + set_tests_properties( + sharded_root_env_t${_threads} + PROPERTIES + TIMEOUT + 900 + ) +endforeach() + +# The fused-sweep, two-pass and real-record cases (fused_cos_sweep_tests.cpp) at fixed teams of two and four: each +# launch audits its own shards' records and agrees with itself bitwise; geometries are compared through complete +# retained maps in tests/test_sharded_openmp.py, never inside one process. +foreach(_threads IN ITEMS 2 4) + monoprop_add_launch_env_case( + fused_env_t${_threads} + "fused_*,two_pass_apply_matches_graph_replay" + ENVIRONMENT "monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE" + ) + set_tests_properties( + fused_env_t${_threads} + PROPERTIES + TIMEOUT + 900 + ) +endforeach() + +# No monoprop_NUM_THREADS at all, whatever the caller's environment holds: the root takes the OpenMP default, so +# nothing between CTest and the library may supply the variable. +monoprop_add_launch_env_case( + sharded_root_env_openmp_default_t3 + "sharded_root_unset_budget_takes_the_openmp_default,sharded_root_owns_t_routed_shards" + ENVIRONMENT "OMP_NUM_THREADS=3;OMP_DYNAMIC=FALSE;monoprop_TEST_EXPECT_TEAM=3;monoprop_TEST_EXPECT_UNSET_BUDGET=3" + ENVIRONMENT_MODIFICATION "monoprop_NUM_THREADS=unset:" +) +set_tests_properties( + sharded_root_env_openmp_default_t3 + PROPERTIES + TIMEOUT + 900 +) + +# Multi-process launches of the sharded runtime: P ranks x T threads each, every rank with an exact team. The +# multi-rank cases run only where monoprop_TEST_SHARDED_RANKS names the world size, so the whole-suite MPI variants +# (default budget) skip them. Binding is left to the OpenMP settings: Open MPI would otherwise bind each rank to one +# core and squeeze its T threads onto it (unknown OMPI_MCA_* variables are harmless under other MPIs). Oversubscription +# is allowed in both Open MPI spellings, so a direct `ctest -L mpi` fits 3- and 4-rank launches onto smaller hosts. +function(monoprop_add_sharded_mpi_launch NAME RANKS THREADS CASES) + cmake_parse_arguments("" "" "" "ENVIRONMENT" ${ARGN}) + add_test( + NAME ${NAME} + COMMAND + ${_monoprop_mpiexec} ${_monoprop_mpiexec_numproc_flag} ${RANKS} + ${MPIEXEC_PREFLAGS} $ + --run_test=${CASES} + ) + set_tests_properties( + ${NAME} + PROPERTIES + LABELS + "mpi;mpi-${RANKS}" + TIMEOUT + 300 + ENVIRONMENT + "OMPI_ALLOW_RUN_AS_ROOT=1;OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1;OMPI_MCA_hwloc_base_binding_policy=none;OMPI_MCA_rmaps_base_oversubscribe=1;PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe;monoprop_NUM_THREADS=${THREADS};OMP_NUM_THREADS=${THREADS};OMP_DYNAMIC=FALSE;monoprop_TEST_EXPECT_TEAM=${THREADS};monoprop_TEST_SHARDED_RANKS=${RANKS};${_ENVIRONMENT}" + ) +endfunction() + +# The physical round (detail/sharded/Exchange.h) between processes, in every MPI build: two and three ranks at +# T = 1, 2 and 4. At world size 1 (the per-case serial runs) the multi-rank cases skip. A timeout is a hang. +if(monoprop_ENABLE_MPI AND _monoprop_mpiexec) + foreach(_ranks IN ITEMS 2 3) + foreach(_threads IN ITEMS 1 2 4) + monoprop_add_sharded_mpi_launch( + sharded_exchange_mpi${_ranks}_t${_threads} ${_ranks} ${_threads} "sharded_exchange_*" + ) + endforeach() + endforeach() +endif() + +# The root over P ranks: two and four ranks (linear routing) at T = 1, 2 and 4; three ranks under splitmix; and three +# ranks under linear routing, which every rank must reject together. A hang is a failure. +if(monoprop_ENABLE_MPI AND _monoprop_mpiexec) + foreach(_ranks IN ITEMS 2 4) + foreach(_threads IN ITEMS 1 2 4) + monoprop_add_sharded_mpi_launch( + sharded_root_mpi${_ranks}_t${_threads} ${_ranks} ${_threads} "sharded_root_multirank_*" + ) + endforeach() + endforeach() + monoprop_add_sharded_mpi_launch( + sharded_root_mpi3_t2_splitmix 3 2 "sharded_root_multirank_*" ENVIRONMENT "monoprop_ROUTING=splitmix" + ) + monoprop_add_sharded_mpi_launch( + sharded_root_mpi3_t2_linear_rejected 3 2 "sharded_root_multirank_rejects_an_unroutable_geometry" + ) + # The serial-against-world equivalence suites, through the public API at P ranks x T. + set( + _monoprop_world_equivalence_cases + "rank_count_energy_within_fp_tolerance,gradient_rank_count_within_fp_tolerance,pauli_rank_count_energy_within_fp_tolerance,mpi_fresh_insert_schrodinger_majorana_serial_world_equiv,mpi_fresh_insert_schrodinger_pauli_serial_world_equiv" + ) + foreach(_ranks IN ITEMS 2 4) + foreach(_threads IN ITEMS 1 2 4) + monoprop_add_sharded_mpi_launch( + sharded_world_equivalence_mpi${_ranks}_t${_threads} ${_ranks} ${_threads} "${_monoprop_world_equivalence_cases}" + ) + endforeach() + endforeach() +endif() + +# Distributed failure and thread-support scenarios, each a separate two-rank launch with a 30 s timeout. Labelled mpi, +# so only the MPI test leg runs them (mpi_failure_driver.cpp lists them). +if(monoprop_ENABLE_MPI AND _monoprop_mpiexec) + add_executable(monoprop_mpi_failure_driver.x mpi_failure_driver.cpp) + target_include_directories( + monoprop_mpi_failure_driver.x + PRIVATE + ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_BINARY_DIR} + ) + target_link_libraries( + monoprop_mpi_failure_driver.x + PRIVATE + monoprop-sanitizers + monoprop-objs + ) + + set(_monoprop_abort_diagnostic "aborting the communicator") + function( + monoprop_add_failure_scenario + NAME + SCENARIO + OPTION + EXPECT + FORBID + ) + cmake_parse_arguments("" "SUCCESS" "" "" ${ARGN}) + add_test( + NAME mpi_failure_${NAME} + COMMAND + ${CMAKE_COMMAND} "-DMPIEXEC=${_monoprop_mpiexec}" + "-DNUMPROC_FLAG=${_monoprop_mpiexec_numproc_flag}" + "-DPREFLAGS=${MPIEXEC_PREFLAGS}" + "-DDRIVER=$" + "-DSCENARIO=${SCENARIO}" "-DOPTION=${OPTION}" "-DEXPECT=${EXPECT}" + "-DFORBID=${FORBID}" "-DEXPECT_SUCCESS=${_SUCCESS}" -DTIMEOUT_SECONDS=30 + -P "${CMAKE_CURRENT_LIST_DIR}/run_mpi_failure_scenario.cmake" + ) + set_tests_properties( + mpi_failure_${NAME} + PROPERTIES + LABELS + "mpi;mpi_failure" + TIMEOUT + 90 + SKIP_RETURN_CODE + 77 + # As for the suite's MPI variants: CI containers may run as root. Binding is left to OpenMP, so a rank's two + # threads are not squeezed onto one core. + ENVIRONMENT + "OMPI_ALLOW_RUN_AS_ROOT=1;OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1;OMPI_MCA_hwloc_base_binding_policy=none;OMPI_MCA_rmaps_base_oversubscribe=1;PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe;OMP_DYNAMIC=FALSE" + ) + endfunction() + + # Every failure scenario must end in the library's abort (or local fail-fast) path, never in an escaped exception. + monoprop_add_failure_scenario( + sharded_worker_throw worker-throw "" "injected replay-finish failure on OpenMP worker 1.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + sharded_before_exchange before-exchange "" "injected traversal failure.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + sharded_active_round active-ticket round "injected cosine failure while the replay round is posted.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + sharded_active_counts active-ticket counts "injected pack failure while the count round is posted.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + sharded_active_local_round active-ticket local-round "injected cosine failure while a caller-local replay round is posted.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + sharded_malformed_receive malformed-receive "" "QueryWire: malformed record.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + # The low-level one-owner-per-rank engine and the exported replay functions, on the sole shard of a T = 1 root. + # Worker 1 of a two-worker kernel team owns the failing range, so the failure starts off the calling thread. + monoprop_add_failure_scenario( + active_ticket active-ticket ticket "injected failure while the replay Ticket.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + active_ticket_cosine_worker active-ticket cosine-worker "cosine range 2 on OpenMP worker 1 while the replay Ticket.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + active_pending active-ticket pending "injected failure while a PendingAlltoallv.*${_monoprop_abort_diagnostic}" "UNGUARDED" + ) + monoprop_add_failure_scenario( + before_exchange_scan_worker before-exchange scan-worker "scan range 1 on OpenMP worker 1.*${_monoprop_abort_diagnostic}" "UNGUARDED|NOT REACHED" + ) + # Resolve workers: the self probe fails before the query exchange; the incoming decode, frozen probe and scatter fail + # after the query round while the peers enter the response round. Worker 1 owns the last range. + foreach(_phase self-probe decode incoming-probe scatter) + string( + REPLACE "-" + "_" + _phase_name + "${_phase}" + ) + monoprop_add_failure_scenario( + resolve_worker_${_phase_name} resolve-worker ${_phase} "${_phase} range [0-9]+ on OpenMP worker 1.*${_monoprop_abort_diagnostic}" "UNGUARDED|NOT REACHED" + ) + endforeach() + + # Thread support: FUNNELED is requested and required. Acceptance runs real distributed operations at two threads per + # rank and must complete; SINGLE is refused at construction, before any communication. + monoprop_add_failure_scenario( + library_initialized_level library-level "" "provided=MPI_THREAD_(FUNNELED|SERIALIZED|MULTIPLE).*thread-support operations completed" "UNGUARDED|${_monoprop_abort_diagnostic}" + SUCCESS + ) + foreach(_level funneled serialized multiple) + string(TOUPPER "${_level}" _upper) + monoprop_add_failure_scenario( + host_initialized_${_level} host-level ${_level} "thread-support operations completed with provided=MPI_THREAD_${_upper}" "UNGUARDED|${_monoprop_abort_diagnostic}" + SUCCESS + ) + endforeach() + monoprop_add_failure_scenario( + insufficient_thread_level_single insufficient-thread-level single "requires MPI_THREAD_FUNNELED.*provides MPI_THREAD_SINGLE" "UNGUARDED|despite" + ) + # Local fail-fast at every supported level: the offending thread must not call MPI_Abort (or anything else in MPI). + foreach(_level funneled serialized multiple) + monoprop_add_failure_scenario( + wrong_thread_entry_${_level} wrong-thread-entry ${_level} "thread that initialized MPI" "${_monoprop_abort_diagnostic}|completed" + ) + endforeach() +endif() + add_subdirectory(link_export_probe) diff --git a/cpp/tests/KernelTestSupport.h b/cpp/tests/KernelTestSupport.h new file mode 100644 index 000000000..26c594687 --- /dev/null +++ b/cpp/tests/KernelTestSupport.h @@ -0,0 +1,357 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// Test-only support for the threaded cosine and fused-apply kernels: a range observer that records which +// OpenMP worker ran each logical range, a synthetic inverted index large enough to span several fold +// blocks, and an independent add-owner count over fused rotation records. +// +// Workers never assert. RangeLog slots are sized by the kernel's prepare() on the calling thread, before +// any worker starts, and each slot is written only by the sole owner of its range; tests inspect them +// after the kernel has joined. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/operator/InvertedIndex.h" + +namespace kernel_test { + +using monoprop::detail::KernelRange; + +inline constexpr size_t kKinds = 9; + +// What one kernel invocation reported for one kind of range. +struct RangeSlots { + size_t prepares = 0; // prepare() calls, on the calling thread + std::vector worker; // omp_get_thread_num() of each range's owner, -1 if never visited + std::vector team; // omp_get_num_threads() seen by that owner + std::vector level; // omp_get_level() seen by that owner (0: not inside any region) + std::vector visits; // visits per range; each element has one writer +}; + +struct RangeLog { + std::array slots{}; + auto operator[](KernelRange kind) -> RangeSlots & { return slots[static_cast(kind)]; } +}; + +// Records the worker that owns each logical range. Optionally throws from one range, on its worker, to +// exercise the helper's capture-and-join path from inside a real kernel body. +struct RecordingObserver { + RangeLog *log; + KernelRange throw_kind = KernelRange::cos_lazy; + size_t throw_range = SIZE_MAX; + + auto prepare(KernelRange kind, size_t ranges) const -> void { + auto &s = (*log)[kind]; + ++s.prepares; + s.worker.assign(ranges, -1); + s.team.assign(ranges, 0); + s.level.assign(ranges, 0); + s.visits.assign(ranges, 0); + } + auto visit(KernelRange kind, size_t range) const -> void { + auto &s = (*log)[kind]; + s.worker[range] = omp_get_thread_num(); + s.team[range] = omp_get_num_threads(); + s.level[range] = omp_get_level(); + ++s.visits[range]; + if (kind == throw_kind && range == throw_range) { + throw std::runtime_error("injected kernel worker failure"); + } + } +}; + +// Keeps every prepare() call, for kernels that run once per window or once per exchange pass (the self probe, +// the incoming decode/probe/scatter): prepare appends a fresh record on the calling thread, and each range's +// owner writes only its own slots of the newest record. Records are inspected after the kernel has joined. +struct PhaseLog { + struct Call { + KernelRange kind; + std::vector worker; // -1 if never visited + std::vector team; + std::vector level; + std::vector visits; + }; + std::vector calls; + + //! The calls of one kind, in order. + [[nodiscard]] auto of(KernelRange kind) const -> std::vector { + std::vector out; + for (const auto &c : calls) { + if (c.kind == kind) { + out.push_back(&c); + } + } + return out; + } +}; + +struct AccumulatingObserver { + PhaseLog *log; + KernelRange throw_kind = KernelRange::cos_lazy; + size_t throw_call = SIZE_MAX; // which call of throw_kind, counted from zero + size_t throw_range = SIZE_MAX; + + auto prepare(KernelRange kind, size_t ranges) const -> void { + log->calls.push_back({kind, + std::vector(ranges, -1), + std::vector(ranges, 0), + std::vector(ranges, 0), + std::vector(ranges, 0)}); + } + auto visit(KernelRange kind, size_t range) const -> void { + auto &c = log->calls.back(); + c.worker[range] = omp_get_thread_num(); + c.team[range] = omp_get_num_threads(); + c.level[range] = omp_get_level(); + ++c.visits[range]; + if (kind == throw_kind && range == throw_range && log->of(kind).size() == throw_call + 1) { + throw std::runtime_error("injected resolve worker failure"); + } + } +}; + +// A PhaseLog call as RangeSlots, so the participation/serial checks below apply to it. +inline auto slots_of(const PhaseLog::Call &c) -> RangeSlots { + return RangeSlots{.prepares = 1, .worker = c.worker, .team = c.team, .level = c.level, .visits = c.visits}; +} + +// Evidence that one kernel's own ranges ran on several workers of a region the kernel opened. +struct Participation { + size_t ranges = 0; + size_t distinct_workers = 0; + int team = 0; + bool every_range_once = false; + bool inside_region = false; // every range saw omp_get_level() >= 1 +}; + +inline auto participation(const RangeSlots &s) -> Participation { + Participation p; + p.ranges = s.worker.size(); + p.every_range_once = std::ranges::all_of(s.visits, [](int v) { return v == 1; }); + p.inside_region = !s.level.empty() && std::ranges::all_of(s.level, [](int l) { return l >= 1; }); + const std::set distinct(s.worker.begin(), s.worker.end()); + p.distinct_workers = distinct.size(); + p.team = s.team.empty() ? 0 : *std::ranges::max_element(s.team); + return p; +} + +// Whether a two-worker team is a fair expectation: a dynamic runtime or a thread limit of one may shrink +// the region, which is then reported, never counted as participation. +inline auto team_may_shrink() -> bool { + return omp_get_dynamic() != 0 || omp_get_thread_limit() < 2; +} + +// Checks on the caller that the kernel's ranges ran inside its own region, each exactly once, on as many +// distinct workers as the region had, and on at least two of them unless the runtime may shrink teams. +inline auto check_participation(const RangeSlots &s, size_t expected_ranges, const char *label) -> void { + const auto p = participation(s); + BOOST_TEST_CONTEXT(label << ": ranges=" << p.ranges << " distinct_workers=" << p.distinct_workers + << " team=" << p.team) { + BOOST_TEST(s.prepares == 1u); + BOOST_TEST(p.ranges == expected_ranges); + BOOST_TEST(p.every_range_once); + BOOST_TEST(p.inside_region); + // A static schedule over at least as many ranges as workers gives every worker a range. + BOOST_TEST(p.distinct_workers == static_cast(p.team)); + if (team_may_shrink()) { + BOOST_WARN_MESSAGE(p.team >= 2, + label << ": multi-worker participation NOT demonstrated (team " << p.team + << " under a dynamic or limited runtime)"); + } + else { + BOOST_TEST(p.team >= 2); + } + } + BOOST_TEST_MESSAGE(label << ": " << p.distinct_workers << " distinct workers over " << p.ranges << " ranges"); +} + +// Checks that a kernel stayed on the calling thread: every range visited once, outside any region. +inline auto check_serial(const RangeSlots &s, size_t expected_ranges, const char *label) -> void { + const auto p = participation(s); + BOOST_TEST_CONTEXT(label) { + BOOST_TEST(s.prepares == 1u); + BOOST_TEST(p.ranges == expected_ranges); + BOOST_TEST(p.every_range_once); + BOOST_TEST(std::ranges::all_of(s.level, [](int l) { return l == 0; })); + } +} + +// A test-side probe, not library behavior: can this runtime give a two-thread region right now? +inline auto runtime_offers_two_workers(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + if (omp_get_thread_limit() < 2) { + boost::test_tools::assertion_result result(false); + result.message() << "OMP_THREAD_LIMIT is below two"; + return result; + } + int team = 0; +#pragma omp parallel num_threads(2) + { +#pragma omp single + team = omp_get_num_threads(); + } + boost::test_tools::assertion_result result(team >= 2); + result.message() << "the OpenMP runtime provided " << team << " worker(s) for a two-thread request"; + return result; +} + +// Deterministic, library-independent pseudo-random words (splitmix64). +struct SplitMix { + uint64_t state; + auto next() -> uint64_t { + uint64_t z = (state += 0x9e3779b97f4a7c15ULL); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; + z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; + return z ^ (z >> 31); + } + auto below(uint64_t n) -> uint64_t { return next() % n; } +}; + +inline constexpr size_t kIndexModes = 8; // 16 columns + +// A synthetic InvertedIndex<8> over `rows` rows with every column tier a fold can meet: dense random +// columns, sparse columns (including rows straddling fold-block edges), a dense-but-sparse column and +// empty columns. Consistent with an operator whose row r touches column c iff bit r of column c is set, +// so its lazily built row parity is well defined. Bits beyond `rows` stay clear, as in the real index. +inline auto make_synthetic_index(size_t rows, uint64_t seed) -> monoprop::detail::InvertedIndex { + using Index = monoprop::detail::InvertedIndex; + Index index; + index.row_count = rows; + const size_t words = index.words(); + const uint64_t last_mask = rows % 64 == 0 ? ~uint64_t{0} : (uint64_t{1} << (rows % 64)) - 1; + SplitMix rng{seed}; + const auto dense = [&](size_t c, int sparsity_shift) { + auto &col = index.cols[c]; + col.is_dense = true; + col.words.assign(words, 0); + for (auto &w : col.words) { + w = rng.next(); + for (int s = 0; s < sparsity_shift; ++s) { + w &= rng.next(); + } + } + if (words != 0) { + col.words.back() &= last_mask; + } + }; + const auto sparse = [&](size_t c, uint64_t one_in) { + auto &col = index.cols[c]; + for (size_t r = 0; r < rows; ++r) { + if (rng.below(one_in) == 0) { + col.set_rows.push_back(static_cast(r)); + } + } + }; + for (size_t c = 0; c < 6; ++c) { + dense(c, 0); // density 1/2 + } + sparse(6, 200); + sparse(7, 997); + // Rows straddling every fold-block edge, so a block boundary splits a sparse column's row list. + { + auto &col = index.cols[8]; + const size_t block_rows = monoprop::detail::kColumnBlockWords * 64; + for (size_t edge = block_rows; edge < rows; edge += block_rows) { + for (size_t r = edge - 3; r < std::min(rows, edge + 3); ++r) { + col.set_rows.push_back(static_cast(r)); + } + } + if (rows != 0) { + col.set_rows.push_back(static_cast(rows - 1)); // the global tail row + } + std::ranges::sort(col.set_rows); + col.set_rows.erase(std::ranges::unique(col.set_rows).begin(), col.set_rows.end()); + } + dense(9, 4); // dense representation, density 1/32 + // Columns 10 and 11 stay empty. + sparse(12, 64); + dense(13, 1); // density 1/4 + sparse(14, 5000); + dense(15, 0); + return index; +} + +// Append rows [index.rows(), new_rows) the way operator growth does: new bits for dense columns (including +// the old tail word's unused bits), ascending new rows for sparse ones. Existing rows keep their columns. +// Does not touch row_parity_; the caller decides whether to drop or keep it. +inline auto grow_synthetic_index(monoprop::detail::InvertedIndex &index, size_t new_rows, uint64_t seed) + -> void { + const size_t old_rows = index.row_count; + index.row_count = new_rows; + const size_t words = index.words(); + SplitMix rng{seed}; + for (auto &col : index.cols) { + if (col.is_dense) { + col.words.resize(words, 0); + for (size_t r = old_rows; r < new_rows; ++r) { + if (rng.below(3) == 0) { + col.words[r >> 6] |= uint64_t{1} << (r & 63U); + } + } + } + else if (!col.set_rows.empty()) { + for (size_t r = old_rows; r < new_rows; ++r) { + if (rng.below(150) == 0) { + col.set_rows.push_back(static_cast(r)); + } + } + } + } +} + +// A Majorana/Pauli generator over the synthetic index's 16 columns. +inline auto generator(std::initializer_list columns) -> monoprop::Monomial { + monoprop::Monomial gen{}; + for (const auto c : columns) { + gen.set(c); + } + return gen; +} + +// Independent add-owner count: every slot a fused contract writes (sources and targets of full +// rotations, local slots of half rotations) mapped to how many records write it. +inline auto add_owner_counts(const monoprop::detail::FusedContract &fc) -> std::map { + std::map owners; + for (const auto &r : fc.hits) { + ++owners[r.src]; + ++owners[r.tgt]; + } + for (const auto &r : fc.inserts) { + ++owners[r.src]; + ++owners[r.tgt]; + } + for (const auto &h : fc.cross_half) { + ++owners[h.local_idx]; + } + return owners; +} + +} // namespace kernel_test diff --git a/cpp/tests/PropagatorTestAccess.h b/cpp/tests/PropagatorTestAccess.h new file mode 100644 index 000000000..b3fda17ad --- /dev/null +++ b/cpp/tests/PropagatorTestAccess.h @@ -0,0 +1,128 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// White-box access for tests only. MonomialPropagator befriends this template but the library never +// defines it, so it adds no production API: tests use it to observe the captured thread budget, to +// inject failures at real mutation and evaluation boundaries through existing members, to inspect the +// root's actual shards and to attach the test-only RootObserver. + +#include +#include +#include +#include + +#include "monoprop/Evolution.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/detail/evolution/layer_build/Common.h" +#include "monoprop/detail/evolution/layer_build/Engine.h" +#include "monoprop/detail/evolution/layer_build/FusedApply.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/sharded/RootObserver.h" +#include "monoprop/detail/sharded/State.h" + +namespace monoprop::detail { + +template +struct PropagatorTestAccess { + using Propagator = MonomialPropagator; + + static auto options(const Propagator &p) -> parallel::Options { return p.parallel_; } + + static auto is_invalid(const Propagator &p) -> bool { return p.invalid_; } + + // The existing protected cutoff predicate: every scan evaluates it, so a throwing one fails a real + // build after the operation has started mutating. + static auto set_cutoff_fn(Propagator &p, CutoffFn fn) -> void { p.cutoff_fn_ = std::move(fn); } + + static auto clone(const Propagator &p) -> std::unique_ptr { return p.clone_(); } + + using Observer = sharded::RootObserver; + + // The root's actual shard states, for private inspection; never a production accessor. + static auto shards(const Propagator &p) -> const sharded::Shards & { return p.shards_; } + + // The root's physical rounds, reused across its multi-rank operations. + static auto rounds(const Propagator &p) -> const sharded::PhysicalRounds * { return p.rounds_.get(); } + + // The (P, T) router the root prepared at construction. + static auto router(const Propagator &p) -> const routing::Router & { return *p.router_; } + + static auto core_term(const Propagator &p) -> double { return p.core_term_; } + + static auto epoch(const Propagator &p) -> size_t { return p.initial_operator_epoch_; } + + // Attach (or, with null, detach) a test observer; copies made while attached inherit it. + static auto observe(Propagator &p, const Observer *observer) -> void { p.observer_ = observer; } + + // Construct through the public constructor's body with `observer` attached, so seeding is observed too. + template + static auto construct_observed(const Observer *observer, Args &&...args) -> std::unique_ptr { + return std::unique_ptr( + new Propagator(typename Propagator::ObservedTag{}, observer, std::forward(args)...)); + } + + // One energy evaluation through the evaluation seam with a caller-local physical round (rounds = nullptr), as an + // external caller of the exported evaluate_shards() runs it, at the root's own geometry and world; the root's + // persistent rounds are not used. The outcome, error included, is returned after the team has joined. + static auto evaluate_with_local_round(Propagator &p, + const VecD ¶ms, + const sharded::EvaluationObserver *observer) -> sharded::EvaluationOutcome { + bool mutation_started = false; + const auto retained = p.prepare_retained_(std::nullopt, mutation_started); + const auto requests = retained.requests(params); + return sharded::evaluate_shards(requests, retained.callbacks, retained.options, false, observer, p.world_); + } + + // One graph-only layer (no coefficients, not appended to the graph) built by the low-level one-owner-per-rank + // engine (build_layer) on the sole shard of a T = 1 root, over the root's communicator, with an explicit kernel + // budget and a test observer on the kernels' ranges, under the root's own operation guard: a failure after the + // layer started counts as a post-mutation failure (invalidation, then mpi::operation_failed). The shard may grow, + // so the object is for inspection only. + template + static auto sole_shard_layer_observed(Propagator &p, + const VecZ &gate, + parallel::Options kernels, + const KernelObserver &observer) -> std::shared_ptr { + return p.run_operation_(true, [&](bool &mutation_started) { + auto &state = p.sole_shard_("sole_shard_layer_observed()"); + const auto gen = indices_to_bitset_checked(gate, 2 * p.logical_num_modes_); + mutation_started = true; + return build_layer(state.op, + gen, + p.cutoff_fn_, + p.lower_atol_, + std::nullopt, + p.upper_atol_, + std::nullopt, + std::nullopt, + state.matched, + p.comm_, + nullptr, + nullptr, + p.schrodinger_, + nullptr, + nullptr, + p.basis_, + kernels, + observer); + }); + } +}; + +} // namespace monoprop::detail diff --git a/cpp/tests/README.md b/cpp/tests/README.md index c69941de5..7c7711c2f 100644 --- a/cpp/tests/README.md +++ b/cpp/tests/README.md @@ -16,10 +16,15 @@ discovers every Boost.Test case and registers it twice: Cases that need multiple ranks check `monoprop::mpi::size(MPI_COMM_WORLD)` and skip (with a message) when run with too few. -The custom `main()` in `unit_tests.cpp` initializes MPI and forces -`monoprop_PARTITIONS=off`, so white-box tests observe the single-partition engine. -A test that needs the partition runtime must pass an explicit `partitions=` argument -(see `partition_equivalence_tests.cpp`). +The launch-time thread policy belongs to the CTest registration, not to the runner: +every discovered case (per case, and in the whole-suite MPI variants) runs with +`monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 OMP_DYNAMIC=FALSE`, so white-box tests +observe one store per rank. Teams of T > 1 are dedicated fresh-process launches +(`*_env_t*`), and `sharded_root_env_openmp_default_t3` and +`openmp_env_root_budget_openmp_default_3` remove `monoprop_NUM_THREADS` on +purpose, so the custom `main()` in `unit_tests.cpp` only initializes MPI and never +supplies the variable. Running the binary by hand uses your environment; set the +variables above to reproduce a CTest case. Nothing reads `monoprop_PARTITIONS`. ## Building Tests @@ -64,9 +69,6 @@ name and cannot address suite-nested cases, tests use flat encoding (`slots_of_string`, `native_bitset`, `jw_basis`), dense Pauli-matrix brute force (`matrix_from_string`, `matmul`, ...), and string helpers. Shared by the Pauli algebra/build-layer tests and the equivalence suites. -- **`ThreadHarness.h`**: `run_comm_threads` — spawn S partition threads over a - transport and capture per-thread exceptions (used by the ShmComm/HybridComm - suites). - **`GraphBuildHarness.h`**: direct Layer/MPGraph construction helpers (`core_with_gate`, `layer_with_gate`, `graph_with_gates`) for white-box MPGraph transform tests. @@ -76,6 +78,12 @@ name and cannot address suite-nested cases, tests use flat - **`dense_query_reference.h`**: the retired dense query record, frozen as the independent oracle for `sparse_query_tests.cpp`. Test-only, and not kept in sync with the wire format. +- **`PropagatorTestAccess.h`**: white-box access that `MonomialPropagator` befriends but the library + never defines. Tests use it to read an object's captured thread budget and validity, to inject + failures through existing members (a throwing cutoff predicate), to inspect the root's actual + shards, router and physical rounds, to attach the test-only `RootObserver`, and to run one layer + of the low-level one-owner-per-rank engine on the sole shard of a T = 1 root + (`sole_shard_layer_observed`, for the failure driver); it adds no public shard interface. - **`TestData.{h,cpp}`**: the `CaseData` struct and msgpack fixture loader. - **`boost-test.cmake` / `boostAddTests.cmake`**: CMake test discovery. @@ -95,7 +103,8 @@ name and cannot address suite-nested cases, tests use flat - **Operator store**: `operator_index_tests.cpp`, `inverted_index_tests.cpp`, `mp_operator_tests.cpp` (MPOperator get_state Pauli/Majorana scoring, get_operator init-map drain, update_initial_operator picture branches, - insert_absent_terms, inverted-index sync, memory estimate, deep copy), + insert_absent_terms, inverted-index sync, memory estimate, deep copy, spare capacity + released in place), `bulk_insert_tests.cpp` (the grouped-prefetch insert vs a one-key-at-a-time reference: table state and enumeration order). - **Layer build / evolution**: `build_graph_tests.cpp`, @@ -112,8 +121,13 @@ name and cannot address suite-nested cases, tests use flat - **Graph / paring**: `pare_graph_tests.cpp`, `mpi_pare.cpp`, `mp_graph_tests.cpp` (MPGraph slice_graph/slice_view transforms, the front_offset lazy-compaction arms, MPGraphView reverse mapping + OOB throw). -- **Transports / distribution**: `shm_comm_tests.cpp`, `hybrid_comm_tests.cpp` - (MPI-only), `partition_equivalence_tests.cpp`, +- **Public surface**: `sharded_api_tests.cpp` (compile-time probes of the + constructor, nested types, extension hooks, exceptions, the removed selector and + headers, each negative probe paired with a positive control). +- **Transports / distribution**: `mpi_alltoallv_tests.cpp` (the real-MPI request + helpers of the low-level engine: `begin_alltoallv`, `PendingAlltoallv`, + `alltoall_counts` and the sparse `PeerPlan`, at world size 1 and in the whole-suite + MPI variants), `flat_exchange_tests.cpp` (`post_flat_alltoallv`/`Ticket`), `mpi_distributed_layer_equivalence.cpp`, `mpi_fresh_insert_equivalence.cpp` (serial↔world equivalence of the Schrödinger fused-resolve fresh-insert arms, Majorana + native Pauli; self-skips at world size 1). @@ -121,10 +135,193 @@ name and cannot address suite-nested cases, tests use flat `update_initial_operator.cpp`, `ctor_validation_tests.cpp` (constructor guard rails + MPGraph bounds). - **Exchange layout preconditions**: `exchange_layout_precondition_tests.cpp` - (the layout width check, driven through ShmComm). + (the layout width check on an ordinary communicator; rank 0 alone runs it on the + world communicator, so a check that communicated would hang). +- **OpenMP worksharing**: `openmp_workshare_tests.cpp` (`parallel::for_blocks`: + exactly-once visits, serial and nested fallbacks, budget and bound validation, + unchanged runtime settings, joined worker exceptions, and actual multi-worker + participation, which is skipped when the runtime cannot provide two workers). + Run it in fresh processes under `OMP_DYNAMIC=TRUE`, `OMP_DYNAMIC=FALSE` and + `OMP_THREAD_LIMIT=1` when changing the helper. +- **Fixed-team phase primitive**: `sharded_team_tests.cpp` covers `detail/sharded/Team.h` + (`run_team`, `phase`, `TeamFailure`): each owner runs once on its own OpenMP worker, the primary is + the calling thread, owners keep their worker across phases, empty owners reach every checkpoint, + writes are visible in the next phase, failures (including non-`std::exception` values) are returned + after the join from the lowest failing worker and suppress later phases, a fast worker's failure + cannot change an earlier checkpoint's decision (also with the interleaving forced: the slow workers read + checkpoint 0's decision only after the fast worker recorded its phase-1 failure, through the test-only + observer of `TeamFailure::checkpoint`), and OpenMP settings are unchanged. Workers only fill + preallocated observation slots; all assertions run after the join. The multi-phase failure cases keep + every worker reaching the same number of checkpoints, so a broken checkpoint shows up as divergent + decisions instead of a deadlock. `cpp/tests/CMakeLists.txt` reruns them as `sharded_team_env_t1`, + `_t2`, `_t4` and `_t4_passive`, fresh processes with `monoprop_NUM_THREADS=OMP_NUM_THREADS=T` and + `OMP_DYNAMIC=FALSE`; cases that need a nonprimary worker skip below T=2 and are left out of the T=1 + launch. The helper assumes exactly T workers, so these cases never run under the limited or dynamic + teams of `openmp_env_kernels_*`, which test `for_blocks`' reduced-team contract. +- **Owner-initialized shard state**: `sharded_state_tests.cpp` covers `detail/sharded/State.h` + (`make_shards`, `seed_shards`, `copy_shards`, `ShardState`). Seeding is checked against a reference + derived in the test from the router, the paired-basis enumeration and the store, and aggregates + against the root at the same launch, built on the test thread outside the team. Further cases cover flat ownership `rank * T + shard` at several + (P, T) geometries, identity handling, empty owners, sparse/dense cache selection, pending entries, + per-shard counts and accounting, exact and independent copies with shared graph cores, the captured + budget, the rank-level validity guard (preservation evidence), and initializer failures on primary, + nonprimary and concurrent owners. `AllocationProbe.cpp` replaces the global `operator new`/`delete` + family for the whole unit-test executable: it counts per-thread allocation, tracks live bytes, and + can make the n-th allocation of one owner throw `std::bad_alloc`. The seed and copy sweeps use it to fail every allocation site in turn and check for a caught exception, + no leak and an intact source. The library calls a test-only observer (`NoShardObserver` in + production) inside each owner's seed or copy body; tests use it to record the executing worker and + its allocated bytes. `sharded_state_env_t1`, `_t2` and `_t4` rerun the cases in fresh exact-team + processes, as for the team primitive. +- **Direct-buffer construction**: `sharded_construction_tests.cpp` covers `detail/sharded/Construction.h` + (`build_graph`, `propagate`, `gather_published`) and the owner-local engine phases it drives + (`LayerBuildEngine::prepare_exchange`, `resolve_published`, `consume_published`). At geometry (1, T) + every shard's rows, IDs, coefficients, caches and graph layers equal the root's at the same launch. + Because both share the phases, the cases also check an independent insertion-order reference, an + independent coefficient-map propagator and the frozen exact energy of `random_exact.msgpack`; global + retained maps across T are compared from separate processes (`tests/test_sharded_openmp.py`). Further + cases cover participation (every + owner's work runs on its own worker inside one team, kernels serial on the owner), the primary-only + traversal for opaque cutoffs and basis changes, empty circuits, operators and owners, identity gates, + the published-view seam (nonzero window bases, empty senders, 255/256/257-query blocks, narrow and + wide positions, the malformed Plain `QueryWire<128>` fixture), same-shard streams beyond one 4096-query + window, and failures (an injected throw in every phase, kernel, decode and publication failures, + concurrent failures, and `AllocationProbe` sweeps) that must join, suppress later phases, report + whether mutation started and leave earlier copies intact. The test observer records visits, + streams and kernel ranges per shard. `sharded_construction_env_t1`, `_t2` and `_t4` rerun the cases in + fresh exact-team processes, as for the team primitive. +- **Snapshot-safe evaluation**: `sharded_evaluation_tests.cpp` covers `detail/sharded/Evaluation.h` + (`evaluate_shards`, `ev_sharded`, `ev_and_grad_sharded`, `replay_shards`, `prepare_retained`, the + owner-local forward replay phases and the ascending-shard folds), the owner-local paring seam + `detail::pare_graph_owner` and coefficient-informed construction (`sharded::build_graph_informed`). At + geometry (1, T) every shard's evolved operator and local term, the energy and every gradient component + equal the root's at the same launch bitwise, for every basis/picture fixture, deep-amplification and + vanishing-cosine parameters, pared and unpared functionals, and partial contraction in both pictures. The + replay and derivative kernels are shared with the low-level evaluator (`detail/evolution/LayerReplay.h`) and + the root drives the same seams, so the cases also check an independent coefficient-map propagator, the frozen exact energy of `random_exact.msgpack`, + single-gate closed-form fits and central finite differences, including the cases of + `tests/test_deep_circuit_gradient.py`. Further cases cover whole self pairs, other-local-shard and + multi-peer layouts, empty owners and no-work layers, the identity counted once, empty parameters and + missing callbacks, repeated parameters and duplicate records, an empty stored pruned mask (replays nothing), + partner staging one block per run (`EvaluationObserver::staging_run`) bitwise equal to the default runs, exact + cosine-set counts (`CosCallbacks::count`) with records reserved once and never reallocated, + interleaved instances, copies and index growth under retained callbacks, participation (every owner's + work runs on its own worker; opaque callbacks run on the primary in an exclusive phase), and failures in + every phase, in callbacks, paring and informed construction, plus `AllocationProbe` sweeps run from a fresh + host thread so every worker starts with cold thread-local scratch. `sharded_evaluation_env_t1`, `_t2` and + `_t4` rerun the cases in fresh exact-team processes, as for the team primitive. +- **Thread budgets and failed-owner rule**: `openmp_runtime_tests.cpp` covers the budget parser and + its capture, per-object budgets through copies, the MPI thread-level decisions (FUNNELED + required) and the initializing-thread guard. It also checks that OpenMP settings and affinity are + unchanged, that old low-level calls default to serial options, and that a failed mutation + invalidates the owner while validation errors do not. `cpp/tests/CMakeLists.txt` reruns some of these cases as + `openmp_env_*` entries, each in a fresh process with a fixed launch environment. +- **Threaded kernels**: `openmp_kernel_tests.cpp` covers the threaded cosine scaling + (`scale_cos_mask`, `scale_cos_lazy`) and fused rotation apply (`apply_fused_contract`): bitwise + agreement of every budget with budget one, small work staying serial, joined worker exceptions, + and participation. Participation is read from a test-only range observer (`KernelTestSupport.h`) + that each kernel calls at the start of its own logical ranges; production code passes the empty + `NoRangeObserver`, so a team opened anywhere else never counts. Lazy folds over a synthetic + multi-block inverted index are checked against the materialised-fold oracle in + `combined_recompute_equivalence.cpp` (cold parity cache, tails, index growth), and fused records + from the real build path (one add-owner per slot, both pictures, cross-rank halves under MPI) in + `fused_cos_sweep_tests.cpp`. `openmp_env_kernels_*` rerun them under `OMP_THREAD_LIMIT=2` and + `OMP_DYNAMIC=TRUE`. +- **Threaded scan**: the `openmp_scan_*` cases compare the threaded bitmap scan + (`fused_find_and_collect` over word ranges, merged in range order) with budget one byte for byte: + every window slot's wire records, sources and values, both self stages, the cosine set and the + fused-sweep coefficients. `ScanTestSupport.h` builds a synthetic operator of four fold blocks per + rank and the scenario matrix (one rank, linear zero and non-zero shifts with non-zero window bases, + dense splitmix at three and four ranks; dense, sparse and empty pivots; Majorana and Pauli; caps, + cutoffs, atol boundaries, capture, fused sweep). They live in `evolution_detail_tests.cpp`, + `pauli_build_layer_tests.cpp`, `exact_upper_atol_rescue.cpp`, `majorana_cutoff_tests.cpp` (opaque + and basis-change cutoffs stay serial), `sparse_query_tests.cpp` (the merge itself) and + `fused_cos_sweep_tests.cpp` (full construction: rows, row IDs, graph layers and coefficients at + budgets 1-4). The scan's participation is read from its own ranges through the same observer. +- **Threaded resolution**: `sparse_resolve_tests.cpp` compares the incoming probe (checked + preparation, decode, `probe_frozen_positions`, predicted IDs, publication) at budgets 2-8 with + budget one over empty/all-hit/all-miss/mixed inputs, 255/256/257-query boundaries, empty senders, + non-zero window bases, narrow and wide positions, spilled rows, escaped counts, Plain and Fused + records and a prebuilt inverted index. It rejects malformed streams (including the noncanonical + escape `QueryWire<128>` `0x8037e`) before any decode, invalid decoded positions before any probe, + keeps the scatter serial for sinks without `parallel_resolve` and for the Schrödinger + `ContractSink`, and checks self windows of 4096 and the publication order across leader/follower + passes. `sparse_query_tests.cpp` covers `QueryWire::checked_extent` and position validation, + `operator_index_tests.cpp` pins the duplicate-key behaviour of bulk insertion, and + `mpi_fresh_insert_equivalence.cpp` compares budgets exactly and reads participation at world size. + `PhaseLog`/`AccumulatingObserver` keep every call of a phase that runs per window or per pass. +- **Distributed failure driver**: `mpi_failure_driver.cpp` is not part of the unit runner (the glob + excludes it). The MPI build compiles it into `monoprop_mpi_failure_driver.x`, and + `run_mpi_failure_scenario.cmake` runs each scenario on two ranks with a 30 s timeout, as the + `mpi_failure_*` CTest entries. The sharded root's scenarios (`mpi_failure_sharded_*`) inject at two + threads per rank through its `RootObserver`. The low-level ones run on the sole shard of a root + launched with one thread per rank: `active_ticket` and `active_ticket_cosine_worker` throw while + an `evolve_step` replay `Ticket` is posted (the latter from worker 1 of the threaded cosine + kernel), `active_pending` while a `PendingAlltoallv` is posted, `before_exchange_scan_worker` and + `resolve_worker_self_probe` throw from a scan or self-probe worker before the query exchange, and + `resolve_worker_decode|incoming_probe|scatter` from the incoming phases after the query round, + while the peer enters the response round. The thread-support scenarios require a clean exit + (`EXPECT_SUCCESS`): `library_initialized_level` (monoprop's own `mpi::init`, which requests + FUNNELED) and `host_initialized_funneled|serialized|multiple` run real operations at two threads per + rank on the world and on a reordered split communicator against one process; + `insufficient_thread_level_single` must be refused and `wrong_thread_entry_*` must fail fast at + FUNNELED, SERIALIZED and MULTIPLE. Each prints the requested and provided level; a provided level + that does not exercise the route ends as a skip ("NOT EXERCISED", exit 77). -New `*.cpp` files are auto-discovered on the next configure — no CMake edit -needed. +New `*.cpp` files are auto-discovered on the next configure, with no CMake edit +needed. A file with its own `main()` must be excluded from the glob, as +`mpi_failure_driver.cpp` is. + +## Sharded OpenMP runtime suites + +There is one runtime, and every suite compiles against it. + +- `sharded_api_tests.cpp` checks the public surface at compile time: no constructor tail, `PartitionChildFactory`, + facade helper, partition-named exception, cached thread-count reader, selector macro or partition/in-process + communicator header, each probe paired with a positive control, and positive controls pin the eleven-argument + constructor, `clone_()`, `apply_initial_operator_()` and the virtual destructor. +- `sharded_root_tests.cpp` covers the root (`ShardedPropagator.inl`): T routed shards at the (P, T) router, + aggregates over every shard with shared metadata counted once, the raw-accessor rule (the actual sole shard at + T = 1, `MultiShardUnsupported` naming the T = 1 launch otherwise, the validity guard first, the object still usable), + the obsolete `monoprop_PARTITIONS` ignored bit for bit, the unset budget taking the OpenMP default, actual-worker + participation in seeding, copies, construction and publication, informed seed replay, propagation, retained + preparation, forward replay, reverse derivatives, contraction and the root's own owner phases, per-shard export and + contraction blocks, initial-operator routing, remapping, ordinary virtual hooks, pre-mutation rejections, + post-mutation invalidation from failures injected in each phase, functional epoch/graph guards, contained + seed/copy failures and an `AllocationProbe` sweep. `sharded_root_env_t1`, `_t2`, `_t4` and `_openmp_default_t3` + rerun it in fresh exact-team processes; its `sharded_root_multirank_*` cases run only in `sharded_root_mpi*` (which + set `monoprop_TEST_SHARDED_RANKS`). +- `fused_cos_sweep_tests.cpp` runs the fused-versus-replay cases at the launch's T, audits the real fused records of + every shard through the propagation seam's test hook (`RootObserver::fused_records`), and checks shard-by-shard + determinism at fixed geometry, also as `fused_env_t2` and `_t4`. +- `openmp_runtime_tests.cpp` keeps its parser and capture cases, the launch-budget entries (`openmp_env_root_*`, + including the unset variable), the thread-level decisions and the invalid-owner cases at the launch's T + (`openmp_env_root_t4` too). +- `sharded_exchange_tests.cpp`: the physical round's single-process contract (including owner-parallel planning and + the replay step's bulk path, symmetric rows and column views, against the entry-wise path) and, in + `sharded_exchange_mpi*`, closed-form blocks between processes. + +Bitwise parity with an independently implemented runtime at the same geometry, and parity across geometries, is +checked from separate processes by `tests/test_sharded_openmp.py` (with `monoprop_TEST_LEGACY_PYTHON` naming a +separately built pre-removal legacy environment), because two class definitions must never share a process. + +Migration ledger, closed by the removal of the partition runtime (S8). Categories: (1) unchanged at launch-time +T = 1, (2) rewritten with private shard inspection at fixed geometry, (3) replaced by separate-process or +global-map coverage, (4) retired with its subject (the deleted runtime), with the named replacement. + +| Suite or registration | Disposition | Coverage now | +| --- | --- | --- | +| `partition_equivalence_tests.cpp`, `partition_group_clone_tests.cpp` | (4) deleted | `sharded_api_tests.cpp` proves the facade and factory absent; ordinary hooks in `sharded_root_virtual_hooks_keep_working`; cross-T maps in Python | +| `shm_comm_tests.cpp`, `hybrid_comm_tests.cpp`, `ThreadHarness.h`, `SlotBlocks.h` | (4) deleted; plain-MPI arms ported | `mpi_alltoallv_tests.cpp` (sparse plans, known counts, empty legs, back-to-back rounds, unroutable plans, request ownership), `flat_exchange_tests.cpp`; in-process verbs have no subject | +| `cpu_topology_tests.cpp` | (4) deleted | no topology code remains; placement is the launcher's | +| `exchange_layout_precondition_tests.cpp` | (2) ShmComm fixture replaced | ordinary communicator; rank-0-only check on the world proves rejection before communication | +| S1-S3 seam suites' legacy-oracle cases (10) and `sharded_{construction,evaluation}_multirank_*` | (3)/(4) retired with the in-process oracle | independent references and the root at the same launch; bitwise per rank against a separately built legacy runtime in `tests/test_sharded_openmp.py` (contraction blocks pin each shard's row order) | +| `openmp_runtime_tests.cpp` one-store cases | (4) | `sharded_root_functional_failures_invalidate_their_owner`; parser, fallback, copy budget and invalid-owner cases kept | +| `fused_cos_sweep_tests.cpp` one-store and cross-budget cases | (3)/(4) | fused shard records and fixed-geometry determinism; kernel budgets in `openmp_scan_*`/`openmp_env_kernels_*`; cross-T maps in Python | +| `mpi_distributed_layer_equivalence.cpp`, `mpi_fresh_insert_equivalence.cpp` hybrid/one-store cases | (4) | `sharded_world_equivalence_mpi*`, `sharded_root_mpi*` | +| `mpi_failure_*` | (2) | sharded scenarios; low-level scenarios migrated to the T = 1 root's sole shard; FUNNELED acceptance added | +| `env_config_tests.cpp` permissive-parser cases | (4) | absence probe in `sharded_api_tests.cpp` | +| Other propagator suites | (1) T = 1 | unchanged assertions on the sole shard | +| `link_export_probe`, `find_package_smoke` | (2) | constructor shape, every operation family, derived hooks, raw-access rule at T = 1 and 2, no selector, the MPI setting, and (MPI) a two-rank comparison with one process | ## MPI Test Configuration diff --git a/cpp/tests/ScanTestSupport.h b/cpp/tests/ScanTestSupport.h new file mode 100644 index 000000000..ad7809116 --- /dev/null +++ b/cpp/tests/ScanTestSupport.h @@ -0,0 +1,517 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +// Test-only support for the threaded bitmap scan: a synthetic operator spanning several fold blocks per rank, +// scan scenarios over windows, routers, pivots, bases and cutoffs, and an exact comparison of two scan +// results stream by stream. Every comparison is on the caller, after the scan has joined. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "KernelTestSupport.h" +#include "monoprop/TypeAliases.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/algebra/AlgebraCommon.h" +#include "monoprop/detail/evolution/layer_build/Scan.h" +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/MPIUtils.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/operator/InvertedIndex.h" +#include "monoprop/detail/operator/MPOperator.h" +#include "monoprop/detail/operator/RowAccess.h" + +namespace scan_test { + +using namespace monoprop; + +inline constexpr size_t kModes = 32; // 64 bits, so a monomial is one word +using Mono = Monomial; +using Op = detail::MPOperator; +using Result = detail::FusedScanResult; + +// Column tiers of the synthetic operator, by bit index (the generator's pivot is its lowest set bit): +inline constexpr size_t kEmptyEnd = 4; // bits [0, 4) are never set: empty fold columns +inline constexpr size_t kSparseEnd = 12; // bits [4, 12) are set with probability 1/256: sparse columns +// bits [12, 64) are set with probability 1/8: dense columns (the promotion threshold is 1/64). + +// Three full fold blocks and a one-word fourth, whose last word holds 37 rows: budgets 2-4 split it into +// 2-4 ranges, and the last range carries the global tail. +inline constexpr size_t kRowsPerRank = (3 * detail::kColumnBlockWords * 64) + 37; + +inline auto draw_term(kernel_test::SplitMix &rng) -> uint64_t { + uint64_t w = 0; + for (size_t b = kEmptyEnd; b < 64; ++b) { + const bool set = b < kSparseEnd ? rng.below(256) == 0 : rng.below(8) == 0; + w |= static_cast(set) << b; + } + return w; +} + +inline auto mono_of(uint64_t word) -> Mono { + Mono m{}; + for (uint64_t x = word; x != 0; x &= x - 1) { + m.set(static_cast(std::countr_zero(x))); + } + return m; +} + +inline auto mono_of_bits(std::initializer_list bits) -> Mono { + Mono m{}; + for (const auto b : bits) { + m.set(b); + } + return m; +} + +inline auto build_op(const std::vector &terms, Basis basis) -> Op { + Op op; + op.basis = basis; + detail::insert_absent_terms( + op, + terms.size(), + [&](size_t k) -> const Mono & { return terms[k]; }, + [&](size_t k, size_t base) { assign_row(*op.store, base + k, terms[k]); }); + return op; +} + +// Distinct terms this rank owns under `router`, `rows` of them, in draw order. Owned-only, as a propagator +// seeds each rank: the scan routes by rank(M) ^ rank_shift(G), which names the owner only for a local M. +inline auto owned_terms(const routing::Router &router, size_t my_rank, size_t rows, uint64_t seed) + -> std::vector { + kernel_test::SplitMix rng{seed}; + std::unordered_set seen; + std::vector terms; + terms.reserve(rows); + while (terms.size() < rows) { + const uint64_t w = draw_term(rng); + if (w == 0 || !seen.insert(w).second) { + continue; + } + const Mono m = mono_of(w); + if (router.flat_world() == 1 || find_rank(m, router) == my_rank) { + terms.push_back(m); + } + } + return terms; +} + +// Signed coefficients in (-1, 1), deterministic. +inline auto coefficients(size_t n, uint64_t seed) -> VecD { + kernel_test::SplitMix rng{seed}; + VecD c(n); + for (auto &v : c) { + v = (static_cast(rng.below(1U << 30)) / static_cast(1U << 29)) - 1.0; + } + return c; +} + +// One scan geometry: a router, the rank the scan runs on, and that rank's operator. +struct Geometry { + const char *label; + size_t ranks; + size_t parts; + bool linear; + size_t my_rank; +}; + +inline auto router_of(const Geometry &g) -> routing::Router { + return routing::Router::for_modes(g.ranks, g.parts, g.linear); +} + +// Operators are expensive to build at this size, so each (geometry, basis) is built once per process. +inline auto geometry_op(const Geometry &g, Basis basis) -> const Op & { + static std::map, Op> cache; + const auto key = std::tuple{g.ranks, g.parts, g.linear, g.my_rank, static_cast(basis)}; + auto it = cache.find(key); + if (it == cache.end()) { + const auto router = router_of(g); + auto op = build_op(owned_terms(router, g.my_rank, kRowsPerRank, 0x5CA11ULL + g.my_rank), basis); + (void)op.inverted_index(); + it = cache.emplace(key, std::move(op)).first; + } + return it->second; +} + +enum class Atol { + none, + lower_equal, // lower atol equal to one term's |sin|·|c|: that term is dropped (<=) + upper_equal, // upper atol equal to one term's |sin|·|c| under a tight cutoff: that term is rescued (>=) + drop_all, // lower atol above every |sin|·|c|: the scan traverses its words but emits nothing +}; + +// Everything one fused_find_and_collect call takes besides the operator and the budget. +struct Scenario { + const char *label; + Basis basis = Basis::Majorana; + Mono gen{}; + CutoffFn cutoff = detail::LengthCutoff{2 * kModes}; + std::optional cap = std::nullopt; // only_rotate_len_k + Atol atol = Atol::none; + bool capture = false; + bool fused_scale = false; // requires capture and no cap + double param = 0.3; +}; + +// A scan's outputs together with the coefficients it may have scaled. +struct Outcome { + Result result; + VecD coeffs; + mpi::SlotWindow window; + size_t my_rank = 0; +}; + +inline constexpr size_t kAtolTerm = 1234; // the term whose |sin|·|c| an atol scenario lands on exactly + +// Runs `s` on `op`, which must be the operator of geometry `g` (or one built the same way). +template +auto run_scan_on(const Op &op, + const Geometry &g, + const Scenario &s, + detail::parallel::Options options, + const Observer &observer = {}) -> Outcome { + const auto router = router_of(g); + const size_t shift = router.rank_shift(s.gen); + const mpi::PeerPlan plan{.sparse = router.is_linear(), .shift = static_cast(shift)}; + Outcome out; + out.window = plan.window(g.my_rank, router.ranks(), router.shards_per_rank()); + out.my_rank = g.my_rank; + out.coeffs = coefficients(op.size(), 0xC0EFFULL + g.my_rank); + const double abs_sin = std::abs(std::sin(2 * s.param)); + std::optional lower; + std::optional upper; + if (s.atol == Atol::lower_equal) { + lower = abs_sin * std::abs(out.coeffs[kAtolTerm]); + } + if (s.atol == Atol::upper_equal) { + upper = abs_sin * std::abs(out.coeffs[kAtolTerm]); + } + if (s.atol == Atol::drop_all) { + lower = 2.0; + } + const auto cut_st = + detail::build_majorana_evolution_cutoff_state(lower, std::cref(out.coeffs), upper, std::optional{s.param}); + const detail::CutoffEvaluator eval(s.cutoff); + double *const sweep = s.fused_scale ? out.coeffs.data() : nullptr; + const double cos_val = std::cos(2 * s.param); + out.result = with_algebra(s.basis, [&]() { + return detail::fused_find_and_collect(op, + s.gen, + eval, + cut_st, + out.coeffs, + s.cap, + out.window, + g.my_rank, + router, + shift, + s.capture, + sweep, + cos_val, + options, + observer); + }); + return out; +} + +template +auto run_scan(const Geometry &g, const Scenario &s, detail::parallel::Options options, const Observer &observer = {}) + -> Outcome { + return run_scan_on(geometry_op(g, s.basis), g, s, options, observer); +} + +// A fresh operator for `g`: its inverted index and row-parity cache have never been built. +inline auto cold_op(const Geometry &g, Basis basis) -> Op { + return build_op(owned_terms(router_of(g), g.my_rank, kRowsPerRank, 0x5CA11ULL + g.my_rank), basis); +} + +inline auto bitwise_equal(std::span a, std::span b) -> bool { + // memcmp must not see an empty vector's null data pointer, even for zero bytes. + return a.size() == b.size() && (a.empty() || std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0); +} + +// The concatenated cosine set, as LayerBuildEngine consumes it. +inline auto concatenated_cos(const Result &r) -> CosMask { + CosMask all; + for (const auto &m : r.cos_blocks) { + all.total_count += m.total_count; + all.blocks.insert(all.blocks.end(), m.blocks.begin(), m.blocks.end()); + } + return all; +} + +// Total queries a result emitted, over every stream. +inline auto emitted(const Result &r) -> size_t { + size_t n = 0; + for (const auto &v : r.leader_src) { + n += v.size(); + } + for (const auto &v : r.follower_src) { + n += v.size(); + } + return n; +} + +// The self stage's logical content only: never its capacity tail. +inline auto check_same_stage(const detail::SelfQueryStage &a, + const detail::SelfQueryStage &b, + const char *which) -> void { + BOOST_TEST_CONTEXT(which) { + BOOST_TEST_REQUIRE(a.size() == b.size()); + BOOST_TEST_REQUIRE(a.positions() == b.positions()); + for (size_t q = 0; q < a.size(); ++q) { + if (a.pos_off[q] != b.pos_off[q] || a.k_of[q] != b.k_of[q] || a.phase_of[q] != b.phase_of[q]) { + BOOST_TEST_REQUIRE(false, "self query " << q << " differs"); + } + } + BOOST_TEST(std::equal(a.pos_flat.begin(), + a.pos_flat.begin() + static_cast(a.positions()), + b.pos_flat.begin())); + } +} + +// Checks one result's own invariants: every populated stream carries the scan's window, value streams are +// empty without capture, self wire stays empty, and the self stages align with the self slot's sources. +inline auto check_shape(const Outcome &o, bool capture) -> void { + const Result &r = o.result; + BOOST_TEST((r.leader_queries.window() == o.window)); + BOOST_TEST((r.follower_queries.window() == o.window)); + BOOST_TEST((r.leader_src.window() == o.window)); + BOOST_TEST((r.follower_src.window() == o.window)); + if (capture) { + BOOST_TEST((r.leader_val.window() == o.window)); + BOOST_TEST((r.follower_val.window() == o.window)); + } + else { + BOOST_TEST(r.leader_val.size() == 0U); + BOOST_TEST(r.follower_val.size() == 0U); + } + if (o.window.contains(o.my_rank)) { + BOOST_TEST(r.leader_queries.at_slot(o.my_rank).empty()); + BOOST_TEST(r.follower_queries.at_slot(o.my_rank).empty()); + BOOST_TEST(r.leader_self.size() == r.leader_src.at_slot(o.my_rank).size()); + BOOST_TEST(r.follower_self.size() == r.follower_src.at_slot(o.my_rank).size()); + if (capture) { + BOOST_TEST(r.leader_self.size() == r.leader_val.at_slot(o.my_rank).size()); + BOOST_TEST(r.follower_self.size() == r.follower_val.at_slot(o.my_rank).size()); + } + } + else { + BOOST_TEST(r.leader_self.size() == 0U); + BOOST_TEST(r.follower_self.size() == 0U); + } + for (const auto wi : o.window.indices()) { + if (o.window.slot(wi) != o.my_rank) { // a remote slot: one wire record, at least one word, per source + BOOST_TEST(r.leader_queries[wi].size() >= r.leader_src[wi].size()); + BOOST_TEST(r.follower_queries[wi].size() >= r.follower_src[wi].size()); + } + if (capture) { + BOOST_TEST(r.leader_val[wi].size() == r.leader_src[wi].size()); + BOOST_TEST(r.follower_val[wi].size() == r.follower_src[wi].size()); + } + } + // Ascending and disjoint across the concatenation, which LayerBuildEngine relies on. + const auto cos = concatenated_cos(r); + size_t bits = 0; + for (size_t k = 0; k < cos.blocks.size(); ++k) { + bits += static_cast(std::popcount(cos.blocks[k].second)); + if (k != 0 && cos.blocks[k - 1].first >= cos.blocks[k].first) { + BOOST_TEST_REQUIRE(false, "cosine blocks not ascending at " << k); + } + } + BOOST_TEST(bits == cos.total_count); +} + +// Exact equality of two scans of the same inputs: every stream of every window slot, both self stages, +// the concatenated cosine set, and the (possibly scaled) coefficients. +inline auto check_same_scan(const Outcome &a, const Outcome &b, bool capture) -> void { + BOOST_TEST_REQUIRE((a.window == b.window)); + check_shape(a, capture); + check_shape(b, capture); + for (const auto wi : a.window.indices()) { + BOOST_TEST_CONTEXT("slot " << a.window.slot(wi)) { + BOOST_TEST((a.result.leader_queries[wi] == b.result.leader_queries[wi])); + BOOST_TEST((a.result.follower_queries[wi] == b.result.follower_queries[wi])); + BOOST_TEST((a.result.leader_src[wi] == b.result.leader_src[wi])); + BOOST_TEST((a.result.follower_src[wi] == b.result.follower_src[wi])); + if (capture) { + BOOST_TEST(bitwise_equal(a.result.leader_val[wi], b.result.leader_val[wi])); + BOOST_TEST(bitwise_equal(a.result.follower_val[wi], b.result.follower_val[wi])); + } + } + } + check_same_stage(a.result.leader_self, b.result.leader_self, "leader_self"); + check_same_stage(a.result.follower_self, b.result.follower_self, "follower_self"); + const auto ca = concatenated_cos(a.result); + const auto cb = concatenated_cos(b.result); + BOOST_TEST(ca.total_count == cb.total_count); + BOOST_TEST((ca.blocks == cb.blocks)); + BOOST_TEST(bitwise_equal(a.coeffs, b.coeffs)); +} + +// FNV-1a over a result's logical content, for comparing one build's scan with another's. +inline auto digest(const Outcome &o, bool capture) -> uint64_t { + uint64_t h = 1469598103934665603ULL; + const auto mix = [&h](uint64_t v) { + for (int k = 0; k < 8; ++k) { + h = (h ^ ((v >> (8 * k)) & 0xFFU)) * 1099511628211ULL; + } + }; + const auto mix_d = [&mix](double d) { mix(std::bit_cast(d)); }; + const Result &r = o.result; + for (const auto wi : o.window.indices()) { + mix(0xABCDEFULL); + for (const auto v : r.leader_queries[wi]) { + mix(v); + } + for (const auto v : r.follower_queries[wi]) { + mix(v); + } + for (const auto v : r.leader_src[wi]) { + mix(v); + } + for (const auto v : r.follower_src[wi]) { + mix(v); + } + if (capture) { + for (const auto v : r.leader_val[wi]) { + mix_d(v); + } + for (const auto v : r.follower_val[wi]) { + mix_d(v); + } + } + } + for (const auto *stage : {&r.leader_self, &r.follower_self}) { + mix(stage->size()); + for (size_t q = 0; q < stage->size(); ++q) { + mix(stage->pos_off[q]); + mix(stage->k_of[q]); + mix(static_cast(static_cast(stage->phase_of[q]))); + } + for (size_t p = 0; p < stage->positions(); ++p) { + mix(stage->pos_flat[p]); + } + } + const auto cos = concatenated_cos(r); + mix(cos.total_count); + for (const auto &[base, bits] : cos.blocks) { + mix(base); + mix(bits); + } + for (const auto c : o.coeffs) { + mix_d(c); + } + return h; +} + +enum class Expect { + output, // the scan emits queries from several ranges + empty_scanned, // the scan traverses its words but emits nothing + not_scanned, // the caller returns before any traversal (no fold columns, or all of them empty) +}; + +// Runs `s` at budget 1 and at budgets 2-4 and requires identical results. Checks on the caller, after each +// scan joined, that the threaded runs split the words into min(budget, fold blocks) ranges that ran inside +// the scan's own region, and that the serial run stayed on the caller in one range. +inline auto check_threaded_matches_serial(const Geometry &g, const Scenario &s, Expect expect) -> void { + using detail::KernelRange; + BOOST_TEST_CONTEXT(g.label << " / " << s.label) { + const size_t words = geometry_op(g, s.basis).inverted_index().words(); + const size_t blocks = detail::logical_ranges(words, detail::kColumnBlockWords); + const bool safe = detail::CutoffEvaluator(s.cutoff).parallel_safe(); + const size_t scanned = expect == Expect::not_scanned ? 0U : 1U; + + kernel_test::RangeLog serial_log; + const auto serial = run_scan(g, s, {.threads = 1}, kernel_test::RecordingObserver{&serial_log}); + kernel_test::check_serial(serial_log[KernelRange::scan], scanned, "serial scan"); + if (expect == Expect::output) { + BOOST_TEST_REQUIRE(emitted(serial.result) > 0U); + // Sources in at least two of the four budget-4 ranges, so the merge really joins pieces. + std::set ranges_hit; + const size_t k = std::min(4, blocks); + for (const auto *streams : {&serial.result.leader_src, &serial.result.follower_src}) { + for (const auto &v : *streams) { + for (const auto src : v) { + for (size_t r = 0; r < k; ++r) { + const auto [lo, hi] = detail::scan_range_words(r, k, words); + if (src >= lo * 64 && src < hi * 64) { + ranges_hit.insert(r); + } + } + } + } + } + BOOST_TEST(ranges_hit.size() >= 2U); + } + else { + BOOST_TEST(emitted(serial.result) == 0U); + } + for (const int threads : {2, 3, 4}) { + BOOST_TEST_CONTEXT("budget " << threads) { + kernel_test::RangeLog log; + const auto threaded = run_scan(g, s, {.threads = threads}, kernel_test::RecordingObserver{&log}); + check_same_scan(serial, threaded, s.capture); + const size_t expected = scanned == 0 ? 0 : (safe ? std::min(threads, blocks) : 1); + if (expected >= 2) { + kernel_test::check_participation(log[KernelRange::scan], expected, "threaded scan"); + } + else { + kernel_test::check_serial(log[KernelRange::scan], expected, "serial fallback scan"); + } + } + } + } +} + +// A generator of the requested pivot tier and parity whose rank shift under `router` is (non)zero, found by +// a deterministic search, so a scenario does not depend on the routing basis the build happens to draw. +inline auto find_generator(const routing::Router &router, + size_t pivot_lo, + size_t pivot_hi, + size_t weight, + bool want_zero_shift, + uint64_t seed) -> Mono { + kernel_test::SplitMix rng{seed}; + for (int attempt = 0; attempt < 100000; ++attempt) { + Mono g{}; + g.set(pivot_lo + rng.below(pivot_hi - pivot_lo)); + const size_t pivot = g.find_first(); + while (g.count() < weight) { + const size_t b = pivot + 1 + rng.below(64 - pivot - 1); + g.set(b); + } + if ((router.rank_shift(g) == 0) == want_zero_shift) { + return g; + } + } + BOOST_FAIL("no generator with the requested shift"); + return {}; +} + +} // namespace scan_test diff --git a/cpp/tests/SlotBlocks.h b/cpp/tests/SlotBlocks.h deleted file mode 100644 index 73544758b..000000000 --- a/cpp/tests/SlotBlocks.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/MPICompat.h" - -// A [P] vector-of-vectors facade over the WindowVec-only exchange verbs, so the transport tests can -// build and check whole-world block arrays. -namespace test_utils { - -template -auto full_window(const std::vector> &blocks) -> monoprop::mpi::WindowVec> { - monoprop::mpi::WindowVec> w(monoprop::mpi::SlotWindow{.base = 0, .count = blocks.size()}); - std::ranges::copy(blocks, w.begin()); - return w; -} - -// Slots outside the round's window come back empty. -template -auto flat_blocks(const monoprop::mpi::WindowVec> &w, size_t world) -> std::vector> { - std::vector> out(world); - for (const auto wi : w.window().indices()) { - out[w.window().slot(wi)] = w[wi]; - } - return out; -} - -template -auto exchange_blocks(const std::vector> &send, - monoprop::mpi::Comm c, - bool skip_self = false, - const std::vector *known_recv_counts = nullptr, - monoprop::mpi::PeerPlan plan = {}) -> std::vector> { - monoprop::mpi::WindowVec> got; - monoprop::mpi::begin_alltoallv(full_window(send), c, skip_self, known_recv_counts, plan).wait_into(got); - return flat_blocks(got, static_cast(monoprop::mpi::size(c))); -} - -} // namespace test_utils diff --git a/cpp/tests/ThreadHarness.h b/cpp/tests/ThreadHarness.h deleted file mode 100644 index 1d8af18f0..000000000 --- a/cpp/tests/ThreadHarness.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include - -namespace test_utils { - -// Body exceptions are captured per-rank and returned for the caller to check: Boost.Test assertions -// are only safe on the main thread. -template -auto run_comm_threads(Comm &comm, int s, Body body) -> std::vector { - std::vector errs(static_cast(s)); - std::vector threads; - threads.reserve(static_cast(s)); - for (int r = 0; r < s; ++r) { - threads.emplace_back([&, r]() { - try { - body(comm, r); - } - catch (...) { - errs[static_cast(r)] = std::current_exception(); - } - }); - } - for (auto &t : threads) { - t.join(); - } - return errs; -} - -} // namespace test_utils diff --git a/cpp/tests/boost-test.cmake b/cpp/tests/boost-test.cmake index f1d4c1a5a..3791b55c3 100644 --- a/cpp/tests/boost-test.cmake +++ b/cpp/tests/boost-test.cmake @@ -23,13 +23,14 @@ if(NOT _monoprop_mpiexec_numproc_flag) set(_monoprop_mpiexec_numproc_flag "-n") endif() +# ENVIRONMENT: VAR=value entries applied to every variant, the per-case `serial` ones and the whole-suite MPI ones. # SERIAL_ENVIRONMENT: VAR=value entries applied to the per-case `serial` variants only. function(discover_tests TARGET) cmake_parse_arguments( "" "" "WORKING_DIRECTORY" - "EXTRA_ARGS;PROPERTIES;SERIAL_ENVIRONMENT" + "EXTRA_ARGS;PROPERTIES;ENVIRONMENT;SERIAL_ENVIRONMENT" ${ARGN} ) @@ -53,6 +54,8 @@ function(discover_tests TARGET) ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_tests-${args_hash}.cmake" ) + # The whole-suite variants run at every rank count in monoprop_MPI_TEST_PROCS, at the launch-time T its caller + # registers (cpp/tests/CMakeLists.txt). if(_monoprop_mpiexec AND monoprop_ENABLE_MPI) set(_enable_mpi_variants "ON") else() @@ -69,6 +72,7 @@ function(discover_tests TARGET) "TEST_EXECUTABLE=$" -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" -D "TEST_PROPERTIES=${_PROPERTIES}" -D + "TEST_ENVIRONMENT=${_ENVIRONMENT}" -D "TEST_SERIAL_ENVIRONMENT=${_SERIAL_ENVIRONMENT}" -D "TEST_LIST=${_TEST_LIST}" -D "CTEST_FILE=${ctest_tests_file}" -D "TEST_ENABLE_MPI_VARIANTS=${_enable_mpi_variants}" -D diff --git a/cpp/tests/boostAddTests.cmake b/cpp/tests/boostAddTests.cmake index 32834ca29..1a5ad5487 100644 --- a/cpp/tests/boostAddTests.cmake +++ b/cpp/tests/boostAddTests.cmake @@ -75,6 +75,9 @@ while(_common_prop_idx LESS _common_prop_len) math(EXPR _common_prop_idx "${_common_prop_idx} + 2") endwhile() +# Every variant's launch environment (the caller's ENVIRONMENT), ahead of what a variant adds. +list(APPEND common_env_list ${TEST_ENVIRONMENT}) + list(APPEND common_labels_list cxx) list(REMOVE_DUPLICATES common_labels_list) list(REMOVE_DUPLICATES common_env_list) diff --git a/cpp/tests/combined_recompute_equivalence.cpp b/cpp/tests/combined_recompute_equivalence.cpp index 6196a0a81..bc66a12a7 100644 --- a/cpp/tests/combined_recompute_equivalence.cpp +++ b/cpp/tests/combined_recompute_equivalence.cpp @@ -24,6 +24,7 @@ #include #include +#include "KernelTestSupport.h" #include "TestUtilities.h" #include "monoprop/detail/evolution/CosineRecompute.h" @@ -113,11 +114,15 @@ BOOST_AUTO_TEST_CASE(combined_scale_cache_equals_recompute) { std::vector a = baseline; std::vector b = baseline; + std::vector threaded = baseline; scale_cos_cached(prepared, a.data(), cos_val); monoprop::detail::scale_cos_lazy(inverted_index, recipe, b.data(), cos_val); + monoprop::detail::scale_cos_lazy(inverted_index, recipe, threaded.data(), cos_val, {.threads = 3}); BOOST_TEST_INFO("layer " << li); BOOST_TEST(std::memcmp(a.data(), b.data(), n * sizeof(double)) == 0); + BOOST_TEST_INFO("layer " << li << " threads 3"); + BOOST_TEST(std::memcmp(a.data(), threaded.data(), n * sizeof(double)) == 0); } // The fixture must actually exercise the odd-|G| parity correction, or the guardrail is hollow. BOOST_TEST(odd_layers > 0u); @@ -341,9 +346,155 @@ BOOST_AUTO_TEST_CASE(lazy_fold_survives_operator_growth) { auto prepared = monoprop::detail::make_fold_cache(sim.mp_op().inverted_index(), gen, scaled_count, kBasis); std::vector expected = baseline; - std::vector actual = baseline; scale_cos_cached(prepared, expected.data(), cos_val); - monoprop::detail::scale_cos_lazy(sim.mp_op().inverted_index(), recipe, actual.data(), cos_val); + for (const int threads : {1, 4}) { + std::vector actual = baseline; + monoprop::detail::scale_cos_lazy(sim.mp_op().inverted_index(), + recipe, + actual.data(), + cos_val, + {.threads = threads}); + BOOST_TEST_INFO("threads " << threads); + BOOST_TEST(std::memcmp(expected.data(), actual.data(), n * sizeof(double)) == 0); + } +} +// --- threaded lazy folds over several fold blocks ----------------------------------------------------------- +// +// The real fixtures above fit in one kColumnBlockWords block, so scale_cos_lazy stays serial on them. These +// use a synthetic InvertedIndex<8> spanning more than three blocks, with dense, sparse, dense-but-sparse and +// empty columns (KernelTestSupport.h), and require every budget to match the materialised-fold oracle +// bitwise. + +namespace { + +struct FoldCase { + const char *label; + Monomial gen; + Basis basis; +}; + +auto fold_cases() -> std::vector { + using kernel_test::generator; + return { + {"majorana even, dense", generator({1, 4}), Basis::Majorana}, + {"majorana even, mixed tiers", generator({0, 2, 9, 13}), Basis::Majorana}, + {"majorana odd, dense + sparse", generator({0, 3, 7}), Basis::Majorana}, + {"majorana odd, sparse + block edges", generator({6, 8, 14}), Basis::Majorana}, + {"majorana even, empty columns", generator({10, 11}), Basis::Majorana}, + {"majorana odd, one empty column (parity only)", generator({10}), Basis::Majorana}, + {"zero generator", generator({}), Basis::Majorana}, + {"pauli J(G), two columns", generator({0, 5}), Basis::Pauli}, + {"pauli J(G), three columns", generator({3, 8, 15}), Basis::Pauli}, + }; +} + +// Threaded first, on a cold parity cache: the kernel must build row parity on the caller before its team +// starts. Only then are the one-thread run and the oracle (whose make_fold_cache warms the cache) computed. +auto check_threaded_lazy_fold(const monoprop::detail::InvertedIndex &index, + const FoldCase &fc, + uint64_t scaled_count) -> void { + constexpr size_t kModes = kernel_test::kIndexModes; + const size_t n = index.rows(); + const auto recipe = monoprop::detail::make_lazy_fold(index, fc.gen, scaled_count, fc.basis); + const std::vector baseline = [n] { + std::vector v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = 1.0 + static_cast(i) * 1e-3; + } + return v; + }(); + const double cos_val = 0.6234; + + std::vector> threaded; + for (const int threads : {2, 3, 4}) { + index.row_parity_.clear(); + threaded.push_back(baseline); + monoprop::detail::scale_cos_lazy(index, recipe, threaded.back().data(), cos_val, {.threads = threads}); + } + index.row_parity_.clear(); + std::vector serial = baseline; + monoprop::detail::scale_cos_lazy(index, recipe, serial.data(), cos_val, {.threads = 1}); + + const auto prepared = monoprop::detail::make_fold_cache(index, fc.gen, scaled_count, fc.basis); + std::vector oracle = baseline; + scale_cos_cached(prepared, oracle.data(), cos_val); + + BOOST_TEST_CONTEXT(fc.label << " rows=" << n << " scaled_count=" << scaled_count) { + BOOST_TEST(std::memcmp(oracle.data(), serial.data(), n * sizeof(double)) == 0); + for (size_t t = 0; t < threaded.size(); ++t) { + BOOST_TEST_INFO("threads " << t + 2); + BOOST_TEST(std::memcmp(oracle.data(), threaded[t].data(), n * sizeof(double)) == 0); + } + // Rows at or beyond scaled_count are never scaled. + size_t scaled_beyond = 0; + for (size_t i = std::min(scaled_count, n); i < n; ++i) { + scaled_beyond += static_cast(serial[i] != baseline[i]); + } + BOOST_TEST(scaled_beyond == 0u); + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(combined_threaded_lazy_fold_matches_oracle) { + constexpr size_t block_rows = monoprop::detail::kColumnBlockWords * 64; + // More than three fold blocks, with 63-, 64- and 65-row tails past a word boundary. + const size_t base = 3 * block_rows + 7 * 64; + for (const size_t tail : {63, 64, 65}) { + const size_t rows = base + tail; + const auto index = kernel_test::make_synthetic_index(rows, 1000 + tail); + size_t odd_folds = 0; + for (const auto &fc : fold_cases()) { + odd_folds += static_cast( + monoprop::algebra_fold_needs_odd_correction(fc.basis, fc.gen)); + // Full; a partial final word; fewer blocks than the index; a single word in the second range; + // nothing at all. + for (const uint64_t scaled_count : {uint64_t{rows}, + uint64_t{rows - 100}, + uint64_t{2 * block_rows + 17}, + uint64_t{block_rows + 1}, + uint64_t{0}}) { + check_threaded_lazy_fold(index, fc, scaled_count); + } + } + // The odd-|G| parity correction and the Pauli fold are both exercised. + BOOST_TEST(odd_folds > 0u); + } +} + +// A retained LazyFold across index growth, threaded: the parity words the workers read must be fetched for +// this call, from the grown index, never retained from before the growth (see LazyFold). +BOOST_AUTO_TEST_CASE(combined_threaded_lazy_fold_survives_index_growth) { + constexpr size_t kModes = kernel_test::kIndexModes; + constexpr size_t block_rows = monoprop::detail::kColumnBlockWords * 64; + const size_t old_rows = 2 * block_rows + 500; + auto index = kernel_test::make_synthetic_index(old_rows, 77); + const auto gen = kernel_test::generator({0, 6, 8}); + const uint64_t scaled_count = old_rows - 3; + const auto recipe = monoprop::detail::make_lazy_fold(index, gen, scaled_count, Basis::Majorana); + + const uint64_t *before = index.row_parity_words(); + BOOST_REQUIRE(before != nullptr); + // Keep the old allocation alive and poisoned, so a stale pointer would be observably wrong. + auto old_row_parity = std::move(index.row_parity_); + for (auto &word : old_row_parity) { + word = ~word; + } + kernel_test::grow_synthetic_index(index, 4 * block_rows + 123, 78); + const size_t n = index.rows(); + + std::vector baseline(n); + for (size_t i = 0; i < n; ++i) { + baseline[i] = 1.0 + static_cast(i) * 1e-3; + } + const double cos_val = 0.6234; + std::vector actual = baseline; + monoprop::detail::scale_cos_lazy(index, recipe, actual.data(), cos_val, {.threads = 4}); + BOOST_TEST(index.row_parity_words() != before); + + const auto prepared = monoprop::detail::make_fold_cache(index, gen, scaled_count, Basis::Majorana); + std::vector expected = baseline; + scale_cos_cached(prepared, expected.data(), cos_val); BOOST_TEST(std::memcmp(expected.data(), actual.data(), n * sizeof(double)) == 0); } diff --git a/cpp/tests/cpu_topology_tests.cpp b/cpp/tests/cpu_topology_tests.cpp deleted file mode 100644 index 678397cc1..000000000 --- a/cpp/tests/cpu_topology_tests.cpp +++ /dev/null @@ -1,464 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Coverage of CpuTopology (hwloc-based topology discovery + thread affinity pinning). -// -// Tests are split into two layers: -// 1. Live smoke tests — exercise enumerate_physical_cores / partition_cpusets / pin_this_thread -// on the actual host topology; these validate end-to-end hwloc integration. -// 2. Policy unit tests — call topo_detail::placement_order with synthetic PhysicalCore vectors -// so the L3-domain interleaving and MPI-rank slicing logic can be checked deterministically -// without depending on live hardware or hwloc. - -#include - -#include -#include -#include -#include -#include -#include - -#if defined(__linux__) -#include -#endif - -#include "monoprop/detail/partition/CpuTopology.h" - -namespace partition = monoprop::detail::partition; -using partition::topo_detail::placement_order; - -/* RAII helper: save and restore the calling thread's CPU affinity around pin_this_thread() calls - * so that CTest is not left pinned to a single PU after the test completes. */ -struct AffinityGuard { -#if defined(__linux__) - cpu_set_t saved_{}; - AffinityGuard() { sched_getaffinity(0, sizeof(saved_), &saved_); } - ~AffinityGuard() { sched_setaffinity(0, sizeof(saved_), &saved_); } -#else - // Off Linux there is no affinity to restore, but the destructor must still be user-declared: - // an empty aggregate makes every `AffinityGuard guard;` look like an unused variable. - AffinityGuard() = default; - ~AffinityGuard() = default; -#endif -}; - -/* ── Live smoke tests ─────────────────────────────────────────────────────── */ - -BOOST_AUTO_TEST_CASE(cpu_topology_enumerate_and_place) { - const auto cores = partition::enumerate_physical_cores(); - - AffinityGuard guard; // save affinity before any potential pin - const auto one = partition::partition_cpusets(/*n=*/1); - BOOST_CHECK(one.size() <= 1u); - if (!one.empty()) { - // A placement only comes back when topology discovery succeeded and pinning is enabled. - BOOST_CHECK(!cores.empty()); - partition::pin_this_thread(one.front()); - // guard restores affinity on scope exit - } - // Pinning is unconditional, so a non-empty core list must produce a non-empty placement. - if (!cores.empty()) { - BOOST_CHECK_EQUAL(one.size(), 1u); - } - - // Oversubscription must always return empty regardless of topology state. - const auto too_many = partition::partition_cpusets(/*n=*/1'000'000); - BOOST_CHECK(too_many.empty()); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_place_co_located_ranks) { - const auto cores = partition::enumerate_physical_cores(); - if (cores.size() < 2) { - return; // need at least two cores for the disjoint-placement check - } - - // Two co-located ranks each requesting one partition. This exercises both placement arms: - // - interleave (group_count ≤ #L3 domains) - // - domain-major slice (group_count > #L3 domains) - const auto rank0 = partition::partition_cpusets(/*n=*/1, /*group_index=*/0, /*group_count=*/2); - const auto rank1 = partition::partition_cpusets(/*n=*/1, /*group_index=*/1, /*group_count=*/2); - if (rank0.empty() || rank1.empty()) { - BOOST_CHECK(rank0.empty()); - BOOST_CHECK(rank1.empty()); - return; - } - BOOST_REQUIRE_EQUAL(rank0.size(), 1u); - BOOST_REQUIRE_EQUAL(rank1.size(), 1u); - // The two placements must be on distinct PUs; sharing would violate the MPI no-starvation - // invariant (one rank's busy-polling collectives cannot starve the other's barrier spins). - BOOST_CHECK(rank0.front().pu != rank1.front().pu); - - // Both arms passed explicitly so neither depends on the host: refuse the shared one, fill the private. - const auto shared_mask = partition::partition_cpusets(/*n=*/cores.size(), - /*group_index=*/1, - /*group_count=*/2, - partition::NodeMask::Shared); - BOOST_CHECK(shared_mask.empty()); - - const auto private_mask = partition::partition_cpusets(/*n=*/cores.size(), - /*group_index=*/1, - /*group_count=*/2, - partition::NodeMask::PerRank); - BOOST_REQUIRE_EQUAL(private_mask.size(), cores.size()); - std::set placed; - for (const auto &set : private_mask) { - placed.insert(set.pu); - } - BOOST_CHECK_EQUAL(placed.size(), cores.size()); - std::set visible; - for (const auto &core : cores) { - visible.insert(core.cpu); - } - for (const int pu : placed) { - BOOST_CHECK(visible.count(pu) == 1); - } -} - -/* ── Policy unit tests (deterministic, no hwloc or live hardware) ─────────── */ - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_interleave_across_l3) { - // 4 cores across 2 L3 domains; single rank receives all. - // by_domain[0] = {0, 4}, by_domain[1] = {2, 6} - // depth-first interleave: 0, 2, 4, 6 - const std::vector cores = {{0, 0}, {2, 1}, {4, 0}, {6, 1}}; - const auto order = placement_order(cores, 4, 0, 1); - BOOST_REQUIRE_EQUAL(order.size(), 4u); - BOOST_CHECK_EQUAL(order[0], 0); - BOOST_CHECK_EQUAL(order[1], 2); - BOOST_CHECK_EQUAL(order[2], 4); - BOOST_CHECK_EQUAL(order[3], 6); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_disjoint_mpi_ranks) { - // 4 cores across 2 L3 domains; 2 co-located ranks each get 1 partition. - // rank0 is dealt domain 0, rank1 is dealt domain 1 ⇒ no shared PU. - const std::vector cores = {{0, 0}, {2, 1}, {4, 0}, {6, 1}}; - const auto r0 = placement_order(cores, 1, 0, 2); - const auto r1 = placement_order(cores, 1, 1, 2); - BOOST_REQUIRE_EQUAL(r0.size(), 1u); - BOOST_REQUIRE_EQUAL(r1.size(), 1u); - BOOST_CHECK(r0.front() != r1.front()); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_domain_major_more_ranks_than_l3) { - // 4 cores in 1 L3 domain; 2 ranks each get 2 partitions (flat domain-major arm). - // order = [0, 2, 4, 6]; rank0 offset=0 → {0,2}, rank1 offset=2 → {4,6}. - const std::vector cores = {{0, 0}, {2, 0}, {4, 0}, {6, 0}}; - const auto r0 = placement_order(cores, 2, 0, 2); - const auto r1 = placement_order(cores, 2, 1, 2); - BOOST_REQUIRE_EQUAL(r0.size(), 2u); - BOOST_REQUIRE_EQUAL(r1.size(), 2u); - - const std::set s0(r0.begin(), r0.end()); - const std::set s1(r1.begin(), r1.end()); - for (const auto cpu : s1) { - BOOST_CHECK(!s0.contains(cpu)); - } -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_insufficient_cores_returns_empty) { - // 2 cores total; 2 ranks × 2 partitions = 4 > 2 ⇒ oversubscription. - const std::vector cores = {{0, 0}, {2, 0}}; - BOOST_CHECK(placement_order(cores, 2, 0, 2).empty()); - BOOST_CHECK(placement_order(cores, 2, 1, 2).empty()); - - // Single rank requesting more cores than exist. - BOOST_CHECK(placement_order(cores, 3, 0, 1).empty()); - - // Empty core list. - BOOST_CHECK(placement_order({}, 1, 0, 1).empty()); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_singleton_l3_domains) { - // 2 cores each in its own singleton domain (no shared L3). - // by_domain[0] = {0}, by_domain[1] = {4}; interleaved: 0, 4. - const std::vector cores = {{0, 0}, {4, 1}}; - const auto order = placement_order(cores, 2, 0, 1); - BOOST_REQUIRE_EQUAL(order.size(), 2u); - BOOST_CHECK_EQUAL(order[0], 0); - BOOST_CHECK_EQUAL(order[1], 4); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_uneven_domains) { - // 3 cores: 2 in domain 0, 1 in domain 1; single rank, 3 partitions. - // by_domain[0] = {0, 4}, by_domain[1] = {2}; interleaved: 0, 2, 4. - const std::vector cores = {{0, 0}, {2, 1}, {4, 0}}; - const auto order = placement_order(cores, 3, 0, 1); - BOOST_REQUIRE_EQUAL(order.size(), 3u); - BOOST_CHECK_EQUAL(order[0], 0); - BOOST_CHECK_EQUAL(order[1], 2); - BOOST_CHECK_EQUAL(order[2], 4); -} - -/* ── The cgroup-placement classification ──────────────────────────────────── */ - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_per_rank_slice_starves_without_collapse) { - // One rank's slice under `srun --cpu-bind=cores`: 2 cores of a 16-core host, one L3 domain. - const std::vector slice = {{6, 0}, {7, 0}}; - - BOOST_CHECK(placement_order(slice, 2, /*group_index=*/3, /*group_count=*/8).empty()); - - // Collapsed to a single group -- what NodeMask::PerRank does -- the same slice places fully. - const auto collapsed = placement_order(slice, 2, /*group_index=*/0, /*group_count=*/1); - BOOST_REQUIRE_EQUAL(collapsed.size(), 2u); - BOOST_CHECK_EQUAL(collapsed[0], 6); - BOOST_CHECK_EQUAL(collapsed[1], 7); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_policy_private_mask_collapses_even_when_the_split_would_fit) { - // 2 ranks x 2 partitions fits these 4 cores, so the collapse is not a fallback: it moves rank 1's cores. - const std::vector cores = {{0, 0}, {2, 1}, {4, 0}, {6, 1}}; - - const auto split = placement_order(cores, 2, /*group_index=*/1, /*group_count=*/2); - BOOST_REQUIRE_EQUAL(split.size(), 2u); - BOOST_CHECK_EQUAL(split[0], 2); - BOOST_CHECK_EQUAL(split[1], 6); - - // What NodeMask::PerRank now passes: the head of this rank's own interleave over both domains. - const auto collapsed = placement_order(cores, 2, /*group_index=*/0, /*group_count=*/1); - BOOST_REQUIRE_EQUAL(collapsed.size(), 2u); - BOOST_CHECK_EQUAL(collapsed[0], 0); - BOOST_CHECK_EQUAL(collapsed[1], 2); -} - -namespace { - -// The flat [n * words] array MPI_Allgather leaves behind, built from per-rank PU-index lists. -auto packed_masks(const std::vector> &pus, size_t words) -> std::vector { - std::vector out(pus.size() * words, 0); - for (size_t r = 0; r < pus.size(); ++r) { - for (const size_t pu : pus[r]) { - out[(r * words) + (pu / 64)] |= uint64_t{1} << (pu % 64); - } - } - return out; -} - -} // namespace - -BOOST_AUTO_TEST_CASE(cpu_topology_masks_disjoint_vs_identical) { - constexpr size_t kWords = partition::kAffinityMaskWords; - - const auto disjoint = packed_masks({{0, 1}, {2, 3}}, kWords); - BOOST_CHECK(partition::masks_are_pairwise_disjoint(disjoint.data(), 2, kWords)); - - const auto identical = packed_masks({{0, 1}, {0, 1}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(identical.data(), 2, kWords)); - - // Partial overlap: "not private" is conservative, since collapsing points every rank at the same cores. - const auto partial = packed_masks({{0, 1}, {1, 2}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(partial.data(), 2, kWords)); - - // An unreadable mask arrives empty and must not be read as "disjoint from everything". - const auto with_empty = packed_masks({{0, 1}, {}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(with_empty.data(), 2, kWords)); - - const auto lone = packed_masks({{0, 1}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(lone.data(), 1, kWords)); - - const auto four_ok = packed_masks({{0}, {1}, {2}, {3}}, kWords); - BOOST_CHECK(partition::masks_are_pairwise_disjoint(four_ok.data(), 4, kWords)); - const auto four_bad = packed_masks({{0}, {1}, {2}, {1}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(four_bad.data(), 4, kWords)); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_masks_span_word_boundaries) { - constexpr size_t kWords = partition::kAffinityMaskWords; - - // A per-word comparison that forgot to loop would answer from word 0 alone. - const auto low_high = packed_masks({{5}, {200}}, kWords); - BOOST_CHECK(partition::masks_are_pairwise_disjoint(low_high.data(), 2, kWords)); - - const auto both_high = packed_masks({{200}, {200}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(both_high.data(), 2, kWords)); - - const auto late_overlap = packed_masks({{1, 3000}, {2, 3000}}, kWords); - BOOST_CHECK(!partition::masks_are_pairwise_disjoint(late_overlap.data(), 2, kWords)); -} - -BOOST_AUTO_TEST_CASE(cpu_topology_affinity_mask_covers_enumerated_cores) { - // The only check that the mask EXCHANGED and the cores PLACED come from one view of the machine. - const auto cores = partition::enumerate_physical_cores(); - if (cores.empty()) { - return; // hwloc loaded no topology at all: there is no second view to agree with. - } - std::vector mine(partition::kAffinityMaskWords, 0); - if (!partition::affinity_mask_words(mine.data(), mine.size())) { - // Not a skip: refusal keys on the HIGHEST allowed PU, and core.cpu is each core's LOWEST sibling. - std::vector wide(partition::kAffinityMaskWords * 64, 0); - BOOST_REQUIRE(partition::affinity_mask_words(wide.data(), wide.size())); - size_t highest = 0; - for (size_t w = wide.size(); w-- > 0;) { - if (wide[w] != 0) { - highest = (w * 64) + static_cast(63 - __builtin_clzll(wide[w])); - break; - } - } - BOOST_CHECK_GE(highest, partition::kAffinityMaskWords * 64); - return; - } - size_t set_bits = 0; - for (const uint64_t w : mine) { - set_bits += static_cast(__builtin_popcountll(w)); - } - BOOST_CHECK(set_bits > 0u); - for (const auto &core : cores) { - const auto pu = static_cast(core.cpu); - BOOST_CHECK((mine[pu / 64] >> (pu % 64)) & 1U); - } -} - -#if defined(__linux__) - -namespace { - -// Confine to the first `k` cores, asserting the narrowing took hold; false only if the kernel refused. -auto confine_to_first(const std::vector &full, size_t k) -> bool { - cpu_set_t mask; - CPU_ZERO(&mask); - for (size_t i = 0; i < k; ++i) { - CPU_SET(full[i].cpu, &mask); - } - if (sched_setaffinity(0, sizeof(mask), &mask) != 0) { - return false; - } - BOOST_REQUIRE_EQUAL(partition::enumerate_physical_cores().size(), k); - return true; -} - -} // namespace - -BOOST_AUTO_TEST_CASE(cpu_topology_per_rank_mask_still_places) { - const auto full = partition::enumerate_physical_cores(); - if (full.empty()) { - return; // hwloc loaded no topology: there is no mask to confine to. - } - // ONE core exercises the collapse, so a single-core runner does not turn this into a free pass. - const size_t k = std::min(full.size(), 2); - - const AffinityGuard guard; - if (!confine_to_first(full, k)) { - return; // the kernel refused the affinity call; nothing below is reachable - } - - // What `srun --cpu-bind=cores` produces: our whole share, told there are eight sibling ranks. - const auto sets = - partition::partition_cpusets(/*n=*/k, /*group_index=*/3, /*group_count=*/8, partition::NodeMask::PerRank); - BOOST_REQUIRE_EQUAL(sets.size(), k); - for (const auto &set : sets) { - // Never pin outside the mask the launcher gave us. - bool inside = false; - for (size_t i = 0; i < k; ++i) { - inside = inside || set.pu == full[i].cpu; - } - BOOST_CHECK(inside); - } - if (k == 2) { - BOOST_CHECK(sets[0].pu != sets[1].pu); - } -} - -BOOST_AUTO_TEST_CASE(cpu_topology_shared_mask_keeps_co_located_ranks_disjoint) { - const auto full = partition::enumerate_physical_cores(); - if (full.size() < 2) { - return; // two ranks cannot hold disjoint cores when there is only one - } - // Two cores, one per rank: a two-physical-core CI runner cannot supply the four the port asked for. - const size_t per_rank = full.size() >= 4 ? 2 : 1; - const size_t shared = 2 * per_rank; - - const AffinityGuard guard; - if (!confine_to_first(full, shared)) { - return; // the kernel refused the affinity call; nothing below is reachable - } - - const auto rank0 = partition::partition_cpusets(/*n=*/per_rank, - /*group_index=*/0, - /*group_count=*/2, - partition::NodeMask::Shared); - const auto rank1 = partition::partition_cpusets(/*n=*/per_rank, - /*group_index=*/1, - /*group_count=*/2, - partition::NodeMask::Shared); - BOOST_REQUIRE_EQUAL(rank0.size(), per_rank); - BOOST_REQUIRE_EQUAL(rank1.size(), per_rank); - for (const auto &a : rank0) { - for (const auto &b : rank1) { - // Two ranks on one core starve each other: busy-polling collectives against barrier spins. - BOOST_CHECK(a.pu != b.pu); - } - } -} - -#endif // __linux__ - -/* ── summarize_masks and the COMMPLACE line ───────────────────────────────── */ - -BOOST_AUTO_TEST_CASE(cpu_topology_summarize_masks) { - constexpr size_t kWords = partition::kAffinityMaskWords; - - // Two ranks holding one CPU each and two ranks sharing the same CPUs differ in exactly node_cpus. - const auto private_ = packed_masks({{0, 1}, {2, 3}}, kWords); - const auto priv = partition::summarize_masks(private_.data(), 2, kWords, 0); - BOOST_REQUIRE(priv.has_value()); - BOOST_CHECK_EQUAL(priv->cpus, 2U); - BOOST_CHECK_EQUAL(priv->node_cpus, 4U); - BOOST_CHECK_EQUAL(priv->cpu_list, "0-3"); - - const auto shared = packed_masks({{0, 1}, {0, 1}}, kWords); - const auto shd = partition::summarize_masks(shared.data(), 2, kWords, 1); - BOOST_REQUIRE(shd.has_value()); - BOOST_CHECK_EQUAL(shd->cpus, 2U); - BOOST_CHECK_EQUAL(shd->node_cpus, 2U); - - // A run crossing a 64-bit word boundary is ONE run: a per-word loop would print "63,64". - const auto cross = packed_masks({{63, 64}}, kWords); - BOOST_CHECK_EQUAL(partition::summarize_masks(cross.data(), 1, kWords, 0).value().cpu_list, "63-64"); - - // No summary rather than a plausible zero: an all-zero row is a mask that did not fit the window. - const auto with_empty = packed_masks({{0, 1}, {}}, kWords); - BOOST_CHECK(!partition::summarize_masks(with_empty.data(), 2, kWords, 0).has_value()); - BOOST_CHECK(!partition::summarize_masks(nullptr, 1, kWords, 0).has_value()); - BOOST_CHECK(!partition::summarize_masks(private_.data(), 2, kWords, 2).has_value()); // self out of range -} - -// The formatter returns a string rather than writing one, so the line is testable without a live rank. -BOOST_AUTO_TEST_CASE(cpu_topology_place_line_reports_every_field) { - const partition::MaskSummary sum{.cpus = 16, .node_cpus = 128, .cpu_list = "0-127"}; - // Whole line, not field lookups: a reordered or unterminated line has to fail too. - BOOST_CHECK_EQUAL(partition::format_place_line(5, 2, 8, "private", sum), - "COMMPLACE rank=5 node_rank=2 node_size=8 masks=private cpus=16 node_cpus=128 " - "cpu_list=0-127\n"); -} - -// 437 identical lines in one 585-test run is noise; a CHANGED line still has to get through. -BOOST_AUTO_TEST_CASE(cpu_topology_place_line_repeats_are_collapsed) { - // Distinctive, so a real COMMPLACE line emitted by some other case cannot be the previous value. - const std::string a = "COMMPLACE test-a\n"; - const std::string b = "COMMPLACE test-b\n"; - BOOST_CHECK(partition::place_line_is_new(a)); - BOOST_CHECK(!partition::place_line_is_new(a)); - BOOST_CHECK(!partition::place_line_is_new(a)); - BOOST_CHECK(partition::place_line_is_new(b)); // a different placement is not a repeat - BOOST_CHECK(!partition::place_line_is_new(b)); - BOOST_CHECK(partition::place_line_is_new(a)); // and neither is coming back to the first one -} - -// The state summarize_masks refuses to classify must SAY unknown rather than print a plausible zero. -BOOST_AUTO_TEST_CASE(cpu_topology_place_line_unknown_is_not_a_verdict) { - BOOST_CHECK_EQUAL(partition::format_place_line(0, 0, 1, "unknown", partition::MaskSummary{}), - "COMMPLACE rank=0 node_rank=0 node_size=1 masks=unknown cpus=0 node_cpus=0 " - "cpu_list=none\n"); -} diff --git a/cpp/tests/ctor_validation_tests.cpp b/cpp/tests/ctor_validation_tests.cpp index d3bb5997f..2080834b9 100644 --- a/cpp/tests/ctor_validation_tests.cpp +++ b/cpp/tests/ctor_validation_tests.cpp @@ -35,9 +35,9 @@ namespace { constexpr size_t N = 8; using MP = MonomialPropagator; -// Construct an invalid partitioned Schrodinger propagator. Its cardinality check must run before -// partition workers start, or a worker's exception is masked by the poisoned shared-memory transport. -auto make_oversized_partitioned_schrodinger() -> MonomialPropagator<64> { +// Construct a Schrodinger propagator whose paired basis cannot be walked or addressed. Its cardinality check runs on +// the caller, before any shard is seeded, so the configuration error is reported as itself. +auto make_oversized_schrodinger() -> MonomialPropagator<64> { return MonomialPropagator<64>(OperatorDict{{VecZ{0, 1}, std::complex(0.0, 1.0)}}, /*cutoff=*/4, VecZ{}, @@ -48,8 +48,7 @@ auto make_oversized_partitioned_schrodinger() -> MonomialPropagator<64> { CutoffType::Length, std::nullopt, /*logical_num_modes=*/64, - Basis::Majorana, - /*partitions=*/2); + Basis::Majorana); } // Construct with the full argument list; individual cases vary just the field(s) under test. @@ -125,12 +124,10 @@ BOOST_AUTO_TEST_CASE(ctor_operator_index_out_of_range_throws) { BOOST_CHECK_THROW(make(op), std::runtime_error); } -BOOST_AUTO_TEST_CASE(ctor_partitioned_schrodinger_cardinality_error_is_not_masked) { - BOOST_CHECK_EXCEPTION(make_oversized_partitioned_schrodinger(), - PropagatorConfigError, - [](const PropagatorConfigError &error) { - return std::string_view{error.what()}.contains("schrodinger_cutoff"); - }); +BOOST_AUTO_TEST_CASE(ctor_oversized_schrodinger_basis_is_rejected_before_seeding) { + BOOST_CHECK_EXCEPTION(make_oversized_schrodinger(), PropagatorConfigError, [](const PropagatorConfigError &error) { + return std::string_view{error.what()}.contains("schrodinger_cutoff"); + }); } // A gate generator index outside the system must throw, not underflow 2*NumModes-1-index into an diff --git a/cpp/tests/env_config_tests.cpp b/cpp/tests/env_config_tests.cpp index 7533727ec..a927ba424 100644 --- a/cpp/tests/env_config_tests.cpp +++ b/cpp/tests/env_config_tests.cpp @@ -21,37 +21,19 @@ using monoprop::config::EnvConfigError; using monoprop::config::RoutingMode; -using monoprop::config::detail::parse_positive_int; using monoprop::config::detail::parse_routing_mode; using monoprop::config::detail::parse_uint64; -BOOST_AUTO_TEST_CASE(env_config_parse_positive_int_null_and_malformed) { - BOOST_CHECK(parse_positive_int(nullptr) == std::nullopt); - BOOST_CHECK(parse_positive_int("") == std::nullopt); - BOOST_CHECK(parse_positive_int("abc") == std::nullopt); - BOOST_CHECK(parse_positive_int("12x") == std::nullopt); // trailing junk rejects, not a partial 12 - BOOST_CHECK(parse_positive_int(" ") == std::nullopt); // strtol consumes ws, then end == text -} - -BOOST_AUTO_TEST_CASE(env_config_parse_positive_int_range) { - BOOST_CHECK(parse_positive_int("0") == std::nullopt); - BOOST_CHECK(parse_positive_int("-5") == std::nullopt); - BOOST_CHECK(parse_positive_int("1000001") == std::nullopt); // above the 1e6 ceiling - BOOST_CHECK(parse_positive_int("1") == std::optional(1)); - BOOST_CHECK(parse_positive_int("42") == std::optional(42)); - BOOST_CHECK(parse_positive_int("1000000") == std::optional(1'000'000)); // inclusive upper bound -} - BOOST_AUTO_TEST_CASE(env_config_settings_cached_singleton) { const auto &a = monoprop::config::get(); const auto &b = monoprop::config::get(); BOOST_CHECK_EQUAL(&a, &b); // Touch a field so the Settings aggregate is actually read. - BOOST_CHECK(a.num_threads == std::nullopt || *a.num_threads >= 1); + BOOST_CHECK(a.route_seed == b.route_seed); } -// Both routing parsers throw where parse_positive_int returns nullopt: a routing knob that -// defaulted silently would change the transport with no diagnostic. +// Both routing parsers throw on a malformed value: a routing knob that defaulted silently would change the transport +// with no diagnostic. BOOST_AUTO_TEST_CASE(env_config_parse_uint64_unset_valid_and_rejected) { BOOST_CHECK(parse_uint64("k", nullptr) == std::nullopt); BOOST_CHECK(parse_uint64("k", "") == std::nullopt); diff --git a/cpp/tests/evolution_detail_tests.cpp b/cpp/tests/evolution_detail_tests.cpp index a673b7ac2..8b8186f7d 100644 --- a/cpp/tests/evolution_detail_tests.cpp +++ b/cpp/tests/evolution_detail_tests.cpp @@ -22,6 +22,7 @@ #include #include +#include "ScanTestSupport.h" #include "monoprop/TypeAliases.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/evolution/CutoffContext.h" @@ -39,6 +40,7 @@ namespace { // resolve_range_ touches only wants_values and self_hit, so the engine drives without the cross-rank sink surface. struct RecordingSink { static constexpr bool wants_values = false; + using Response = TermIndex; // named by the engine's exchange phases, which this case never calls std::vector> hits; // (src, found) auto self_hit(size_t src, size_t found, int /*phase*/, double /*v_src*/) -> void { hits.emplace_back(src, found); } }; @@ -148,13 +150,11 @@ BOOST_AUTO_TEST_CASE(self_resolve_mark_bounded_by_combined_size) { matched.begin_gate(op.size()); detail::LayerBuildEngine<8, RecordingSink> eng(op, - mpi::Comm{}, /*R_=*/1, /*my_rank_=*/0, matched, combined_size, RecordingSink{}, - mpi::PeerPlan{}, mpi::SlotWindow{.base = 0, .count = 1}); // The self leg is staged as positions, never encoded, so this feeds the stage the scan would fill. using Eng = detail::LayerBuildEngine<8, RecordingSink>; @@ -223,3 +223,174 @@ BOOST_AUTO_TEST_CASE(cutoff_context_is_below_sin) { BOOST_TEST(ctx.is_below_sin(0.1)); BOOST_TEST(!ctx.is_below_sin(1.0)); } + +// --- threaded bitmap scan (fused_find_and_collect over logical word ranges) ------------------------------------ + +// Ranges are contiguous runs of whole fold blocks, nonempty, ascending with the range ID, covering every +// word, and depend only on the word count and the budget. +BOOST_AUTO_TEST_CASE(openmp_scan_ranges_partition_words_in_order) { + constexpr size_t kBlock = detail::kColumnBlockWords; + for (const size_t words : + {size_t{1}, kBlock - 1, kBlock, kBlock + 1, (3 * kBlock) + 1, 7 * kBlock, size_t{100003}}) { + for (const int threads : {1, 2, 3, 4, 7, 96}) { + const size_t ranges = detail::scan_ranges(words, {.threads = threads}); + BOOST_TEST_CONTEXT("words=" << words << " threads=" << threads) { + BOOST_TEST(ranges == std::min(threads, detail::logical_ranges(words, kBlock))); + size_t next = 0; + for (size_t r = 0; r < ranges; ++r) { + const auto [lo, hi] = detail::scan_range_words(r, ranges, words); + BOOST_TEST(lo == next); + BOOST_TEST(hi > lo); + BOOST_TEST((lo % kBlock) == 0U); + next = hi; + } + BOOST_TEST(next == words); + } + } + } +} + +// Majorana scans at budgets 2-4 against budget 1, byte for byte, across windows (one rank; linear routing +// with zero and non-zero shifts, including a two-slot window at a non-zero base; dense splitmix at four and +// three ranks), dense/sparse/empty pivots, odd generators with empty fold columns, length caps, structural +// cutoffs (including cutoff 0), lower/upper atol boundaries, scans that emit nothing, value capture and the +// fused cos sweep. only_rotate_len_k must be positive (0 is rejected before any scan). +BOOST_AUTO_TEST_CASE(openmp_scan_ranges_match_serial_majorana) { + using scan_test::Atol; + using scan_test::Expect; + using scan_test::Scenario; + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + const auto dense_even = scan_test::mono_of_bits({14, 20, 33, 47}); + const auto sparse_even = scan_test::mono_of_bits({5, 40}); + const auto dense_odd = scan_test::mono_of_bits({13, 29, 52}); + const auto empty_odd = scan_test::mono_of_bits({0, 1, 2}); + const auto empty_even = scan_test::mono_of_bits({0, 3}); + const std::vector> single{ + {{.label = "dense pivot", .gen = dense_even}, Expect::output}, + {{.label = "sparse pivot", .gen = sparse_even}, Expect::output}, + {{.label = "odd generator", .gen = dense_odd}, Expect::output}, + {{.label = "odd generator, empty fold columns", .gen = empty_odd}, Expect::output}, + {{.label = "even generator, empty fold columns", .gen = empty_even}, Expect::not_scanned}, + {{.label = "lower atol above every term", .gen = dense_even, .atol = Atol::drop_all}, Expect::empty_scanned}, + {{.label = "lower atol above every term, fused", + .gen = dense_even, + .atol = Atol::drop_all, + .capture = true, + .fused_scale = true}, + Expect::empty_scanned}, + {{.label = "length cutoff 0", .gen = dense_even, .cutoff = detail::LengthCutoff<32>{0}}, Expect::empty_scanned}, + {{.label = "length cutoff 0, upper atol rescue", + .gen = dense_odd, + .cutoff = detail::LengthCutoff<32>{0}, + .atol = Atol::upper_equal, + .capture = true}, + Expect::output}, + {{.label = "cap 6", .gen = dense_even, .cap = 6}, Expect::output}, + {{.label = "cap 6, capture", .gen = dense_odd, .cap = 6, .capture = true}, Expect::output}, + {{.label = "length cutoff 5", .gen = dense_even, .cutoff = detail::LengthCutoff<32>{5}}, Expect::output}, + {{.label = "support cutoff 4", .gen = sparse_even, .cutoff = detail::SupportCutoff<32>{4}}, Expect::output}, + {{.label = "lower atol at a term", .gen = dense_even, .atol = Atol::lower_equal}, Expect::output}, + {{.label = "lower atol at a term, capture", .gen = dense_odd, .atol = Atol::lower_equal, .capture = true}, + Expect::output}, + {{.label = "upper atol rescue", + .gen = dense_even, + .cutoff = detail::LengthCutoff<32>{2}, + .atol = Atol::upper_equal}, + Expect::output}, + {{.label = "capture", .gen = sparse_even, .capture = true}, Expect::output}, + {{.label = "fused cos sweep", .gen = dense_even, .capture = true, .fused_scale = true}, Expect::output}, + {{.label = "fused cos sweep, odd, lower atol", + .gen = dense_odd, + .atol = Atol::lower_equal, + .capture = true, + .fused_scale = true}, + Expect::output}, + }; + for (const auto &[s, e] : single) { + scan_test::check_threaded_matches_serial(solo, s, e); + } + + const std::vector multi{ + {"linear R=4, zero shift", 4, 1, true, 2}, + {"linear R=4, non-zero shift", 4, 1, true, 0}, + {"linear R=2 S=2, non-zero shift", 2, 2, true, 1}, + {"linear R=2 S=2, zero shift", 2, 2, true, 1}, + {"splitmix R=4", 4, 1, false, 3}, + {"splitmix R=3", 3, 1, false, 1}, + }; + for (size_t gi = 0; gi < multi.size(); ++gi) { + const auto &g = multi[gi]; + const auto router = scan_test::router_of(g); + const bool want_zero = gi == 0 || gi == 3 || !g.linear; + const auto dense = g.linear ? scan_test::find_generator(router, 12, 30, 4, want_zero, 11 + gi) : dense_even; + const auto odd = g.linear ? scan_test::find_generator(router, 12, 30, 3, want_zero, 23 + gi) : dense_odd; + const auto sparse = g.linear ? scan_test::find_generator(router, 4, 12, 2, want_zero, 37 + gi) : sparse_even; + if (g.linear) { + BOOST_TEST_REQUIRE((router.rank_shift<32>(dense) == 0U) == want_zero); + } + const std::vector cases{ + {.label = "dense pivot", .gen = dense}, + {.label = "sparse pivot, capture", .gen = sparse, .capture = true}, + {.label = "odd generator, lower atol", .gen = odd, .atol = Atol::lower_equal}, + {.label = "odd generator, empty fold columns", .gen = empty_odd}, + {.label = "fused cos sweep", .gen = dense, .capture = true, .fused_scale = true}, + {.label = "cap 6, capture", .gen = odd, .cap = 6, .capture = true}, + }; + for (const auto &s : cases) { + scan_test::check_threaded_matches_serial(g, s, Expect::output); + } + } +} + +// A threaded scan as the first user of a fresh operator: the inverted index and the odd-generator row-parity +// cache are both cold, so building either lazily inside a worker would race (and TSan reports it). The scan +// must prepare both on the caller and still match a serial scan of an identical operator. +BOOST_AUTO_TEST_CASE(openmp_scan_prepares_cold_caches_on_the_caller) { + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + for (const auto &gen : {scan_test::mono_of_bits({13, 29, 52}), scan_test::mono_of_bits({0, 1, 2})}) { + const scan_test::Scenario s{.label = "odd, cold", .gen = gen, .capture = true}; + const auto threaded_op = scan_test::cold_op(solo, Basis::Majorana); + kernel_test::RangeLog log; + const auto threaded = + scan_test::run_scan_on(threaded_op, solo, s, {.threads = 4}, kernel_test::RecordingObserver{&log}); + BOOST_TEST(log[detail::KernelRange::scan].worker.size() == 4U); + const auto serial_op = scan_test::cold_op(solo, Basis::Majorana); + const auto serial = scan_test::run_scan_on(serial_op, solo, s, {.threads = 1}); + scan_test::check_same_scan(serial, threaded, true); + } +} + +// Worker participation in the scan's own range bodies, at the budget of the ranges' count. +BOOST_AUTO_TEST_CASE(openmp_scan_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + const scan_test::Scenario s{.label = "dense pivot", .gen = scan_test::mono_of_bits({14, 20, 33, 47})}; + kernel_test::RangeLog log; + (void)scan_test::run_scan(solo, s, {.threads = 4}, kernel_test::RecordingObserver{&log}); + kernel_test::check_participation(log[detail::KernelRange::scan], 4, "scan"); +} + +// A worker failure inside one scan range is joined, then rethrown on the caller; the other ranges are not +// merged into anything the caller can observe. +BOOST_AUTO_TEST_CASE(openmp_scan_worker_failure_joins_before_rethrow) { + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + for (const bool fused : {false, true}) { + const scan_test::Scenario s{.label = "failure", + .gen = scan_test::mono_of_bits({14, 20, 33, 47}), + .capture = fused, + .fused_scale = fused}; + for (const size_t failing : {size_t{0}, size_t{2}}) { + kernel_test::RangeLog log; + BOOST_CHECK_THROW( + (void)scan_test::run_scan(solo, + s, + {.threads = 3}, + kernel_test::RecordingObserver{&log, detail::KernelRange::scan, failing}), + std::runtime_error); + const auto &slots = log[detail::KernelRange::scan]; + BOOST_TEST(slots.prepares == 1U); + BOOST_TEST(slots.visits.size() == 3U); + BOOST_TEST(slots.visits[failing] == 1); + } + } +} diff --git a/cpp/tests/evolved_operator_coefficients_tests.cpp b/cpp/tests/evolved_operator_coefficients_tests.cpp index ef5b5af29..1e7358d3b 100644 --- a/cpp/tests/evolved_operator_coefficients_tests.cpp +++ b/cpp/tests/evolved_operator_coefficients_tests.cpp @@ -27,8 +27,9 @@ #include "monoprop/algebra/AlgebraCommon.h" #include "monoprop/detail/mpi/MPICompat.h" -// evolved_operator_coefficients() probes the operator index with caller-supplied keys instead of -// enumerating it. Oracle throughout: evolved_operator_terms(params, 0.0), which enumerates. +// evolved_operator_coefficients() probes every shard's operator index with caller-supplied keys instead of +// enumerating it. Oracle throughout: evolved_operator_terms(params, 0.0), which enumerates, at the same shard count. +// Every case runs at the launch's T; cpp/tests/CMakeLists.txt also reruns the suite at T = 2 and 4. namespace { @@ -38,7 +39,7 @@ using namespace test_utils; constexpr size_t kNumModes = 8; constexpr unsigned int kCutoff = 8; -auto majorana_sim(const CaseData &data, size_t partitions = 1) -> MonomialPropagator { +auto majorana_sim(const CaseData &data) -> MonomialPropagator { return MonomialPropagator(data.hamiltonian, kCutoff, data.initial_state, @@ -49,20 +50,19 @@ auto majorana_sim(const CaseData &data, size_t partitions = 1) -> MonomialPropag CutoffType::Length, std::nullopt, kNumModes, - Basis::Majorana, - partitions); + Basis::Majorana); } // A propagator whose graph is built and ready to contract. -auto built_sim(const CaseData &data, size_t partitions = 1) -> MonomialPropagator { - auto sim = majorana_sim(data, partitions); +auto built_sim(const CaseData &data) -> MonomialPropagator { + auto sim = majorana_sim(data); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); return sim; } // A Schrodinger-picture propagator: the state's identity amplitude is an ordinary index row there, // so the empty key resolves through the probe rather than through core_term(). -auto built_schrodinger_sim(const CaseData &data, size_t partitions = 1) -> MonomialPropagator { +auto built_schrodinger_sim(const CaseData &data) -> MonomialPropagator { auto sim = MonomialPropagator(data.hamiltonian, kCutoff, data.initial_state, @@ -73,8 +73,7 @@ auto built_schrodinger_sim(const CaseData &data, size_t partitions = 1) -> Monom CutoffType::Length, std::nullopt, kNumModes, - Basis::Majorana, - partitions); + Basis::Majorana); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); return sim; } @@ -92,7 +91,7 @@ auto keys_of(const std::vector>> &terms) -> // Querying every enumerated term must reproduce the enumerated coefficients exactly: same // contraction, same decode, only the index access differs. -BOOST_AUTO_TEST_CASE(coefficients_match_enumerated_terms) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_coefficients_match_enumerated_terms) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_sim(data); @@ -110,7 +109,7 @@ BOOST_AUTO_TEST_CASE(coefficients_match_enumerated_terms) { } // A monomial the operator does not carry reads back as exactly 0, not as noise and not as a throw. -BOOST_AUTO_TEST_CASE(absent_term_reads_back_zero) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_absent_term_reads_back_zero) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_sim(data); @@ -138,7 +137,7 @@ BOOST_AUTO_TEST_CASE(absent_term_reads_back_zero) { } // The result is positional: out[q] belongs to keys[q] whatever order the keys arrive in. -BOOST_AUTO_TEST_CASE(query_order_is_preserved) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_query_order_is_preserved) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_sim(data); @@ -166,33 +165,9 @@ BOOST_AUTO_TEST_CASE(query_order_is_preserved) { } } -// Partitions hash-split the operator, so a key resolves in exactly one of them. The probe must -// still see the whole operator, matching the single-partition run term for term. -BOOST_AUTO_TEST_CASE(partitioned_probe_matches_single_partition) { - const auto data = load_case_data("random_exact.msgpack"); - auto ref = built_sim(data); - - const auto terms = ref.evolved_operator_terms(data.parameters, 0.0); - BOOST_REQUIRE(!terms.empty()); - const auto keys = keys_of(terms); - const auto expected = ref.evolved_operator_coefficients(data.parameters, keys); - - for (const size_t partitions : {size_t{2}, size_t{4}}) { - auto sim = built_sim(data, partitions); - const auto coeffs = sim.evolved_operator_coefficients(data.parameters, keys); - BOOST_REQUIRE_EQUAL(coeffs.size(), expected.size()); - for (size_t q = 0; q < expected.size(); ++q) { - BOOST_TEST_CONTEXT("partitions=" << partitions << " query " << q) { - BOOST_TEST(near(coeffs[q].real(), expected[q].real())); - BOOST_TEST(near(coeffs[q].imag(), expected[q].imag())); - } - } - } -} - // The keys are user input, so the checked encode is mandatory: an out-of-range slot must throw // rather than write past the monomial (indices_to_bitset is noexcept and would corrupt memory). -BOOST_AUTO_TEST_CASE(out_of_range_slot_index_throws) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_out_of_range_slot_index_throws) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_sim(data); @@ -201,7 +176,7 @@ BOOST_AUTO_TEST_CASE(out_of_range_slot_index_throws) { } // An empty query list is a no-op, not a degenerate probe. -BOOST_AUTO_TEST_CASE(empty_query_list_returns_empty) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_empty_query_list_returns_empty) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_sim(data); @@ -213,7 +188,7 @@ BOOST_AUTO_TEST_CASE(empty_query_list_returns_empty) { // the empty query has to resolve to core_term() for the two APIs to agree term for term. The // fixture Hamiltonian has no identity term, so inject one -- otherwise core_term() is 0 and the // check cannot tell the core term apart from the absent-term answer. -BOOST_AUTO_TEST_CASE(empty_term_resolves_to_core_term) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_empty_term_resolves_to_core_term) { auto data = load_case_data("random_exact.msgpack"); data.hamiltonian[VecZ{}] = std::complex{0.75, 0.0}; auto sim = built_sim(data); @@ -233,7 +208,7 @@ BOOST_AUTO_TEST_CASE(empty_term_resolves_to_core_term) { // The Schrodinger picture is the motivating case: reading a handful of amplitudes out of an evolved // state. There is no core term to divert the identity there -- it is an ordinary index row carrying // the state's identity amplitude -- so the empty query resolves through the probe like any other. -BOOST_AUTO_TEST_CASE(schrodinger_coefficients_match_enumerated_terms) { +BOOST_AUTO_TEST_CASE(evolved_coefficients_schrodinger_coefficients_match_enumerated_terms) { const auto data = load_case_data("random_exact.msgpack"); auto sim = built_schrodinger_sim(data); @@ -258,66 +233,18 @@ BOOST_AUTO_TEST_CASE(schrodinger_coefficients_match_enumerated_terms) { BOOST_TEST(probed.at(0).imag() == identity->second.imag()); } -// The two axes above cross: this is the only configuration where the identity is a hash-partitioned -// index row, so it exercises the empty key against the concurrent per-partition probe and merge. -BOOST_AUTO_TEST_CASE(partitioned_schrodinger_probe_matches_single_partition) { +// The query is a list, not a set: a repeated key is answered once per occurrence. Worth pinning because the probe +// visits every shard, so a key counted twice would show up here first. +BOOST_AUTO_TEST_CASE(evolved_coefficients_repeated_key_is_answered_once_per_occurrence) { const auto data = load_case_data("random_exact.msgpack"); - auto ref = built_schrodinger_sim(data); - - const auto terms = ref.evolved_operator_terms(data.parameters, 0.0); - BOOST_REQUIRE(!terms.empty()); - const auto keys = keys_of(terms); - BOOST_REQUIRE(std::any_of(keys.begin(), keys.end(), [](const auto &key) { return key.empty(); })); - const auto expected = ref.evolved_operator_coefficients(data.parameters, keys); - - for (const size_t partitions : {size_t{2}, size_t{4}}) { - auto sim = built_schrodinger_sim(data, partitions); - const auto coeffs = sim.evolved_operator_coefficients(data.parameters, keys); - BOOST_REQUIRE_EQUAL(coeffs.size(), expected.size()); - for (size_t q = 0; q < expected.size(); ++q) { - BOOST_TEST_CONTEXT("partitions=" << partitions << " query " << q) { - BOOST_TEST(near(coeffs[q].real(), expected[q].real())); - BOOST_TEST(near(coeffs[q].imag(), expected[q].imag())); - } - } - } -} - -// core_term_ is replicated on every partition, so the empty key must be assigned from core_term() -// -- routed through partition 0 -- and not summed over the partitions the way an index row is. -BOOST_AUTO_TEST_CASE(partitioned_empty_term_resolves_to_core_term) { - auto data = load_case_data("random_exact.msgpack"); - data.hamiltonian[VecZ{}] = std::complex{0.75, 0.0}; - - for (const size_t partitions : {size_t{1}, size_t{2}, size_t{4}}) { - auto sim = built_sim(data, partitions); - const auto coeffs = sim.evolved_operator_coefficients(data.parameters, {VecZ{}}); - BOOST_TEST_CONTEXT("partitions=" << partitions) { - BOOST_REQUIRE_EQUAL(coeffs.size(), 1u); - BOOST_TEST(coeffs[0].real() == 0.75); - BOOST_TEST(coeffs[0].imag() == 0.0); - } - } -} - -// The query is a list, not a set: a repeated key is answered once per occurrence. Worth pinning -// because the partitioned merge accumulates, so a key counted twice would show up here first. -BOOST_AUTO_TEST_CASE(repeated_key_is_answered_once_per_occurrence) { - const auto data = load_case_data("random_exact.msgpack"); - - const auto terms = built_sim(data).evolved_operator_terms(data.parameters, 0.0); + auto sim = built_sim(data); + const auto terms = sim.evolved_operator_terms(data.parameters, 0.0); BOOST_REQUIRE(!terms.empty()); const auto &key = terms.front().first; const auto expected = terms.front().second; - - for (const size_t partitions : {size_t{1}, size_t{2}}) { - auto sim = built_sim(data, partitions); - const auto coeffs = sim.evolved_operator_coefficients(data.parameters, {key, VecZ{}, key}); - BOOST_TEST_CONTEXT("partitions=" << partitions) { - BOOST_REQUIRE_EQUAL(coeffs.size(), 3u); - BOOST_TEST(near(coeffs[0].real(), expected.real())); - BOOST_TEST(near(coeffs[0].imag(), expected.imag())); - BOOST_TEST(coeffs[2] == coeffs[0]); - } - } + const auto coeffs = sim.evolved_operator_coefficients(data.parameters, {key, VecZ{}, key}); + BOOST_REQUIRE_EQUAL(coeffs.size(), 3u); + BOOST_TEST(coeffs[0].real() == expected.real()); + BOOST_TEST(coeffs[0].imag() == expected.imag()); + BOOST_TEST(coeffs[2] == coeffs[0]); } diff --git a/cpp/tests/exact_upper_atol_rescue.cpp b/cpp/tests/exact_upper_atol_rescue.cpp index 3243d965d..574c3bb66 100644 --- a/cpp/tests/exact_upper_atol_rescue.cpp +++ b/cpp/tests/exact_upper_atol_rescue.cpp @@ -18,6 +18,7 @@ #include #include +#include "ScanTestSupport.h" #include "TestUtilities.h" // upper_atol rescue: a structural cutoff of 0 rejects every non-identity partner, and upper_atol = 0 @@ -77,3 +78,37 @@ MAKE_ZERO_CUTOFF_RESCUE_TEST(zero_cutoff_upper_atol_zero_is_exact_lih, LihFixtur MAKE_ZERO_CUTOFF_RESCUE_TEST(zero_cutoff_upper_atol_zero_is_exact_lih, LihFixture, World) #undef MAKE_ZERO_CUTOFF_RESCUE_TEST + +// The atol boundaries decide per term from the pre-scale coefficient in whichever range owns it: a lower atol +// equal to a term's |sin|·|c| drops it, an upper atol equal to it rescues a partner the cutoff rejects. The +// threaded scan must make exactly the serial decisions, with and without value capture and the fused sweep. +BOOST_AUTO_TEST_CASE(openmp_scan_atol_boundaries_match_serial) { + using scan_test::Atol; + using scan_test::Expect; + const auto gen = scan_test::mono_of_bits({14, 20, 33, 47}); + const auto tight = monoprop::detail::LengthCutoff<32>{2}; + for (const scan_test::Geometry g : + {scan_test::Geometry{"one rank", 1, 1, false, 0}, scan_test::Geometry{"splitmix R=4", 4, 1, false, 1}}) { + for (const bool capture : {false, true}) { + scan_test::check_threaded_matches_serial( + g, + {.label = "lower atol at a term", .gen = gen, .atol = Atol::lower_equal, .capture = capture}, + Expect::output); + scan_test::check_threaded_matches_serial(g, + {.label = "upper atol rescue", + .gen = gen, + .cutoff = tight, + .atol = Atol::upper_equal, + .capture = capture}, + Expect::output); + } + scan_test::check_threaded_matches_serial(g, + {.label = "upper atol rescue, fused cos sweep", + .gen = gen, + .cutoff = tight, + .atol = Atol::upper_equal, + .capture = true, + .fused_scale = true}, + Expect::output); + } +} diff --git a/cpp/tests/exchange_layout_precondition_tests.cpp b/cpp/tests/exchange_layout_precondition_tests.cpp index 1015db8f7..d89acbc89 100644 --- a/cpp/tests/exchange_layout_precondition_tests.cpp +++ b/cpp/tests/exchange_layout_precondition_tests.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Preconditions on a layer's exchange layout. Driven through ShmComm, so it needs no ranks. +// Preconditions on a layer's exchange layout, on an ordinary communicator (or the MPI-off stub). #include @@ -21,32 +21,44 @@ #include "monoprop/detail/mpi/Comm.h" #include "monoprop/detail/mpi/Exchange.h" #include "monoprop/detail/mpi/MPICompat.h" -#include "monoprop/detail/mpi/ShmComm.h" using monoprop::mpi::CollectiveArgumentError; using monoprop::mpi::Comm; -using monoprop::mpi::ShmComm; -BOOST_AUTO_TEST_CASE(exchange_layout_width_is_checked) { - // Sized 4 but driven by one thread: the throw must precede any collective, or this hangs. - ShmComm world(4); - const Comm comm = Comm::make_shm(&world, /*rank=*/0); - BOOST_REQUIRE_EQUAL(monoprop::mpi::size(comm), 4); +namespace { - const std::vector too_short(3, 0); - BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(too_short, comm), CollectiveArgumentError); +auto require_width_checks(const Comm &comm) -> void { + const int ranks = monoprop::mpi::size(comm); + BOOST_REQUIRE_GE(ranks, 1); + for (const int width : {0, ranks - 1, ranks + 1, ranks + 3}) { + if (width < 0 || width == ranks) { + continue; + } + BOOST_TEST_CONTEXT("width " << width << " on " << ranks << " rank(s)") { + const std::vector layout(static_cast(width), 0); + BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(layout, comm), CollectiveArgumentError); + } + } + const std::vector exact(static_cast(ranks), 0); + BOOST_CHECK_NO_THROW(monoprop::mpi::check_exchange_layout_width(exact, comm)); +} - const std::vector too_long(5, 0); - BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(too_long, comm), CollectiveArgumentError); +} // namespace - const std::vector empty; - BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(empty, comm), CollectiveArgumentError); +BOOST_AUTO_TEST_CASE(exchange_layout_width_is_checked) { + require_width_checks(Comm(MPI_COMM_SELF)); } -BOOST_AUTO_TEST_CASE(exchange_layout_of_the_right_width_is_accepted) { - ShmComm world(1); - const Comm comm = Comm::make_shm(&world, /*rank=*/0); - - const std::vector exact(1, 0); - BOOST_CHECK_NO_THROW(monoprop::mpi::check_exchange_layout_width(exact, comm)); +// The width check runs before any communication: only rank 0 calls it here, so a collective inside would leave rank 0 +// waiting for peers that never join (a hang, which the MPI variants' timeout reports as a failure). In the MPI-off +// build and in the per-case serial runs the world has one rank. +BOOST_AUTO_TEST_CASE(exchange_layout_width_is_rejected_before_communication) { + const Comm world(MPI_COMM_WORLD); + if (monoprop::mpi::rank(world) == 0) { + const std::vector too_wide(static_cast(monoprop::mpi::size(world)) + 2, 0); + BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(too_wide, world), CollectiveArgumentError); + const std::vector too_short(static_cast(monoprop::mpi::size(world)) - 1, 0); + BOOST_CHECK_THROW(monoprop::mpi::check_exchange_layout_width(too_short, world), CollectiveArgumentError); + } + require_width_checks(world); } diff --git a/cpp/tests/find_package_smoke/CMakeLists.txt b/cpp/tests/find_package_smoke/CMakeLists.txt index 3cfa5e754..c7af7eaf3 100644 --- a/cpp/tests/find_package_smoke/CMakeLists.txt +++ b/cpp/tests/find_package_smoke/CMakeLists.txt @@ -5,5 +5,22 @@ set(CMAKE_CXX_EXTENSIONS OFF) find_package(monoprop CONFIG REQUIRED) -add_executable(smoke ../link_export_probe/link_export_probe.cpp) +add_executable( + smoke + ../link_export_probe/link_export_probe.cpp + ../link_export_probe/sharded_state_chain.cpp + ../link_export_probe/sharded_construction_chain.cpp + ../link_export_probe/sharded_evaluation_chain.cpp +) target_link_libraries(smoke PRIVATE monoprop::monoprop) + +# The MPI setting the caller expects of this package; defaults to what the package reports. Pass +# -DSMOKE_EXPECT_MPI=ON/OFF to check an installed package independently. +if(NOT DEFINED SMOKE_EXPECT_MPI) + set(SMOKE_EXPECT_MPI "${monoprop_ENABLE_MPI}") +endif() +target_compile_definitions( + smoke + PRIVATE + monoprop_EXPECT_MPI=$ +) diff --git a/cpp/tests/fused_cos_sweep_tests.cpp b/cpp/tests/fused_cos_sweep_tests.cpp index a8d36ea23..ddefb63ce 100644 --- a/cpp/tests/fused_cos_sweep_tests.cpp +++ b/cpp/tests/fused_cos_sweep_tests.cpp @@ -14,12 +14,37 @@ #include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include + +#include + +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/RootObserver.h" + +#include "KernelTestSupport.h" +#include "PropagatorTestAccess.h" +#include "ScanTestSupport.h" #include "TestUtilities.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/detail/evolution/layer_build/FusedApply.h" // The fused cos sweep's one deliberate FP deviation from the two-pass path (≤1 ulp per hit endpoint, // from resolve's stored·(1/cos) recovery), against the build_graph()+replay evaluation as oracle. +// +// A budget is a shard count, which no case changes inside a process: the fused-versus-replay cases run at the launch's +// T (1 per case, 2 and 4 in fused_env_t*), the real fused records are read per shard from the propagation seam itself, +// and fixed-geometry runs must agree bitwise shard by shard. Across geometries only complete retained maps are +// comparable (tests/test_sharded_openmp.py). namespace { @@ -47,6 +72,28 @@ auto graph_energy(const CaseData &data, const SimulatorConfig &cfg) -> double { return fn(data.parameters); } +// The budgets a case may use in this process: only the launch's, captured by every root. +auto budgets() -> std::vector { + return {monoprop::detail::parallel::capture_thread_budget().threads}; +} + +template +auto build_root(const CaseData &data, const SimulatorConfig &cfg, int budget) -> MonomialPropagator { + MonomialPropagator sim(data.hamiltonian, + static_cast(2 * NumModes), + data.initial_state, + cfg.schrodinger_cutoff, + cfg.comm, + cfg.atol, + cfg.upper_atol, + cfg.cutoff_type, + cfg.basis_change, + NumModes, + Basis::Majorana); + BOOST_TEST_REQUIRE(monoprop::detail::PropagatorTestAccess::options(sim).threads == budget); + return sim; +} + void check_agreement(const CaseData &data, const SimulatorConfig &cfg, const char *label) { const double inplace = inplace_energy(data, cfg); const double graph = graph_energy(data, cfg); @@ -56,6 +103,80 @@ void check_agreement(const CaseData &data, const SimulatorConfig &cfg, const cha } } +// check_agreement on roots at the launch's T, with the same tolerances. +void check_root_agreement(const CaseData &data, const SimulatorConfig &cfg, int budget, const char *label) { + constexpr size_t N = ExampleDataFix::n_modes; + auto inplace_sim = build_root(data, cfg, budget); + inplace_sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); + const double inplace = inplace_sim.expectation_value_functional(std::nullopt)(VecD{}); + auto graph_sim = build_root(data, cfg, budget); + graph_sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const double graph = graph_sim.expectation_value_functional(std::nullopt)(data.parameters); + BOOST_TEST_CONTEXT(label << " budget=" << budget << " inplace=" << inplace << " graph=" << graph) { + BOOST_CHECK_SMALL(inplace - graph, kAgreeAtol); + BOOST_CHECK_SMALL(inplace - data.actual_expval, kExactAtol); + } +} + +// --- a workload large enough for the threaded kernels --------------------------------------------------------- + +constexpr size_t kBig = 10; // 20 Majoranas +using BigAccess = monoprop::detail::PropagatorTestAccess; + +// A synthetic Hermitian operator of 16k even-weight terms and gates of weights 2-4 (odd included), large +// enough that fused records and replayed folds span several logical ranges. One gate sits at theta = pi/4, +// where the fused sweep recovers pre-cos values through 1/cos ~ 1.6e16. +auto big_case() -> CaseData { + CaseData data; + data.num_modes = kBig; + kernel_test::SplitMix rng{2026}; + const auto draw_indices = [&rng](size_t weight) { + VecZ idx; + while (idx.size() < weight) { + const size_t m = rng.below(2 * kBig); + if (std::ranges::find(idx, m) == idx.end()) { + idx.push_back(m); + } + } + std::ranges::sort(idx); + return idx; + }; + while (data.hamiltonian.size() < 16000) { + const auto idx = draw_indices(2 * (1 + rng.below(3))); + const double x = static_cast(rng.below(1U << 20)) / (1U << 20) - 0.5; + data.hamiltonian[idx] = x * hermitian_coefficient(indices_to_bitset(idx)); + } + data.initial_state = {0, 2, 4, 6, 8}; + constexpr size_t gates = 9; + for (size_t g = 0; g < gates; ++g) { + data.majoranas.push_back(draw_indices(2 + rng.below(3))); + data.param_inds.push_back(g); + data.gen_coeffs.push_back(1.0); + data.parameters.push_back(g == 4 ? std::numbers::pi / 4 : 0.1 + static_cast(rng.below(900)) / 1000.0); + } + // Repeated generators: rotating again by G targets the M*G terms the first rotation inserted, so both + // pictures produce hit records as well as inserts. + data.majoranas[6] = data.majoranas[2]; + data.majoranas[8] = data.majoranas[3]; + return data; +} + +auto big_root(const CaseData &data, bool schrodinger, int budget, MPI_Comm comm) -> MonomialPropagator { + return build_root( + data, + SimulatorConfig{.schrodinger_cutoff = schrodinger ? std::optional(2 * kBig) : std::nullopt, + .comm = comm}, + budget); +} + +auto bitwise_equal(const VecD &a, const VecD &b) -> bool { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0; +} + +auto comm_size(MPI_Comm comm) -> int { + return mpi::size(mpi::Comm(comm)); +} + } // namespace BOOST_FIXTURE_TEST_CASE(fused_sweep_matches_graph_replay_heisenberg, ExampleDataFix) { @@ -77,3 +198,438 @@ BOOST_FIXTURE_TEST_CASE(fused_sweep_matches_graph_replay_schrodinger, ExampleDat BOOST_FIXTURE_TEST_CASE(fused_sweep_matches_graph_replay_schrodinger_atol, ExampleDataFix) { check_agreement(data, SimulatorConfig{.schrodinger_cutoff = 2 * n_modes, .atol = 1e-10}, "schrodinger atol=1e-10"); } + +// The two-pass apply (a length cap at the full width truncates nothing but disables the fused sweep) against +// graph replay, with the same tolerances: the insert snapshots must be gathered before the cosine phase. +BOOST_FIXTURE_TEST_CASE(two_pass_apply_matches_graph_replay, ExampleDataFix) { + constexpr size_t cap = 2 * n_modes; + for (const bool schrodinger : {false, true}) { + for (const int budget : budgets()) { + const SimulatorConfig cfg{.schrodinger_cutoff = + schrodinger ? std::optional(2 * n_modes) : std::nullopt}; + auto inplace_sim = build_root(data, cfg, budget); + inplace_sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, cap); + const double inplace = inplace_sim.expectation_value_functional(std::nullopt)(VecD{}); + auto graph_sim = build_root(data, cfg, budget); + graph_sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, std::nullopt, cap); + const double graph = graph_sim.expectation_value_functional(std::nullopt)(data.parameters); + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger << " budget=" << budget << " inplace=" << inplace + << " graph=" << graph) { + BOOST_CHECK_SMALL(inplace - graph, kAgreeAtol); + BOOST_CHECK_SMALL(inplace - data.actual_expval, kExactAtol); + } + } + } +} + +// The same fused-versus-replay agreement at the launch's T shards. This fixture is small; the big-workload cases below +// cover many-gate records. +BOOST_FIXTURE_TEST_CASE(fused_sweep_matches_graph_replay_at_the_launch_team, ExampleDataFix) { + for (const int budget : budgets()) { + check_root_agreement(data, SimulatorConfig{}, budget, "heisenberg"); + check_root_agreement(data, SimulatorConfig{.atol = 1e-10}, budget, "heisenberg atol=1e-10"); + check_root_agreement(data, SimulatorConfig{.schrodinger_cutoff = 2 * n_modes}, budget, "schrodinger"); + } +} + +namespace { + +namespace sharded = monoprop::detail::sharded; + +// One shard's fused records for one gate, copied from the propagation seam just before production applied them. +struct CapturedGate { + size_t step = 0; + int worker = -1; + monoprop::detail::FusedContract fc; + CosMask cos; + VecD before; + double apply_angle = 0.0; + bool schrodinger = false; + bool fused_scale = false; +}; + +// Copies every owner's real records. Each owner appends to its own shard's list only; the test reads them after the +// propagation's team has joined. +class RecordCapture final : public sharded::RootObserver { +public: + explicit RecordCapture(size_t shards) : gates_(shards) {} + auto fused_records(size_t step, size_t shard, const sharded::FusedGateView &gate) const -> void override { + gates_.at(shard).push_back(CapturedGate{.step = step, + .worker = omp_get_thread_num(), + .fc = gate.records, + .cos = gate.cos, + .before = VecD(gate.coeffs.begin(), gate.coeffs.end()), + .apply_angle = gate.apply_angle, + .schrodinger = gate.schrodinger, + .fused_scale = gate.fused_scale}); + } + [[nodiscard]] auto gates() const -> const std::vector> & { return gates_; } + +private: + mutable std::vector> gates_; +}; + +auto picture_of(const monoprop::detail::MPOperator &op, bool schrodinger) -> const VecD & { + return schrodinger ? op.state_coeffs : op.op_coeffs; +} + +auto apply_captured(const CapturedGate &gate, int threads, kernel_test::RangeLog *log) -> VecD { + auto fc = gate.fc; + VecD out = gate.before; + if (log != nullptr) { + monoprop::detail::apply_fused_contract(fc, + out, + gate.cos, + gate.apply_angle, + gate.schrodinger, + gate.fused_scale, + {threads}, + kernel_test::RecordingObserver{log}); + } + else { + monoprop::detail::apply_fused_contract(fc, + out, + gate.cos, + gate.apply_angle, + gate.schrodinger, + gate.fused_scale, + {threads}); + } + return out; +} + +} // namespace + +// Records from the root's real propagation, read per shard from the seam itself, in both pictures and in the +// fused-scale and two-pass (length cap) modes: every destination slot of a shard has exactly one add-owner across +// hits, inserts and cross-owner halves; each owner applies its records on its own worker; replaying a gate's records +// gives the next gate's starting coefficients and, after the last gate, the shard's picture, bitwise; and the apply +// kernel at budgets 2-4 (run here, outside the team) matches budget 1 bitwise. Halves appear exactly when there is +// another flat owner (P x T > 1). Under an MPI launch each rank audits its own shards. +BOOST_AUTO_TEST_CASE(fused_shard_records_have_one_add_owner_and_apply_exactly) { + const auto data = big_case(); + const auto launch = budgets().front(); + const auto flat_world = static_cast(comm_size(MPI_COMM_WORLD)) * static_cast(launch); + for (const bool schrodinger : {false, true}) { + for (const bool length_cap : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger << " length_cap=" << length_cap << " T=" << launch) { + auto sim = big_root(data, schrodinger, launch, MPI_COMM_WORLD); + const auto &shards = BigAccess::shards(sim); + const RecordCapture capture(shards.size()); + BigAccess::observe(sim, &capture); + const std::optional cap = length_cap ? std::optional(2 * kBig) : std::nullopt; + sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, cap); + BigAccess::observe(sim, nullptr); + + size_t duplicate_owners = 0; + size_t hits = 0; + size_t inserts = 0; + size_t halves = 0; + size_t foreign_workers = 0; + size_t replay_mismatches = 0; + size_t budget_mismatches = 0; + size_t max_gate_records = 0; + size_t max_apply_ranges = 0; + bool fused_seen = false; + bool two_pass_seen = false; + for (size_t t = 0; t < shards.size(); ++t) { + const auto &gates = capture.gates()[t]; + BOOST_TEST_REQUIRE(gates.size() == data.majoranas.size()); + for (size_t g = 0; g < gates.size(); ++g) { + const auto &gate = gates[g]; + BOOST_TEST(gate.step == g); + foreign_workers += static_cast(gate.worker != static_cast(t)); + for (const auto &[slot, count] : kernel_test::add_owner_counts(gate.fc)) { + duplicate_owners += static_cast(count != 1); + } + hits += gate.fc.hits.size(); + inserts += gate.fc.inserts.size(); + halves += gate.fc.cross_half.size(); + const size_t records = gate.fc.hits.size() + gate.fc.inserts.size() + gate.fc.cross_half.size(); + (gate.fused_scale ? fused_seen : two_pass_seen) = true; + const VecD serial = apply_captured(gate, 1, nullptr); + for (const int threads : {2, 3, 4}) { + kernel_test::RangeLog log; + budget_mismatches += + static_cast(!bitwise_equal(apply_captured(gate, threads, &log), serial)); + if (records > max_gate_records) { + max_gate_records = records; + max_apply_ranges = log[monoprop::detail::KernelRange::fused_apply].worker.size(); + } + } + // What production applied: the last result is the shard's picture, and a two-pass gate + // starts from the previous result, extended by the rows it inserted (a fused sweep scales + // its sources during traversal first). + if (g + 1 == gates.size() || !gates[g + 1].fused_scale) { + const VecD &next = + g + 1 < gates.size() ? gates[g + 1].before : picture_of(shards[t]->op, schrodinger); + replay_mismatches += static_cast( + next.size() < serial.size() + || std::memcmp(next.data(), serial.data(), serial.size() * sizeof(double)) != 0); + } + } + } + BOOST_TEST_MESSAGE("records: hits=" << hits << " inserts=" << inserts << " halves=" << halves); + BOOST_TEST(duplicate_owners == 0U); + BOOST_TEST(foreign_workers == 0U); + BOOST_TEST(replay_mismatches == 0U); + BOOST_TEST(budget_mismatches == 0U); + BOOST_TEST(hits + inserts > 0U); + BOOST_TEST((halves > 0U) == (flat_world > 1)); + BOOST_TEST(max_apply_ranges == monoprop::detail::logical_ranges(max_gate_records, 1024)); + BOOST_TEST(fused_seen == !length_cap); + BOOST_TEST(two_pass_seen == length_cap); + } + } + } +} + +// At a fixed geometry the root is deterministic shard by shard: two roots propagate (fused and two-pass) and +// build and replay a graph to the same rows in the same row-ID order, bitwise coefficients and contraction blocks, +// and the same energies. Every row sits on the shard its flat owner names, and no key is on two shards. +BOOST_AUTO_TEST_CASE(fused_shards_are_deterministic_at_fixed_geometry) { + const auto data = big_case(); + const auto launch = budgets().front(); + const auto rank = static_cast(mpi::rank(mpi::Comm(MPI_COMM_WORLD))); + for (const bool schrodinger : {false, true}) { + for (const bool length_cap : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger << " length_cap=" << length_cap << " T=" << launch) { + const std::optional cap = length_cap ? std::optional(2 * kBig) : std::nullopt; + auto first = big_root(data, schrodinger, launch, MPI_COMM_WORLD); + auto second = big_root(data, schrodinger, launch, MPI_COMM_WORLD); + first.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, cap); + second.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, cap); + const auto &a = BigAccess::shards(first); + const auto &b = BigAccess::shards(second); + const auto &router = BigAccess::router(first); + std::set keys; + size_t misrouted = 0; + for (size_t t = 0; t < a.size(); ++t) { + BOOST_TEST(a[t]->op.store->size() == b[t]->op.store->size()); + for (size_t i = 0; i < a[t]->op.store->size() && i < b[t]->op.store->size(); ++i) { + BOOST_TEST( + (materialize_row(*a[t]->op.store, i) == materialize_row(*b[t]->op.store, i))); + } + BOOST_TEST(bitwise_equal(picture_of(a[t]->op, schrodinger), picture_of(b[t]->op, schrodinger))); + a[t]->op.store->for_each([&](const auto &mono, size_t) { + misrouted += static_cast(router.dest(mono) != rank * a.size() + t); + keys.insert(bitset_to_indices(mono)); + }); + } + BOOST_TEST(misrouted == 0U); + BOOST_TEST(keys.size() == first.size()); + BOOST_TEST(first.expectation_value(VecD{}) == second.expectation_value(VecD{})); + } + } + BOOST_TEST_CONTEXT("graph replay, schrodinger=" << schrodinger << " T=" << launch) { + auto first = big_root(data, schrodinger, launch, MPI_COMM_WORLD); + auto second = big_root(data, schrodinger, launch, MPI_COMM_WORLD); + first.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + second.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + BOOST_TEST(bitwise_equal(first.contract_partially(data.parameters, false), + second.contract_partially(data.parameters, false))); + BOOST_TEST(first.expectation_value(data.parameters) == second.expectation_value(data.parameters)); + const auto [value, gradient] = first.expectation_value_and_gradient(data.parameters); + const auto [other, other_gradient] = second.expectation_value_and_gradient(data.parameters); + BOOST_TEST(value == other); + BOOST_TEST(bitwise_equal(gradient, other_gradient)); + } + } +} + +// --- threaded bitmap scan: fused sweep and full construction ------------------------------------------------- + +// The fused cos sweep scales each anticommuting source exactly once, in whichever range owns it, after the +// scan captured its pre-scale value; every other coefficient stays untouched. Budgets 2-4 against the +// independently derived expectation, not only against budget 1. +BOOST_AUTO_TEST_CASE(openmp_scan_fused_sweep_scales_each_source_once) { + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + for (const auto &gen : {scan_test::mono_of_bits({14, 20, 33, 47}), scan_test::mono_of_bits({13, 29, 52})}) { + const scan_test::Scenario plain{.label = "capture", .gen = gen, .capture = true}; + // Without a cap, the non-fused scan's cosine set is every anticommuting source. + const auto reference = scan_test::run_scan(solo, plain, {.threads = 1}); + const auto anti = scan_test::concatenated_cos(reference.result); + BOOST_TEST_REQUIRE(anti.total_count > 0U); + const double cos_val = std::cos(2 * plain.param); + VecD expect = reference.coeffs; + for (const auto &[base, bits] : anti.blocks) { + for (uint64_t m = bits; m != 0; m &= m - 1) { + const size_t i = base + static_cast(std::countr_zero(m)); + expect[i] = reference.coeffs[i] * cos_val; + } + } + scan_test::Scenario fused = plain; + fused.fused_scale = true; + for (const int threads : {1, 2, 3, 4}) { + BOOST_TEST_CONTEXT("budget " << threads) { + const auto out = scan_test::run_scan(solo, fused, {.threads = threads}); + BOOST_TEST(scan_test::bitwise_equal(out.coeffs, expect)); + BOOST_TEST(out.result.cos_blocks.size() >= 1U); + BOOST_TEST(scan_test::concatenated_cos(out.result).total_count == 0U); // the sweep replaces the set + // Captured values are the pre-scale coefficients, and the same queries are emitted. + const auto &a = reference.result.leader_val.at_slot(0); + const auto &b = out.result.leader_val.at_slot(0); + BOOST_TEST(scan_test::bitwise_equal(a, b)); + BOOST_TEST((reference.result.leader_src.at_slot(0) == out.result.leader_src.at_slot(0))); + BOOST_TEST((reference.result.follower_src.at_slot(0) == out.result.follower_src.at_slot(0))); + } + } + } +} + +namespace { + +constexpr size_t kWide = 16; // 32 Majoranas +using WideAccess = monoprop::detail::PropagatorTestAccess; + +// About 90k even-weight terms, so the scan's operator spans at least two fold blocks from the first gate and +// more as gates insert partners; gates of weights 2-4 with two repeats, and one at theta = pi/4. +auto wide_case() -> CaseData { + CaseData data; + data.num_modes = kWide; + kernel_test::SplitMix rng{0x5CA2ULL}; + const auto draw_indices = [&rng](size_t weight) { + VecZ idx; + while (idx.size() < weight) { + const size_t m = rng.below(2 * kWide); + if (std::ranges::find(idx, m) == idx.end()) { + idx.push_back(m); + } + } + std::ranges::sort(idx); + return idx; + }; + while (data.hamiltonian.size() < 90000) { + const auto idx = draw_indices(2 * (1 + rng.below(3))); + const double x = static_cast(rng.below(1U << 20)) / (1U << 20) - 0.5; + data.hamiltonian[idx] = x * hermitian_coefficient(indices_to_bitset(idx)); + } + data.initial_state = {0, 2, 4, 6, 8, 10, 12, 14}; + constexpr size_t gates = 6; + for (size_t g = 0; g < gates; ++g) { + data.majoranas.push_back(draw_indices(2 + rng.below(3))); + data.param_inds.push_back(g); + data.gen_coeffs.push_back(1.0); + data.parameters.push_back(g == 2 ? std::numbers::pi / 4 : 0.1 + static_cast(rng.below(900)) / 1000.0); + } + data.majoranas[4] = data.majoranas[1]; + return data; +} + +// The root at the launch's T. +auto wide_root(const CaseData &data, bool schrodinger, MPI_Comm comm) -> MonomialPropagator { + return MonomialPropagator(data.hamiltonian, + /*cutoff=*/10, + data.initial_state, + schrodinger ? std::optional(2 * kWide) : std::nullopt, + comm, + /*lower_atol=*/1e-12, + std::nullopt, + CutoffType::Length, + std::nullopt, + kWide, + Basis::Majorana); +} + +// Rows in row-ID order: a store's IDs are its insertion order, so equal rows mean equal IDs. +auto same_store_rows(const monoprop::detail::OperatorIndex &sa, const monoprop::detail::OperatorIndex &sb) + -> bool { + if (sa.size() != sb.size()) { + return false; + } + for (size_t i = 0; i < sa.size(); ++i) { + if (materialize_row(sa, i) != materialize_row(sb, i)) { + return false; + } + } + return true; +} + +auto same_core(const LayerCore &a, const LayerCore &b) -> bool { + const auto &x = a.cross_rank; + const auto &y = b.cross_rank; + if (x.occupied.size() != y.occupied.size()) { + return false; + } + for (size_t k = 0; k < x.occupied.size(); ++k) { + if (x.occupied[k].slot != y.occupied[k].slot || x.occupied[k].sin_send_count != y.occupied[k].sin_send_count + || x.occupied[k].in_count != y.occupied[k].in_count) { + return false; + } + } + return x.sin_send_indices == y.sin_send_indices + && x.sin_recv_phases.uses_binary_phases == y.sin_recv_phases.uses_binary_phases + && x.sin_recv_phases.total_count == y.sin_recv_phases.total_count + && x.sin_recv_phases.phase_words == y.sin_recv_phases.phase_words + && x.sin_recv_phases.phase_values == y.sin_recv_phases.phase_values && x.world_size == y.world_size + && x.self_pos == y.self_pos && x.self_offset == y.self_offset && a.generator_words == b.generator_words + && a.scaled_count == b.scaled_count && a.param_index == b.param_index + && std::bit_cast(a.gen_coeff) == std::bit_cast(b.gen_coeff) + && a.gate_index == b.gate_index; +} + +auto same_graph(const MPGraph &a, const MPGraph &b) -> bool { + if (a.layers() != b.layers()) { + return false; + } + for (size_t l = 0; l < a.layers(); ++l) { + const auto &la = a.get_layer(l); + const auto &lb = b.get_layer(l); + if (!same_core(la.core(), lb.core()) || (la.pruned_cos() == nullptr) != (lb.pruned_cos() == nullptr)) { + return false; + } + } + return true; +} + +} // namespace + +// Full construction at the launch's fixed geometry, shard by shard: two roots build the same graph on every shard +// (every layer's endpoints, phases, slots, generator and scaled_count), the same rows in the same row-ID order and the +// same coefficients and contraction blocks, and propagate (fused sweep and the two-pass length-cap path) to the same +// rows and coefficients, in both pictures. Under an MPI launch each rank compares its own shards. Changing T changes +// ownership, so other geometries are compared through complete retained maps (tests/test_sharded_openmp.py). +BOOST_AUTO_TEST_CASE(fused_wide_shards_match_at_fixed_geometry) { + const auto data = wide_case(); + for (const bool schrodinger : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger) { + auto first = wide_root(data, schrodinger, MPI_COMM_WORLD); + auto second = wide_root(data, schrodinger, MPI_COMM_WORLD); + first.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + second.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + BOOST_TEST_MESSAGE("graph-built rows on this rank: " << first.size() + << ", layers: " << first.graph_layers()); + const auto &a = WideAccess::shards(first); + const auto &b = WideAccess::shards(second); + BOOST_TEST_REQUIRE(a.size() == b.size()); + for (size_t t = 0; t < a.size(); ++t) { + BOOST_TEST_CONTEXT("shard " << t) { + BOOST_TEST(same_store_rows(*a[t]->op.store, *b[t]->op.store)); + BOOST_TEST(same_graph(a[t]->graph, b[t]->graph)); + const auto &pa = schrodinger ? a[t]->op.state_coeffs : a[t]->op.op_coeffs; + const auto &pb = schrodinger ? b[t]->op.state_coeffs : b[t]->op.op_coeffs; + BOOST_TEST(bitwise_equal(pa, pb)); + } + } + BOOST_TEST(bitwise_equal(first.contract_partially(data.parameters, false), + second.contract_partially(data.parameters, false))); + for (const bool cap : {false, true}) { + auto x = wide_root(data, schrodinger, MPI_COMM_WORLD); + auto y = wide_root(data, schrodinger, MPI_COMM_WORLD); + const std::optional k = cap ? std::optional(2 * kWide) : std::nullopt; + x.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, k); + y.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters, k); + const auto &xs = WideAccess::shards(x); + const auto &ys = WideAccess::shards(y); + for (size_t t = 0; t < xs.size(); ++t) { + BOOST_TEST_CONTEXT("cap=" << cap << " shard " << t) { + BOOST_TEST(same_store_rows(*xs[t]->op.store, *ys[t]->op.store)); + const auto &pa = schrodinger ? xs[t]->op.state_coeffs : xs[t]->op.op_coeffs; + const auto &pb = schrodinger ? ys[t]->op.state_coeffs : ys[t]->op.op_coeffs; + BOOST_TEST(bitwise_equal(pa, pb)); + } + } + } + } + } +} diff --git a/cpp/tests/hybrid_comm_tests.cpp b/cpp/tests/hybrid_comm_tests.cpp deleted file mode 100644 index 2910881ee..000000000 --- a/cpp/tests/hybrid_comm_tests.cpp +++ /dev/null @@ -1,1262 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// HybridComm transport equivalence: R MPI ranks x S in-process partitions must behave as one flat P=R*S -// SPMD world, with only partition 0 touching MPI, exactly as PartitionGroup drives it. -// -// The staging layout never reaches a caller, so no case here distinguishes a consistently transposed -// tiling from the shipped one; only the bit-identity comment in HybridComm.h pins that choice. - -#include - -#ifdef monoprop_ENABLE_MPI - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "SlotBlocks.h" -#include "ThreadHarness.h" -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/HybridComm.h" -#include "monoprop/detail/mpi/MPICompat.h" - -using monoprop::mpi::Comm; -using monoprop::mpi::HybridComm; - -namespace { - -template -auto run_hybrid(int s, Body body) -> std::vector { - HybridComm hyb(MPI_COMM_WORLD, s); - return test_utils::run_comm_threads(hyb, s, body); -} - -auto world_size() -> int { - int n = 0; - MPI_Comm_size(MPI_COMM_WORLD, &n); - return n; -} -auto world_rank() -> int { - int r = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &r); - return r; -} - -} // namespace - -BOOST_AUTO_TEST_CASE(hybrid_comm_flat_size_and_rank) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - for (const int S : {1, 2, 3}) { - std::vector seen_size(static_cast(S), -1); - std::vector seen_rank(static_cast(S), -1); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - seen_size[static_cast(u)] = monoprop::mpi::size(c); - seen_rank[static_cast(u)] = monoprop::mpi::rank(c); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int u = 0; u < S; ++u) { - BOOST_CHECK_EQUAL(seen_size[static_cast(u)], R * S); - BOOST_CHECK_EQUAL(seen_rank[static_cast(u)], world_rank() * S + u); - } - } -} - -// Each partition contributes its global id, so the expected total is sum_{g(P) * (P - 1) / 2.0; - std::vector got(static_cast(S), -1.0); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const double mine = static_cast(monoprop::mpi::rank(c)); - got[static_cast(u)] = monoprop::mpi::allreduce_sum(mine, c); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int u = 0; u < S; ++u) { - BOOST_CHECK_EQUAL(got[static_cast(u)], expected); - } - } -} - -// begin_alltoallv must deliver each source's block contiguously in ascending global source order with -// tags intact (Resolve.h's positional pairing). With no known recv counts this drives the fused verb. -BOOST_AUTO_TEST_CASE(hybrid_comm_alltoallv_source_order_and_tags) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - for (const int S : {1, 2, 3}) { - const int P = R * S; - // recv[u] = what this rank's partition u received, indexed by global source. - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - const int len = g % 3 + 1; - std::vector> send(static_cast(P)); - for (int d = 0; d < P; ++d) { - for (int j = 0; j < len; ++j) { - send[static_cast(d)].push_back(g * 1000 + j); - } - } - const auto out = test_utils::exchange_blocks(send, c); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int u = 0; u < S; ++u) { - const auto &out = recv[static_cast(u)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int src = 0; src < P; ++src) { - const int len = src % 3 + 1; - const auto &blk = out[static_cast(src)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), len); - for (int j = 0; j < len; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], src * 1000 + j); - } - } - } - } -} - -// Back-to-back alltoallvs with varying counts (zeros, growth, shrink) on one HybridComm: staging and -// offset-table reuse (a stale staged byte surfaces as a wrong tag), and the no-trailing-barrier rule. -BOOST_AUTO_TEST_CASE(hybrid_comm_repeated_alltoallv_varying_sizes) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - const int S = 3; - const int P = R * S; - const int rounds = 30; - std::atomic failures{0}; - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - for (int round = 0; round < rounds; ++round) { - // Every 7th round is "big": pushes the staging HWM up so later rounds expose stale bytes. - const auto len_of = [&](int src) { - return (round % 7 == 6) ? (src % 3 + 1) * 17 : (src + round) % 4; // includes 0 - }; - const int my_len = len_of(g); - std::vector> send(static_cast(P)); - for (int d = 0; d < P; ++d) { - for (int j = 0; j < my_len; ++j) { - send[static_cast(d)].push_back(g * 100000 + round * 100 + j); - } - } - const auto out = test_utils::exchange_blocks(send, c); - if (static_cast(out.size()) != P) { - failures.fetch_add(1); - continue; - } - for (int src = 0; src < P; ++src) { - const int len = len_of(src); - const auto &blk = out[static_cast(src)]; - if (static_cast(blk.size()) != len) { - failures.fetch_add(1); - continue; - } - for (int j = 0; j < len; ++j) { - if (blk[static_cast(j)] != src * 100000 + round * 100 + j) { - failures.fetch_add(1); - } - } - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_EQUAL(failures.load(), 0); -} - -// alltoallv_resolve driven directly: it folds the count round into the payload verb's barriered -// windows and sizes recv itself. -BOOST_AUTO_TEST_CASE(hybrid_comm_alltoallv_resolve_fused) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - const int S = 3; - const int P = R * S; - const int rounds = 30; - std::atomic failures{0}; - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector recv; // reused across rounds (staging + recv HWM) - std::vector rc(static_cast(P)), rd(static_cast(P)); - for (int round = 0; round < rounds; ++round) { - const auto len_of = [&](int src) { - return (round % 7 == 6) ? (src % 3 + 1) * 17 : (src + round) % 4; // includes 0 - }; - const int my_len = len_of(g); - std::vector send; - std::vector sc(static_cast(P)), sd(static_cast(P)); - int off = 0; - for (int d = 0; d < P; ++d) { - sc[static_cast(d)] = my_len; - sd[static_cast(d)] = off; - for (int j = 0; j < my_len; ++j) { - send.push_back(g * 100000 + round * 100 + j); - } - off += my_len; - } - hyb.alltoallv_resolve(u, - {.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = recv, - .recv_counts = rc.data(), - .recv_displs = rd.data()}, - monoprop::mpi::datatype::get()); - int expected_total = 0; - for (int src = 0; src < P; ++src) { - const int len = len_of(src); - if (rc[static_cast(src)] != len || rd[static_cast(src)] != expected_total) { - failures.fetch_add(1); - } - for (int j = 0; j < len; ++j) { - if (recv[static_cast(expected_total + j)] != src * 100000 + round * 100 + j) { - failures.fetch_add(1); - } - } - expected_total += len; - } - if (static_cast(recv.size()) != expected_total) { - failures.fetch_add(1); - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_EQUAL(failures.load(), 0); -} - -namespace { - -// Counts depending on both ends of the leg; every 5th round is a high-water round, so the smaller -// rounds after it run over stale staged bytes. -auto pair_count(int g, int d, int round) -> int { - if (round % 5 == 4) { - return (g * 3 + d * 7) % 11 + 1; - } - return (g * 2 + d * 3 + round) % 4; -} - -// Unique per (source, destination, index), so a block from the wrong source cannot match. -auto pair_tag(int g, int d, int j) -> int { - return (g * 100 + d) * 1000 + j; -} - -} // namespace - -// The only direct coverage of HybridComm::alltoallv, and the only count matrix varying along both -// indices, which is what pins the indexing of col_sum_ and recv_col_. -BOOST_AUTO_TEST_CASE(hybrid_comm_alltoallv_pairwise_counts) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - const int rounds = 12; - for (const int S : {1, 2, 3}) { - const int P = R * S; - std::atomic failures{0}; - std::atomic payload_checks{0}; - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector sc(static_cast(P)), sd(static_cast(P)); - std::vector rc(static_cast(P)), rd(static_cast(P)); - std::vector send; - std::vector recv; // reassigned per round; stage_recv_'s HWM is what carries stale bytes - for (int round = 0; round < rounds; ++round) { - send.clear(); - int so = 0; - int ro = 0; - for (int d = 0; d < P; ++d) { - const int n = pair_count(g, d, round); - sc[static_cast(d)] = n; - sd[static_cast(d)] = so; - so += n; - for (int j = 0; j < n; ++j) { - send.push_back(pair_tag(g, d, j)); - } - const int m = pair_count(d, g, round); // the transpose: what d sends me - rc[static_cast(d)] = m; - rd[static_cast(d)] = ro; - ro += m; - } - recv.assign(static_cast(ro), -1); - const auto args = monoprop::mpi::FlatAlltoallvArgs{.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = recv.data(), - .recv_counts = rc.data(), - .recv_displs = rd.data()} - .bytes(); - hyb.alltoallv(u, args, monoprop::mpi::datatype::get()); - for (int src = 0; src < P; ++src) { - const int m = rc[static_cast(src)]; - for (int j = 0; j < m; ++j) { - payload_checks.fetch_add(1); - if (recv[static_cast(rd[static_cast(src)] + j)] != pair_tag(src, g, j)) { - failures.fetch_add(1); - } - } - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - // Assertions inside count-dependent loops can all be skipped, so count the payload comparisons. - BOOST_CHECK_GT(payload_checks.load(), 0); - BOOST_CHECK_EQUAL(failures.load(), 0); - } -} - -// The same pairwise counts through the fused verb, whose recv staging is sized from the count matrix -// rather than from published rows. rc/rd are outputs here, and are checked. -BOOST_AUTO_TEST_CASE(hybrid_comm_alltoallv_resolve_pairwise_counts) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - const int rounds = 12; - for (const int S : {1, 2, 3}) { - const int P = R * S; - std::atomic failures{0}; - // Two counters: the layout checks run unconditionally, so one combined counter would pass on them. - std::atomic layout_checks{0}; - std::atomic payload_checks{0}; - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector sc(static_cast(P)), sd(static_cast(P)); - std::vector rc(static_cast(P)), rd(static_cast(P)); - std::vector send; - std::vector recv; // reused across rounds (staging + recv HWM) - for (int round = 0; round < rounds; ++round) { - send.clear(); - int so = 0; - for (int d = 0; d < P; ++d) { - const int n = pair_count(g, d, round); - sc[static_cast(d)] = n; - sd[static_cast(d)] = so; - so += n; - for (int j = 0; j < n; ++j) { - send.push_back(pair_tag(g, d, j)); - } - } - hyb.alltoallv_resolve(u, - {.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = recv, - .recv_counts = rc.data(), - .recv_displs = rd.data()}, - monoprop::mpi::datatype::get()); - int total = 0; - for (int src = 0; src < P; ++src) { - const int m = pair_count(src, g, round); - layout_checks.fetch_add(1); - if (rc[static_cast(src)] != m || rd[static_cast(src)] != total) { - failures.fetch_add(1); - } - for (int j = 0; j < m; ++j) { - payload_checks.fetch_add(1); - if (recv[static_cast(total + j)] != pair_tag(src, g, j)) { - failures.fetch_add(1); - } - } - total += m; - } - layout_checks.fetch_add(1); - if (static_cast(recv.size()) != total) { - failures.fetch_add(1); - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_GT(layout_checks.load(), 0); - BOOST_CHECK_GT(payload_checks.load(), 0); - BOOST_CHECK_EQUAL(failures.load(), 0); - } -} - -BOOST_AUTO_TEST_CASE(hybrid_comm_allreduce_sum_inplace_global) { - if (world_size() < 2) { - return; - } - const int R = world_size(); - for (const int S : {1, 2, 3}) { - const int P = R * S; - // Lengths straddle the slice-partition edges: shorter than S, partial cache lines, many lines. - for (const size_t N : {size_t{1}, size_t{5}, size_t{8 * 3 + 3}, size_t{257}}) { - std::vector> res(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector v(N); - for (size_t k = 0; k < N; ++k) { - v[k] = static_cast(g + 1) * 0.25 + static_cast(k); - } - hyb.allreduce_sum_inplace(u, v.data(), N); - res[static_cast(u)] = v; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - // sum over g of ((g+1)*0.25 + k) = P(P+1)/8 + P*k - for (int u = 0; u < S; ++u) { - BOOST_REQUIRE_EQUAL(res[static_cast(u)].size(), N); - for (size_t k = 0; k < N; ++k) { - const double expect = - static_cast(P) * (P + 1) / 8.0 + static_cast(P) * static_cast(k); - BOOST_CHECK_CLOSE(res[static_cast(u)][k], expect, 1e-12); - BOOST_CHECK_EQUAL(res[static_cast(u)][k], res[0][k]); // bit-identical - } - } - } - } -} - -// Poison releases barrier waiters on every rank; the test completing at all proves that. Partition 0 poisons -// before entering a collective, so no rank is committed to MPI and the partition-0 guard deliberately does -// not fire. Poisoning inside a collective calls MPI_Abort instead, so it cannot be a ctest case. -BOOST_AUTO_TEST_CASE(hybrid_comm_poison_releases_waiters) { - if (world_size() < 2) { - return; - } - for (const int S : {2, 3}) { - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - if (u == 0) { - hyb.poison(); - return; - } - std::vector send(static_cast(world_size() * S), 1); - std::vector got(static_cast(world_size() * S)); - hyb.alltoall_counts(u, send.data(), got.data()); - }); - BOOST_CHECK(errs[0] == nullptr); - for (int u = 1; u < S; ++u) { - BOOST_REQUIRE(errs[static_cast(u)] != nullptr); - BOOST_CHECK_THROW(std::rethrow_exception(errs[static_cast(u)]), monoprop::mpi::ShmCommPoisoned); - } - } -} - -// The pairing is derived independently on both ends, so it must be an involution: whoever I send to -// sends back to me. A plan on which it fails deadlocks rather than mis-delivers, and no transport case -// below can distinguish the two. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_peer_is_an_involution) { - for (const int shift : {0, 1, 2, 3, 5, 8, 13, 255}) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - BOOST_REQUIRE(!plan.dense()); - BOOST_REQUIRE_EQUAL(plan.count(256), 1); - for (int me = 0; me < 256; ++me) { - const int peer = plan.peer(me, 0); - BOOST_REQUIRE_EQUAL(plan.peer(peer, 0), me); - BOOST_REQUIRE(plan.contains(me, peer)); - BOOST_REQUIRE(plan.contains(peer, me)); - BOOST_REQUIRE(!plan.contains(me, peer ^ 1)); // the peer set is a singleton - } - } - const monoprop::mpi::PeerPlan dense_plan{}; - BOOST_REQUIRE(dense_plan.dense()); - BOOST_REQUIRE_EQUAL(dense_plan.count(7), 7); - for (int k = 0; k < 7; ++k) { - BOOST_REQUIRE_EQUAL(dense_plan.peer(3, k), k); // dense ignores `me`: every rank is a peer - BOOST_REQUIRE(dense_plan.contains(3, k)); - } -} - -// A sparse PeerPlan replaces the collectives with point-to-point over the one peer the plan names, so -// the two failure modes it can have are DROPPED data and a HANG -- neither of which a dense-path test -// can see. Every rank derives the same pairing from the same shift, and a block whose destination is -// not the peer must be empty: send only to the plan's peer and check the delivery is exactly that. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_delivers_only_to_its_peers) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; // the XOR pairing needs a power-of-two rank count - } - for (const int S : {1, 2, 3}) { - const int P = R * S; - for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(world_rank(), 0); - BOOST_REQUIRE_EQUAL(plan.count(R), 1); // sparse is always pairwise - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector> send(static_cast(P)); - for (int t = 0; t < S; ++t) { - auto &blk = send[static_cast(peer * S + t)]; - for (int j = 0; j <= t; ++j) { - blk.push_back(g * 1000 + t * 10 + j); - } - } - const auto out = test_utils::exchange_blocks(send, - c, - /*skip_self=*/false, - /*known_recv_counts=*/nullptr, - plan); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - // XOR is an involution, so whoever I send to sends to me: my sources are exactly `peer`'s - // partitions, and the block from (peer, su) to my partition t has t+1 entries. - for (int t = 0; t < S; ++t) { - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int src = 0; src < P; ++src) { - const auto &blk = out[static_cast(src)]; - if (src / S != peer) { - BOOST_CHECK(blk.empty()); // a non-peer must not appear at all - continue; - } - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), t + 1); - for (int j = 0; j <= t; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], src * 1000 + t * 10 + j); - } - } - } - } - } -} - -// Same contract on the S == 1 world, which takes the pure-MPI branch (MPI_Ialltoallv vs Isend/Irecv) -// rather than HybridComm's staged one, and on the KNOWN-recv-counts path the response round uses. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_on_the_plain_mpi_path) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - Comm c{MPI_COMM_WORLD}; - for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(world_rank(), 0); - std::vector> send(static_cast(R)); - for (int j = 0; j < 4; ++j) { - send[static_cast(peer)].push_back(world_rank() * 1000 + j); - } - // Unknown recv layout: the counts round is point-to-point too. - const auto out = test_utils::exchange_blocks(send, c, false, nullptr, plan); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - for (int src = 0; src < R; ++src) { - if (src != peer) { - BOOST_CHECK(out[static_cast(src)].empty()); - continue; - } - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), 4); - for (int j = 0; j < 4; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], src * 1000 + j); - } - } - // Known recv layout (the response round): counts are the transpose, so peer-only again. - std::vector known(static_cast(R), 0); - known[static_cast(peer)] = 4; - const auto out2 = test_utils::exchange_blocks(send, c, false, &known, plan); - BOOST_REQUIRE_EQUAL(static_cast(out2.size()), R); - BOOST_CHECK(out2[static_cast(peer)] == out[static_cast(peer)]); - } -} - -// known_recv_counts is CALLER-supplied, so it can carry a count for a rank the plan does not name -- -// the response round's transpose is only as masked as whatever produced it. No receive is ever posted -// for a non-peer, so an unmasked count sizes recv_buffer for bytes nothing writes and wait_into would -// hand that slot to the caller as data. Both the plain-MPI and the staged HybridComm path must drop it. -BOOST_AUTO_TEST_CASE(hybrid_comm_known_recv_counts_are_masked_through_the_plan) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - constexpr int kReal = 4; - constexpr int kBogus = 7; // what a stale or unmasked transpose would claim a non-peer is sending - for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(world_rank(), 0); - const int bad = (peer + 1) % R; // the peer set is exactly {peer} - BOOST_REQUIRE(bad != peer); - - // S == 1: the plain-MPI Isend/Irecv branch of begin_alltoallv. - { - Comm c{MPI_COMM_WORLD}; - std::vector> send(static_cast(R)); - for (int j = 0; j < kReal; ++j) { - send[static_cast(peer)].push_back(world_rank() * 1000 + j); - } - std::vector known(static_cast(R), 0); - known[static_cast(peer)] = kReal; - known[static_cast(bad)] = kBogus; - const auto out = test_utils::exchange_blocks(send, c, false, &known, plan); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - BOOST_CHECK(out[static_cast(bad)].empty()); // unmasked, this holds kBogus elements - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), kReal); - for (int j = 0; j < kReal; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], peer * 1000 + j); - } - } - - // S == 2: the same array through HybridComm's staged alltoallv. - constexpr int S = 2; - const int P = R * S; - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector> send(static_cast(P)); - std::vector known(static_cast(P), 0); - for (int t = 0; t < S; ++t) { - for (int j = 0; j < kReal; ++j) { - send[static_cast((peer * S) + t)].push_back((g * 1000) + j); - } - known[static_cast((peer * S) + t)] = kReal; - known[static_cast((bad * S) + t)] = kBogus; - } - const auto out = test_utils::exchange_blocks(send, c, false, &known, plan); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int su = 0; su < S; ++su) { - BOOST_CHECK(out[static_cast((bad * S) + su)].empty()); - const auto &blk = out[static_cast((peer * S) + su)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), kReal); - for (int j = 0; j < kReal; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], (((peer * S) + su) * 1000) + j); - } - } - } - } -} - -namespace { - -// Varies along BOTH ends and hits 0, so a block landing on the wrong peer or the wrong partition -// changes a length, not just a value. -auto sparse_count(int src, int dst) -> int { - return ((src * 3) + (dst * 5)) % 4; -} -auto sparse_tag(int src, int dst, int j) -> int { - return (((src * 128) + dst) * 1000) + j; -} - -} // namespace - -// A zero-count leg is where a send/recv posting asymmetry deadlocks rather than mis-delivers: both ends -// must skip on the SAME value. Nothing above ever sends an empty block over a real message, so force -// one -- the lower-numbered end of every pair sends nothing while its peer sends four. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_with_an_empty_leg) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - constexpr int kLen = 4; - for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered separately - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(me, 0); - BOOST_REQUIRE(peer != me); - const int my_len = me < peer ? 0 : kLen; // exactly one end of the pair is silent - const int peer_len = peer < me ? 0 : kLen; - - { - Comm c{MPI_COMM_WORLD}; - std::vector> send(static_cast(R)); - for (int j = 0; j < my_len; ++j) { - send[static_cast(peer)].push_back(sparse_tag(me, peer, j)); - } - const auto out = test_utils::exchange_blocks(send, c, false, nullptr, plan); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), peer_len); - for (int j = 0; j < peer_len; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], sparse_tag(peer, me, j)); - } - } - - for (const int S : {1, 2}) { - const int P = R * S; - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector> send(static_cast(P)); - for (int t = 0; t < S; ++t) { - const int d = (peer * S) + t; - for (int j = 0; j < my_len; ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - const auto out = test_utils::exchange_blocks(send, c, false, nullptr, plan); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int su = 0; su < S; ++su) { - const auto &blk = out[static_cast((peer * S) + su)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), peer_len); - for (int j = 0; j < peer_len; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag((peer * S) + su, g, j)); - } - } - } - } - } -} - -// shift == 0 makes every rank its own and only peer, so the sparse path's self slot is the whole -// exchange -- and skip_self then removes the one leg that would have moved anything for a partition. -// The remaining S-1 in-rank legs must still arrive. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_skip_self_at_shift_zero) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = 0}; - BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); - - { - Comm c{MPI_COMM_WORLD}; // the self peer is the ONLY peer, and skip_self drops it - std::vector> send(static_cast(R)); - for (int j = 0; j < 5; ++j) { - send[static_cast(me)].push_back(sparse_tag(me, me, j)); - } - const auto out = test_utils::exchange_blocks(send, c, /*skip_self=*/true, nullptr, plan); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - for (const auto &blk : out) { - BOOST_CHECK(blk.empty()); - } - } - - for (const int S : {2, 3}) { - const int P = R * S; - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - Comm c = Comm::make_hybrid(&hyb, u); - const int g = monoprop::mpi::rank(c); - std::vector> send(static_cast(P)); - for (int t = 0; t < S; ++t) { - const int d = (me * S) + t; - for (int j = 0; j <= t; ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - const auto out = test_utils::exchange_blocks(send, c, /*skip_self=*/true, nullptr, plan); - recv[static_cast(u)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int src = 0; src < P; ++src) { - const bool in_rank = (src / S) == me; - const int want = (in_rank && src != g) ? t + 1 : 0; // own slot dropped by skip_self - BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), want); - for (int j = 0; j < want; ++j) { - BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], sparse_tag(src, g, j)); - } - } - } - } -} - -// Engine.h's run_exchange posts both its rounds on ONE communicator under kFlatPayloadTag, and the -// non-overtaking argument in Pairwise.h is what says a round-2 receive cannot match a round-1 send. -// Two back-to-back sparse rounds, the second on the transpose of the first's counts, assert it. -BOOST_AUTO_TEST_CASE(hybrid_comm_sparse_plan_back_to_back_rounds) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - Comm c{MPI_COMM_WORLD}; - for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(me, 0); - const int len = 3 + (me % 2); // asymmetric, so a swapped round is a length mismatch - - std::vector> q(static_cast(R)); - for (int j = 0; j < len; ++j) { - q[static_cast(peer)].push_back(sparse_tag(me, peer, j)); - } - const auto q_out = test_utils::exchange_blocks(q, c, false, nullptr, plan); - - // Round 2 immediately, same comm and tag, sized from round 1's transpose -- the response shape. - std::vector known(static_cast(R), 0); - known[static_cast(peer)] = len; - std::vector> r(static_cast(R)); - const int back = static_cast(q_out[static_cast(peer)].size()); - for (int j = 0; j < back; ++j) { - r[static_cast(peer)].push_back(q_out[static_cast(peer)][static_cast(j)] + 7); - } - const auto r_out = test_utils::exchange_blocks(r, c, false, &known, plan); - - BOOST_REQUIRE_EQUAL(back, 3 + (peer % 2)); - BOOST_REQUIRE_EQUAL(static_cast(r_out[static_cast(peer)].size()), len); - for (int j = 0; j < len; ++j) { - BOOST_CHECK_EQUAL(r_out[static_cast(peer)][static_cast(j)], sparse_tag(me, peer, j) + 7); - } - for (int src = 0; src < R; ++src) { - if (src != peer) { - BOOST_CHECK(r_out[static_cast(src)].empty()); - } - } - } -} - -namespace { - -// One fused-resolve round under `plan`: flattens send_blocks, runs the verb, and re-splits the payload -// by global source using the resolved counts, so a case compares delivered BLOCKS rather than offsets. -auto resolve_round(HybridComm &hyb, - int u, - int P, - const std::vector> &send_blocks, - monoprop::mpi::PeerPlan plan) -> std::vector> { - std::vector send; - std::vector sc(static_cast(P)), sd(static_cast(P)); - for (int d = 0; d < P; ++d) { - const auto &blk = send_blocks[static_cast(d)]; - sc[static_cast(d)] = static_cast(blk.size()); - sd[static_cast(d)] = static_cast(send.size()); - send.insert(send.end(), blk.begin(), blk.end()); - } - std::vector recv; - std::vector rc(static_cast(P)), rd(static_cast(P)); - hyb.alltoallv_resolve(u, - {.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = recv, - .recv_counts = rc.data(), - .recv_displs = rd.data()}, - monoprop::mpi::datatype::get(), - plan); - std::vector> out(static_cast(P)); - for (int src = 0; src < P; ++src) { - const auto off = static_cast(rd[static_cast(src)]); - const auto n = static_cast(rc[static_cast(src)]); - out[static_cast(src)].assign(recv.begin() + static_cast(off), - recv.begin() + static_cast(off + n)); - } - return out; -} - -// Every leg INTO an odd rank is empty, so a multi-peer plan always has a peer whose payload legs are -// skipped entirely while its S*S-int count block still travels. Both ends read the same function of -// (src, dst), which is what keeps the skip symmetric. -auto muted_count(int src, int dst, int S) -> int { - return (dst / S) % 2 == 1 ? 0 : sparse_count(src, dst); -} - -} // namespace - -// The fused resolve POSTS its count round in B1→B2 and drains it in B3→B4, with every partition's -// pack_send_ running underneath. Only the sparse arm splits -- plan.dense() is a blocking MPI_Alltoall -// -- so drive the SAME peer-masked layout through both arms and require the delivered blocks to agree -// element for element. A dropped or torn count block changes a length here, not just a value. -BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_matches_the_dense_arm) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; // the XOR pairing needs a power-of-two rank count - } - const int me = world_rank(); - for (int shift = 0; shift < R; ++shift) { - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - BOOST_REQUIRE_EQUAL(plan.count(R), 1); - for (const int S : {1, 2, 3}) { - const int P = R * S; - // Peer-masked, so the dense plan carries the identical bytes: its non-peer blocks are - // empty rather than absent, and a dense count of zero and a sparse absence must agree. - const auto fill = [&](int g) { - std::vector> send(static_cast(P)); - const int b = plan.peer(me, 0); - for (int t = 0; t < S; ++t) { - const int d = (b * S) + t; - for (int j = 0; j < muted_count(g, d, S); ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - return send; - }; - std::vector>> dense(static_cast(S)); - std::vector>> sparse(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - const int g = (me * S) + u; - dense[static_cast(u)] = resolve_round(hyb, u, P, fill(g), {}); - sparse[static_cast(u)] = resolve_round(hyb, u, P, fill(g), plan); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &got = sparse[static_cast(t)]; - const auto &want_dense = dense[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(got.size()), P); - BOOST_REQUIRE_EQUAL(static_cast(want_dense.size()), P); - for (int src = 0; src < P; ++src) { - const int want = plan.contains(me, src / S) ? muted_count(src, g, S) : 0; - const auto &blk = got[static_cast(src)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), want); - BOOST_REQUIRE_EQUAL(static_cast(want_dense[static_cast(src)].size()), want); - for (int j = 0; j < want; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag(src, g, j)); - BOOST_CHECK_EQUAL(blk[static_cast(j)], - want_dense[static_cast(src)][static_cast(j)]); - } - } - } - } - } -} - -// shift == 0 at full bits makes this rank its own and only peer, so the count round posts NOTHING: -// sparse_pairwise memcpys the S*S-int block in place and the B3→B4 wait must be a no-op rather than a -// wait on stale requests. The S^2 in-rank payload legs still have to arrive. -BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_self_peer_only) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = 0}; - BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); - for (const int S : {1, 2, 3}) { - const int P = R * S; - std::vector>> recv(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - const int g = (me * S) + u; - std::vector> send(static_cast(P)); - for (int t = 0; t < S; ++t) { - const int d = (me * S) + t; - for (int j = 0; j <= t; ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - recv[static_cast(u)] = resolve_round(hyb, u, P, send, plan); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &out = recv[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int src = 0; src < P; ++src) { - const int want = (src / S) == me ? t + 1 : 0; // block (su -> t) has t+1 entries - const auto &blk = out[static_cast(src)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), want); - for (int j = 0; j < want; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag(src, g, j)); - } - } - } - } -} - -// A peer with nothing to send is where the split can hang: pack_send_ copies zero bytes in B2→B3 while -// the count round is still live, and the payload round posts no leg for it at all. Back-to-back rounds, -// the second silent, so the second also runs over the first's staging high-water bytes. -BOOST_AUTO_TEST_CASE(hybrid_comm_resolve_split_count_round_zero_count_peer) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - for (int shift = 1; shift < R; ++shift) { // shift 0 is the self peer, covered above - const monoprop::mpi::PeerPlan plan{.sparse = true, .shift = shift}; - const int peer = plan.peer(me, 0); - BOOST_REQUIRE(peer != me); - const int my_len = me < peer ? 0 : 4; // exactly one end of the pair is silent - const int peer_len = peer < me ? 0 : 4; - for (const int S : {1, 2}) { - const int P = R * S; - std::vector>> loud(static_cast(S)); - std::vector>> quiet(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - const int g = (me * S) + u; - std::vector> send(static_cast(P)); - for (int t = 0; t < S; ++t) { - const int d = (peer * S) + t; - for (int j = 0; j < my_len; ++j) { - send[static_cast(d)].push_back(sparse_tag(g, d, j)); - } - } - loud[static_cast(u)] = resolve_round(hyb, u, P, send, plan); - quiet[static_cast(u)] = - resolve_round(hyb, u, P, std::vector>(static_cast(P)), plan); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int t = 0; t < S; ++t) { - const int g = (me * S) + t; - const auto &out = loud[static_cast(t)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), P); - for (int su = 0; su < S; ++su) { - const auto &blk = out[static_cast((peer * S) + su)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), peer_len); - for (int j = 0; j < peer_len; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], sparse_tag((peer * S) + su, g, j)); - } - } - // The all-silent round: every resolved count is zero, so a stale staged byte cannot hide. - for (const auto &blk : quiet[static_cast(t)]) { - BOOST_CHECK(blk.empty()); - } - } - } - } -} - -// derived_wire_plan_ is what lets alltoallv narrow its own wire when the caller cannot: only partition 0 -// reaches MPI, and ITS row may be the empty one while a sibling partition holds the rank's only traffic. -// Reading the peer set off partition 0's row alone resolves to the SELF peer, whose legs are all zero -- -// every block is then dropped with no hang to show for it. So this case puts the traffic where partition -// 0 cannot see it, and checks the payload arrives carrying its source's global id. -// -// The multi-peer and lossy-plan refusals in the same window are NOT exercised here: they reach MPI -// through guard_partition0_, which turns a throw into MPI_Abort, so a case for them would take the -// runner down with it. -BOOST_AUTO_TEST_CASE(hybrid_comm_derived_wire_plan_reads_every_partitions_row) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; // the XOR pairing needs a power-of-two rank count - } - const int me = world_rank(); - const int peer = me ^ 1; // shift 1, so every rank derives the same pairing - constexpr int kLen = 5; - - for (const int S : {2, 4}) { - const int P = R * S; - const int carrier = S - 1; // never partition 0 -- that is the whole point of the case - std::vector> got(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - const int g = (me * S) + u; - std::vector counts(static_cast(P), 0); - const std::vector displs(static_cast(P), 0); - std::vector send; - if (u == carrier) { - counts[static_cast((peer * S) + carrier)] = kLen; - for (int j = 0; j < kLen; ++j) { - send.push_back((g * 1000) + j); - } - } - // Symmetric layout: my peer's carrier partition sends me exactly what I send it, which is - // what the derived plan requires and what lets the recv rows name the peer set. One block, so - // both displacements are zero. - std::vector recv(static_cast(u == carrier ? kLen : 0), -1); - const monoprop::mpi::AlltoallvArgs args{.send = reinterpret_cast(send.data()), - .send_counts = counts.data(), - .send_displs = displs.data(), - .recv = reinterpret_cast(recv.data()), - .recv_counts = counts.data(), - .recv_displs = displs.data(), - .elem = sizeof(int)}; - hyb.alltoallv(u, args, MPI_INT, monoprop::mpi::PeerPlan{}, /*derive_wire_plan=*/true); - got[static_cast(u)] = recv; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int u = 0; u < S; ++u) { - if (u != carrier) { - BOOST_CHECK(got[static_cast(u)].empty()); - continue; - } - // The values name their sender, so this fails on a plan that named the wrong peer as well as - // on one that named none. - const int src = (peer * S) + carrier; - BOOST_REQUIRE_EQUAL(static_cast(got[static_cast(u)].size()), kLen); - for (int j = 0; j < kLen; ++j) { - BOOST_CHECK_EQUAL(got[static_cast(u)][static_cast(j)], (src * 1000) + j); - } - } - } -} - -// A layer that spans SEVERAL peer ranks -- a legacy graph, or one built by hand -- must still replay. -// The rank-uniform gate picks the transport; the peer set is only a narrowing, so a rank whose rows do -// not pin one peer down keeps the full set and posts point-to-point over all of it. Before, this shape -// aborted the job from inside guard_partition0_. -BOOST_AUTO_TEST_CASE(hybrid_comm_derived_wire_plan_falls_back_to_every_peer) { - const int R = world_size(); - if (R < 4 || (R & (R - 1)) != 0) { - return; // two distinct non-self peers is the point, so R = 2 cannot show it - } - const int me = world_rank(); - // XOR by 1 and by 2: an involution each, so the count matrix stays symmetric and every rank spans - // exactly two peers -- which is what makes this a valid layout the narrowing simply cannot use. - // Ascending, because the displacements below are a prefix sum over slots and the checks read blocks - // back in that order. - const int peers[2] = {std::min(me ^ 1, me ^ 2), std::max(me ^ 1, me ^ 2)}; - constexpr int kLen = 3; - - for (const int S : {1, 2}) { - const int P = R * S; - std::vector> got(static_cast(S)); - auto errs = run_hybrid(S, [&](HybridComm &hyb, int u) { - const int g = (me * S) + u; - std::vector counts(static_cast(P), 0); - std::vector displs(static_cast(P), 0); - for (const int b : peers) { - counts[static_cast((b * S) + u)] = kLen; - } - int run = 0; - for (int d = 0; d < P; ++d) { - displs[static_cast(d)] = run; - run += counts[static_cast(d)]; - } - std::vector send(static_cast(run)); - for (const int b : peers) { - const int d = (b * S) + u; - for (int j = 0; j < kLen; ++j) { - send[static_cast(displs[static_cast(d)] + j)] = (g * 1000) + j; - } - } - std::vector recv(static_cast(run), -1); - const monoprop::mpi::AlltoallvArgs args{.send = reinterpret_cast(send.data()), - .send_counts = counts.data(), - .send_displs = displs.data(), - .recv = reinterpret_cast(recv.data()), - .recv_counts = counts.data(), - .recv_displs = displs.data(), - .elem = sizeof(int)}; - hyb.alltoallv(u, args, MPI_INT, monoprop::mpi::PeerPlan{}, /*derive_wire_plan=*/true); - got[static_cast(u)] = recv; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - // Both legs arrive, each carrying its own sender's global id, so a dropped peer fails here. - for (int u = 0; u < S; ++u) { - const auto &out = got[static_cast(u)]; - BOOST_REQUIRE_EQUAL(static_cast(out.size()), 2 * kLen); - for (int k = 0; k < 2; ++k) { - const int src = (peers[k] * S) + u; - for (int j = 0; j < kLen; ++j) { - BOOST_CHECK_EQUAL(out[static_cast((k * kLen) + j)], (src * 1000) + j); - } - } - } - } -} - -// A sparse plan is a public aggregate, so a shift outside the rank-index width reaches the transports -// as an out-of-range vector index and an invalid MPI rank. Every sparse entry point refuses it first, -// and it must be an exception rather than an assert: release builds are where it would be indexed. -BOOST_AUTO_TEST_CASE(hybrid_comm_unroutable_peer_plan_is_refused) { - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const monoprop::mpi::PeerPlan bad{.sparse = true, .shift = R}; - const int P = R; - std::vector> send(static_cast(P)); - Comm c{MPI_COMM_WORLD}; - BOOST_CHECK_THROW( - static_cast(monoprop::mpi::begin_alltoallv(test_utils::full_window(send), c, false, nullptr, bad)), - std::invalid_argument); - std::vector counts(static_cast(P), 0); - std::vector got(static_cast(P), 0); - BOOST_CHECK_THROW(monoprop::mpi::alltoall_counts(counts.data(), got.data(), P, c, bad), std::invalid_argument); -} - -// MPI reads the payload buffers until the requests retire, so the handle that owns them must not be -// copyable (two owners, one set of requests) and must not be destroyable while they are live. -BOOST_AUTO_TEST_CASE(hybrid_comm_pending_alltoallv_owns_its_requests) { - using Pending = monoprop::mpi::PendingAlltoallv; - static_assert(!std::is_copy_constructible_v); - static_assert(!std::is_copy_assignable_v); - static_assert(std::is_nothrow_move_constructible_v); - static_assert(std::is_nothrow_move_assignable_v); - - // Dropping a handle without waiting must still complete the transfer, not free the buffers under it. - const int R = world_size(); - if (R < 2 || (R & (R - 1)) != 0) { - return; - } - const int me = world_rank(); - Comm c{MPI_COMM_WORLD}; - std::vector> send(static_cast(R)); - for (int d = 0; d < R; ++d) { - send[static_cast(d)] = {(me * 1000) + d}; - } - { - auto h = monoprop::mpi::begin_alltoallv(test_utils::full_window(send), c); - auto moved = std::move(h); // the requests travel with the buffers MPI is reading - monoprop::mpi::WindowVec> got; - moved.wait_into(got); - const auto out = test_utils::flat_blocks(got, static_cast(R)); - BOOST_REQUIRE_EQUAL(static_cast(out.size()), R); - for (int src = 0; src < R; ++src) { - BOOST_REQUIRE_EQUAL(out[static_cast(src)].size(), 1U); - BOOST_CHECK_EQUAL(out[static_cast(src)][0], (src * 1000) + me); - } - } - // Same round, never waited on: the destructor has to drain it or the next collective mismatches. - { - static_cast(monoprop::mpi::begin_alltoallv(test_utils::full_window(send), c)); - } - MPI_Barrier(MPI_COMM_WORLD); -} - -#endif // monoprop_ENABLE_MPI diff --git a/cpp/tests/link_export_probe/CMakeLists.txt b/cpp/tests/link_export_probe/CMakeLists.txt index fab629bed..66caf6bf9 100644 --- a/cpp/tests/link_export_probe/CMakeLists.txt +++ b/cpp/tests/link_export_probe/CMakeLists.txt @@ -6,7 +6,13 @@ # ld64 errors on undefined symbols by default, so no flag is needed there ("-Wl,-undefined,error" is # accepted but deprecated, and warns). -add_executable(monoprop_link_export_probe.x link_export_probe.cpp) +add_executable( + monoprop_link_export_probe.x + link_export_probe.cpp + sharded_state_chain.cpp + sharded_construction_chain.cpp + sharded_evaluation_chain.cpp +) # monoprop-sanitizers is a no-op interface library outside a sanitizer build. Under one, libmonoprop.so # is ASan-instrumented; without this, the probe executable itself is not, so the ASan runtime reaches @@ -20,6 +26,13 @@ target_link_libraries( monoprop ) +# (j): the consumer must inherit exactly the build's MPI setting through the imported target. +target_compile_definitions( + monoprop_link_export_probe.x + PRIVATE + monoprop_EXPECT_MPI=$ +) + if(NOT APPLE) target_link_options(monoprop_link_export_probe.x PRIVATE "-Wl,--no-undefined") endif() @@ -31,3 +44,31 @@ set_tests_properties( LABELS "unit" ) +# A fixed team of two, so the multi-shard paths (and the raw-access rejection) run here. +set_tests_properties( + monoprop_link_export_probe + PROPERTIES + ENVIRONMENT + "monoprop_NUM_THREADS=2;OMP_NUM_THREADS=2;OMP_DYNAMIC=FALSE" +) +# Several ranks of the same consumer: the multi-rank root chain (l) runs only here, at T = 1 and T = 2. +if(monoprop_ENABLE_MPI AND _monoprop_mpiexec) + foreach(_threads IN ITEMS 1 2) + add_test( + NAME monoprop_link_export_probe_mpi2_t${_threads} + COMMAND + ${_monoprop_mpiexec} ${_monoprop_mpiexec_numproc_flag} 2 + ${MPIEXEC_PREFLAGS} $ + ) + set_tests_properties( + monoprop_link_export_probe_mpi2_t${_threads} + PROPERTIES + LABELS + "mpi;mpi-2" + TIMEOUT + 300 + ENVIRONMENT + "OMPI_ALLOW_RUN_AS_ROOT=1;OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1;OMPI_MCA_hwloc_base_binding_policy=none;OMPI_MCA_rmaps_base_oversubscribe=1;PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe;monoprop_NUM_THREADS=${_threads};OMP_NUM_THREADS=${_threads};OMP_DYNAMIC=FALSE;monoprop_TEST_EXPECT_RANKS=2" + ) + endforeach() +endif() diff --git a/cpp/tests/link_export_probe/link_export_probe.cpp b/cpp/tests/link_export_probe/link_export_probe.cpp index f579e42c1..494977002 100644 --- a/cpp/tests/link_export_probe/link_export_probe.cpp +++ b/cpp/tests/link_export_probe/link_export_probe.cpp @@ -24,26 +24,90 @@ // but it does not run any of it. main() below actually drives both chains implicated by the bug report: // (a) the graph-building / Schrodinger path (detail/graph_encoding/MPGraphEncodingStorage.h), via // build_graph(), graph_memory_usage(), and expectation_value_and_gradient(). -// (b) the partition path (detail/partition/CpuTopology.h), via a partitions > 1 construction, which is -// the only way to make PartitionGroup actually place and pin partition-worker threads. +// (b) the constructor's shape -- the eleven mathematical arguments construct, no partition, factory, thread or shard +// argument does, and PartitionChildFactory is gone -- checked at compile time, then a construction that works. +// It also checks the installed usage requirements a consumer inherits: +// (c) the OpenMP worksharing template (detail/parallel/Workshare.h), instantiated in this translation +// unit, so OpenMP compile and link flags must reach the consumer through the imported target alone; +// (d) the Pauli basis, alongside the Majorana graph chain in (a); +// (e) the ordinary constructor, whose template calls the out-of-line capture_thread_budget(), +// require_thread_support() and require_initializing_thread(), and operations reaching the exported evolve and +// derivative entry points with their trailing Options; plus the raw-access rule (the sole shard at T = 1, +// MultiShardUnsupported at T > 1, after which the object still answers). +// (f) the fixed-team phase primitive (detail/sharded/Team.h), instantiated here like (c): its region, +// barriers and masked construct must compile and link through the imported target's OpenMP flags. +// (g) owner-initialized shard state (detail/sharded/State.h) in sharded_state_chain.cpp, whose only monoprop +// include is that header: seeding, owner copies, counts and a failing initializer through the imported target. +// (h) direct-buffer construction (detail/sharded/Construction.h) in sharded_construction_chain.cpp, whose only +// monoprop include is that header: one-team propagation and graph construction over the captured budget's +// shards through the imported target. +// (i) sharded evaluation (detail/sharded/Evaluation.h) in sharded_evaluation_chain.cpp, whose first monoprop include +// is that header: a coefficient-informed extension replaying its seed in-team, a retained functional, and runtime +// calls to the exported ev_sharded() and ev_and_grad_sharded() through the imported target. +// (j) the inherited configuration: no runtime selector reaches the consumer, and it sees monoprop_ENABLE_MPI exactly +// when it expected an MPI package (monoprop_EXPECT_MPI, when set). The root runs every public operation family +// and its exports and aggregates. +// (k) an ordinary derived class -- virtual clone_() and update_initial_operator() overrides, the protected +// apply_initial_operator_() -- used through the base interface. +// (l) MPI builds launched on several ranks: the root on MPI_COMM_WORLD against the same root on MPI_COMM_SELF -- +// energies, gradients, global term counts and every exported key -- with T-dependent raw access. #include "monoprop/MonomialPropagator.h" #include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/parallel/Workshare.h" +#include "monoprop/detail/sharded/Team.h" +#include +#include #include +#include +#include +#include #include #include +#include #include +#ifdef monoprop_SHARDED_OPENMP_PROTOTYPE +#error "monoprop::monoprop carries monoprop_SHARDED_OPENMP_PROTOTYPE: the removed runtime selector came back" +#endif + +#if defined(monoprop_EXPECT_MPI) +#if monoprop_EXPECT_MPI && !defined(monoprop_ENABLE_MPI) +#error "expected an MPI package, but monoprop::monoprop did not carry monoprop_ENABLE_MPI" +#elif !monoprop_EXPECT_MPI && defined(monoprop_ENABLE_MPI) +#error "expected an MPI-off package, but monoprop::monoprop carries monoprop_ENABLE_MPI" +#endif +#endif + +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/MPIUtils.h" + // Forces every member function of MonomialPropagator to be compiled for these two // representative widths, regardless of which ones main() below happens to call. template class monoprop::MonomialPropagator<2>; template class monoprop::MonomialPropagator<6>; +// Defined in sharded_state_chain.cpp. +auto run_sharded_state_chain() -> bool; +// Defined in sharded_construction_chain.cpp. +auto run_sharded_construction_chain() -> bool; +// Defined in sharded_evaluation_chain.cpp. +auto run_sharded_evaluation_chain() -> bool; + namespace { using namespace monoprop; +template +concept HasPartitionFactory = requires { typename P::PartitionChildFactory; }; + // Drives Engine::finish() -> LayerBuildSink::finalize() -> build_layer_storage_unified(), plus the // graph-memory and gradient accessors that read the resulting PackedCrossRankStorage / exchange layout. auto run_graph_build_chain() -> void { @@ -76,28 +140,343 @@ auto run_graph_build_chain() -> void { grad.size()); } -// Drives PartitionGroup's constructor: partitions > 1 is required for the facade to actually exist, so -// enumerate_physical_cores / affinity_mask_words / summarize_masks / format_place_line / partition_cpusets / -// pin_this_thread all run for real (not merely compiled) on the master threads it spawns. -auto run_partition_chain() -> void { +// Without the imported target's OpenMP compile flags the pragmas in Workshare.h would silently compile +// out, so check for them explicitly rather than rely on a link failure. +#ifndef _OPENMP +#error "monoprop::monoprop did not propagate the OpenMP compile flags to this consumer" +#endif + +// Instantiates for_blocks here: its region, and the omp_* calls it makes, are compiled and linked in +// the consumer. The body only writes its own element; library calls stay outside the region. +auto run_openmp_workshare_chain() -> void { + constexpr size_t kBlocks = 257; + std::vector squares(kBlocks, 0); + monoprop::detail::parallel::for_blocks(kBlocks, {.threads = 2}, [&](size_t b) { squares[b] = b * b; }); + size_t sum = 0; + for (const auto s : squares) { + sum += s; + } + const size_t expected = (kBlocks - 1) * kBlocks * (2 * kBlocks - 1) / 6; + std::println(stderr, "[link_export_probe] openmp chain: blocks={} sum={} expected={}", kBlocks, sum, expected); + if (sum != expected) { + std::println(stderr, "[link_export_probe] openmp chain: FAILED"); + std::exit(1); + } +} + +// Teams at the captured budget: in the first, every owner publishes in one phase and reads its neighbour's value in +// the next; in the second, the last owner fails and its exception must come back as the returned error. +auto run_sharded_team_chain() -> void { + namespace sharded = monoprop::detail::sharded; + const auto options = monoprop::detail::parallel::capture_thread_budget(); + const auto threads = static_cast(options.threads); + std::vector published(threads, 0); + std::vector seen(threads, 0); + const auto ok = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + if (sharded::phase(failure, shard, [&] { published[shard] = shard + 1; })) { + sharded::phase(failure, shard, [&] { seen[shard] = published[(shard + 1) % threads]; }); + } + }); + const auto failed = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { + if (shard + 1 == threads) { + throw std::runtime_error("expected"); + } + }); + }); + auto correct = !ok && static_cast(failed); + for (size_t shard = 0; shard < threads; ++shard) { + correct = correct && seen[shard] == (shard + 1) % threads + 1; + } + std::println(stderr, "[link_export_probe] sharded team chain: team={} correct={}", threads, correct); + if (!correct) { + std::println(stderr, "[link_export_probe] sharded team chain: FAILED"); + std::exit(1); + } +} + +// The Pauli basis: slots 2q/2q+1 are qubit q's x/z planes, so {0, 1} is Z on qubit 0 and {0} is X. +auto run_pauli_chain() -> void { + constexpr size_t kQubits = 3; + OperatorDict observable; + observable[VecZ{0, 1}] = std::complex{1.0, 0.0}; + + MonomialPropagator sim(observable, + /*cutoff=*/2 * kQubits, + VecZ{}, + /*schrodinger_cutoff=*/std::nullopt, + MPI_COMM_SELF, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::nullopt, + CutoffType::Support, + /*basis_change=*/std::nullopt, + /*logical_num_modes=*/kQubits, + Basis::Pauli); + sim.propagate(std::vector{{0}}, VecZ{0}, VecD{1.0}, VecD{0.3}); + + std::println(stderr, "[link_export_probe] pauli chain: size={}", sim.size()); +} + +// The constructor has no partition, factory, thread or shard argument at all. +auto run_constructor_shape_chain() -> void { constexpr size_t kModes = 6; OperatorDict ham; ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; - MonomialPropagator sim(ham, - 2 * kModes, - VecZ{0, 1}, - /*schrodinger_cutoff=*/std::nullopt, - MPI_COMM_SELF, - /*lower_atol=*/std::nullopt, - /*upper_atol=*/std::nullopt, - CutoffType::Length, - /*basis_change=*/std::nullopt, - /*logical_num_modes=*/kModes, - Basis::Majorana, - /*partitions=*/2); + using MP = MonomialPropagator; + using Factory = std::function(mpi::Comm)>; + constexpr auto constructible_with = []() { + return std::is_constructible_v, + mpi::Comm, + std::optional, + std::optional, + CutoffType, + std::optional>, + size_t, + Basis, + Tail...>; + }; + static_assert(constructible_with.template operator()<>()); + static_assert(!constructible_with.template operator()()); + static_assert(!constructible_with.template operator()()); + static_assert(!constructible_with.template operator()()); + static_assert(!HasPartitionFactory); + MP sim(ham, + 2 * kModes, + VecZ{0, 1}, + /*schrodinger_cutoff=*/std::nullopt, + MPI_COMM_SELF, + /*lower_atol=*/std::nullopt, + /*upper_atol=*/std::nullopt, + CutoffType::Length, + /*basis_change=*/std::nullopt, + /*logical_num_modes=*/kModes, + Basis::Majorana); + std::println(stderr, + "[link_export_probe] constructor chain: no partition surface; eleven-argument size={}", + sim.size()); +} - std::println(stderr, "[link_export_probe] partition chain: size={}", sim.size()); +auto run_raw_access_chain() -> void { + constexpr size_t kModes = 2; + OperatorDict ham; + ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; + MonomialPropagator sim(ham, 2 * kModes, VecZ{0, 1}, std::nullopt, MPI_COMM_SELF); + sim.build_graph(std::vector{{0, 2}, {1, 3}}, VecZ{0, 1}, VecD{1.0, 1.0}); + const auto [value, grad] = sim.expectation_value_and_gradient(VecD{0.1, 0.2}); + const auto budget = monoprop::detail::parallel::capture_thread_budget(); + // Raw access is the sole shard's at T = 1 and rejected above it; a rejection changes nothing. + bool raw_ok = false; + try { + raw_ok = sim.mp_op().size() == sim.size() && sim.indexing().size() == sim.size() + && sim.graph().layers() == sim.graph_layers() && sim.graph_data().size() == sim.graph_layers() + && budget.threads == 1; + } + catch (const MultiShardUnsupported &) { + raw_ok = budget.threads > 1 && sim.expectation_value(VecD{0.1, 0.2}) == value; + } + std::println(stderr, + "[link_export_probe] raw access chain: budget={} value={} grad_size={} raw_access_rule={}", + budget.threads, + value, + grad.size(), + raw_ok); + if (!raw_ok || grad.size() != 2) { + std::println(stderr, "[link_export_probe] raw access chain: FAILED"); + std::exit(1); + } +} + +// The integrated root through the imported target only: every public operation family, checked against itself +// (graph against graph-free propagation, direct against retained evaluation, unique exported keys). +auto run_sharded_root_chain() -> bool { + constexpr size_t kModes = 6; + OperatorDict ham; + ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; + ham[VecZ{2, 3}] = std::complex{0.0, 0.5}; + ham[VecZ{0, 3, 4, 7}] = std::complex{0.25, 0.0}; + ham[VecZ{}] = std::complex{1.5, 0.0}; + const VecZ state{0, 1, 2}; + const std::vector gates{{0, 2}, {1, 4}, {3, 5}, {2, 7}, {0, 9}}; + const VecZ mapping{0, 1, 0, 2, 1}; + const VecD gen{1.0, -0.5, 1.0, 0.75, 1.0}; + const VecD params{0.3, -0.2, 0.7}; + const auto make = [&] { return MonomialPropagator(ham, 2 * kModes, state, std::nullopt, MPI_COMM_SELF); }; + auto graph = make(); + graph.build_graph(gates, mapping, gen); + auto direct = make(); + direct.propagate(gates, mapping, gen, params); + const double energy = graph.expectation_value(params); + const auto [value, gradient] = graph.expectation_value_and_gradient(params); + const auto pared = graph.expectation_value_and_gradient_functional(1e-3)(params); + auto informed = make(); + informed.build_graph(gates, mapping, gen, std::nullopt, params); + auto copy = graph; + copy.update_initial_operator(OperatorDict{{VecZ{0, 1}, std::complex{0.0, 2.0}}}); + const auto terms = graph.evolved_operator_terms(params, 0.0); + std::vector keys; + for (const auto &[key, coeff] : terms) { + keys.push_back(key); + } + std::ranges::sort(keys); + const bool unique = std::ranges::adjacent_find(keys) == keys.end(); + const auto contracted = graph.contract_partially(params, true); + const double after = graph.expectation_value(VecD{}); + const double reference = direct.expectation_value(VecD{}); + const auto close = [](double a, double b) { return std::abs(a - b) <= 1e-10 * (1.0 + std::abs(a)); }; + const auto threads = monoprop::detail::parallel::capture_thread_budget().threads; + const bool correct = close(energy, reference) && energy == value && gradient.size() == params.size() + && close(pared.first, energy) && close(after, energy) && unique && terms.size() == graph.size() + && contracted.size() == graph.size() && graph.graph_layers() == 0 + && copy.expectation_value(params) != energy && informed.graph_layers() == gates.size(); + std::println(stderr, + "[link_export_probe] sharded root chain: team={} energy={} terms={} correct={}", + threads, + energy, + terms.size(), + correct); + return correct; +} + +// An ordinary extension through the imported target: virtual clone_() and update_initial_operator() overrides, and the +// protected apply_initial_operator_() returning this rank's share, all used through the base interface. +class DerivedRoot final : public MonomialPropagator<6> { +public: + using MonomialPropagator<6>::MonomialPropagator; + DerivedRoot(const DerivedRoot &) = default; + auto update_initial_operator(const OperatorDict &op_dict) -> void override { + ++updates; + share = apply_initial_operator_(op_dict).first.size(); + } + size_t updates = 0; + size_t share = 0; + +protected: + auto clone_() const -> std::unique_ptr> override { + return std::make_unique(*this); + } +}; + +class CloneProbe final : public MonomialPropagator<6> { +public: + static auto clone_of(const MonomialPropagator<6> &p) -> std::unique_ptr> { + return (p.*(&CloneProbe::clone_))(); + } +}; + +auto hook_operator() -> OperatorDict { + OperatorDict ham; + ham[VecZ{0, 1}] = std::complex{0.0, 1.0}; + ham[VecZ{2, 3}] = std::complex{0.0, 0.5}; + ham[VecZ{0, 3, 4, 7}] = std::complex{0.25, 0.0}; + ham[VecZ{1, 2, 5, 6}] = std::complex{-0.125, 0.0}; + return ham; +} + +auto run_derived_hooks_chain(MPI_Comm comm) -> bool { + const auto ham = hook_operator(); + DerivedRoot sim(ham, 12, VecZ{0, 1, 2}, std::nullopt, comm); + sim.build_graph(std::vector{{0, 2}, {1, 4}, {3, 5}}, VecZ{0, 1, 0}, VecD{1.0, -0.5, 1.0}); + const VecD params{0.3, -0.2}; + const double energy = sim.expectation_value(params); + const auto clone = CloneProbe::clone_of(sim); + MonomialPropagator<6> &base = sim; + OperatorDict scaled; + for (const auto &[key, value] : ham) { + scaled[key] = value * 2.0; + } + base.update_initial_operator(scaled); + const auto global_share = mpi::allreduce_sum(sim.share, mpi::Comm(comm)); + const bool correct = + dynamic_cast(clone.get()) != nullptr && sim.updates == 1 && global_share == ham.size() + && clone->expectation_value(params) == energy + && std::abs(sim.expectation_value(params) - (2.0 * energy)) <= 1e-12 * (1.0 + std::abs(energy)); + std::println(stderr, + "[link_export_probe] derived hooks chain: ranks={} share={} global_share={} correct={}", + mpi::size(mpi::Comm(comm)), + sim.share, + global_share, + correct); + return correct; +} + +// Several ranks: the root on the world communicator against one process's root, through the imported target. +auto run_multirank_root_chain() -> bool { + const auto world = mpi::Comm(MPI_COMM_WORLD); + const auto ranks = mpi::size(world); + // A launch that names its rank count must get it: a single-rank run cannot stand in for this chain. + if (const char *expected = std::getenv("monoprop_TEST_EXPECT_RANKS"); + expected != nullptr && std::atoi(expected) != ranks) { + std::println(stderr, "[link_export_probe] multirank root chain: expected {} ranks, got {}", expected, ranks); + return false; + } + if (ranks < 2) { + std::println(stderr, "[link_export_probe] multirank root chain: skipped on one rank"); + return true; + } + const auto ham = hook_operator(); + const std::vector gates{{0, 2}, {1, 4}, {3, 5}, {2, 7}, {0, 9}, {6, 11}}; + const VecZ mapping{0, 1, 0, 2, 1, 2}; + const VecD gen{1.0, -0.5, 1.0, 0.75, 1.0, 0.5}; + const VecD params{0.3, -0.2, 0.7}; + const auto make = [&](MPI_Comm comm) { return MonomialPropagator<6>(ham, 12, VecZ{0, 1, 2}, std::nullopt, comm); }; + auto distributed = make(MPI_COMM_WORLD); + auto single = make(MPI_COMM_SELF); + distributed.build_graph(gates, mapping, gen); + single.build_graph(gates, mapping, gen); + const auto close = [](double a, double b) { + return std::abs(a - b) <= 1e-9 + (1e-7 * std::max(std::abs(a), std::abs(b))); + }; + const auto [dv, dg] = distributed.expectation_value_and_gradient(params); + const auto [sv, sg] = single.expectation_value_and_gradient(params); + bool correct = close(dv, sv) && dg.size() == sg.size(); + for (size_t i = 0; correct && i < dg.size(); ++i) { + correct = close(dg[i], sg[i]); + } + const auto local = distributed.evolved_operator_terms(params, 0.0); + std::map> reference; + for (const auto &[key, coeff] : single.evolved_operator_terms(params, 0.0)) { + reference.emplace(key, coeff); + } + size_t unknown = 0; + for (const auto &[key, coeff] : local) { + const auto it = reference.find(key); + unknown += it == reference.end() || !close(it->second.real(), coeff.real()) ? 1 : 0; + } + const auto global_terms = mpi::allreduce_sum(local.size(), world); + const auto global_size = mpi::allreduce_sum(distributed.size(), world); + const auto global_unknown = mpi::allreduce_sum(unknown, world); + correct = correct && global_terms == reference.size() && global_size == single.size() && global_unknown == 0; + // Raw access depends on T alone, whatever the rank count. + const auto threads = monoprop::detail::parallel::capture_thread_budget().threads; + bool raw = false; + try { + raw = distributed.mp_op().size() == distributed.size() && threads == 1; + } + catch (const MultiShardUnsupported &) { + raw = threads > 1; + } + const bool hooks = run_derived_hooks_chain(MPI_COMM_WORLD); + std::println(stderr, + "[link_export_probe] multirank root chain: ranks={} team={} energy={} reference={} global_terms={} " + "reference_terms={} global_size={} reference_size={} unknown={} raw_access_rule={}", + ranks, + threads, + dv, + sv, + global_terms, + reference.size(), + global_size, + single.size(), + global_unknown, + raw); + correct = correct && raw && hooks; + std::println(stderr, "[link_export_probe] multirank root chain: correct={}", correct); + return correct; } } // namespace @@ -105,7 +484,35 @@ auto run_partition_chain() -> void { auto main() -> int { monoprop::mpi::init(); run_graph_build_chain(); - run_partition_chain(); + run_constructor_shape_chain(); + run_openmp_workshare_chain(); + run_sharded_team_chain(); + if (!run_sharded_state_chain()) { + std::println(stderr, "[link_export_probe] sharded state chain: FAILED"); + return 1; + } + if (!run_sharded_construction_chain()) { + std::println(stderr, "[link_export_probe] sharded construction chain: FAILED"); + return 1; + } + if (!run_sharded_evaluation_chain()) { + std::println(stderr, "[link_export_probe] sharded evaluation chain: FAILED"); + return 1; + } + run_pauli_chain(); + run_raw_access_chain(); + if (!run_sharded_root_chain()) { + std::println(stderr, "[link_export_probe] sharded root chain: FAILED"); + return 1; + } + if (!run_derived_hooks_chain(MPI_COMM_SELF)) { + std::println(stderr, "[link_export_probe] derived hooks chain: FAILED"); + return 1; + } + if (!run_multirank_root_chain()) { + std::println(stderr, "[link_export_probe] multirank root chain: FAILED"); + return 1; + } monoprop::mpi::finalize(); std::println(stderr, "[link_export_probe] OK"); return 0; diff --git a/cpp/tests/link_export_probe/sharded_construction_chain.cpp b/cpp/tests/link_export_probe/sharded_construction_chain.cpp new file mode 100644 index 000000000..be2ee7a87 --- /dev/null +++ b/cpp/tests/link_export_probe/sharded_construction_chain.cpp @@ -0,0 +1,120 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Chain (h) of link_export_probe.cpp, in its own translation unit so that detail/sharded/Construction.h is the first +// and only monoprop header: an installed consumer compiles it self-contained, with OpenMP flags from the imported +// target. + +#include "monoprop/detail/sharded/Construction.h" + +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/ThreadBudget.h" + +namespace { + +constexpr size_t kModes = 4; +using Mono = monoprop::Monomial; + +// Slot indices map to bits as for every generator the library receives. +auto mono(const monoprop::VecZ &slots) -> Mono { + return monoprop::indices_to_bitset(slots); +} + +} // namespace + +// Seeds a Heisenberg operator on the captured budget's shards, propagates one gate through the direct-buffer team +// and checks the rotated pair against its closed form, then builds a two-gate graph on fresh shards. +auto run_sharded_construction_chain() -> bool { + namespace sharded = monoprop::detail::sharded; + const auto options = monoprop::detail::parallel::capture_thread_budget(); + const auto threads = static_cast(options.threads); + const auto router = monoprop::routing::make_router(1, threads); + + // H = 1.0 * i g0 g1; the gate G = g1 g2 rotates it into the partner g0 g2. + monoprop::OperatorDict op; + op[monoprop::VecZ{0, 1}] = std::complex{0.0, 1.0}; + const monoprop::VecZ state{0, 1}; + const auto seed = sharded::OperatorSeed{.initial_operator = op, + .initial_state = state, + .router = router, + .basis = monoprop::Basis::Majorana, + .logical_num_modes = kModes, + .paired = std::nullopt, + .inline_width = 4}; + const monoprop::CutoffFn cutoff = monoprop::detail::LengthCutoff{2 * kModes, kModes}; + const auto ctx = sharded::ConstructionContext{.cutoff_fn = cutoff, .router = router}; + + auto shards = sharded::seed_shards(options, seed, 0); + const std::vector gates{mono({1, 2})}; + const std::vector angles{0.3}; + const auto propagated = + sharded::propagate(options, + shards, + ctx, + {.generators = gates, .mapped_params = angles, .only_rotate_len_k = {}}); + double norm = 0.0; + size_t rows = 0; + for (const auto &s : shards) { + for (const double c : s->op.op_coeffs) { + norm += c * c; + } + rows += s->size(); + } + + auto graph_shards = sharded::seed_shards(options, seed, 0); + const std::vector circuit{mono({1, 2}), mono({0, 3})}; + const std::vector mapping{0, 1}; + const std::vector coeffs{1.0, 1.0}; + const std::vector gate_indices{0, 1}; + const auto built = sharded::build_graph(options, + graph_shards, + ctx, + {.generators = circuit, + .parameter_mapping = mapping, + .gen_coeffs = coeffs, + .gate_indices = gate_indices, + .only_rotate_len_k = {}}); + for (const auto &outcome : {propagated, built}) { + if (outcome.error) { + try { + std::rethrow_exception(outcome.error); + } + catch (const std::exception &e) { + std::println(stderr, "[link_export_probe] sharded construction chain: {}", e.what()); + } + } + } + size_t layers = 0; + for (const auto &s : graph_shards) { + layers += s->graph.layers(); + } + + // A rotation preserves the coefficient norm: cos^2 + sin^2 of the one source term. + const auto correct = !propagated.error && propagated.mutation_started && !built.error && rows == 2 + && std::abs(norm - 1.0) < 1e-12 && layers == 2 * threads; + std::println(stderr, + "[link_export_probe] sharded construction chain: shards={} rows={} norm={} layers={} correct={}", + threads, + rows, + norm, + layers, + correct); + return correct; +} diff --git a/cpp/tests/link_export_probe/sharded_evaluation_chain.cpp b/cpp/tests/link_export_probe/sharded_evaluation_chain.cpp new file mode 100644 index 000000000..7cbfccef3 --- /dev/null +++ b/cpp/tests/link_export_probe/sharded_evaluation_chain.cpp @@ -0,0 +1,169 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Chain (i) of link_export_probe.cpp, in its own translation unit so that detail/sharded/Evaluation.h is the first +// monoprop header: an installed consumer compiles it self-contained, with OpenMP flags from the imported target, and +// calls the exported sharded evaluators across the shared-library boundary. + +#include "monoprop/detail/sharded/Evaluation.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/Construction.h" + +namespace { + +constexpr size_t kModes = 4; +using Mono = monoprop::Monomial; + +auto mono(const monoprop::VecZ &slots) -> Mono { + return monoprop::indices_to_bitset(slots); +} + +auto report(const char *what, const std::exception_ptr &error) -> bool { + if (!error) { + return true; + } + try { + std::rethrow_exception(error); + } + catch (const std::exception &e) { + std::println(stderr, "[link_export_probe] sharded evaluation chain: {}: {}", what, e.what()); + } + return false; +} + +} // namespace + +// Builds a two-gate graph on the captured budget's shards (structurally, then extended coefficient-informed, which +// replays the existing graph in-team as its seed), retains a functional, and checks ev_and_grad_sharded() against +// ev_sharded() and a central difference of it. +auto run_sharded_evaluation_chain() -> bool { + namespace sharded = monoprop::detail::sharded; + const auto options = monoprop::detail::parallel::capture_thread_budget(); + const auto threads = static_cast(options.threads); + const auto router = monoprop::routing::make_router(1, threads); + + // H = i g0 g1 + 0.5 i g2 g3 + 0.25 (identity); state with modes 0 and 1 occupied. + monoprop::OperatorDict op; + op[monoprop::VecZ{0, 1}] = std::complex{0.0, 1.0}; + op[monoprop::VecZ{2, 3}] = std::complex{0.0, 0.5}; + op[monoprop::VecZ{}] = std::complex{0.25, 0.0}; + const monoprop::VecZ state{0, 1}; + const auto seed = sharded::OperatorSeed{.initial_operator = op, + .initial_state = state, + .router = router, + .basis = monoprop::Basis::Majorana, + .logical_num_modes = kModes, + .paired = std::nullopt, + .inline_width = 4}; + const monoprop::CutoffFn cutoff = monoprop::detail::LengthCutoff{2 * kModes, kModes}; + const auto ctx = sharded::ConstructionContext{.cutoff_fn = cutoff, .router = router}; + auto shards = sharded::seed_shards(options, seed, 0); + + const std::vector first{mono({1, 2}), mono({0, 3})}; + const std::vector first_mapping{0, 1}; + const std::vector ones{1.0, 1.0}; + const std::vector first_gates{0, 1}; + bool ok = report("build_graph", + sharded::build_graph(options, + shards, + ctx, + {.generators = first, + .parameter_mapping = first_mapping, + .gen_coeffs = ones, + .gate_indices = first_gates, + .only_rotate_len_k = {}}) + .error); + + // Extend coefficient-informed: gate 2 is driven by parameter 2; the existing layers replay at parameters 0 and 1. + const std::vector params{0.3, -0.2, 0.45}; + const std::vector second{mono({2, 5})}; + const std::vector second_mapping{2}; + const std::vector second_coeff{0.5}; + const std::vector second_gates{2}; + const std::vector build_angle{params[2] * 0.5}; + // Heisenberg seed angles: the existing layers' mapped parameters in replay order (optimizer order reversed). + const std::vector seed_angles{params[1], params[0]}; + ok = ok + && report("build_graph_informed", + sharded::build_graph_informed(options, + shards, + ctx, + {.graph = {.generators = second, + .parameter_mapping = second_mapping, + .gen_coeffs = second_coeff, + .gate_indices = second_gates, + .only_rotate_len_k = {}}, + .mapped_params = build_angle, + .seed_params = std::span(seed_angles)}) + .error); + + // Optimizer order is the reverse of the graph's layer order. + const size_t layers = shards.front()->graph.layers(); + monoprop::VecZ mapping(layers); + monoprop::VecD gen(layers); + for (size_t l = 0; l < layers; ++l) { + const auto t = shards.front()->graph.get_layer_traversal(l); + mapping[layers - 1 - l] = t.param_index(); + gen[layers - 1 - l] = t.gen_coeff(); + } + auto retained = sharded::prepare_retained(options, + shards, + {.core_term = 0.25, + .parameter_mapping = mapping, + .gen_coeffs = gen, + .pare_threshold = {}, + .basis = monoprop::Basis::Majorana, + .schrodinger = false, + .rank = 0}); + ok = ok && report("prepare_retained", retained.error) && retained.retained.has_value(); + if (!ok) { + std::println(stderr, "[link_export_probe] sharded evaluation chain: shards={} correct=false", threads); + return false; + } + const auto &r = *retained.retained; + const auto energy = [&](const monoprop::VecD &p) { + const auto requests = r.requests(p); + return sharded::ev_sharded(requests, r.callbacks, options, MPI_COMM_SELF); + }; + const monoprop::VecD point(params.begin(), params.end()); + const auto requests = r.requests(point); + const auto [value, grad] = sharded::ev_and_grad_sharded(requests, r.callbacks, options, MPI_COMM_SELF); + double worst = 0.0; + for (size_t i = 0; i < grad.size(); ++i) { + auto plus = point; + auto minus = point; + plus[i] += 1e-6; + minus[i] -= 1e-6; + worst = std::max(worst, std::abs(grad[i] - ((energy(plus) - energy(minus)) / 2e-6))); + } + const bool correct = layers == 3 && grad.size() == 3 && std::abs(value - energy(point)) < 1e-12 && worst < 1e-6; + std::println(stderr, + "[link_export_probe] sharded evaluation chain: shards={} layers={} value={} worst_fd={} correct={}", + threads, + layers, + value, + worst, + correct); + return correct; +} diff --git a/cpp/tests/link_export_probe/sharded_state_chain.cpp b/cpp/tests/link_export_probe/sharded_state_chain.cpp new file mode 100644 index 000000000..af0266317 --- /dev/null +++ b/cpp/tests/link_export_probe/sharded_state_chain.cpp @@ -0,0 +1,82 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Chain (g) of link_export_probe.cpp, in its own translation unit so that detail/sharded/State.h is the first and +// only monoprop header: an installed consumer compiles it self-contained, with OpenMP flags from the imported target. + +#include "monoprop/detail/sharded/State.h" + +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/ThreadBudget.h" + +// Seeds both pictures at the captured budget, copies the Heisenberg shards on their owners and checks the counts +// against the unsharded totals, then checks that an initializer failure reaches the caller. +auto run_sharded_state_chain() -> bool { + namespace sharded = monoprop::detail::sharded; + constexpr size_t kModes = 4; + const auto options = monoprop::detail::parallel::capture_thread_budget(); + const auto threads = static_cast(options.threads); + const auto router = monoprop::routing::make_router(1, threads); + + monoprop::OperatorDict op; + op[monoprop::VecZ{}] = std::complex{1.5, 0.0}; + op[monoprop::VecZ{0, 1}] = std::complex{0.0, 1.0}; + op[monoprop::VecZ{2, 3}] = std::complex{0.0, -0.5}; + op[monoprop::VecZ{1, 2}] = std::complex{0.0, 0.25}; + const monoprop::VecZ state{0, 1}; + const auto core = sharded::validate_initial_operator(op, monoprop::Basis::Majorana, kModes); + + const auto heisenberg = sharded::OperatorSeed{.initial_operator = op, + .initial_state = state, + .router = router, + .basis = monoprop::Basis::Majorana, + .logical_num_modes = kModes, + .paired = std::nullopt, + .inline_width = 4}; + const auto seeded = sharded::seed_shards(options, heisenberg, 0); + const auto copied = sharded::copy_shards(options, seeded); + + auto schrodinger = heisenberg; + schrodinger.paired = sharded::paired_basis_bounds(3, kModes, router.flat_world()); + const auto paired = sharded::seed_shards(options, schrodinger, 0); + + auto failed = false; + try { + (void)sharded::make_shards(options, [&](size_t shard) { + if (shard + 1 == threads) { + throw std::runtime_error("expected"); + } + return std::make_unique>(monoprop::detail::MPOperator{}, false); + }); + } + catch (const std::runtime_error &) { + failed = true; + } + + const auto correct = core == 1.5 && sharded::total_size(seeded) == 3 && sharded::total_size(copied) == 3 + && sharded::total_size(paired) == monoprop::paired_op_size(2, kModes) + && sharded::operator_memory_usage(copied).total_bytes() > 0 && failed; + std::println(stderr, + "[link_export_probe] sharded state chain: shards={} heisenberg={} schrodinger={} correct={}", + threads, + sharded::total_size(seeded), + sharded::total_size(paired), + correct); + return correct; +} diff --git a/cpp/tests/majorana_cutoff_tests.cpp b/cpp/tests/majorana_cutoff_tests.cpp index 3790d14f4..8b4f1c9d5 100644 --- a/cpp/tests/majorana_cutoff_tests.cpp +++ b/cpp/tests/majorana_cutoff_tests.cpp @@ -28,11 +28,13 @@ #include #include +#include "ScanTestSupport.h" #include "monoprop/TypeAliases.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/core/Monomial.h" #include "monoprop/detail/evolution/layer_build/PartnerMerge.h" +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" using namespace monoprop; using cd = std::complex; @@ -205,6 +207,12 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_evaluator_dispatch_and_popcount) { BOOST_TEST((opaque_ev.support_cutoff() == nullptr)); BOOST_TEST(!opaque_ev.max_slot_bound().has_value()); + // Only the typed built-in cutoffs may run in scan workers. A stateless lambda is still opaque: the + // evaluator cannot see what it does, and constness of the std::function says nothing about it. + BOOST_TEST(length_ev.parallel_safe()); + BOOST_TEST(support_ev.parallel_safe()); + BOOST_TEST(!opaque_ev.parallel_safe()); + // passes_with_popcount: pc <= cutoff short-circuits to true; otherwise it equals a direct eval. Monomial unpaired; // length 4, not paired unpaired.set(0); @@ -356,6 +364,43 @@ BOOST_AUTO_TEST_CASE(majorana_cutoff_generate_paired_op_matches_the_enumeration) } } +// An opaque cutoff keeps the scan on the calling thread at any budget, in one range, with the same result; +// so does the basis-change cutoff, whose closure is conservatively treated as opaque. +BOOST_AUTO_TEST_CASE(openmp_scan_opaque_cutoff_stays_serial) { + using scan_test::Expect; + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + const auto gen = scan_test::mono_of_bits({14, 20, 33, 47}); + const CutoffFn<32> opaque = [](const Monomial<32> &m) { return m.count() <= 6; }; + BOOST_TEST(!detail::CutoffEvaluator<32>(opaque).parallel_safe()); + scan_test::check_threaded_matches_serial(solo, + {.label = "opaque lambda", .gen = gen, .cutoff = opaque}, + Expect::output); + scan_test::check_threaded_matches_serial( + solo, + {.label = "opaque lambda, fused", .gen = gen, .cutoff = opaque, .capture = true, .fused_scale = true}, + Expect::output); + + // Identity basis change: the predicate is a real basis-change closure, and its decisions equal the plain + // length cutoff's, so the serial fallback must also reproduce the typed, threaded scan. + MonomialList<32> identity; + for (size_t b = 0; b < 64; ++b) { + Monomial<32> row{}; + row.set(63 - b); // change_basis reads row 2N-1-pos for bit pos + identity.push_back(row); + } + const auto changed = detail::cutoff_function_basis_change<32>(CutoffType::Length, 6, identity); + BOOST_TEST(!detail::CutoffEvaluator<32>(changed).parallel_safe()); + scan_test::check_threaded_matches_serial(solo, + {.label = "basis change", .gen = gen, .cutoff = changed}, + Expect::output); + const auto typed = scan_test::run_scan(solo, + {.label = "typed", .gen = gen, .cutoff = detail::LengthCutoff<32>{6}}, + {.threads = 4}); + const auto via_basis = + scan_test::run_scan(solo, {.label = "basis change", .gen = gen, .cutoff = changed}, {.threads = 4}); + scan_test::check_same_scan(typed, via_basis, false); +} + namespace { // passes_positions agrees with passes_with_popcount on every partner, including inactive-prefix and diff --git a/cpp/tests/mp_operator_tests.cpp b/cpp/tests/mp_operator_tests.cpp index 2c822c381..aec9131e8 100644 --- a/cpp/tests/mp_operator_tests.cpp +++ b/cpp/tests/mp_operator_tests.cpp @@ -21,10 +21,16 @@ #include #include +#include #include #include #include +#ifdef __linux__ +#include +#include +#endif + #include "monoprop/MonomialPropagator.h" #include "monoprop/algebra/Algebra.h" #include "monoprop/detail/mpi/MPICompat.h" @@ -400,6 +406,59 @@ BOOST_AUTO_TEST_CASE(mp_operator_breakdown_keeps_init_operator_entries_out_of_to BOOST_CHECK_EQUAL(acc.total_bytes(), 120U); } +// Resident pages of [first, last) by mincore(); only pages wholly inside the range are counted. +#ifdef __linux__ +auto resident_pages(const double *first, const double *last) -> size_t { + const auto page = static_cast(sysconf(_SC_PAGESIZE)); + const auto begin = (reinterpret_cast(first) + page - 1) & ~(page - 1); + const auto end = reinterpret_cast(last) & ~(page - 1); + if (begin >= end) { + return 0; + } + std::vector marks((end - begin) / page); + BOOST_REQUIRE_EQUAL(mincore(reinterpret_cast(begin), end - begin, marks.data()), 0); + return static_cast(std::ranges::count_if(marks, [](unsigned char m) { return (m & 1U) != 0; })); +} +#endif + +// A large vector's spare capacity is returned to the system in place: no copy (the block stays where it is), the +// values stay, and the accounting reports the entries alone, as after a shrinking copy. A small vector is still copied +// down to its size. +BOOST_AUTO_TEST_CASE(mp_operator_release_slack_returns_large_spare_capacity_in_place) { + detail::MPOperator<8> op; + constexpr size_t capacity = size_t{1} << 20; // 8 MiB of doubles + constexpr size_t live = capacity / 2 + 3; + op.op_coeffs.resize(capacity, 1.5); // every page of the block resident + op.op_coeffs.resize(live); + for (size_t i = 0; i < live; ++i) { + op.op_coeffs[i] = static_cast(i); + } + op.state_vals_.reserve(64); + op.state_vals_.assign(5, -1.0); + const double *block = op.op_coeffs.data(); +#ifdef __linux__ + BOOST_TEST_REQUIRE(resident_pages(block + live, block + op.op_coeffs.capacity()) > 0U); +#endif + + op.release_slack(); + + BOOST_TEST(op.op_coeffs.size() == live); + bool kept = true; + for (size_t i = 0; i < live; ++i) { + kept = kept && op.op_coeffs[i] == static_cast(i); + } + BOOST_TEST(kept); + BOOST_TEST(op.state_vals_.capacity() == 5U); + BOOST_TEST(op.slack_copy_bytes() == 0U); + const auto breakdown = detail::estimate_memory_usage<8>(op); + BOOST_TEST(breakdown.op_coeffs_bytes == live * sizeof(double)); + BOOST_TEST(breakdown.state_coeffs_bytes == 5 * sizeof(double)); +#ifdef __linux__ + BOOST_TEST(op.op_coeffs.data() == block); + BOOST_TEST(resident_pages(block + live, block + op.op_coeffs.capacity()) == 0U); +#endif +} + BOOST_AUTO_TEST_CASE(mp_operator_copy_constructor_clones_store_and_coeffs) { auto op = build_indexed_op({indices_to_bitset<8>({0, 1}), indices_to_bitset<8>({2, 3})}); op.initial_state = {0}; diff --git a/cpp/tests/mpi_alltoallv_tests.cpp b/cpp/tests/mpi_alltoallv_tests.cpp new file mode 100644 index 000000000..f93c63204 --- /dev/null +++ b/cpp/tests/mpi_alltoallv_tests.cpp @@ -0,0 +1,373 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * The real-MPI request helpers of the low-level one-owner-per-rank engine (detail/mpi/MPICompat.h): begin_alltoallv, + * PendingAlltoallv, alltoall_counts and the PeerPlan narrowing, on an ordinary communicator. In the MPI-off build and + * in the per-case serial runs the world has one rank and the stub self-copies; the cases that need peers run in the + * whole-suite MPI variants. Ported from the plain-MPI arms of the in-process transport suites removed with the + * partition runtime; their in-process (ShmComm/HybridComm) arms have no surviving subject. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/MPICompat.h" + +using monoprop::mpi::Comm; +using monoprop::mpi::PeerPlan; + +namespace { + +auto world_size() -> int { + return monoprop::mpi::size(Comm(MPI_COMM_WORLD)); +} + +auto world_rank() -> int { + return monoprop::mpi::rank(Comm(MPI_COMM_WORLD)); +} + +// The sparse XOR pairing needs a power-of-two world of at least two ranks. +auto pairable(int ranks) -> bool { + return ranks >= 2 && (ranks & (ranks - 1)) == 0; +} + +template +auto full_window(const std::vector> &blocks) -> monoprop::mpi::WindowVec> { + monoprop::mpi::WindowVec> w(monoprop::mpi::SlotWindow{.base = 0, .count = blocks.size()}); + std::ranges::copy(blocks, w.begin()); + return w; +} + +// Slots outside the round's window come back empty. +template +auto flat_blocks(const monoprop::mpi::WindowVec> &w, size_t world) -> std::vector> { + std::vector> out(world); + for (const auto wi : w.window().indices()) { + out[w.window().slot(wi)] = w[wi]; + } + return out; +} + +template +auto exchange_blocks(const std::vector> &send, + const Comm &c, + bool skip_self = false, + const std::vector *known_recv_counts = nullptr, + PeerPlan plan = {}) -> std::vector> { + monoprop::mpi::WindowVec> got; + monoprop::mpi::begin_alltoallv(full_window(send), c, skip_self, known_recv_counts, plan).wait_into(got); + return flat_blocks(got, static_cast(monoprop::mpi::size(c))); +} + +auto sparse_tag(int src, int dst, int j) -> int { + return (((src * 128) + dst) * 1000) + j; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(mpi_alltoallv_sparse_plan_peer_is_an_involution) { + for (const int shift : {0, 1, 2, 3, 5, 8, 13, 255}) { + const PeerPlan plan{.sparse = true, .shift = shift}; + BOOST_REQUIRE(!plan.dense()); + BOOST_REQUIRE_EQUAL(plan.count(256), 1); + for (int me = 0; me < 256; ++me) { + const int peer = plan.peer(me, 0); + BOOST_REQUIRE_EQUAL(plan.peer(peer, 0), me); + BOOST_REQUIRE(plan.contains(me, peer)); + BOOST_REQUIRE(plan.contains(peer, me)); + BOOST_REQUIRE(!plan.contains(me, peer ^ 1)); // the peer set is a singleton + } + } + const PeerPlan dense_plan{}; + BOOST_REQUIRE(dense_plan.dense()); + BOOST_REQUIRE_EQUAL(dense_plan.count(7), 7); + for (int k = 0; k < 7; ++k) { + BOOST_REQUIRE_EQUAL(dense_plan.peer(3, k), k); // dense ignores `me`: every rank is a peer + BOOST_REQUIRE(dense_plan.contains(3, k)); + } +} + +// alltoall_counts is a transpose: recv[s] on rank r is what s declared it sends to r. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_counts_transpose) { + const int ranks = world_size(); + const int me = world_rank(); + std::vector send(static_cast(ranks)); + for (int t = 0; t < ranks; ++t) { + send[static_cast(t)] = (me * 100) + t; + } + std::vector got(static_cast(ranks), -1); + monoprop::mpi::alltoall_counts(send.data(), got.data(), ranks, Comm(MPI_COMM_WORLD)); + for (int s = 0; s < ranks; ++s) { + BOOST_CHECK_EQUAL(got[static_cast(s)], (s * 100) + me); + } +} + +// Each source's block arrives contiguously in ascending source order, which Resolve.h's positional pairing relies on. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_dense_source_order_and_tags) { + const int ranks = world_size(); + const int me = world_rank(); + const Comm c(MPI_COMM_WORLD); + std::vector> send(static_cast(ranks)); + for (int t = 0; t < ranks; ++t) { + for (int j = 0; j <= me; ++j) { + send[static_cast(t)].push_back((me * 1000) + j); + } + } + const auto got = exchange_blocks(send, c); + BOOST_REQUIRE_EQUAL(static_cast(got.size()), ranks); + for (int s = 0; s < ranks; ++s) { + const auto &blk = got[static_cast(s)]; + BOOST_REQUIRE_EQUAL(static_cast(blk.size()), s + 1); // source s sent s + 1 + for (int j = 0; j <= s; ++j) { + BOOST_CHECK_EQUAL(blk[static_cast(j)], (s * 1000) + j); + } + } + // skip_self: the self slot is neither sent nor received; every other source arrives intact. + const auto skipped = exchange_blocks(send, c, /*skip_self=*/true); + for (int s = 0; s < ranks; ++s) { + BOOST_CHECK_EQUAL(skipped[static_cast(s)].size(), s == me ? 0U : static_cast(s + 1)); + } +} + +// A sparse plan replaces the collectives with point-to-point over the one peer the plan names, so its failure modes +// are dropped data and a hang. Unknown recv layout (a pairwise count round) and the known layout of a response round. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_sparse_plan_delivers_only_to_its_peer) { + const int ranks = world_size(); + if (!pairable(ranks)) { + return; + } + const Comm c(MPI_COMM_WORLD); + for (int shift = 0; shift < ranks; ++shift) { + const PeerPlan plan{.sparse = true, .shift = shift}; + const int peer = plan.peer(world_rank(), 0); + std::vector> send(static_cast(ranks)); + for (int j = 0; j < 4; ++j) { + send[static_cast(peer)].push_back((world_rank() * 1000) + j); + } + const auto out = exchange_blocks(send, c, false, nullptr, plan); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), ranks); + for (int src = 0; src < ranks; ++src) { + if (src != peer) { + BOOST_CHECK(out[static_cast(src)].empty()); + continue; + } + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(src)].size()), 4); + for (int j = 0; j < 4; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(src)][static_cast(j)], (src * 1000) + j); + } + } + std::vector known(static_cast(ranks), 0); + known[static_cast(peer)] = 4; + const auto out2 = exchange_blocks(send, c, false, &known, plan); + BOOST_REQUIRE_EQUAL(static_cast(out2.size()), ranks); + BOOST_CHECK(out2[static_cast(peer)] == out[static_cast(peer)]); + } +} + +// known_recv_counts is caller-supplied and can name a rank the plan does not: no receive is posted for it, so an +// unmasked count would hand the caller unwritten bytes as data. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_known_recv_counts_are_masked_through_the_plan) { + const int ranks = world_size(); + if (!pairable(ranks)) { + return; + } + constexpr int kReal = 4; + constexpr int kBogus = 7; + const Comm c(MPI_COMM_WORLD); + for (int shift = 0; shift < ranks; ++shift) { + const PeerPlan plan{.sparse = true, .shift = shift}; + const int peer = plan.peer(world_rank(), 0); + const int bad = (peer + 1) % ranks; + BOOST_REQUIRE(bad != peer); + std::vector> send(static_cast(ranks)); + for (int j = 0; j < kReal; ++j) { + send[static_cast(peer)].push_back((world_rank() * 1000) + j); + } + std::vector known(static_cast(ranks), 0); + known[static_cast(peer)] = kReal; + known[static_cast(bad)] = kBogus; + const auto out = exchange_blocks(send, c, false, &known, plan); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), ranks); + BOOST_CHECK(out[static_cast(bad)].empty()); + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), kReal); + for (int j = 0; j < kReal; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], (peer * 1000) + j); + } + } +} + +// A zero-count leg is where a send/recv posting asymmetry deadlocks: both ends must skip on the same value. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_sparse_plan_with_an_empty_leg) { + const int ranks = world_size(); + if (!pairable(ranks)) { + return; + } + const int me = world_rank(); + constexpr int kLen = 4; + const Comm c(MPI_COMM_WORLD); + for (int shift = 1; shift < ranks; ++shift) { + const PeerPlan plan{.sparse = true, .shift = shift}; + const int peer = plan.peer(me, 0); + BOOST_REQUIRE(peer != me); + const int my_len = me < peer ? 0 : kLen; + const int peer_len = peer < me ? 0 : kLen; + std::vector> send(static_cast(ranks)); + for (int j = 0; j < my_len; ++j) { + send[static_cast(peer)].push_back(sparse_tag(me, peer, j)); + } + const auto out = exchange_blocks(send, c, false, nullptr, plan); + BOOST_REQUIRE_EQUAL(static_cast(out[static_cast(peer)].size()), peer_len); + for (int j = 0; j < peer_len; ++j) { + BOOST_CHECK_EQUAL(out[static_cast(peer)][static_cast(j)], sparse_tag(peer, me, j)); + } + } +} + +// Shift zero makes every rank its own and only peer, and skip_self then drops that one leg. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_sparse_plan_skip_self_at_shift_zero) { + const int ranks = world_size(); + if (!pairable(ranks)) { + return; + } + const int me = world_rank(); + const PeerPlan plan{.sparse = true, .shift = 0}; + BOOST_REQUIRE_EQUAL(plan.peer(me, 0), me); + std::vector> send(static_cast(ranks)); + for (int j = 0; j < 5; ++j) { + send[static_cast(me)].push_back(sparse_tag(me, me, j)); + } + const auto out = exchange_blocks(send, Comm(MPI_COMM_WORLD), /*skip_self=*/true, nullptr, plan); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), ranks); + for (const auto &blk : out) { + BOOST_CHECK(blk.empty()); + } +} + +// Engine.h's run_exchange posts both its rounds on one communicator under one tag; MPI's non-overtaking order is what +// keeps a round-2 receive from matching a round-1 send. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_sparse_plan_back_to_back_rounds) { + const int ranks = world_size(); + if (!pairable(ranks)) { + return; + } + const int me = world_rank(); + const Comm c(MPI_COMM_WORLD); + for (int shift = 0; shift < ranks; ++shift) { + const PeerPlan plan{.sparse = true, .shift = shift}; + const int peer = plan.peer(me, 0); + const int len = 3 + (me % 2); + std::vector> q(static_cast(ranks)); + for (int j = 0; j < len; ++j) { + q[static_cast(peer)].push_back(sparse_tag(me, peer, j)); + } + const auto q_out = exchange_blocks(q, c, false, nullptr, plan); + std::vector known(static_cast(ranks), 0); + known[static_cast(peer)] = len; + std::vector> r(static_cast(ranks)); + const int back = static_cast(q_out[static_cast(peer)].size()); + for (int j = 0; j < back; ++j) { + r[static_cast(peer)].push_back(q_out[static_cast(peer)][static_cast(j)] + 7); + } + const auto r_out = exchange_blocks(r, c, false, &known, plan); + BOOST_REQUIRE_EQUAL(back, 3 + (peer % 2)); + BOOST_REQUIRE_EQUAL(static_cast(r_out[static_cast(peer)].size()), len); + for (int j = 0; j < len; ++j) { + BOOST_CHECK_EQUAL(r_out[static_cast(peer)][static_cast(j)], sparse_tag(me, peer, j) + 7); + } + for (int src = 0; src < ranks; ++src) { + if (src != peer) { + BOOST_CHECK(r_out[static_cast(src)].empty()); + } + } + } +} + +// A shift outside the rank-index width would index out of range and name an invalid MPI rank: refused first, as an +// exception, before anything is posted. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_unroutable_peer_plan_is_refused) { + const int ranks = world_size(); + const PeerPlan bad{.sparse = true, .shift = ranks}; + std::vector> send(static_cast(ranks)); + const Comm c(MPI_COMM_WORLD); + BOOST_CHECK_THROW(static_cast(monoprop::mpi::begin_alltoallv(full_window(send), c, false, nullptr, bad)), + std::invalid_argument); + std::vector counts(static_cast(ranks), 0); + std::vector got(static_cast(ranks), 0); + BOOST_CHECK_THROW(monoprop::mpi::alltoall_counts(counts.data(), got.data(), ranks, c, bad), std::invalid_argument); +} + +// MPI reads the payload buffers until the requests retire, so the owning handle is move-only and drains when dropped. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_pending_owns_its_requests) { + using Pending = monoprop::mpi::PendingAlltoallv; + static_assert(!std::is_copy_constructible_v); + static_assert(!std::is_copy_assignable_v); + static_assert(std::is_nothrow_move_constructible_v); + static_assert(std::is_nothrow_move_assignable_v); + + const int ranks = world_size(); + const int me = world_rank(); + const Comm c(MPI_COMM_WORLD); + std::vector> send(static_cast(ranks)); + for (int d = 0; d < ranks; ++d) { + send[static_cast(d)] = {(me * 1000) + d}; + } + { + auto h = monoprop::mpi::begin_alltoallv(full_window(send), c); + auto moved = std::move(h); // the requests travel with the buffers MPI is reading + monoprop::mpi::WindowVec> got; + moved.wait_into(got); + const auto out = flat_blocks(got, static_cast(ranks)); + BOOST_REQUIRE_EQUAL(static_cast(out.size()), ranks); + for (int src = 0; src < ranks; ++src) { + BOOST_REQUIRE_EQUAL(out[static_cast(src)].size(), 1U); + BOOST_CHECK_EQUAL(out[static_cast(src)][0], (src * 1000) + me); + } + } + // Never waited on: the destructor drains it, or the next collective would mismatch. + { + static_cast(monoprop::mpi::begin_alltoallv(full_window(send), c)); + } + // Move assignment drains the destination's own round before adopting the source's. + { + auto first = monoprop::mpi::begin_alltoallv(full_window(send), c); + auto second = monoprop::mpi::begin_alltoallv(full_window(send), c); + first = std::move(second); + monoprop::mpi::WindowVec> got; + first.wait_into(got); + BOOST_CHECK_EQUAL(flat_blocks(got, static_cast(ranks)).size(), static_cast(ranks)); + } + BOOST_CHECK_EQUAL(monoprop::mpi::allreduce_sum(1, c), ranks); +} + +// The physical reductions are ordinary MPI sums; every rank ends with the same value. +BOOST_AUTO_TEST_CASE(mpi_alltoallv_allreduce_sums_agree) { + const int ranks = world_size(); + const int me = world_rank(); + const Comm c(MPI_COMM_WORLD); + BOOST_CHECK_EQUAL(monoprop::mpi::allreduce_sum(static_cast(me) + 1, c), + static_cast(ranks) * (static_cast(ranks) + 1) / 2); + monoprop::VecD values{static_cast(me) + 0.5, 1.0, 0.0}; + monoprop::mpi::allreduce_sum_inplace(values, c); + BOOST_CHECK_CLOSE(values[0], static_cast(ranks) * static_cast(ranks) / 2.0, 1e-12); + BOOST_CHECK_EQUAL(values[1], static_cast(ranks)); + BOOST_CHECK_EQUAL(values[2], 0.0); +} diff --git a/cpp/tests/mpi_distributed_layer_equivalence.cpp b/cpp/tests/mpi_distributed_layer_equivalence.cpp index 152eb2b75..3b667c8e8 100644 --- a/cpp/tests/mpi_distributed_layer_equivalence.cpp +++ b/cpp/tests/mpi_distributed_layer_equivalence.cpp @@ -25,9 +25,9 @@ #include "monoprop/MonomialPropagator.h" #include "monoprop/detail/mpi/MPICompat.h" -// Single-rank (self) vs multi-rank (world) equivalence of energy, gradient, native-Pauli energy and -// the MPI x partition hybrid. Oracle: the self run; the only expected difference is summation order, -// covered by near()'s kFpRtol = 1e-7. +// Single-rank (self) vs multi-rank (world) equivalence of energy, gradient and native-Pauli energy. Oracle: the self +// run; the only expected difference is summation order, covered by near()'s kFpRtol = 1e-7. The same public calls also +// run on the root at P ranks x the launch's T (sharded_world_equivalence_mpi*, cpp/tests/CMakeLists.txt). namespace { @@ -106,8 +106,8 @@ BOOST_AUTO_TEST_CASE(gradient_rank_count_within_fp_tolerance) { } } -// Native Pauli engine across rank counts. The same owner hash and cross-rank resolve path drive the -// intra-process partition runtime, so this guards both. +// Native Pauli engine across rank counts. The same owner hash and cross-owner resolve path drive the shards of one +// rank, so this guards both. constexpr size_t kPauliQ = 6; @@ -160,44 +160,4 @@ BOOST_AUTO_TEST_CASE(pauli_rank_count_energy_within_fp_tolerance) { BOOST_TEST(near(e_serial, e_world)); } -// MPI x partition hybrid: partitions=S under R ranks builds the HybridComm flat R*S world, which only changes -// allreduce association, so the energy must match serial and the global term count must be exactly -// invariant. Explicit partitions= wins over the suite's monoprop_PARTITIONS=off, so this is the sole case -// exercising the hybrid transport end to end. - -auto run_energy_partitioned(const TestInputs& inputs, MPI_Comm comm, size_t partitions) -> std::pair { - MonomialPropagator sim(inputs.data.hamiltonian, - kCutoff, - inputs.data.initial_state, - std::nullopt, - comm, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - partitions); - sim.build_graph(inputs.data.majoranas, inputs.data.param_inds, inputs.data.gen_coeffs); - auto fn = sim.expectation_value_functional(); - const double e = fn(inputs.data.parameters); - return {e, sim.size()}; -} - -BOOST_AUTO_TEST_CASE(hybrid_mpi_partition_energy_and_size_equivalence) { - if (mpi::size(MPI_COMM_WORLD) < 2) { - BOOST_TEST_MESSAGE("Skipping hybrid case: requires at least 2 ranks."); - return; - } - const auto inputs = load_inputs(); - const auto [e_serial, n_serial] = run_energy_partitioned(inputs, MPI_COMM_SELF, 1); - const auto [e_hybrid, n_local] = run_energy_partitioned(inputs, MPI_COMM_WORLD, 2); - // Each rank's facade holds only its local partitions; the global term count is the cross-rank sum. - const size_t n_hybrid_global = mpi::allreduce_sum(n_local, MPI_COMM_WORLD); - BOOST_TEST_MESSAGE("serial=" << e_serial << " (n=" << n_serial << ") hybrid R*2=" << e_hybrid - << " (global n=" << n_hybrid_global << ")"); - BOOST_TEST(near(e_serial, e_hybrid)); - BOOST_CHECK_EQUAL(n_serial, n_hybrid_global); -} - } // namespace diff --git a/cpp/tests/mpi_failure_driver.cpp b/cpp/tests/mpi_failure_driver.cpp new file mode 100644 index 000000000..7d2b57d73 --- /dev/null +++ b/cpp/tests/mpi_failure_driver.cpp @@ -0,0 +1,629 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Launcher-supervised distributed scenarios. Its own main(), outside the unit runner: every failure scenario ends the +// whole job. run_mpi_failure_scenario.cmake launches it on two ranks with a 30 s timeout and requires either a prompt +// nonzero exit carrying the expected diagnostic, or (the thread-support acceptance scenarios) a clean exit carrying the +// expected completion line. +// +// Rank 0 fails while its peers follow the normal phase. A failure the library does not handle escapes to main(), which +// models a host that catches it and shuts down: rank 0 finalizes while its peers are still inside the operation, so the +// job hangs until the timeout. Every failure scenario therefore passes only through the library's own abort or +// fail-fast path. The sharded root runs at two OpenMP threads per rank and is injected through its test-only +// RootObserver on rank 0 (rank 1 for the malformed receive): +// +// worker-throw worker 1 throws in a replay finish of an evaluation +// before-exchange the primary throws in the first gate's traversal, before any physical round +// active-ticket round the primary throws in a cosine pass while the replay round is posted +// active-ticket counts the primary throws while packing queries, with the count round posted +// active-ticket local-round as `round`, through the exported evaluation seam with a caller-local round +// (no persistent rounds): the live failure must reach the abort before that +// round's draining destructor runs +// malformed-receive rank 1 rewrites its outgoing query blocks; rank 0's owners must refuse them +// +// The low-level one-owner-per-rank engine and the exported replay functions, on the sole shard of a root launched with +// one thread per rank (raw access is available at T = 1, also over several ranks), with explicit kernel budgets: +// +// active-ticket ticket|cosine-worker a failure (on the caller, or on worker 1 of the threaded cosine kernel) while +// evolve_step's replay Ticket is posted +// active-ticket pending a failure while a begin_alltoallv PendingAlltoallv is posted +// before-exchange scan-worker a worker of the threaded scan throws before the first query exchange +// resolve-worker self-probe a worker of the threaded self probe throws before the first query exchange +// resolve-worker decode|incoming-probe|scatter +// a worker of the threaded incoming phases throws after the query round, while +// the peers enter the response round +// +// Thread support. The library requests MPI_THREAD_FUNNELED and requires at least FUNNELED from whoever initialized +// MPI; only the initializing thread, as the primary of the library's team, calls MPI. Each line below records the +// requested and the provided level; a launch whose provided level does not exercise the route ends with +// "NOT EXERCISED" (exit 77, reported as skipped): +// +// library-level the library initializes MPI; real operations complete (success expected) +// host-level funneled|serialized|multiple +// the host initializes MPI at that level; real operations complete on the world and +// on a reordered split communicator (success expected; exactly FUNNELED for +// funneled) +// insufficient-thread-level single host-initialized MPI below FUNNELED is refused at construction +// wrong-thread-entry funneled|serialized|multiple +// a guarded entry called from a non-initializing std::thread fails fast locally + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "PropagatorTestAccess.h" +#include "monoprop/Evolution.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/OperationFailure.h" +#include "monoprop/detail/sharded/Exchange.h" +#include "monoprop/detail/sharded/RootObserver.h" + +namespace { + +using namespace monoprop; + +constexpr size_t kModes = 6; +using Propagator = MonomialPropagator; +using Access = detail::PropagatorTestAccess; + +// Exit code CTest reads as "skipped": the MPI library's provided level does not exercise the scenario's route. +constexpr int kNotExercised = 77; + +auto say(int rank, std::string_view text) -> void { + std::fputs(std::format("[driver] rank {}: {}\n", rank, text).c_str(), stderr); + std::fflush(stderr); +} + +auto world_rank() -> int { + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + return rank; +} + +// Twenty terms anticommuting with the {0, 1} generator, so both ranks own work under any hash split. +auto hamiltonian() -> OperatorDict { + OperatorDict ham; + for (size_t j = 2; j < 2 * kModes; ++j) { + ham[VecZ{0, j}] = std::complex{0.0, 0.1 * static_cast(j)}; + ham[VecZ{1, j}] = std::complex{0.0, 0.05 * static_cast(j)}; + } + return ham; +} + +const std::vector kGates{{0, 1}, {2, 3}, {1, 4}}; +const VecZ kMapping{0, 1, 2}; +const VecD kGenCoeffs{1.0, 1.0, 1.0}; +const VecD kParams{0.3, 0.2, 0.1}; + +// The budget is captured when a root is constructed, from the launch's monoprop_NUM_THREADS. +auto set_team(int threads) -> void { + ::setenv("monoprop_NUM_THREADS", std::to_string(threads).c_str(), 1); +} + +// --- the sharded root, injected through its observer --------------------------------------------------------------- + +// Throws once, on `rank` only, at the armed visit of the sharded root's seams; or rewrites outgoing queries. +class Injector final : public detail::sharded::RootObserver { +public: + struct Arm { + bool construction = false; + int work = 0; + size_t shard = 0; + const char *what = ""; + }; + Injector(int rank, std::optional arm, bool corrupt) : rank_(rank), arm_(arm), corrupt_(corrupt) {} + + auto visit(detail::sharded::EvaluationWork work, size_t /*step*/, size_t shard) const -> void override { + fire(false, static_cast(work), shard); + } + auto construction(detail::sharded::ConstructionWork work, size_t /*step*/, size_t shard) const -> void override { + fire(true, static_cast(work), shard); + } + auto queries_packed(size_t /*step*/, size_t shard, detail::sharded::PhysicalExchange &round) const + -> void override { + if (!corrupt_) { + return; + } + // Every word all ones: a record whose phase field is not ternary. + for (size_t k = 0; k < round.peers().size(); ++k) { + for (size_t t = 0; t < round.threads(); ++t) { + for (auto &word : round.send_block(shard, k, t)) { + word = ~size_t{0}; + } + } + } + } + +private: + auto fire(bool construction, int work, size_t shard) const -> void { + if (arm_ && arm_->construction == construction && arm_->work == work && arm_->shard == shard) { + throw std::runtime_error( + std::format("injected {} on OpenMP worker {} of rank {}", arm_->what, omp_get_thread_num(), rank_)); + } + } + + int rank_; + std::optional arm_; + bool corrupt_; +}; + +auto make_root(const detail::sharded::RootObserver *observer, MPI_Comm comm = MPI_COMM_WORLD) + -> std::unique_ptr { + return Access::construct_observed(observer, + hamiltonian(), + static_cast(2 * kModes), + VecZ{0, 1}, + std::optional{}, + mpi::Comm(comm), + std::optional{}, + std::optional{}, + CutoffType::Length, + std::optional>{}, + kModes, + Basis::Majorana); +} + +auto sharded_scenario(int rank, std::string_view scenario, std::string_view option) -> void { + using CW = detail::sharded::ConstructionWork; + using EW = detail::sharded::EvaluationWork; + std::optional arm; + bool corrupt = false; + if (scenario == "worker-throw") { + arm = Injector::Arm{false, static_cast(EW::finish), 1, "replay-finish failure"}; + } + else if (scenario == "before-exchange") { + arm = Injector::Arm{true, static_cast(CW::traverse), 0, "traversal failure before any physical round"}; + } + else if (scenario == "active-ticket" && option == "round") { + arm = Injector::Arm{false, static_cast(EW::cosine), 0, "cosine failure while the replay round is posted"}; + } + else if (scenario == "active-ticket" && option == "counts") { + arm = Injector::Arm{true, static_cast(CW::pack), 0, "pack failure while the count round is posted"}; + } + else if (scenario == "active-ticket" && option == "local-round") { + arm = Injector::Arm{false, + static_cast(EW::cosine), + 0, + "cosine failure while a caller-local replay round is posted"}; + } + else if (scenario == "malformed-receive") { + corrupt = rank == 1; + } + set_team(2); + const Injector injector(rank, rank == 0 ? arm : std::nullopt, corrupt); + auto sim = make_root(&injector); + say(rank, std::format("{} local terms, team {}", sim->size(), Access::options(*sim).threads)); + sim->build_graph(kGates, kMapping, kGenCoeffs); + if (option == "local-round") { + const auto outcome = Access::evaluate_with_local_round(*sim, kParams, &injector); + if (outcome.error) { + std::rethrow_exception(outcome.error); // a caller that got the failure back; its peer is still in a round + } + return; + } + (void)sim->expectation_value(kParams); +} + +// --- the low-level engine and replay functions on the sole shard of a T = 1 root ----------------------------------- + +auto make_sole_shard_root() -> std::unique_ptr { + set_team(1); + return make_root(nullptr); +} + +auto active_ticket(int rank) -> void { + auto sim = make_sole_shard_root(); + sim->build_graph(kGates, kMapping, kGenCoeffs); + VecD coeffs = sim->mp_op().get_operator(); + const auto view = sim->graph().replay_view(); + // Splitmix replays through MPI_Ialltoallv, so the Ticket always holds a posted request here. + const detail::LayerCosScale scale = [rank](size_t, double *, double) { + if (rank == 0) { + throw std::runtime_error("injected failure while the replay Ticket is in flight"); + } + }; + say(rank, "posting the replay exchange"); + evolve_step(coeffs, view, 0.3, 0, MPI_COMM_WORLD, scale); +} + +// Throws from one logical range of a real kernel, on whichever worker owns it. +struct ThrowingRangeObserver { + auto prepare(detail::KernelRange, size_t) const noexcept -> void {} + auto visit(detail::KernelRange, size_t range) const -> void { + if (range == 2) { + throw std::runtime_error(std::format("injected failure in cosine range {} on OpenMP worker {} while the " + "replay Ticket is in flight", + range, + omp_get_thread_num())); + } + } +}; + +auto active_ticket_cosine_worker(int rank) -> void { + auto sim = make_sole_shard_root(); + sim->build_graph(kGates, kMapping, kGenCoeffs); + VecD coeffs = sim->mp_op().get_operator(); + const auto view = sim->graph().replay_view(); + // A stored mask of four kernel ranges, so the kernel really opens a team; the real layer is tiny. + CosMask mask; + for (size_t w = 0; w < 4 * detail::cos_mask_range_blocks; ++w) { + mask.blocks.emplace_back(w * 64, ~uint64_t{0}); + mask.total_count += 64; + } + auto scratch = std::make_shared(mask.blocks.size() * 64, 1.0); + const detail::LayerCosScale scale = [rank, mask, scratch](size_t, double *, double v) { + if (rank == 0) { + detail::scale_cos_mask(scratch->data(), mask, v, {.threads = 2}, ThrowingRangeObserver{}); + } + }; + say(rank, "posting the replay exchange"); + evolve_step(coeffs, view, 0.3, 0, MPI_COMM_WORLD, scale); +} + +auto active_pending(int rank, int size) -> void { + const mpi::SlotWindow window{.base = 0, .count = static_cast(size)}; + mpi::WindowVec> send(window); + for (const auto wi : window.indices()) { + send[wi].assign(4, static_cast(rank)); + } + auto pending = mpi::begin_alltoallv(send, MPI_COMM_WORLD); + say(rank, std::format("PendingAlltoallv posted {} request(s)", pending.posted)); + if (rank == 0) { + mpi::guard_distributed(MPI_COMM_WORLD, [] { + throw std::runtime_error("injected failure while a PendingAlltoallv is in flight"); + }); + } + mpi::WindowVec> recv; + pending.wait_into(recv); +} + +// Enough terms that each rank's store spans several fold blocks, so the threaded kernels really open a two-worker team. +constexpr size_t kWideModes = 16; +using WideAccess = detail::PropagatorTestAccess; + +auto make_wide_sole_shard_root() -> std::unique_ptr> { + OperatorDict ham; + uint64_t state = 0x5CA3ULL; + while (ham.size() < 200000) { + VecZ idx; + while (idx.size() < 6) { // weight 6: C(32, 6) ~ 9e5 distinct terms, and an imaginary Hermitian coefficient + state = (state * 6364136223846793005ULL) + 1442695040888963407ULL; + const size_t m = (state >> 33) % (2 * kWideModes); + if (std::ranges::find(idx, m) == idx.end()) { + idx.push_back(m); + } + } + std::ranges::sort(idx); + ham[idx] = std::complex{0.0, 1e-3}; + } + set_team(1); + return WideAccess::construct_observed(nullptr, + ham, + static_cast(2 * kWideModes), + VecZ{0, 1}, + std::optional{}, + mpi::Comm(MPI_COMM_WORLD), + std::optional{}, + std::optional{}, + CutoffType::Length, + std::optional>{}, + kWideModes, + Basis::Majorana); +} + +// Throws from one word range of the real threaded scan, on whichever worker owns it. +struct ThrowingScanObserver { + int rank; + auto prepare(detail::KernelRange, size_t) const noexcept -> void {} + auto visit(detail::KernelRange kind, size_t range) const -> void { + if (rank == 0 && kind == detail::KernelRange::scan && range == 1) { + throw std::runtime_error( + std::format("injected failure in scan range {} on OpenMP worker {}", range, omp_get_thread_num())); + } + } +}; + +auto scan_worker(int rank) -> void { + auto sim = make_wide_sole_shard_root(); + say(rank, std::format("{} local terms, {} words", sim->size(), sim->mp_op().inverted_index().words())); + (void)WideAccess::sole_shard_layer_observed(*sim, VecZ{0, 5}, {.threads = 2}, ThrowingScanObserver{rank}); + if (rank == 0) { + say(rank, "INJECTION NOT REACHED: the scan ran in fewer than two ranges"); + } +} + +// Throws from the last logical range of one resolve phase on rank 0, so with a two-worker team the failure starts on +// worker 1, off the calling thread. Only the first call of that phase with two or more ranges throws. +struct ThrowingResolveObserver { + int rank; + detail::KernelRange kind; + const char *label; + std::shared_ptr ranges = std::make_shared(0); + + auto prepare(detail::KernelRange k, size_t n) const noexcept -> void { + if (k == kind && *ranges < 2) { + *ranges = n; + } + } + auto visit(detail::KernelRange k, size_t range) const -> void { + if (rank == 0 && k == kind && *ranges >= 2 && range + 1 == *ranges) { + throw std::runtime_error( + std::format("injected failure in {} range {} on OpenMP worker {}", label, range, omp_get_thread_num())); + } + } +}; + +auto resolve_worker(int rank, std::string_view phase) -> void { + auto sim = make_wide_sole_shard_root(); + const auto kind = phase == "self-probe" ? detail::KernelRange::self_probe + : phase == "decode" ? detail::KernelRange::decode + : phase == "scatter" ? detail::KernelRange::scatter + : detail::KernelRange::incoming_probe; + say(rank, std::format("{} local terms", sim->size())); + (void)WideAccess::sole_shard_layer_observed(*sim, + VecZ{0, 5}, + {.threads = 2}, + ThrowingResolveObserver{rank, kind, phase.data()}); + if (rank == 0) { + say(rank, std::format("INJECTION NOT REACHED: the {} phase ran in fewer than two ranges", phase)); + } +} + +// --- thread support ------------------------------------------------------------------------------------------------ + +auto close_to(double a, double b) -> bool { + return std::abs(a - b) <= 1e-9 + (1e-7 * std::max(std::abs(a), std::abs(b))); +} + +// Real distributed work at two threads per rank on `comm`: construction, graph build, energies, gradients, retained +// functionals, contraction, export, a copy, an initial-operator update and graph-free propagation; every result is +// checked against a single-process root of the same budget, so the physical rounds really carried data. Every MPI call +// inside the library comes from this (the initializing) thread as its team's primary. Returns the energy. +auto run_operations(int rank, MPI_Comm comm, std::string_view label) -> double { + set_team(2); + auto reference = make_root(nullptr, MPI_COMM_SELF); + reference->build_graph(kGates, kMapping, kGenCoeffs); + const auto [ref_energy, ref_gradient] = reference->expectation_value_and_gradient(kParams); + + auto sim = make_root(nullptr, comm); + sim->build_graph(kGates, kMapping, kGenCoeffs); + const double energy = sim->expectation_value(kParams); + const auto [energy_g, gradient] = sim->expectation_value_and_gradient(kParams); + const auto functional = sim->expectation_value_and_gradient_functional(); + const auto [energy_f, gradient_f] = functional(kParams); + const auto pared = sim->expectation_value_functional(1e-10); + const double energy_p = pared(kParams); + const auto contracted = sim->contract_partially(kParams, false); + const auto terms = sim->evolved_operator_terms(kParams, 0.0); + const Propagator copy(*sim); + const double energy_c = Propagator(copy).expectation_value(kParams); + + auto fresh = make_root(nullptr, comm); + fresh->propagate(kGates, kMapping, kGenCoeffs, kParams); + auto fresh_reference = make_root(nullptr, MPI_COMM_SELF); + fresh_reference->propagate(kGates, kMapping, kGenCoeffs, kParams); + const double propagated = fresh->expectation_value(VecD{}); + const double propagated_ref = fresh_reference->expectation_value(VecD{}); + + auto updated = make_root(nullptr, comm); + updated->update_initial_operator(hamiltonian()); + updated->build_graph(kGates, kMapping, kGenCoeffs); + const double energy_u = updated->expectation_value(kParams); + + bool ok = close_to(energy, ref_energy) && close_to(energy_g, ref_energy) && close_to(energy_f, ref_energy) + && close_to(energy_p, ref_energy) && close_to(energy_c, ref_energy) && close_to(energy_u, ref_energy) + && close_to(propagated, propagated_ref) && gradient.size() == ref_gradient.size() + && gradient_f.size() == ref_gradient.size(); + for (size_t i = 0; ok && i < ref_gradient.size(); ++i) { + ok = close_to(gradient[i], ref_gradient[i]) && close_to(gradient_f[i], ref_gradient[i]); + } + const size_t global_terms = mpi::allreduce_sum(terms.size(), mpi::Comm(comm)); + const size_t global_size = mpi::allreduce_sum(sim->size(), mpi::Comm(comm)); + ok = + ok && global_terms == reference->size() && global_size == reference->size() && contracted.size() == sim->size(); + if (!ok) { + throw std::runtime_error(std::format("{}: results differ from the single-process reference (energy {} vs {})", + label, + energy, + ref_energy)); + } + say(rank, + std::format("{}: energy {:.17g}, {} local of {} global terms, team {}", + label, + energy, + sim->size(), + global_size, + Access::options(*sim).threads)); + return energy; +} + +auto thread_support_operations(int rank) -> void { + const double world = run_operations(rank, MPI_COMM_WORLD, "world"); + // A supplied communicator whose rank order is the reverse of the world's: ownership moves, results do not. + int size = 1; + MPI_Comm_size(MPI_COMM_WORLD, &size); + MPI_Comm reordered = MPI_COMM_NULL; + MPI_Comm_split(MPI_COMM_WORLD, 0, size - 1 - rank, &reordered); + const double split = run_operations(rank, reordered, "reordered split communicator"); + MPI_Comm_free(&reordered); + if (!close_to(world, split)) { + throw std::runtime_error("the reordered communicator changed the energy"); + } +} + +auto insufficient_level(int rank) -> void { + set_team(2); + // The constructor validates the provided level before it communicates. + const auto sim = make_root(nullptr); + say(rank, std::format("constructed with {} terms despite the insufficient level", sim->size())); +} + +auto wrong_thread(int rank) -> void { + set_team(2); + auto sim = make_root(nullptr); + sim->build_graph(kGates, kMapping, kGenCoeffs); + if (rank == 0) { + std::thread host([&sim] { (void)sim->expectation_value(kParams); }); + host.join(); + } + else { + (void)sim->expectation_value(kParams); + } +} + +auto parse_level(std::string_view name) -> std::optional { + if (name == "single") { + return MPI_THREAD_SINGLE; + } + if (name == "funneled") { + return MPI_THREAD_FUNNELED; + } + if (name == "serialized") { + return MPI_THREAD_SERIALIZED; + } + if (name == "multiple") { + return MPI_THREAD_MULTIPLE; + } + return std::nullopt; +} + +} // namespace + +auto main(int argc, char **argv) -> int { + if (argc < 2) { + std::fputs("usage: monoprop_mpi_failure_driver.x [option]\n", stderr); + return 2; + } + const std::string_view scenario = argv[1]; + const std::string_view option = argc > 2 ? argv[2] : ""; + // Dense collectives on every exchange, so each peer is committed to the round rank 0 abandons. + ::setenv("monoprop_ROUTING", "splitmix", 1); + + const bool host_owned = + scenario == "insufficient-thread-level" || scenario == "wrong-thread-entry" || scenario == "host-level"; + std::optional requested; + if (host_owned) { + requested = parse_level(option); + if (!requested) { + std::fputs("a host-initialized scenario needs a thread level\n", stderr); + return 2; + } + int provided = MPI_THREAD_SINGLE; + MPI_Init_thread(&argc, &argv, *requested, &provided); + } + else { + mpi::init(&argc, &argv); + } + const int rank = world_rank(); + int size = 1; + MPI_Comm_size(MPI_COMM_WORLD, &size); + int provided = MPI_THREAD_SINGLE; + MPI_Query_thread(&provided); + say(rank, + std::format("scenario={} {} size={} MPI {} requested={} provided={}", + scenario, + option, + size, + host_owned ? "host-initialized" : "library-initialized", + requested ? mpi::thread_level_name(*requested) : mpi::thread_level_name(mpi::required_thread_level), + mpi::thread_level_name(provided))); + + const auto not_exercised = [&](std::string_view why) { + say(rank, + std::format("NOT EXERCISED: requested {} but the MPI library provided {}; {}", + mpi::thread_level_name(requested.value_or(mpi::required_thread_level)), + mpi::thread_level_name(provided), + why)); + MPI_Finalize(); + return kNotExercised; + }; + if (scenario == "insufficient-thread-level" && mpi::thread_level_satisfies(provided, mpi::required_thread_level)) { + return not_exercised("the refusal needs a level below the required one"); + } + if (scenario == "host-level" && provided != *requested) { + return not_exercised("the acceptance run needs exactly the requested level"); + } + + try { + if (scenario == "insufficient-thread-level") { + insufficient_level(rank); + } + else if (scenario == "wrong-thread-entry") { + wrong_thread(rank); + } + else if (scenario == "host-level" || scenario == "library-level") { + thread_support_operations(rank); + say(rank, + std::format("thread-support operations completed with provided={}", mpi::thread_level_name(provided))); + } + else if (scenario == "active-ticket" && option == "local-round") { + sharded_scenario(rank, scenario, option); + } + else if (scenario == "active-ticket" && option == "ticket") { + active_ticket(rank); + } + else if (scenario == "active-ticket" && option == "cosine-worker") { + active_ticket_cosine_worker(rank); + } + else if (scenario == "active-ticket" && option == "pending") { + active_pending(rank, size); + } + else if (scenario == "before-exchange" && option == "scan-worker") { + scan_worker(rank); + } + else if (scenario == "resolve-worker" + && (option == "self-probe" || option == "decode" || option == "incoming-probe" + || option == "scatter")) { + resolve_worker(rank, option); + } + else if (scenario == "worker-throw" || scenario == "before-exchange" || scenario == "active-ticket" + || scenario == "malformed-receive") { + sharded_scenario(rank, scenario, option); + } + else { + say(rank, std::format("unknown scenario '{}' '{}'", scenario, option)); + MPI_Abort(MPI_COMM_WORLD, 2); + } + } + catch (const std::exception &e) { + say(rank, std::format("UNGUARDED: an exception escaped the library: {}", e.what())); + // A host that handled the error and shut down; peers still inside the operation hang. + MPI_Finalize(); + return 3; + } + + // The next collective of the peers' normal phase. + MPI_Barrier(MPI_COMM_WORLD); + say(rank, "scenario completed without a failure"); + MPI_Finalize(); + return 0; +} diff --git a/cpp/tests/mpi_fresh_insert_equivalence.cpp b/cpp/tests/mpi_fresh_insert_equivalence.cpp index 2973131b6..69dbb50e4 100644 --- a/cpp/tests/mpi_fresh_insert_equivalence.cpp +++ b/cpp/tests/mpi_fresh_insert_equivalence.cpp @@ -16,18 +16,25 @@ // fresh partner is state-scored (majorana_state_phase / pauli_state_phase) rather than left at 0. The // Heisenberg R>1 resolve/apply paths are already covered by exact_upper_atol_rescue and // mpi_distributed_layer_equivalence. Only runs at world >= 2. Oracle: serial<->world equivalence -- -// the deterministic base+j miss-prefix must sum the same terms at any rank count, to near()'s rtol. +// the deterministic base+j miss-prefix must sum the same terms at any rank count, to near()'s rtol. The same public +// calls also drive the root at P ranks x the launch's T (sharded_world_equivalence_mpi*, cpp/tests/CMakeLists.txt). #include +#include +#include #include +#include #include #include #include +#include "KernelTestSupport.h" #include "PauliTestOracle.h" +#include "PropagatorTestAccess.h" #include "TestUtilities.h" #include "monoprop/MonomialPropagator.h" +#include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/detail/mpi/MPICompat.h" namespace { @@ -39,7 +46,7 @@ using pauli_oracle::slots_of_string; // schrodinger_cutoff engages the picture; a low structural cutoff plus the upper_atol = 0 rescue // forces most partners to be fresh inserts, so the miss arm runs on nearly every partner. template -auto run_schrodinger_majorana(const CaseData& data, MPI_Comm comm) -> double { +auto run_schrodinger_majorana(const CaseData &data, MPI_Comm comm) -> double { MonomialPropagator sim(data.hamiltonian, /*cutoff=*/2U, data.initial_state, diff --git a/cpp/tests/mpi_utils_tests.cpp b/cpp/tests/mpi_utils_tests.cpp index bba4f5dc3..946174e37 100644 --- a/cpp/tests/mpi_utils_tests.cpp +++ b/cpp/tests/mpi_utils_tests.cpp @@ -28,6 +28,7 @@ #include "monoprop/detail/evolution/CutoffContext.h" #include "monoprop/detail/evolution/layer_build/Scan.h" #include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/MPICompat.h" #include "monoprop/detail/mpi/MPIUtils.h" #include "monoprop/detail/mpi/Routing.h" #include "monoprop/detail/operator/MPOperator.h" @@ -320,3 +321,25 @@ BOOST_AUTO_TEST_CASE(mpi_utils_paired_enumeration_partitions_by_find_rank) { } } } + +#ifdef monoprop_ENABLE_MPI +// The transport decision is agreed once per communicator and cached on it as an attribute. Warm calls +// repeat the agreed answer, and a duplicated or freshly created communicator agrees on its own instead of +// inheriting an entry keyed by a recycled raw handle. Every rank runs the same call sequence. +BOOST_AUTO_TEST_CASE(mpi_utils_routes_pairwise_is_cached_per_communicator) { + const bool cold = mpi::routes_pairwise(MPI_COMM_WORLD); + BOOST_TEST(mpi::routes_pairwise(MPI_COMM_WORLD) == cold); + for (int round = 0; round < 2; ++round) { + MPI_Comm dup = MPI_COMM_NULL; + MPI_Comm_dup(MPI_COMM_WORLD, &dup); + BOOST_TEST(mpi::routes_pairwise(dup) == cold); + BOOST_TEST(mpi::routes_pairwise(dup) == cold); + MPI_Comm_free(&dup); + } + int size = 1; + MPI_Comm_size(MPI_COMM_WORLD, &size); + if (size == 1) { + BOOST_TEST(!cold); // no peer to pair with + } +} +#endif diff --git a/cpp/tests/openmp_kernel_tests.cpp b/cpp/tests/openmp_kernel_tests.cpp new file mode 100644 index 000000000..41793f141 --- /dev/null +++ b/cpp/tests/openmp_kernel_tests.cpp @@ -0,0 +1,536 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The threaded cosine and fused-apply kernels: exact agreement with their one-thread execution, and +// kernel-specific worker participation. Participation is read from a test-only range observer that each +// kernel calls at the start of each of its own logical ranges (KernelTestSupport.h); a team created +// anywhere else does not count. Lazy-fold equivalence against the materialised-fold oracle lives in +// combined_recompute_equivalence.cpp; fused records from the real build path in fused_cos_sweep_tests.cpp. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "KernelTestSupport.h" +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/evolution/layer_build/FusedApply.h" +#include "monoprop/detail/parallel/Options.h" + +namespace { + +using namespace monoprop; +using kernel_test::RangeLog; +using kernel_test::RecordingObserver; +using monoprop::detail::KernelRange; +using monoprop::detail::parallel::Options; + +constexpr auto kBudgets = {1, 2, 3, 4}; + +// Distinct, non-degenerate values, so a missed, extra or doubled scaling shows up bitwise. +auto distinct_values(size_t n) -> VecD { + VecD v(n); + for (size_t i = 0; i < n; ++i) { + v[i] = 1.0 + static_cast(i) * 1e-3; + } + return v; +} + +auto bitwise_equal(const VecD &a, const VecD &b) -> bool { + return a.size() == b.size() && std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0; +} + +// Equal up to the rounding of one multiply-add whose terms are bounded by |original| + 1 (|cos|, |sin| and every +// snapshot value are at most one here): the difference a separately compiled fused multiply-add can make. +auto equal_up_to_contraction(const VecD &a, const VecD &b, const VecD &original) -> bool { + if (a.size() != b.size() || a.size() != original.size()) { + return false; + } + constexpr double eps = std::numeric_limits::epsilon(); + for (size_t i = 0; i < a.size(); ++i) { + if (std::abs(a[i] - b[i]) > 2 * eps * (std::abs(original[i]) + 1.0)) { + return false; + } + } + return true; +} + +// The plan's mask: 4097 words, alternating bits, and only bit zero in the final word. +auto plan_mask() -> CosMask { + CosMask mask; + for (size_t w = 0; w < 4097; ++w) { + const auto bits = w == 4096 ? uint64_t{1} : uint64_t{0x5555555555555555}; + mask.blocks.emplace_back(w * 64, bits); + mask.total_count += static_cast(std::popcount(bits)); + } + return mask; +} + +// Every row below `rows` selected, one block per word, the last word partial when rows % 64 != 0. +auto full_mask(size_t rows) -> CosMask { + CosMask mask; + for (size_t base = 0; base < rows; base += 64) { + const size_t n = std::min(64, rows - base); + const uint64_t bits = n == 64 ? ~uint64_t{0} : (uint64_t{1} << n) - 1; + mask.blocks.emplace_back(base, bits); + mask.total_count += n; + } + return mask; +} + +// Selected indices of a mask, independent of the kernel under test. +auto selected(const CosMask &mask, size_t n) -> std::vector { + std::vector sel(n, 0); + for (const auto &[base, bits] : mask.blocks) { + for (size_t t = 0; t < 64; ++t) { + if ((bits >> t) & 1U) { + sel[base + t] = 1; + } + } + } + return sel; +} + +// Every budget against the one-thread kernel, and the one-thread kernel against a hand oracle that +// multiplies exactly the selected coefficients and leaves the rest untouched. +auto check_mask_budgets(const CosMask &mask, size_t n, const char *label) -> void { + constexpr double cos_val = 0.625; + const VecD original = distinct_values(n); + VecD oracle = original; + const auto sel = selected(mask, n); + for (size_t i = 0; i < n; ++i) { + if (sel[i] != 0) { + oracle[i] = original[i] * cos_val; + } + } + VecD serial = original; + monoprop::detail::scale_cos_mask(serial.data(), mask, cos_val, Options{.threads = 1}); + BOOST_TEST_CONTEXT(label) { + BOOST_TEST(bitwise_equal(serial, oracle)); + for (const int threads : kBudgets) { + VecD threaded = original; + monoprop::detail::scale_cos_mask(threaded.data(), mask, cos_val, Options{.threads = threads}); + BOOST_TEST_INFO("threads " << threads); + BOOST_TEST(bitwise_equal(threaded, serial)); + } + // More workers than ranges. + VecD many = original; + monoprop::detail::scale_cos_mask(many.data(), mask, cos_val, Options{.threads = 64}); + BOOST_TEST(bitwise_equal(many, serial)); + } +} + +// --- hand-built fused records ------------------------------------------------------------------------------- + +// A contract whose destinations are a shuffled permutation of distinct slots: hits and inserts own two slots +// each, halves one. The two-pass cosine mask covers every destination plus a padding region no record +// touches (anticommuting terms whose rotation produced no record), so it spans several mask ranges; a few +// slots are in neither. +struct HandContract { + monoprop::detail::FusedContract fc; + CosMask cos; + size_t slots = 0; +}; + +auto hand_contract(size_t n_hits, size_t n_inserts, size_t n_halves, uint64_t seed) -> HandContract { + HandContract h; + const size_t record_slots = 2 * (n_hits + n_inserts) + n_halves + 97; + constexpr size_t pad_words = 2100; + h.slots = record_slots + pad_words * 64; + std::vector perm(record_slots); + for (size_t i = 0; i < record_slots; ++i) { + perm[i] = i; + } + kernel_test::SplitMix rng{seed}; + for (size_t i = record_slots - 1; i > 0; --i) { + std::swap(perm[i], perm[rng.below(i + 1)]); + } + size_t next = 0; + const auto value = [&rng] { return static_cast(rng.below(1U << 20)) / (1U << 19) - 1.0; }; + const auto phase = [&rng] { return rng.below(2) == 0 ? int32_t{1} : int32_t{-1}; }; + for (size_t k = 0; k < n_hits; ++k) { + h.fc.hits.push_back( + {.src = perm[next], .tgt = perm[next + 1], .v_src = value(), .v_tgt = value(), .phase = phase()}); + next += 2; + } + for (size_t k = 0; k < n_inserts; ++k) { + // v_tgt stays zero: Schrödinger gathers it in the apply, Heisenberg inserts are born at zero. + h.fc.inserts.push_back({.src = perm[next], .tgt = perm[next + 1], .v_src = value(), .phase = phase()}); + next += 2; + } + for (size_t k = 0; k < n_halves; ++k) { + h.fc.cross_half.push_back( + {.local_idx = perm[next], .v_partner = value(), .phase_signed = phase(), .is_insert = rng.below(3) == 0}); + ++next; + } + std::vector selected_slots(perm.begin(), perm.begin() + static_cast(next)); + for (size_t s = record_slots; s < h.slots; s += 2) { + selected_slots.push_back(s); + } + std::ranges::sort(selected_slots); + CosineWordBuilder builder; + for (const auto s : selected_slots) { + builder.push_index(s); + } + h.cos = builder.finish(); + return h; +} + +// Read at run time, so the oracle and the kernel both take std::cos/std::sin from the math library; a +// compile-time constant could be folded with a differently rounded result. +auto rotation_angle() -> double { + static volatile double angle = 0.37; + return angle; +} + +// Independent oracle: the pre-threading serial apply_fused_contract, verbatim in its phase order, record order +// and arithmetic, over a copy of the contract. +auto reference_apply(const HandContract &h, const VecD &coeffs, bool schrodinger, bool fused_scale) -> VecD { + auto fc = h.fc; + VecD out = coeffs; + const double param = rotation_angle(); + if (schrodinger) { + for (auto &r : fc.inserts) { + r.v_tgt = out[r.tgt]; + } + } + const double cos_val = std::cos(2 * param); + const double sin_val = std::sin(2 * param); + double *const c = out.data(); + if (!fused_scale) { + for (const auto &[base, bits] : h.cos.blocks) { + for (size_t t = 0; t < 64; ++t) { + if ((bits >> t) & 1U) { + c[base + t] *= cos_val; + } + } + } + } + const size_t n_hit = fc.hits.size(); + for (size_t k = 0; k < n_hit + fc.inserts.size(); ++k) { + const bool is_insert = k >= n_hit; + const auto &r = is_insert ? fc.inserts[k - n_hit] : fc.hits[k]; + c[r.src] += sin_val * static_cast(-r.phase) * r.v_tgt; + if (fused_scale && is_insert) { + c[r.tgt] = cos_val * c[r.tgt] + sin_val * static_cast(r.phase) * r.v_src; + } + else { + c[r.tgt] += sin_val * static_cast(r.phase) * r.v_src; + } + } + for (const auto &half : fc.cross_half) { + if (fused_scale && half.is_insert) { + c[half.local_idx] = + cos_val * c[half.local_idx] + sin_val * static_cast(half.phase_signed) * half.v_partner; + } + else { + c[half.local_idx] += sin_val * static_cast(half.phase_signed) * half.v_partner; + } + } + return out; +} + +auto apply_copy(const HandContract &h, + const VecD &coeffs, + bool schrodinger, + bool fused_scale, + int threads, + const RecordingObserver *observer = nullptr) -> VecD { + auto fc = h.fc; + VecD out = coeffs; + const double param = rotation_angle(); + if (observer != nullptr) { + monoprop::detail::apply_fused_contract(fc, out, h.cos, param, schrodinger, fused_scale, {threads}, *observer); + } + else { + monoprop::detail::apply_fused_contract(fc, out, h.cos, param, schrodinger, fused_scale, {threads}); + } + return out; +} + +} // namespace + +// --- stored-mask cosine ------------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE(openmp_cos_mask_exact) { + const CosMask mask = plan_mask(); + VecD s(4097 * 64, 1.25), p = s; + monoprop::detail::scale_cos_mask(s.data(), mask, 0.625, {.threads = 1}); + monoprop::detail::scale_cos_mask(p.data(), mask, 0.625, {.threads = 3}); + BOOST_CHECK_EQUAL_COLLECTIONS(s.begin(), s.end(), p.begin(), p.end()); + check_mask_budgets(mask, 4097 * 64, "plan mask, 4097 words"); +} + +BOOST_AUTO_TEST_CASE(openmp_cos_mask_edge_cases) { + check_mask_budgets(CosMask{}, 128, "empty mask"); + for (const size_t tail : {63, 64, 65}) { + const size_t rows = 2048 * 64 + tail; + BOOST_TEST_CONTEXT("tail " << tail) { + check_mask_budgets(full_mask(rows), rows, "full mask with a partial tail"); + } + } + // Exactly one range (serial whatever the budget), and one block over it (two ranges). + check_mask_budgets(full_mask(1024 * 64), 1024 * 64, "1024 blocks"); + check_mask_budgets(full_mask(1024 * 64 + 1), 1024 * 64 + 1, "1025 blocks"); + // Sparse blocks: bases need not be consecutive, only ascending and distinct. + CosMask sparse; + for (size_t w = 0; w < 5000; w += 3) { + sparse.blocks.emplace_back(w * 64, uint64_t{0x8000000000000001}); + sparse.total_count += 2; + } + check_mask_budgets(sparse, 5000 * 64, "sparse blocks"); +} + +BOOST_AUTO_TEST_CASE(openmp_cos_mask_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + const CosMask mask = plan_mask(); + VecD c(4097 * 64, 1.0); + RangeLog log; + monoprop::detail::scale_cos_mask(c.data(), mask, 0.5, Options{.threads = 4}, RecordingObserver{&log}); + kernel_test::check_participation(log[KernelRange::cos_mask], 5, "scale_cos_mask budget 4"); +} + +BOOST_AUTO_TEST_CASE(openmp_cos_mask_small_or_serial_work_stays_on_the_caller) { + VecD c(4097 * 64, 1.0); + RangeLog one_thread; + monoprop::detail::scale_cos_mask(c.data(), plan_mask(), 0.5, Options{.threads = 1}, RecordingObserver{&one_thread}); + kernel_test::check_serial(one_thread[KernelRange::cos_mask], 5, "budget 1"); + RangeLog tiny; + monoprop::detail::scale_cos_mask(c.data(), + full_mask(1024 * 64), + 0.5, + Options{.threads = 4}, + RecordingObserver{&tiny}); + kernel_test::check_serial(tiny[KernelRange::cos_mask], 1, "one range, budget 4"); + RangeLog empty; + monoprop::detail::scale_cos_mask(c.data(), CosMask{}, 0.5, Options{.threads = 4}, RecordingObserver{&empty}); + kernel_test::check_serial(empty[KernelRange::cos_mask], 0, "empty mask, budget 4"); +} + +// --- lazy cosine --------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(openmp_cos_lazy_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + // Three full fold blocks plus a partial fourth. + const size_t rows = 3 * monoprop::detail::kColumnBlockWords * 64 + 1000; + const auto index = kernel_test::make_synthetic_index(rows, 11); + const auto gen = kernel_test::generator({0, 3, 7}); // odd: exercises the caller-prepared row parity + const auto recipe = monoprop::detail::make_lazy_fold(index, gen, rows, Basis::Majorana); + VecD c = distinct_values(rows); + RangeLog log; + monoprop::detail::scale_cos_lazy(index, + recipe, + c.data(), + 0.5, + Options{.threads = 4}, + RecordingObserver{&log}); + kernel_test::check_participation(log[KernelRange::cos_lazy], 4, "scale_cos_lazy budget 4"); +} + +BOOST_AUTO_TEST_CASE(openmp_cos_lazy_small_or_serial_work_stays_on_the_caller) { + const size_t rows = 3 * monoprop::detail::kColumnBlockWords * 64 + 1000; + const auto index = kernel_test::make_synthetic_index(rows, 12); + const auto gen = kernel_test::generator({1, 4}); + const auto recipe = monoprop::detail::make_lazy_fold(index, gen, rows, Basis::Majorana); + VecD c = distinct_values(rows); + RangeLog one_thread; + monoprop::detail::scale_cos_lazy(index, + recipe, + c.data(), + 0.5, + Options{.threads = 1}, + RecordingObserver{&one_thread}); + kernel_test::check_serial(one_thread[KernelRange::cos_lazy], 4, "budget 1"); + // A partial scaled_count inside the first block: one range, serial at any budget. + const auto short_recipe = + monoprop::detail::make_lazy_fold(index, gen, 1000, Basis::Majorana); + RangeLog tiny; + monoprop::detail::scale_cos_lazy(index, + short_recipe, + c.data(), + 0.5, + Options{.threads = 4}, + RecordingObserver{&tiny}); + kernel_test::check_serial(tiny[KernelRange::cos_lazy], 1, "one range, budget 4"); +} + +// --- fused apply ------------------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE(openmp_fused_apply_exact_on_disjoint_records) { + // Several record ranges in every stream, with partial final ranges. + const auto h = hand_contract(3 * 1024 + 17, 2 * 1024 + 5, 1500, 7); + const auto owners = kernel_test::add_owner_counts(h.fc); + BOOST_TEST(std::ranges::all_of(owners, [](const auto &kv) { return kv.second == 1; })); + const VecD coeffs = distinct_values(h.slots); + for (const bool schrodinger : {false, true}) { + for (const bool fused_scale : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger << " fused_scale=" << fused_scale) { + const VecD serial = apply_copy(h, coeffs, schrodinger, fused_scale, 1); + BOOST_TEST(!bitwise_equal(serial, coeffs)); + // The oracle is compiled separately, so where a slot is `cos·c + sin·φ·v` (fused-scale insert + // arms) the compiler may contract a different product into a fused multiply-add. Every other + // arm is a single multiply-add and must match bitwise. + const VecD reference = reference_apply(h, coeffs, schrodinger, fused_scale); + if (fused_scale) { + BOOST_TEST(equal_up_to_contraction(serial, reference, coeffs)); + } + else { + BOOST_TEST(bitwise_equal(serial, reference)); + } + for (const int threads : {2, 3, 4, 64}) { + BOOST_TEST_INFO("threads " << threads); + BOOST_TEST(bitwise_equal(apply_copy(h, coeffs, schrodinger, fused_scale, threads), serial)); + } + // Slots no record or mask touches keep their values. + const auto sel = selected(h.cos, h.slots); + for (size_t i = 0; i < h.slots; ++i) { + if (sel[i] == 0 && !owners.contains(i)) { + BOOST_TEST(serial[i] == coeffs[i]); + } + } + } + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_fused_apply_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + const auto h = hand_contract(3 * 1024 + 17, 2 * 1024 + 5, 1500, 8); + const VecD coeffs = distinct_values(h.slots); + RangeLog log; + const RecordingObserver observer{&log}; + (void)apply_copy(h, coeffs, /*schrodinger=*/true, /*fused_scale=*/false, 4, &observer); + const size_t records = h.fc.hits.size() + h.fc.inserts.size() + h.fc.cross_half.size(); + kernel_test::check_participation(log[KernelRange::fused_gather], 3, "insert gather budget 4"); + kernel_test::check_participation(log[KernelRange::cos_mask], + monoprop::detail::logical_ranges(h.cos.blocks.size(), 1024), + "two-pass cosine phase budget 4"); + kernel_test::check_participation(log[KernelRange::fused_apply], + monoprop::detail::logical_ranges(records, 1024), + "rotation records budget 4"); +} + +BOOST_AUTO_TEST_CASE(openmp_fused_apply_small_or_serial_work_stays_on_the_caller) { + const auto h = hand_contract(3 * 1024 + 17, 2 * 1024 + 5, 1500, 9); + const VecD coeffs = distinct_values(h.slots); + RangeLog one_thread; + const RecordingObserver serial_observer{&one_thread}; + (void)apply_copy(h, coeffs, true, false, 1, &serial_observer); + const size_t records = h.fc.hits.size() + h.fc.inserts.size() + h.fc.cross_half.size(); + kernel_test::check_serial(one_thread[KernelRange::fused_gather], 3, "gather budget 1"); + kernel_test::check_serial(one_thread[KernelRange::fused_apply], + monoprop::detail::logical_ranges(records, 1024), + "records budget 1"); + + // Fused-scale Heisenberg: no gather and no cosine phase at all. + RangeLog heisenberg; + const RecordingObserver heisenberg_observer{&heisenberg}; + (void)apply_copy(h, coeffs, false, true, 4, &heisenberg_observer); + BOOST_TEST(heisenberg[KernelRange::fused_gather].prepares == 0u); + BOOST_TEST(heisenberg[KernelRange::cos_mask].prepares == 0u); + + const auto tiny = hand_contract(300, 200, 100, 10); + const VecD tiny_coeffs = distinct_values(tiny.slots); + RangeLog small; + const RecordingObserver small_observer{&small}; + (void)apply_copy(tiny, tiny_coeffs, true, false, 4, &small_observer); + kernel_test::check_serial(small[KernelRange::fused_gather], 1, "tiny gather budget 4"); + kernel_test::check_serial(small[KernelRange::fused_apply], 1, "tiny records budget 4"); + + HandContract empty; + empty.slots = 16; + RangeLog none; + const RecordingObserver none_observer{&none}; + const VecD untouched = apply_copy(empty, distinct_values(16), true, false, 4, &none_observer); + BOOST_TEST(bitwise_equal(untouched, distinct_values(16))); + kernel_test::check_serial(none[KernelRange::fused_apply], 0, "empty contract budget 4"); +} + +// --- worker failures inside the kernels' own ranges -------------------------------------------------------- + +// A worker failure inside a kernel range is captured, every worker joins, and the original exception reaches +// the caller; no range runs twice. Which other ranges ran is unspecified. +BOOST_AUTO_TEST_CASE(openmp_kernel_worker_failures_join_before_rethrow) { + const auto check = [](const RangeLog &log, KernelRange kind, const char *label) { + const auto &s = log.slots[static_cast(kind)]; + BOOST_TEST_CONTEXT(label) { + BOOST_TEST(s.visits.at(2) == 1); + BOOST_TEST(std::ranges::all_of(s.visits, [](int v) { return v <= 1; })); + } + }; + for (const int threads : {1, 4}) { + BOOST_TEST_CONTEXT("threads " << threads) { + VecD c(4097 * 64, 1.0); + RangeLog mask_log; + BOOST_CHECK_THROW(monoprop::detail::scale_cos_mask(c.data(), + plan_mask(), + 0.5, + Options{.threads = threads}, + RecordingObserver{&mask_log, KernelRange::cos_mask, 2}), + std::runtime_error); + check(mask_log, KernelRange::cos_mask, "scale_cos_mask"); + + const size_t rows = 3 * monoprop::detail::kColumnBlockWords * 64 + 1000; + const auto index = kernel_test::make_synthetic_index(rows, 13); + const auto recipe = + monoprop::detail::make_lazy_fold(index, + kernel_test::generator({0, 3, 7}), + rows, + Basis::Majorana); + VecD lazy = distinct_values(rows); + RangeLog lazy_log; + BOOST_CHECK_THROW(monoprop::detail::scale_cos_lazy( + index, + recipe, + lazy.data(), + 0.5, + Options{.threads = threads}, + RecordingObserver{&lazy_log, KernelRange::cos_lazy, 2}), + std::runtime_error); + check(lazy_log, KernelRange::cos_lazy, "scale_cos_lazy"); + + const auto h = hand_contract(3 * 1024 + 17, 2 * 1024 + 5, 1500, 14); + for (const auto kind : {KernelRange::fused_gather, KernelRange::cos_mask, KernelRange::fused_apply}) { + RangeLog fused_log; + const RecordingObserver observer{&fused_log, kind, 2}; + BOOST_CHECK_THROW((void)apply_copy(h, distinct_values(h.slots), true, false, threads, &observer), + std::runtime_error); + check(fused_log, kind, "apply_fused_contract"); + } + } + } +} + +// The debug-build owner check agrees with the independent count, and rejects a slot written twice: a hit's +// target reused as a half's local slot, and a source shared by two inserts. +BOOST_AUTO_TEST_CASE(openmp_fused_add_owner_check_detects_a_shared_slot) { + auto h = hand_contract(3 * 1024 + 17, 2 * 1024 + 5, 1500, 15); + BOOST_TEST(monoprop::detail::fused_add_owners_unique(h.fc)); + BOOST_TEST(monoprop::detail::fused_add_owners_unique(monoprop::detail::FusedContract{})); + auto shared_half = h.fc; + shared_half.cross_half.back().local_idx = shared_half.hits.front().tgt; + BOOST_TEST(!monoprop::detail::fused_add_owners_unique(shared_half)); + auto shared_insert = h.fc; + shared_insert.inserts.back().src = shared_insert.inserts.front().src; + BOOST_TEST(!monoprop::detail::fused_add_owners_unique(shared_insert)); +} diff --git a/cpp/tests/openmp_runtime_tests.cpp b/cpp/tests/openmp_runtime_tests.cpp new file mode 100644 index 000000000..d71c4bb6f --- /dev/null +++ b/cpp/tests/openmp_runtime_tests.cpp @@ -0,0 +1,556 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Construction-time thread budgets, their per-object plumbing, and the invalid-owner rule. +// +// Budgets are captured from monoprop_NUM_THREADS when an object is constructed. Cases that need a specific +// launch environment are also registered as separate fresh-process CTest entries (see CMakeLists.txt). The +// pure parser and capture cases set the variable around one call with ScopedEnv. Every propagator captures the budget, +// and its T is its shard count, so no integration case changes the budget inside a process: those cases run at the +// launch's T. These tests check what reaches each object; kernel-level parallel execution is covered by +// openmp_kernel_tests.cpp. + +#include + +#include +#ifdef __linux__ +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "PropagatorTestAccess.h" +#include "TestUtilities.h" +#include "monoprop/Evolution.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/evolution/layer_build/FusedApply.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/OperationFailure.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/parallel/Workshare.h" + +namespace { + +using namespace monoprop; +using monoprop::detail::parallel::capture_thread_budget; +using monoprop::detail::parallel::Options; +using monoprop::detail::parallel::resolve_thread_budget; + +constexpr size_t kNumModes = 8; +constexpr unsigned int kCutoff = 4; +constexpr const char *kBudgetVariable = "monoprop_NUM_THREADS"; + +using Access = monoprop::detail::PropagatorTestAccess; +using Propagator = MonomialPropagator; + +// Sets (or unsets, for nullopt) one environment variable for a scope, then restores it. +class ScopedEnv { +public: + ScopedEnv(const char *name, std::optional value) : name_(name) { + if (const char *old = std::getenv(name)) { + old_ = std::string(old); + } + if (value) { + ::setenv(name, value->c_str(), 1); + } + else { + ::unsetenv(name); + } + } + ScopedEnv(const ScopedEnv &) = delete; + auto operator=(const ScopedEnv &) -> ScopedEnv & = delete; + ~ScopedEnv() { + if (old_) { + ::setenv(name_, old_->c_str(), 1); + } + else { + ::unsetenv(name_); + } + } + +private: + const char *name_; + std::optional old_; +}; + +// A distinct type, so a test can tell the original exception from anything the library substitutes. +class InjectedFailure : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// The root at the launch's budget. +auto make_root(const test_utils::CaseData &data) -> Propagator { + return Propagator(data.hamiltonian, kCutoff, data.initial_state, std::nullopt, MPI_COMM_SELF); +} + +auto load() -> test_utils::CaseData { + return test_utils::load_case_data("random_exact.msgpack"); +} + +auto build_all(Propagator &sim, const test_utils::CaseData &data) -> void { + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); +} + +// What the launch environment should resolve to, independently of the library's capture. +auto expected_launch_budget() -> int { + if (const char *expected = std::getenv("monoprop_TEST_EXPECT_BUDGET")) { + return std::stoi(expected); + } + const char *configured = std::getenv(kBudgetVariable); + return resolve_thread_budget(configured ? std::optional(configured) : std::nullopt, + omp_get_max_threads()) + .threads; +} + +struct RuntimeSettings { + int max_threads = 0; + int dynamic = 0; + int max_active_levels = 0; + int thread_limit = 0; + omp_sched_t schedule = omp_sched_static; + int chunk = 0; + omp_proc_bind_t proc_bind = omp_proc_bind_false; + + auto operator==(const RuntimeSettings &) const -> bool = default; +}; + +auto runtime_settings() -> RuntimeSettings { + RuntimeSettings s; + s.max_threads = omp_get_max_threads(); + s.dynamic = omp_get_dynamic(); + s.max_active_levels = omp_get_max_active_levels(); + s.thread_limit = omp_get_thread_limit(); + omp_get_schedule(&s.schedule, &s.chunk); + s.proc_bind = omp_get_proc_bind(); + return s; +} + +auto binding_disabled(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + boost::test_tools::assertion_result result(omp_get_proc_bind() == omp_proc_bind_false); + result.message() << "OpenMP binding is enabled (OMP_PROC_BIND); run with OMP_PROC_BIND=false"; + return result; +} + +// Every state-consuming entry an invalid owner must refuse, each run on its own. +template +auto for_each_state_entry(Propagator &sim, const test_utils::CaseData &data, Check check) -> void { + check("build_graph", [&] { sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); }); + check("propagate", [&] { sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); }); + check("expectation_value", [&] { (void)sim.expectation_value(data.parameters); }); + check("expectation_value_and_gradient", [&] { (void)sim.expectation_value_and_gradient(data.parameters); }); + check("expectation_value_functional", [&] { (void)sim.expectation_value_functional(); }); + check("expectation_value_and_gradient_functional", [&] { (void)sim.expectation_value_and_gradient_functional(); }); + check("contract_partially", [&] { (void)sim.contract_partially(data.parameters, false); }); + check("evolved_operator_terms", [&] { (void)sim.evolved_operator_terms(data.parameters, 0.0); }); + check("update_initial_operator", [&] { sim.update_initial_operator(data.hamiltonian); }); + check("set_parameter_mapping", [&] { sim.set_parameter_mapping(sim.parameter_mapping()); }); + check("update_cutoff", [&] { sim.update_cutoff(kCutoff); }); + check("size", [&] { (void)sim.size(); }); + check("graph_size", [&] { (void)sim.graph_size(); }); + check("graph_data", [&] { (void)sim.graph_data(); }); + check("copy", [&] { const Propagator copy(sim); }); + check("clone", [&] { (void)Access::clone(sim); }); +} + +auto require_invalid(Propagator &sim, const test_utils::CaseData &data) -> void { + for_each_state_entry(sim, data, [](const char *what, auto &&call) { + BOOST_TEST_CONTEXT(what) { + BOOST_CHECK_THROW(call(), InvalidPropagatorError); + } + }); +} + +} // namespace + +// --- resolve_thread_budget: the pure parser -------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(openmp_budget_absent_uses_runtime_default) { + BOOST_TEST(resolve_thread_budget(std::nullopt, 1).threads == 1); + BOOST_TEST(resolve_thread_budget(std::nullopt, 3).threads == 3); +} + +BOOST_AUTO_TEST_CASE(openmp_budget_absent_rejects_a_nonpositive_runtime_default) { + BOOST_CHECK_THROW((void)resolve_thread_budget(std::nullopt, 0), std::invalid_argument); + BOOST_CHECK_THROW((void)resolve_thread_budget(std::nullopt, -1), std::invalid_argument); +} + +BOOST_AUTO_TEST_CASE(openmp_budget_explicit_value_overrides_the_default) { + BOOST_TEST(resolve_thread_budget("8", 2).threads == 8); + // A valid explicit value wins even over an invalid fallback. + BOOST_TEST(resolve_thread_budget("1", 0).threads == 1); + BOOST_TEST(resolve_thread_budget("1", -1).threads == 1); + // Leading zeros are harmless, however many. + BOOST_TEST(resolve_thread_budget("0008", 2).threads == 8); + BOOST_TEST(resolve_thread_budget("0000000000000000000000000000000000000003", 1).threads == 3); +} + +BOOST_AUTO_TEST_CASE(openmp_budget_rejects_malformed_values) { + for (const std::string_view text : + {"", "0", "00", "-1", "-0", "abc", "2,3", " 3", "3 ", "+3", "\t3", "3\n", "0x10", "1e3", "3.0", "3a"}) { + BOOST_TEST_CONTEXT("value=\"" << text << "\"") { + BOOST_CHECK_EXCEPTION((void)resolve_thread_budget(text, 2), + std::invalid_argument, + [](const std::invalid_argument &e) { + return std::string_view(e.what()).find(kBudgetVariable) != std::string_view::npos; + }); + } + } + // Non-ASCII digits (Arabic-Indic three, U+0663) are not decimal digits here. + BOOST_CHECK_THROW((void)resolve_thread_budget("\xd9\xa3", 2), std::invalid_argument); +} + +BOOST_AUTO_TEST_CASE(openmp_budget_accepts_int_max_without_creating_a_team) { + // Parsing only: no region is opened, so no INT_MAX-sized team is ever requested. + BOOST_TEST(resolve_thread_budget("2147483647", 1).threads == INT_MAX); + BOOST_TEST(resolve_thread_budget("0002147483647", 1).threads == INT_MAX); +} + +BOOST_AUTO_TEST_CASE(openmp_budget_rejects_overflow) { + for (const std::string_view text : + {"2147483648", "4294967297", "18446744073709551617", "99999999999999999999999"}) { + BOOST_TEST_CONTEXT("value=" << text) { + BOOST_CHECK_THROW((void)resolve_thread_budget(text, 2), std::invalid_argument); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_budget_has_no_arbitrary_thread_cap) { + // The removed partition runtime's parser silently ignored anything above one million. + BOOST_TEST(resolve_thread_budget("1000001", 1).threads == 1000001); +} + +// --- capture_thread_budget: one environment read per call --------------------------------------------- + +BOOST_AUTO_TEST_CASE(openmp_capture_reads_the_environment) { + { + const ScopedEnv env(kBudgetVariable, "5"); + BOOST_TEST(capture_thread_budget().threads == 5); + } + { + const ScopedEnv env(kBudgetVariable, std::nullopt); + BOOST_TEST(capture_thread_budget().threads == omp_get_max_threads()); + } + { + // Present but empty is an error, never the fallback. + const ScopedEnv env(kBudgetVariable, ""); + BOOST_CHECK_THROW((void)capture_thread_budget(), std::invalid_argument); + } + { + const ScopedEnv env(kBudgetVariable, "two"); + BOOST_CHECK_THROW((void)capture_thread_budget(), std::invalid_argument); + } +} + +// --- per-object capture ------------------------------------------------------------------------------------ + +// Registered again as fresh processes with monoprop_NUM_THREADS=1, =3, and unset with OMP_NUM_THREADS=3. +BOOST_AUTO_TEST_CASE(openmp_root_budget_matches_the_launch_environment) { + const auto data = load(); + const auto sim = make_root(data); + BOOST_TEST(Access::options(sim).threads == expected_launch_budget()); + // The shard count is the budget. + BOOST_TEST(Access::shards(sim).size() == static_cast(expected_launch_budget())); + // The unset-variable launch: nothing supplied the variable, so the OpenMP default decided. + if (std::getenv("monoprop_TEST_EXPECT_UNSET_BUDGET") != nullptr) { + BOOST_TEST(std::getenv(kBudgetVariable) == nullptr); + BOOST_TEST(Access::options(sim).threads == omp_get_max_threads()); + } +} + +// The root reads the variable once, at construction: copies and clones never reread it, so even an unparseable +// value set afterwards reaches neither them nor the source, while a new construction does read it. No second budget +// is ever created in this process. +BOOST_AUTO_TEST_CASE(openmp_root_captures_once_and_copies_preserve_it) { + const auto data = load(); + auto sim = make_root(data); + const int team = Access::options(sim).threads; + build_all(sim, data); + const double energy = sim.expectation_value(data.parameters); + const ScopedEnv env(kBudgetVariable, "not-a-budget"); + const Propagator copy(sim); + BOOST_TEST(Access::options(copy).threads == team); + BOOST_TEST(Access::shards(copy).size() == static_cast(team)); + BOOST_TEST(Access::options(*Access::clone(sim)).threads == team); + BOOST_TEST(sim.expectation_value(data.parameters) == energy); + BOOST_TEST(Propagator(copy).expectation_value(data.parameters) == energy); + BOOST_CHECK_THROW((void)make_root(data), PropagatorConfigError); +} + +BOOST_AUTO_TEST_CASE(openmp_invalid_budget_is_a_config_error_at_construction) { + const auto data = load(); + for (const char *value : {"0", "abc", "", "-2", "2147483648"}) { + const ScopedEnv env(kBudgetVariable, value); + BOOST_TEST_CONTEXT("monoprop_NUM_THREADS=\"" << value << "\"") { + BOOST_CHECK_EXCEPTION((void)make_root(data), PropagatorConfigError, [](const PropagatorConfigError &e) { + return std::string_view(e.what()).find(kBudgetVariable) != std::string_view::npos; + }); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_propagator_work_leaves_openmp_settings_unchanged) { + const auto data = load(); + const auto before = runtime_settings(); + { + auto sim = make_root(data); + build_all(sim, data); + (void)sim.expectation_value_and_gradient(data.parameters); + const auto copy = sim; + (void)sim.contract_partially(data.parameters, true); + } + BOOST_TEST((runtime_settings() == before)); +} + +// Registered again with OMP_PROC_BIND=false; skipped where the runtime was asked to bind. +BOOST_AUTO_TEST_CASE(openmp_propagator_work_does_not_change_affinity, + *boost::unit_test::precondition(binding_disabled)) { +#ifdef __linux__ + cpu_set_t before; + CPU_ZERO(&before); + BOOST_REQUIRE(sched_getaffinity(0, sizeof(before), &before) == 0); + { + const auto data = load(); + auto sim = make_root(data); + build_all(sim, data); + (void)sim.expectation_value_and_gradient(data.parameters); + monoprop::detail::parallel::for_blocks(8, Access::options(sim), [](size_t) {}); + } + cpu_set_t after; + CPU_ZERO(&after); + BOOST_REQUIRE(sched_getaffinity(0, sizeof(after), &after) == 0); + BOOST_TEST(CPU_EQUAL(&before, &after)); +#else + BOOST_TEST_MESSAGE("affinity masks are only compared on Linux"); +#endif +} + +// --- old low-level call sites keep compiling and default to serial Options --------------------------------- + +BOOST_AUTO_TEST_CASE(openmp_low_level_calls_default_to_serial_options) { + const auto data = load(); + auto sim = make_root(data); + build_all(sim, data); + const auto &graph = sim.graph(); + const auto view = graph.replay_view(); + const auto mapped = map_params(data.parameters, sim.parameter_mapping(), VecD(sim.graph_layers(), 1.0), 1.0, true); + const auto cos = build_cos_callbacks(sim.mp_op().inverted_index(), view, Basis::Majorana); + const auto cos_threaded = + build_cos_callbacks(sim.mp_op().inverted_index(), view, Basis::Majorana, Options{.threads = 3}); + + const VecD op = sim.mp_op().get_operator(); + const VecD by_default = evolve_operator(VecD(op), view, mapped, MPI_COMM_SELF, cos.scale); + const VecD explicit_serial = evolve_operator(VecD(op), view, mapped, MPI_COMM_SELF, cos.scale, Options{}); + const VecD threaded = + evolve_operator(VecD(op), view, mapped, MPI_COMM_SELF, cos_threaded.scale, Options{.threads = 3}); + BOOST_CHECK_EQUAL_COLLECTIONS(by_default.begin(), by_default.end(), explicit_serial.begin(), explicit_serial.end()); + BOOST_CHECK_EQUAL_COLLECTIONS(by_default.begin(), by_default.end(), threaded.begin(), threaded.end()); + + VecD s1(op.size(), 0.5), h1 = by_default, s2 = s1, h2 = h1; + const size_t last = sim.graph_layers() - 1; + const LayerAngle angle{.gen_coeff = 1.0, .param = mapped[last]}; + const double d1 = state_operator_derivative_local(s1, h1, view, last, angle, MPI_COMM_SELF, cos.accumulate); + const double d2 = state_operator_derivative_local(s2, + h2, + view, + last, + angle, + MPI_COMM_SELF, + cos_threaded.accumulate, + {}, + Options{.threads = 3}); + BOOST_TEST(d1 == d2); + BOOST_CHECK_EQUAL_COLLECTIONS(h1.begin(), h1.end(), h2.begin(), h2.end()); + + CosMask mask; + mask.blocks.emplace_back(0, uint64_t{0b1011}); + mask.total_count = 3; + VecD a(64, 2.0), b = a; + monoprop::detail::scale_cos_mask(a.data(), mask, 0.5); + monoprop::detail::scale_cos_mask(b.data(), mask, 0.5, Options{.threads = 3}); + BOOST_CHECK_EQUAL_COLLECTIONS(a.begin(), a.end(), b.begin(), b.end()); + + EvalState state = EvalState::dense(VecD(op.size(), 0.25)); + const auto mapping = sim.parameter_mapping(); + const VecD gens(mapping.size(), 1.0); + const EvalRequest serial_request{.e_core = 0.0, + .state = state, + .op = op, + .parameter_mapping = mapping, + .gen_coeffs = gens, + .graph = view, + .params = data.parameters}; + BOOST_TEST(serial_request.parallel.threads == 1); +} + +// --- invalid-owner enforcement ------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE(openmp_failed_build_invalidates_the_owner) { + const auto data = load(); + const size_t half = data.majoranas.size() / 2; + BOOST_REQUIRE(half > 0); + const std::vector first(data.majoranas.begin(), data.majoranas.begin() + static_cast(half)); + const std::vector rest(data.majoranas.begin() + static_cast(half), data.majoranas.end()); + const VecZ first_map(data.param_inds.begin(), data.param_inds.begin() + static_cast(half)); + const VecZ rest_map(data.param_inds.begin() + static_cast(half), data.param_inds.end()); + const VecD first_gen(data.gen_coeffs.begin(), data.gen_coeffs.begin() + static_cast(half)); + const VecD rest_gen(data.gen_coeffs.begin() + static_cast(half), data.gen_coeffs.end()); + + auto sim = make_root(data); + sim.build_graph(first, first_map, first_gen); + const VecD params(expected_num_params(sim.parameter_mapping()), 0.3); + const double before = sim.expectation_value(params); + Propagator early(sim); + auto retained = sim.expectation_value_functional(); + + Access::set_cutoff_fn(sim, [](const Monomial &) -> bool { + throw InjectedFailure("injected cutoff failure"); + }); + // The original exception reaches a single-rank caller unchanged. + BOOST_CHECK_THROW(sim.build_graph(rest, rest_map, rest_gen), InjectedFailure); + BOOST_TEST(Access::is_invalid(sim)); + + require_invalid(sim, data); + BOOST_CHECK_THROW((void)retained(params), InvalidPropagatorError); + // Immutable configuration stays readable. + BOOST_TEST(sim.cutoff() == kCutoff); + BOOST_TEST(!sim.schrodinger()); + // An independent earlier copy is unaffected. + BOOST_TEST(test_utils::near(early.expectation_value(params), before)); +} + +BOOST_AUTO_TEST_CASE(openmp_validation_errors_leave_the_owner_usable) { + const auto data = load(); + auto sim = make_root(data); + build_all(sim, data); + const double before = sim.expectation_value(data.parameters); + const auto check_usable = [&](const char *what) { + BOOST_TEST_CONTEXT(what) { + BOOST_TEST(!Access::is_invalid(sim)); + BOOST_TEST(test_utils::near(sim.expectation_value(data.parameters), before)); + } + }; + + BOOST_CHECK_THROW(sim.build_graph(data.majoranas, VecZ{0}, data.gen_coeffs), std::exception); + check_usable("mismatched mapping length"); + + // A generator index past the system: rejected before any gate runs, even behind a valid gate. + std::vector bad_gates{data.majoranas.front(), VecZ{0, 2 * kNumModes}}; + BOOST_CHECK_THROW(sim.build_graph(bad_gates, VecZ{0, 0}, VecD{1.0, 1.0}), std::exception); + check_usable("out-of-range generator index in build_graph"); + auto fresh = make_root(data); + BOOST_CHECK_THROW(fresh.propagate(bad_gates, VecZ{0, 0}, VecD{1.0, 1.0}, VecD{0.1}), std::exception); + BOOST_TEST(!Access::is_invalid(fresh)); + + BOOST_CHECK_THROW(sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters), std::exception); + check_usable("propagate on a stored graph"); + + BOOST_CHECK_THROW((void)sim.expectation_value(VecD{0.1}), std::exception); + check_usable("wrong parameter length"); + auto retained = sim.expectation_value_functional(); + BOOST_CHECK_THROW((void)retained(VecD{0.1}), std::exception); + check_usable("wrong parameter length through a retained functional"); + + sim.update_upper_atol(0.5); + BOOST_CHECK_THROW(sim.update_lower_atol(1.0), PropagatorConfigError); + sim.update_upper_atol(std::nullopt); + check_usable("crossed atol pair"); + BOOST_CHECK_THROW(sim.set_parameter_mapping(VecZ{0}), std::exception); + check_usable("wrong mapping length"); + + OperatorDict bad_operator = data.hamiltonian; + bad_operator[VecZ{0, 2 * kNumModes}] = 1.0; + BOOST_CHECK_THROW(sim.update_initial_operator(bad_operator), std::exception); + check_usable("out-of-range initial-operator index"); + // Weight 2 * kNumModes is past the cutoff, so no shard holds the term: Heisenberg rejects it before the epoch + // moves. + VecZ absent_term(2 * kNumModes); + std::iota(absent_term.begin(), absent_term.end(), 0); + BOOST_CHECK_THROW(sim.update_initial_operator(OperatorDict{{absent_term, 1.0}}), std::runtime_error); + check_usable("initial-operator term absent from the operator"); + BOOST_TEST(test_utils::near(retained(data.parameters), before)); + BOOST_TEST(test_utils::near(retained(data.parameters), before)); +} + +BOOST_AUTO_TEST_CASE(openmp_invalid_owner_can_still_be_destroyed) { + const auto data = load(); + auto owner = std::make_unique(make_root(data)); + build_all(*owner, data); + Access::set_cutoff_fn(*owner, [](const Monomial &) -> bool { throw InjectedFailure("injected"); }); + BOOST_CHECK_THROW(build_all(*owner, data), InjectedFailure); + BOOST_CHECK_NO_THROW(owner.reset()); +} + +// --- failure boundary helper ----------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(openmp_operation_failed_rethrows_the_original_on_one_rank) { + const auto error = std::make_exception_ptr(InjectedFailure("original")); + BOOST_CHECK_EXCEPTION(mpi::operation_failed(MPI_COMM_SELF, error), InjectedFailure, [](const InjectedFailure &e) { + return std::string_view(e.what()) == "original"; + }); + int calls = 0; + BOOST_CHECK_THROW(mpi::guard_distributed(MPI_COMM_SELF, + [&] { + ++calls; + throw InjectedFailure("guarded"); + }), + InjectedFailure); + BOOST_TEST(calls == 1); + BOOST_TEST(mpi::guard_distributed(MPI_COMM_SELF, [] { return 7; }) == 7); +} + +#ifdef monoprop_ENABLE_MPI +BOOST_AUTO_TEST_CASE(openmp_thread_level_decisions_are_pure) { + const int levels[] = {MPI_THREAD_SINGLE, MPI_THREAD_FUNNELED, MPI_THREAD_SERIALIZED, MPI_THREAD_MULTIPLE}; + for (const int provided : levels) { + BOOST_TEST_CONTEXT("provided=" << mpi::thread_level_name(provided)) { + BOOST_TEST(mpi::thread_level_satisfies(provided, MPI_THREAD_SERIALIZED) + == (provided >= MPI_THREAD_SERIALIZED)); + BOOST_TEST(mpi::thread_level_satisfies(provided, MPI_THREAD_FUNNELED) == (provided >= MPI_THREAD_FUNNELED)); + // FUNNELED is the requested and required level: SINGLE is refused, anything at or above FUNNELED serves. + BOOST_TEST(mpi::thread_level_satisfies(provided, mpi::required_thread_level) + == (provided != MPI_THREAD_SINGLE)); + } + } + // Only the initializing thread, as the library team's primary, ever calls MPI. + BOOST_TEST(mpi::required_thread_level == MPI_THREAD_FUNNELED); + BOOST_TEST(std::string(mpi::thread_level_name(MPI_THREAD_FUNNELED)) == "MPI_THREAD_FUNNELED"); +} + +BOOST_AUTO_TEST_CASE(openmp_initialized_mpi_meets_the_required_level) { + int provided = MPI_THREAD_SINGLE; + MPI_Query_thread(&provided); + BOOST_TEST_MESSAGE("MPI provided thread support: " << mpi::thread_level_name(provided)); + BOOST_TEST(mpi::thread_level_satisfies(provided, mpi::required_thread_level)); + BOOST_CHECK_NO_THROW(mpi::require_thread_support()); + int is_main = 0; + MPI_Is_thread_main(&is_main); + BOOST_TEST(is_main != 0); + // The initializing thread passes the guard; a wrong thread is covered by mpi_failure_driver. + mpi::require_initializing_thread(); +} +#endif diff --git a/cpp/tests/openmp_workshare_tests.cpp b/cpp/tests/openmp_workshare_tests.cpp new file mode 100644 index 000000000..89093f475 --- /dev/null +++ b/cpp/tests/openmp_workshare_tests.cpp @@ -0,0 +1,324 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Workers never call BOOST_TEST: each block writes only its own preallocated element (or an atomic), +// and every assertion runs on the caller after for_blocks has returned. + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/Workshare.h" + +using monoprop::detail::parallel::for_blocks; +using monoprop::detail::parallel::Options; + +namespace { + +// The OpenMP runtime settings (ICVs) that a library helper must never change. +struct RuntimeSettings { + int max_threads = 0; + int dynamic = 0; + int max_active_levels = 0; + int thread_limit = 0; + omp_sched_t schedule = omp_sched_static; + int chunk = 0; + omp_proc_bind_t proc_bind = omp_proc_bind_false; + int level = 0; + + auto operator==(const RuntimeSettings &) const -> bool = default; +}; + +auto runtime_settings() -> RuntimeSettings { + RuntimeSettings s; + s.max_threads = omp_get_max_threads(); + s.dynamic = omp_get_dynamic(); + s.max_active_levels = omp_get_max_active_levels(); + s.thread_limit = omp_get_thread_limit(); + omp_get_schedule(&s.schedule, &s.chunk); + s.proc_bind = omp_get_proc_bind(); + s.level = omp_get_level(); + return s; +} + +// A test-side probe, not library behavior: can this runtime give a two-thread region right now? +auto runtime_offers_two_workers(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + if (omp_get_thread_limit() < 2) { + return {false}; + } + int team = 0; +#pragma omp parallel num_threads(2) + { +#pragma omp single + team = omp_get_num_threads(); + } + boost::test_tools::assertion_result result(team >= 2); + result.message() << "the OpenMP runtime provided " << team << " worker(s) for a two-thread request"; + return result; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(openmp_blocks_visit_once) { + std::vector visits(19); + for_blocks(19, {.threads = 3}, [&](size_t b) { ++visits[b]; }); + for (const auto n : visits) { + BOOST_TEST(n == 1); + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_visit_once_across_counts_and_budgets) { + for (const size_t count : {size_t{0}, size_t{1}, size_t{19}}) { + for (const int threads : {1, 2, 3}) { + std::vector visits(count, 0); + std::vector worker(count, -1); + for_blocks(count, {.threads = threads}, [&](size_t b) { + ++visits[b]; + worker[b] = omp_get_thread_num(); + }); + const auto requested = std::min(static_cast(threads), count); + for (size_t b = 0; b < count; ++b) { + BOOST_TEST(visits[b] == 1, "count=" << count << " threads=" << threads << " block=" << b); + BOOST_TEST(worker[b] >= 0); + BOOST_TEST(static_cast(worker[b]) < requested); + } + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_more_requested_workers_than_blocks) { + constexpr size_t count = 2; + std::vector visits(count, 0); + std::vector team(count, 0); + std::vector worker(count, -1); + for_blocks(count, {.threads = 8}, [&](size_t b) { + ++visits[b]; + team[b] = omp_get_num_threads(); + worker[b] = omp_get_thread_num(); + }); + for (size_t b = 0; b < count; ++b) { + BOOST_TEST(visits[b] == 1); + // The region requests min(threads, count) workers, so no idle third worker is created. + BOOST_TEST(team[b] >= 1); + BOOST_TEST(team[b] <= static_cast(count)); + BOOST_TEST(worker[b] < static_cast(count)); + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_budget_one_runs_ascending_on_the_caller) { + constexpr size_t count = 19; + const auto caller = std::this_thread::get_id(); + std::vector order; + bool on_caller = true; + int level = -1; + for_blocks(count, {.threads = 1}, [&](size_t b) { + order.push_back(b); + on_caller = on_caller && std::this_thread::get_id() == caller; + level = omp_get_level(); + }); + BOOST_REQUIRE_EQUAL(order.size(), count); + for (size_t b = 0; b < count; ++b) { + BOOST_TEST(order[b] == b); + } + BOOST_TEST(on_caller); + BOOST_TEST(level == 0); // no OpenMP region was opened +} + +// The serial path runs the body directly, so its first exception ends the loop at once. +BOOST_AUTO_TEST_CASE(openmp_blocks_budget_one_propagates_at_the_failing_block) { + std::vector order; + BOOST_CHECK_EXCEPTION(for_blocks(19, + {.threads = 1}, + [&](size_t b) { + order.push_back(b); + if (b == 5) { + throw std::runtime_error("block 5"); + } + }), + std::runtime_error, + [](const std::runtime_error &e) { return std::string(e.what()) == "block 5"; }); + const std::vector expected{0, 1, 2, 3, 4, 5}; + BOOST_CHECK_EQUAL_COLLECTIONS(order.begin(), order.end(), expected.begin(), expected.end()); +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_nested_invocation_runs_serially_in_order) { + constexpr size_t outer = 4; + constexpr size_t inner = 5; + std::vector visits(outer * inner, 0); + std::vector> order(outer); + std::vector outer_level(outer, -1); + std::vector inner_level(outer * inner, -1); + for_blocks(outer, {.threads = 2}, [&](size_t o) { + outer_level[o] = omp_get_level(); + for_blocks(inner, {.threads = 3}, [&](size_t i) { + ++visits[o * inner + i]; + order[o].push_back(i); // one writer: the inner loop runs on outer block o's worker + inner_level[o * inner + i] = omp_get_level(); + }); + }); + for (size_t o = 0; o < outer; ++o) { + BOOST_REQUIRE_EQUAL(order[o].size(), inner); + for (size_t i = 0; i < inner; ++i) { + BOOST_TEST(visits[o * inner + i] == 1); + BOOST_TEST(order[o][i] == i); + // No nested region: the inner body runs at the outer body's nesting level. + BOOST_TEST(inner_level[o * inner + i] == outer_level[o]); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_nested_invocation_propagates_at_the_failing_block) { + std::vector> order(2); + BOOST_CHECK_THROW(for_blocks(2, + {.threads = 2}, + [&](size_t o) { + for_blocks(6, {.threads = 3}, [&](size_t i) { + order[o].push_back(i); + if (i == 2) { + throw std::runtime_error("inner"); + } + }); + }), + std::runtime_error); + // Each outer block that ran stopped its serial inner loop at the failing block. + for (const auto &seen : order) { + BOOST_TEST(seen.size() <= 3U); + for (size_t i = 0; i < seen.size(); ++i) { + BOOST_TEST(seen[i] == i); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_invalid_budget_is_rejected_before_work) { + for (const int threads : {0, -1, INT_MIN}) { + for (const size_t count : {size_t{0}, size_t{5}}) { + int calls = 0; + BOOST_CHECK_THROW(for_blocks(count, {.threads = threads}, [&](size_t) { ++calls; }), std::invalid_argument); + BOOST_TEST(calls == 0); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_unrepresentable_count_is_rejected_before_work) { + const auto first_unrepresentable = static_cast(std::numeric_limits::max()) + 1U; + for (const size_t count : {first_unrepresentable, std::numeric_limits::max()}) { + for (const int threads : {1, 3}) { + int calls = 0; + BOOST_CHECK_THROW(for_blocks(count, {.threads = threads}, [&](size_t) { ++calls; }), std::length_error); + BOOST_TEST(calls == 0); + } + } +} + +BOOST_AUTO_TEST_CASE(openmp_blocks_leave_runtime_settings_unchanged) { + const auto before = runtime_settings(); + std::vector sink(64, 0); + for_blocks(sink.size(), {.threads = 1}, [&](size_t b) { sink[b] = 1; }); + for_blocks(sink.size(), {.threads = 3}, [&](size_t b) { sink[b] = 2; }); + for_blocks(sink.size(), {.threads = 1000}, [&](size_t b) { sink[b] = 3; }); + for_blocks(4, {.threads = 2}, [&](size_t o) { + for_blocks(16, {.threads = 4}, [&](size_t i) { sink[o * 16 + i] = 4; }); + }); + BOOST_CHECK_THROW(for_blocks(8, {.threads = 3}, [](size_t) { throw std::runtime_error("fail"); }), + std::runtime_error); + BOOST_CHECK_THROW(for_blocks(8, {.threads = 0}, [](size_t) {}), std::invalid_argument); + const auto after = runtime_settings(); + BOOST_TEST((before == after)); + BOOST_TEST(after.max_threads == before.max_threads); + BOOST_TEST(after.dynamic == before.dynamic); + BOOST_TEST(after.max_active_levels == before.max_active_levels); + BOOST_TEST(std::count(sink.begin(), sink.end(), 4) == 64); +} + +// A worker failure is caught inside the region, every worker joins, and only then does the caller +// see the exception. After a failure no particular subset of blocks is required to have run. +BOOST_AUTO_TEST_CASE(openmp_blocks_worker_exception_is_rethrown_after_join) { + constexpr size_t count = 19; + std::atomic in_flight{0}; + std::vector finished(count, 0); + // Block 7 opens the second worker's static chunk (19 blocks over 3 workers: 7/6/6), so the + // exception is raised while the other workers are still sleeping inside their bodies. + constexpr size_t failing = 7; + int in_flight_at_catch = -1; + try { + for_blocks(count, {.threads = 3}, [&](size_t b) { + in_flight.fetch_add(1); + if (b == failing) { + in_flight.fetch_sub(1); + throw std::runtime_error("block 7"); + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + finished[b] = 1; + in_flight.fetch_sub(1); + }); + BOOST_FAIL("for_blocks returned normally after a worker threw"); + } + catch (const std::runtime_error &e) { + in_flight_at_catch = in_flight.load(); + BOOST_TEST(std::string(e.what()) == "block 7"); + } + BOOST_TEST(in_flight_at_catch == 0); + BOOST_TEST(finished[failing] == 0); +} + +// Every block throws; worker 0 always receives block 0 first under a static schedule, whatever the +// actual team size, so the first populated slot is worker 0's and it holds block 0's exception. +BOOST_AUTO_TEST_CASE(openmp_blocks_first_populated_worker_slot_is_rethrown) { + for (const int threads : {2, 3}) { + BOOST_CHECK_EXCEPTION( + for_blocks(19, {.threads = threads}, [](size_t b) { throw std::runtime_error(std::to_string(b)); }), + std::runtime_error, + [](const std::runtime_error &e) { return std::string(e.what()) == "0"; }); + } +} + +// Test-only worker observation: each block records its actual OpenMP worker ID. A request for four +// workers is not evidence of participation; the recorded IDs are. +BOOST_AUTO_TEST_CASE(openmp_blocks_multiple_workers_participate, + *boost::unit_test::precondition(runtime_offers_two_workers)) { + constexpr size_t count = 64; + constexpr int threads = 4; + std::vector worker(count, -1); + std::vector team(count, 0); + for_blocks(count, {.threads = threads}, [&](size_t b) { + worker[b] = omp_get_thread_num(); + team[b] = omp_get_num_threads(); + }); + const auto actual = *std::max_element(team.begin(), team.end()); + const std::set distinct(worker.begin(), worker.end()); + BOOST_TEST(actual >= 1); + BOOST_TEST(actual <= threads); + // A static schedule with count >= team hands every actual worker at least one block. + BOOST_TEST(distinct.size() == static_cast(actual)); + BOOST_TEST(*distinct.begin() == 0); + BOOST_TEST(*distinct.rbegin() == actual - 1); + BOOST_WARN_MESSAGE( + actual >= 2, + "multi-worker participation NOT demonstrated: the runtime gave this region " << actual << " worker(s)"); + BOOST_TEST_MESSAGE("openmp_blocks_multiple_workers_participate: observed " + << distinct.size() << " distinct workers of " << threads << " requested"); +} diff --git a/cpp/tests/operator_index_tests.cpp b/cpp/tests/operator_index_tests.cpp index 73a08165d..1650435c1 100644 --- a/cpp/tests/operator_index_tests.cpp +++ b/cpp/tests/operator_index_tests.cpp @@ -14,9 +14,11 @@ #include +#include #include #include #include +#include #include #include @@ -24,6 +26,7 @@ #include "monoprop/algebra/MajoranaAlgebra.h" // indices_to_bitset #include "monoprop/core/Monomial.h" #include "monoprop/detail/operator/OperatorIndex.h" +#include "monoprop/detail/parallel/Options.h" using namespace monoprop; using namespace monoprop::detail; @@ -222,3 +225,95 @@ BOOST_AUTO_TEST_CASE(find_batch_on_empty_store_is_all_missing) { BOOST_TEST(out[i] == Store::kNotFound); } } + +// Shared invariant 14, pinned: bulk insertion does not deduplicate. Two rows holding the same key are both +// indexed, and a lookup returns the first one inserted (it sits earlier on the probe chain). Production never +// relies on this -- query keys are distinct by construction -- but a replacement index must either keep it or +// change it as an explicit, reviewed contract change. The options overload is the Task 11 publication seam; +// the packed index ignores it and publishes serially, identically. +BOOST_AUTO_TEST_CASE(bulk_insert_duplicate_keys_are_both_indexed) { + const auto dup = bs({2, 7, 9}); + const auto other = bs({1, 4}); + const std::array keys{dup, other, dup}; + for (const int threads : {1, 4}) { + for (const bool hashed : {false, true}) { + Store s; + const size_t base = s.grow_rows_geometric(keys.size()); + for (size_t k = 0; k < keys.size(); ++k) { + s.set(base + k, keys[k]); + } + if (hashed) { + s.bulk_insert_hashed( + keys.size(), + base, + [&](size_t k) { + std::vector pos; + s.for_each_position(base + k, [&](size_t b) { pos.push_back(static_cast(b)); }); + return Store::fold_hash_positions(pos); + }, + parallel::Options{.threads = threads}); + } + else { + s.bulk_insert(keys.size(), base, [&](size_t k) -> const MSet & { return keys[k]; }); + } + BOOST_TEST_CONTEXT("threads=" << threads << " hashed=" << hashed) { + BOOST_TEST(s.size() == 3u); + size_t indexed = 0; + std::vector rows_for_dup; + s.for_each([&](const MSet &key, size_t row) { + ++indexed; + if (key == dup) { + rows_for_dup.push_back(row); + } + }); + BOOST_TEST(indexed == 3u); + std::ranges::sort(rows_for_dup); + BOOST_TEST((rows_for_dup == std::vector{0, 2})); + BOOST_REQUIRE(s.find(dup).has_value()); + BOOST_TEST(*s.find(dup) == 0u); + BOOST_TEST(*s.find(other) == 1u); + } + } + } +} + +// The frozen probe slices one shared position buffer into blocks: absolute offsets, sliced counts and outputs. +// A sliced call must give exactly what the whole call gives for the same queries, hashes included. +BOOST_AUTO_TEST_CASE(find_batch_positions_on_sliced_spans_matches_the_whole_call) { + Store s; + for (size_t i = 0; i < 120; ++i) { + const auto key = bs({i / 60, 4 + (i % 60)}); + s.push_back(key); + s.emplace(key, i); + } + std::vector flat{63, 63, 63}; // unreferenced prefix: offsets are absolute + std::vector off; + std::vector k_of; + for (size_t i = 0; i < 200; ++i) { + const auto key = (i % 3 == 0) ? bs({0, 1, 2 + (i % 20)}) : bs({(i / 60) % 2, 4 + (i % 60)}); + off.push_back(flat.size()); + uint32_t k = 0; + for (size_t b = key.find_first(); b < key.size(); b = key.find_next(b)) { + flat.push_back(static_cast(b)); + ++k; + } + k_of.push_back(k); + } + std::vector whole(off.size()); + std::vector whole_hash(off.size()); + s.find_batch_positions(flat, off, k_of, whole, whole_hash); + std::vector sliced(off.size(), 7); + std::vector sliced_hash(off.size(), 7); + for (size_t lo = 0; lo < off.size(); lo += 37) { + const size_t m = std::min(37, off.size() - lo); + s.find_batch_positions(flat, + std::span(off).subspan(lo, m), + std::span(k_of).subspan(lo, m), + std::span(sliced).subspan(lo, m), + std::span(sliced_hash).subspan(lo, m)); + } + BOOST_TEST(sliced == whole, boost::test_tools::per_element()); + BOOST_TEST(sliced_hash == whole_hash, boost::test_tools::per_element()); + BOOST_TEST(std::ranges::count(whole, Store::kNotFound) > 0); + BOOST_TEST(std::ranges::count_if(whole, [](size_t v) { return v != Store::kNotFound; }) > 0); +} diff --git a/cpp/tests/pare_graph_tests.cpp b/cpp/tests/pare_graph_tests.cpp index f92e9bc8b..7a2448e0f 100644 --- a/cpp/tests/pare_graph_tests.cpp +++ b/cpp/tests/pare_graph_tests.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "TestUtilities.h" @@ -26,6 +27,7 @@ #include "monoprop/detail/evolution/CosineRecompute.h" #include "monoprop/detail/graph/MPGraphLayers.h" #include "monoprop/detail/operator/InvertedIndex.h" +#include "monoprop/detail/pare/PareGraph.h" using namespace test_utils; using namespace monoprop; @@ -142,6 +144,54 @@ BOOST_AUTO_TEST_CASE(pare_graph_emits_expected_layer_kinds) { BOOST_TEST(pared.get_layer(marked_layer).pruned_cos() != static_cast(nullptr)); } +// The owner sweep hands its provider one buffer for every layer, and an in-place fold replaces whatever that buffer +// held: the reused-buffer path folds exactly what a fresh mask does, and every stored set is allocated at its size. +BOOST_AUTO_TEST_CASE(pare_graph_owner_folds_into_one_reused_buffer) { + const auto data = load_case_data("random_exact.msgpack"); + SimulatorConfig cfg{.comm = MPI_COMM_SELF}; + auto sim = build_simulator(data, cfg); + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const auto &graph = sim.graph(); + const auto &inverted_index = sim.mp_op().inverted_index(); + const size_t rows = sim.mp_op().size(); + BOOST_REQUIRE(graph.layers() > 1); + + // An in-place fold into a buffer holding another layer's blocks equals the fresh fold. + for (size_t i = 0; i < graph.layers(); ++i) { + CosMask reused{.blocks = {{0, ~uint64_t{0}}, {64, 1}}, .total_count = 65}; + monoprop::detail::full_cos_mask_into(inverted_index, + graph.get_layer_traversal(i), + monoprop::Basis::Majorana, + reused); + const CosMask fresh = recompute_cos(inverted_index, graph.get_layer_traversal(i)); + BOOST_TEST(reused.blocks == fresh.blocks, "layer " << i); + BOOST_TEST(reused.total_count == fresh.total_count, "layer " << i); + } + + // Keep only row 0, one past the index space marked in layer 0's set: the sweep prunes, so sets get stored. + std::set buffers; + const auto provider = [&](size_t i, CosMask &out) { + buffers.insert(&out); + monoprop::detail::full_cos_mask_into(inverted_index, + graph.get_layer_traversal(i), + monoprop::Basis::Majorana, + out); + out.blocks.emplace_back(((rows >> 6) + 1) << 6, uint64_t{1}); + ++out.total_count; + }; + const auto pared = + monoprop::detail::pare_graph_owner(graph, VecZ{0}, rows + 128, /*schrodinger=*/false, 0, provider); + BOOST_TEST(buffers.size() == 1U); + size_t stored = 0; + for (size_t i = 0; i < pared.layers(); ++i) { + if (const CosMask *pruned = pared.get_layer(i).pruned_cos(); pruned != nullptr) { + ++stored; + BOOST_TEST(pruned->blocks.capacity() == pruned->blocks.size(), "layer " << i); + } + } + BOOST_TEST(stored == pared.layers()); +} + BOOST_AUTO_TEST_CASE(pare_graph_energy_matches_unpared) { const auto data = load_case_data("random_exact.msgpack"); SimulatorConfig cfg{.comm = MPI_COMM_SELF}; diff --git a/cpp/tests/partition_equivalence_tests.cpp b/cpp/tests/partition_equivalence_tests.cpp deleted file mode 100644 index fd7f40cfa..000000000 --- a/cpp/tests/partition_equivalence_tests.cpp +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "PauliTestOracle.h" -#include "TestUtilities.h" -#include "monoprop/MonomialPropagator.h" -#include "monoprop/algebra/MajoranaAlgebra.h" -#include "monoprop/detail/mpi/MPICompat.h" - -// Intra-process partition equivalence: a propagator with partitions>1 (S partition propagators over an in-process -// ShmComm) must match the ordinary single-partition propagator within fp accumulation tolerance. -// Oracle: the S=1 run. Pure std::thread, so this runs in the serial build with no mpiexec. - -namespace { - -using namespace monoprop; -using namespace test_utils; -using pauli_oracle::slots_of_string; - -constexpr size_t kNumModes = 8; -constexpr unsigned int kCutoff = 4; - -auto majorana_sim(const CaseData &data, size_t partitions, std::optional lower_atol = std::nullopt) - -> MonomialPropagator { - return MonomialPropagator(data.hamiltonian, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - lower_atol, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - partitions); -} - -// Schrodinger's initial rows are the GLOBAL paired basis, hash-partitioned across the partitions -- -// the only construction input that is not already slot-local, and so the only one a mis-sharded seed -// can break. At kNumModes=8 with sc=kCutoff+2 that is sum C(8,0..3) = 93 terms. -auto majorana_schrodinger_sim(const CaseData &data, size_t partitions) -> MonomialPropagator { - return MonomialPropagator(data.hamiltonian, - kCutoff, - data.initial_state, - /*schrodinger_cutoff=*/kCutoff + 2, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - partitions); -} - -BOOST_AUTO_TEST_CASE(partition_majorana_energy_matches_across_partition_counts) { - const auto data = load_case_data("random_exact.msgpack"); - auto ref = majorana_sim(data, 1); - ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const double e1 = ref.expectation_value(data.parameters); - - for (const size_t S : {size_t{2}, size_t{4}}) { - auto sim = majorana_sim(data, S); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const double e = sim.expectation_value(data.parameters); - BOOST_TEST_CONTEXT("partitions=" << S << " e=" << e << " ref=" << e1) { - BOOST_TEST(near(e1, e)); - } - // Aggregated operator size is partition-count invariant (terms are hash-partitioned, no overlap). - BOOST_CHECK_EQUAL(ref.size(), sim.size()); - } -} - -BOOST_AUTO_TEST_CASE(partition_majorana_gradient_matches_across_partition_counts) { - const auto data = load_case_data("random_exact.msgpack"); - auto ref = majorana_sim(data, 1); - ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const auto g1 = ref.expectation_value_and_gradient(data.parameters).second; - - for (const size_t S : {size_t{2}, size_t{4}}) { - auto sim = majorana_sim(data, S); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const auto g = sim.expectation_value_and_gradient(data.parameters).second; - BOOST_REQUIRE_EQUAL(g1.size(), g.size()); - for (size_t i = 0; i < g1.size(); ++i) { - BOOST_TEST_CONTEXT("partitions=" << S << " grad idx=" << i) { - BOOST_TEST(near(g1[i], g[i])); - } - } - } -} - -BOOST_AUTO_TEST_CASE(partition_majorana_propagate_then_expectation_matches) { - const auto data = load_case_data("random_exact.msgpack"); - auto run = [&](size_t S) { - auto sim = majorana_sim(data, S); - sim.propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); - return std::pair{sim.expectation_value({}), sim.size()}; - }; - const auto [e1, n1] = run(1); - for (const size_t S : {size_t{2}, size_t{4}}) { - const auto [e, n] = run(S); - BOOST_TEST_CONTEXT("partitions=" << S) { - BOOST_TEST(near(e1, e)); - BOOST_CHECK_EQUAL(n1, n); - } - } -} - -// Two independent S=4 runs are bit-identical: ShmComm sums in ascending rank order and each partition is -// deterministic, so a given partition count has no run-to-run jitter. -BOOST_AUTO_TEST_CASE(partition_energy_is_deterministic) { - const auto data = load_case_data("random_exact.msgpack"); - auto energy_s4 = [&] { - auto sim = majorana_sim(data, 4); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - return sim.expectation_value(data.parameters); - }; - BOOST_CHECK_EQUAL(energy_s4(), energy_s4()); -} - -// contract_partially() hands back raw coefficients positioned by the owning partition's own indexing, -// so a facade's array is the per-partition blocks concatenated in partition order. What IS invariant is -// the multiset: the partitions are disjoint and cover every term. Sorting both sides is the only -// comparison the API's contract supports — see the note on contract_partially(). -BOOST_AUTO_TEST_CASE(partition_contract_partially_matches_as_a_multiset) { - const auto data = load_case_data("random_exact.msgpack"); - auto sorted_coeffs = [&](size_t S) { - auto sim = majorana_sim(data, S); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - auto coeffs = sim.contract_partially(data.parameters, /*inplace=*/false); - std::ranges::sort(coeffs); - return coeffs; - }; - const auto ref = sorted_coeffs(1); - for (const size_t S : {size_t{2}, size_t{4}}) { - const auto coeffs = sorted_coeffs(S); - BOOST_REQUIRE_EQUAL(ref.size(), coeffs.size()); - for (size_t i = 0; i < ref.size(); ++i) { - BOOST_TEST_CONTEXT("partitions=" << S << " sorted idx=" << i) { - BOOST_TEST(near(ref[i], coeffs[i])); - } - } - } -} - -// The raw per-partition accessors have no facade reading: the facade's own graph_/mp_op_ are never -// populated, so returning them would hand a C++ consumer empty state that looks valid. -BOOST_AUTO_TEST_CASE(partition_raw_accessors_reject_a_facade) { - const auto data = load_case_data("random_exact.msgpack"); - auto sim = majorana_sim(data, 4); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - BOOST_CHECK_THROW(static_cast(sim.graph()), std::runtime_error); - BOOST_CHECK_THROW(static_cast(sim.mp_op()), std::runtime_error); - BOOST_CHECK_THROW(static_cast(sim.indexing()), std::runtime_error); - BOOST_CHECK_THROW(static_cast(sim.graph_data()), std::runtime_error); - - auto solo = majorana_sim(data, 1); - solo.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - BOOST_CHECK_GT(solo.graph().layers(), 0U); - BOOST_CHECK_EQUAL(solo.graph_data().size(), solo.graph_layers()); -} - -// A facade owns no operator, so a setter that stopped there would leave every partition on its old -// configuration and the run would silently ignore the new value. The cutoff truncates structurally -// during the graph build, so it shows up in the aggregated term count. LiH is used rather than the -// random_exact fixture, which is small enough that no setting truncates anything. The tightened -// oracle must itself differ from the wide run, else the last assertion would hold vacuously. -BOOST_AUTO_TEST_CASE(partition_setters_reach_every_partition) { - constexpr size_t kLihModes = LihFixture::n_modes; - const auto data = load_case_data("lih_fermionic_spin_exact.msgpack"); - - auto build = [&](unsigned int cutoff, unsigned int updated_cutoff) { - MonomialPropagator sim(data.hamiltonian, - cutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kLihModes, - Basis::Majorana, - /*partitions=*/4); - if (cutoff != updated_cutoff) { - sim.update_cutoff(updated_cutoff); - BOOST_CHECK_EQUAL(sim.cutoff(), updated_cutoff); - } - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - return sim.size(); - }; - - const size_t n_wide = build(2 * kLihModes, 2 * kLihModes); - const size_t n_tight = build(4, 4); - BOOST_REQUIRE_NE(n_wide, n_tight); - // Constructed wide, then tightened: only the partitions' own cutoff can produce the tight count. - BOOST_CHECK_EQUAL(n_tight, build(2 * kLihModes, 4)); -} - -BOOST_AUTO_TEST_CASE(partition_deep_copy_matches) { - const auto data = load_case_data("random_exact.msgpack"); - auto sim = majorana_sim(data, 4); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const double e = sim.expectation_value(data.parameters); - - MonomialPropagator copy(sim); // clones the partition group (fresh threads + ShmComm) - const double e_copy = copy.expectation_value(data.parameters); - BOOST_CHECK_EQUAL(e, e_copy); - BOOST_CHECK_EQUAL(sim.size(), copy.size()); -} - -constexpr size_t kNq = 6; - -auto pauli_sim(const std::map &obs, size_t partitions) -> MonomialPropagator { - OperatorDict init; - for (const auto &[p, c] : obs) { - init[slots_of_string(p)] = std::complex(c, 0.0); - } - return MonomialPropagator(init, - /*cutoff=*/kNq, - /*initial_state=*/{}, - std::nullopt, - MPI_COMM_SELF, - /*lower_atol=*/1e-12, - std::nullopt, - CutoffType::Support, - std::nullopt, - kNq, - Basis::Pauli, - partitions); -} - -auto run_pauli_energy(size_t partitions) -> std::pair { - std::map obs; - obs["ZIIIII"] = 1.0; - obs["IIZZII"] = 0.5; - auto sim = pauli_sim(obs, partitions); - - std::vector gens; - VecZ pmap; - VecD gcoeffs; - size_t p = 0; - for (size_t q = 0; q < kNq; ++q) { - std::string s(kNq, 'I'); - s[q] = 'X'; - gens.push_back(slots_of_string(s)); - pmap.push_back(p++); - gcoeffs.push_back(1.0); - } - for (size_t q = 0; q + 1 < kNq; ++q) { - std::string s(kNq, 'I'); - s[q] = 'Z'; - s[q + 1] = 'Z'; - gens.push_back(slots_of_string(s)); - pmap.push_back(p++); - gcoeffs.push_back(1.0); - } - VecD params(p, 0.3); - sim.propagate(gens, pmap, gcoeffs, params); - return {sim.expectation_value({}), sim.size()}; -} - -BOOST_AUTO_TEST_CASE(partition_pauli_energy_matches_across_partition_counts) { - const auto [e1, n1] = run_pauli_energy(1); - for (const size_t S : {size_t{2}, size_t{4}}) { - const auto [e, n] = run_pauli_energy(S); - BOOST_TEST_CONTEXT("pauli partitions=" << S << " e=" << e << " ref=" << e1) { - BOOST_TEST(near(e1, e)); - BOOST_CHECK_EQUAL(n1, n); - } - } -} - -} // namespace - -// A partition factory that throws must surface the exception, not std::terminate: the ctor starts the -// master threads before building the partitions on them (first-touch locality), so the unwind has to join -// already-started threads. Every MonomialPropagator ctor validation reaches this path. -BOOST_AUTO_TEST_CASE(partition_factory_exception_propagates_without_terminate) { - const auto data = load_case_data("random_exact.msgpack"); - // logical_num_modes = 0 is rejected by each partition's own constructor, on its own master thread. - BOOST_CHECK_THROW(MonomialPropagator(data.hamiltonian, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - /*logical_num_modes=*/0, - Basis::Majorana, - /*partitions=*/4), - std::runtime_error); - - // An out-of-range operator index takes the same path, and the group stays usable afterwards. - auto bad_op = data.hamiltonian; - bad_op[VecZ{2 * kNumModes}] = std::complex(1.0, 0.0); - BOOST_CHECK_THROW(MonomialPropagator(bad_op, - kCutoff, - data.initial_state, - std::nullopt, - MPI_COMM_SELF, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - kNumModes, - Basis::Majorana, - /*partitions=*/4), - std::runtime_error); - BOOST_CHECK_NO_THROW(majorana_sim(data, 4)); -} - -// Reads the constructor's output with no graph in between: the aggregate initial basis must be every -// paired term exactly once at any partition count. This is what a seed that keeps the wrong share, or -// that generates the global set per partition and then over- or under-filters, gets wrong. -BOOST_AUTO_TEST_CASE(partition_schrodinger_initial_basis_is_partition_count_invariant) { - const auto data = load_case_data("random_exact.msgpack"); - constexpr unsigned int kSchrodingerCutoff = kCutoff + 2; - const size_t expected = paired_op_size(kSchrodingerCutoff / 2 + kSchrodingerCutoff % 2, kNumModes); - - for (const size_t S : {size_t{1}, size_t{2}, size_t{4}}) { - BOOST_TEST_CONTEXT("partitions=" << S) { - BOOST_CHECK_EQUAL(majorana_schrodinger_sim(data, S).size(), expected); - } - } -} - -BOOST_AUTO_TEST_CASE(partition_schrodinger_energy_matches_across_partition_counts) { - const auto data = load_case_data("random_exact.msgpack"); - auto ref = majorana_schrodinger_sim(data, 1); - ref.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const double e1 = ref.expectation_value(data.parameters); - - for (const size_t S : {size_t{2}, size_t{4}}) { - auto sim = majorana_schrodinger_sim(data, S); - sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); - const double e = sim.expectation_value(data.parameters); - BOOST_TEST_CONTEXT("partitions=" << S << " e=" << e << " ref=" << e1) { - BOOST_TEST(near(e1, e)); - } - BOOST_CHECK_EQUAL(ref.size(), sim.size()); - } -} diff --git a/cpp/tests/partition_group_clone_tests.cpp b/cpp/tests/partition_group_clone_tests.cpp deleted file mode 100644 index 64de3e0ff..000000000 --- a/cpp/tests/partition_group_clone_tests.cpp +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include - -#include "TestUtilities.h" -#include "monoprop/MonomialPropagator.h" -#include "monoprop/detail/mpi/MPICompat.h" - -// A derived MonomialPropagator must keep its own type through partitioning: both its partition -// children (built via the injected child_factory) and clones of a partitioned facade (built via the -// virtual clone_() hook) have to come out as the derived type, not the base. - -namespace { - -using namespace monoprop; -using namespace test_utils; - -constexpr size_t kNumModes = 8; -constexpr unsigned int kCutoff = 4; - -// A toy subclass exercising both extension points: `child_factory` builds more of itself, and -// `clone_()` overrides the base's default (which would otherwise slice a copy down to Base). -template -class DerivedPropagator : public MonomialPropagator { -public: - using Base = MonomialPropagator; - - DerivedPropagator(const OperatorDict &initial_operator, - unsigned int cutoff, - const VecZ &initial_state, - mpi::Comm comm, - size_t partitions) - : Base(initial_operator, - cutoff, - initial_state, - std::nullopt, - comm, - std::nullopt, - std::nullopt, - CutoffType::Length, - std::nullopt, - NumModes, - Basis::Majorana, - partitions, - typename Base::PartitionChildFactory{[=](mpi::Comm partition_comm) -> std::unique_ptr { - return std::make_unique(initial_operator, - cutoff, - initial_state, - partition_comm, - /*partitions=*/1); - }}) {} - - DerivedPropagator(const DerivedPropagator &) = default; - - // Test-only passthrough: is_partition_facade() is protected on Base. - auto is_facade() const -> bool { return this->is_partition_facade(); } - - auto children_are_all_derived() -> bool { - const auto flags = - this->map_partitions_([](Base &p) { return dynamic_cast(&p) != nullptr; }); - return std::ranges::all_of(flags, [](bool ok) { return ok; }); - } - - auto indexed_children_are_all_derived() -> bool { - const auto flags = this->map_partitions_indexed_( - [](int, Base &p) { return dynamic_cast(&p) != nullptr; }); - return std::ranges::all_of(flags, [](bool ok) { return ok; }); - } - - // Makes partition 1 throw while the others are stuck in a collective, so they can leave only via - // ShmCommPoisoned. Partition 1 throws after a first, completed collective; the others then wait in a - // second one that partition 1 never joins. - auto throw_with_poisoned_peers() -> void { - (void)this->map_partitions_([](Base &p) { return static_cast(p).root_or_poisoned_(); }); - } - -protected: - auto clone_() const -> std::unique_ptr override { return std::make_unique(*this); } - -private: - auto root_or_poisoned_() -> int { - (void)mpi::allreduce_sum(1, this->comm_); - if (mpi::rank(this->comm_) == 1) { - throw std::logic_error("root cause"); - } - return mpi::allreduce_sum(1, this->comm_); - } -}; - -} // namespace - -BOOST_AUTO_TEST_CASE(partition_facade_children_are_derived_type) { - const auto data = load_case_data("random_exact.msgpack"); - DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); - - BOOST_TEST(sim.is_facade()); - BOOST_TEST(sim.children_are_all_derived()); - BOOST_TEST(sim.indexed_children_are_all_derived()); -} - -BOOST_AUTO_TEST_CASE(partition_facade_copy_children_are_derived_type) { - const auto data = load_case_data("random_exact.msgpack"); - DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/4); - - DerivedPropagator copy(sim); // exercises PartitionGroup's copy ctor -> clone_() - BOOST_TEST(copy.is_facade()); - BOOST_TEST(copy.children_are_all_derived()); -} - -// The facade rethrows the exception that caused a failure, not the ShmCommPoisoned it triggered elsewhere, -// even when the poisoned partition has the lower rank. Here partition 0 gets ShmCommPoisoned and partition 1 -// the logic_error that caused it. -BOOST_AUTO_TEST_CASE(partition_facade_rethrows_root_cause_over_poison) { - const auto data = load_case_data("random_exact.msgpack"); - DerivedPropagator sim(data.hamiltonian, kCutoff, data.initial_state, MPI_COMM_SELF, /*partitions=*/2); - - BOOST_CHECK_THROW(sim.throw_with_poisoned_peers(), std::logic_error); -} diff --git a/cpp/tests/pauli_build_layer_tests.cpp b/cpp/tests/pauli_build_layer_tests.cpp index af069fcd7..5f8575068 100644 --- a/cpp/tests/pauli_build_layer_tests.cpp +++ b/cpp/tests/pauli_build_layer_tests.cpp @@ -26,6 +26,7 @@ #include #include "PauliTestOracle.h" +#include "ScanTestSupport.h" #include "monoprop/MonomialPropagator.h" #include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/algebra/PauliAlgebra.h" @@ -452,3 +453,56 @@ BOOST_AUTO_TEST_CASE(pauli_build_layer_replay_fold_consumers) { BOOST_TEST(e_contract == e_graph, boost::test_tools::tolerance(1e-9)); BOOST_TEST(e_jw == e_graph, boost::test_tools::tolerance(1e-9)); } + +// Pauli scans fold J(G) = pair_swap(G) but split on the untransformed G's lowest bit. Threaded ranges at +// budgets 2-4 must reproduce budget 1 exactly, including a J(G) that lands on empty columns while the +// pivot does not, and dense/sparse splitmix and linear windows. +BOOST_AUTO_TEST_CASE(openmp_scan_pauli_ranges_match_serial) { + using scan_test::Expect; + using scan_test::Scenario; + const auto pauli = Basis::Pauli; + // {1, 20}: J(G) = {0, 21}, so the fold reads empty column 0 while the pivot is bit 1 (also empty). + // {12, 3}: pivot 3 is empty but J(G) = {13, 2} reads dense column 13 -- the pivot is not a fold column. + const std::vector> single{ + {{.label = "dense J(G)", .basis = pauli, .gen = scan_test::mono_of_bits({14, 21, 33, 46})}, Expect::output}, + {{.label = "odd-weight G", .basis = pauli, .gen = scan_test::mono_of_bits({13, 28, 51})}, Expect::output}, + {{.label = "sparse pivot", .basis = pauli, .gen = scan_test::mono_of_bits({6, 40})}, Expect::output}, + {{.label = "J(G) on an empty column", .basis = pauli, .gen = scan_test::mono_of_bits({1, 20})}, Expect::output}, + {{.label = "pivot outside J(G)", .basis = pauli, .gen = scan_test::mono_of_bits({3, 12})}, Expect::output}, + {{.label = "empty fold columns", .basis = pauli, .gen = scan_test::mono_of_bits({0, 1})}, Expect::not_scanned}, + {{.label = "capture, cap 5", + .basis = pauli, + .gen = scan_test::mono_of_bits({14, 21}), + .cap = 5, + .capture = true}, + Expect::output}, + {{.label = "fused cos sweep", + .basis = pauli, + .gen = scan_test::mono_of_bits({13, 28, 51}), + .capture = true, + .fused_scale = true}, + Expect::output}, + {{.label = "support cutoff, lower atol", + .basis = pauli, + .gen = scan_test::mono_of_bits({14, 21, 33, 46}), + .cutoff = detail::SupportCutoff<32>{4}, + .atol = scan_test::Atol::lower_equal}, + Expect::output}, + }; + const scan_test::Geometry solo{"one rank", 1, 1, false, 0}; + for (const auto &[s, e] : single) { + scan_test::check_threaded_matches_serial(solo, s, e); + } + for (const scan_test::Geometry g : + {scan_test::Geometry{"linear R=4", 4, 1, true, 1}, scan_test::Geometry{"splitmix R=3", 3, 1, false, 2}}) { + const auto router = scan_test::router_of(g); + const auto gen = g.linear ? scan_test::find_generator(router, 12, 30, 3, false, 99) : single[1].first.gen; + scan_test::check_threaded_matches_serial(g, + {.label = "odd-weight G", .basis = pauli, .gen = gen}, + Expect::output); + scan_test::check_threaded_matches_serial( + g, + {.label = "fused cos sweep", .basis = pauli, .gen = gen, .capture = true, .fused_scale = true}, + Expect::output); + } +} diff --git a/cpp/tests/run_mpi_failure_scenario.cmake b/cpp/tests/run_mpi_failure_scenario.cmake new file mode 100644 index 000000000..5658232a0 --- /dev/null +++ b/cpp/tests/run_mpi_failure_scenario.cmake @@ -0,0 +1,101 @@ +# Runs one mpi_failure_driver scenario under the MPI launcher and checks how the job ends. +# +# Inputs (-D): MPIEXEC, NUMPROC_FLAG, PREFLAGS (MPIEXEC_PREFLAGS, may be empty), DRIVER, SCENARIO, OPTION +# (may be empty), EXPECT (regex that must appear in the combined output), FORBID (optional regex that must +# not), EXPECT_SUCCESS (optional; ON for an acceptance scenario), TIMEOUT_SECONDS (default 30). +# +# A failure scenario passes on a nonzero exit before the timeout with EXPECT in the output; an acceptance +# scenario (EXPECT_SUCCESS) passes on exit 0 before the timeout with EXPECT in the output. A timeout is a +# failure, never a pass: it means some rank was left blocked. The launcher must end the peers of a rank that +# exits abnormally (Open MPI's default); the wrong-thread case depends on that policy. "NOT EXERCISED" in the +# output (the MPI library's provided thread level does not exercise the route) is reported as a skip. + +if(NOT DEFINED TIMEOUT_SECONDS) + set(TIMEOUT_SECONDS 30) +endif() + +set( + _command + "${MPIEXEC}" + "${NUMPROC_FLAG}" + 2 + ${PREFLAGS} + "${DRIVER}" + "${SCENARIO}" +) +if(NOT "${OPTION}" STREQUAL "") + list(APPEND _command "${OPTION}") +endif() + +string(TIMESTAMP _start "%s" UTC) +execute_process( + COMMAND + ${_command} + TIMEOUT ${TIMEOUT_SECONDS} + RESULT_VARIABLE _result + OUTPUT_VARIABLE _stdout + ERROR_VARIABLE _stderr +) +string(TIMESTAMP _stop "%s" UTC) +math(EXPR _elapsed "${_stop} - ${_start}") + +message("command: ${_command}") +message( + "elapsed: ${_elapsed} s (timeout ${TIMEOUT_SECONDS} s); result: ${_result}" +) +message("--- stdout ---\n${_stdout}--- stderr ---\n${_stderr}--------------") + +set(_output "${_stdout}${_stderr}") +if(_result MATCHES "timeout") + message( + FATAL_ERROR + "FAILED: the job did not terminate within ${TIMEOUT_SECONDS} s" + ) +endif() +if(_output MATCHES "NOT EXERCISED") + message( + "NOT EXERCISED: the MPI library supplied more thread support than requested" + ) + if(CMAKE_VERSION VERSION_LESS 3.29) + message(FATAL_ERROR "NOT EXERCISED (CMake < 3.29 cannot report a skip)") + endif() + cmake_language(EXIT 77) +endif() +if(EXPECT_SUCCESS) + if(NOT "${_result}" STREQUAL "0") + message( + FATAL_ERROR + "FAILED: the acceptance scenario exited with ${_result}, not 0" + ) + endif() +elseif("${_result}" STREQUAL "0") + message( + FATAL_ERROR + "FAILED: the job exited 0; the injected failure did not end it" + ) +endif() +if(NOT _output MATCHES "${EXPECT}") + message(FATAL_ERROR "FAILED: expected diagnostic /${EXPECT}/ not found") +endif() +if( + DEFINED + FORBID + AND + NOT + "${FORBID}" + STREQUAL + "" + AND + _output + MATCHES + "${FORBID}" +) + message(FATAL_ERROR "FAILED: forbidden output /${FORBID}/ found") +endif() +if(EXPECT_SUCCESS) + message("PASSED: exit 0 after ${_elapsed} s with the expected completion") +else() + message( + "PASSED: nonzero exit after ${_elapsed} s with the expected diagnostic" + ) +endif() diff --git a/cpp/tests/sharded_api_tests.cpp b/cpp/tests/sharded_api_tests.cpp new file mode 100644 index 000000000..902ab1fa2 --- /dev/null +++ b/cpp/tests/sharded_api_tests.cpp @@ -0,0 +1,225 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * The public C++ surface of MonomialPropagator after the removal of the partition runtime, checked at compile time. + * + * There is one runtime: no partition count, child factory, facade helper, partition-named exception, cached thread + * count or replacement thread/shard argument, and no development selector. Every negative probe is paired with a + * positive control that applies the same probe to a test-local type which does have the member, so a probe that is + * false for an unrelated reason (a typo, an access error, a missing header) fails instead of passing silently. + * Positive controls on the propagator itself pin the constructor and the ordinary extension hooks that must survive. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/MonomialPropagator.h" +#include "monoprop/detail/EnvConfig.h" + +// The removed development selector must not reach the library, the tests or an installed consumer. +#ifdef monoprop_SHARDED_OPENMP_PROTOTYPE +#error "monoprop_SHARDED_OPENMP_PROTOTYPE is defined: the removed runtime selector came back" +#endif + +// The removed runtime's headers are gone from the include path. +#if __has_include("monoprop/detail/partition/PartitionGroup.h") \ + || __has_include("monoprop/detail/partition/CpuTopology.h") +#error "a partition-runtime header is back on the include path" +#endif +#if __has_include("monoprop/detail/mpi/ShmComm.h") \ + || __has_include("monoprop/detail/mpi/HybridComm.h") \ + || __has_include("monoprop/detail/mpi/PartitionBarrier.h") \ + || __has_include("monoprop/detail/mpi/CpuRelax.h") +#error "an in-process communicator or custom-barrier header is back on the include path" +#endif +// Positive control: the include path itself works. +#if !__has_include("monoprop/detail/sharded/Team.h") +#error "the include-path probe cannot see a header that exists" +#endif + +// Declared, never defined, by this test: complete only if the library defines them. +namespace monoprop { +class MultiPartitionUnsupported; +class PartitionCountMismatch; +} // namespace monoprop + +namespace { + +using namespace monoprop; + +using MP = MonomialPropagator<8>; + +template +concept Complete = requires { sizeof(T); }; + +// The process-cached environment settings hold only the routing knobs; the budget is captured, strictly, at +// construction (ThreadBudget.h). +template +concept HasCachedThreadCount = requires(const S &s) { s.num_threads; }; + +template +concept HasCachedRouting = requires(const S &s) { s.routing_mode; }; + +template +concept HasPartitionFactory = requires { typename P::PartitionChildFactory; }; + +template +concept HasLayerData = requires { typename P::LayerData; }; + +// `Target` constructed from the eleven mathematical arguments, followed by `Tail`. +template +constexpr bool kConstructible = std::is_constructible_v, + mpi::Comm, + std::optional, + std::optional, + CutoffType, + std::optional>, + size_t, + Basis, + Tail...>; + +using RemovedFactory = std::function(mpi::Comm)>; + +/* + * Protected members are visible from a derived class only, so the probes are static members of a class derived from + * the one probed. `Self` keeps each requirement dependent: a missing member is then a false requirement, not a hard + * error. + */ +template +struct ProbeOf : Base { + using Base::Base; + + template + static constexpr bool kFacade = requires(Self &p) { p.is_partition_facade(); } || requires(const Self &p) { + p.first_partition_(); + } || requires(Self &p) { p.for_each_partition_([](MP &) {}); } || requires(Self &p) { + p.map_partitions_([](MP &) { return 0; }); + } || requires(Self &p) { p.map_partitions_indexed_([](int, MP &) { return 0; }); } || requires(Self &p) { + p.concat_partitions_([](MP &) { return VecD{}; }); + } || requires(const Self &p) { + p.sum_partitions_([](const MP &) { return size_t{0}; }); + } || requires(const Self &p) { p.fold_partitions_([](const MP &) { return size_t{0}; }, [](size_t &, size_t) {}); }; + + template + static constexpr bool kClone = requires(const Self &p) { + { p.clone_() } -> std::same_as>; + }; + + template + static constexpr bool kInitialOperator = requires(Self &p, const OperatorDict &op) { + { p.apply_initial_operator_(op) } -> std::same_as, VecD>>; + }; +}; + +using Probe = ProbeOf; + +// Positive controls: test-local types that do have what the probes look for. +struct FacadeBase : MP { + using MP::MP; + +protected: + auto is_partition_facade() -> bool { return true; } +}; +using FacadeControl = ProbeOf; + +struct TailControl { + TailControl(const OperatorDict &, + unsigned int, + const VecZ &, + std::optional, + mpi::Comm, + std::optional, + std::optional, + CutoffType, + std::optional>, + size_t, + Basis, + size_t = 0, + RemovedFactory = nullptr) {} + using PartitionChildFactory = RemovedFactory; +}; + +struct SettingsControl { + std::optional num_threads; + std::optional routing_mode; +}; + +// --- constructor --------------------------------------------------------------------------------------------------- + +static_assert(kConstructible, "the eleven-argument constructor must stay"); +static_assert(std::is_constructible_v, + mpi::Comm>, + "the defaulted tail must stay defaulted"); +static_assert(!kConstructible, "no partition count"); +static_assert(!kConstructible, "no child factory"); +static_assert(!kConstructible); +// No replacement thread or shard count of any integral type. +static_assert(!kConstructible); +static_assert(!kConstructible); +static_assert(kConstructible && kConstructible + && kConstructible, + "positive control for the constructor probes"); + +// --- nested types and helpers -------------------------------------------------------------------------------------- + +static_assert(HasLayerData); +static_assert(!HasPartitionFactory, "PartitionChildFactory is removed"); +static_assert(HasPartitionFactory, "positive control for the nested-type probe"); +static_assert(!Probe::kFacade<>, "no partition-facade helper may remain"); +static_assert(FacadeControl::kFacade<>, "positive control for the facade probe"); +static_assert(Probe::kClone<>, "the virtual clone hook stays"); +static_assert(Probe::kInitialOperator<>, "the protected initial-operator hook stays"); +static_assert(std::has_virtual_destructor_v); + +// --- environment --------------------------------------------------------------------------------------------------- + +static_assert(HasCachedRouting); +static_assert(!HasCachedThreadCount, "no cached, permissive thread-count reader"); +static_assert(HasCachedThreadCount, "positive control for the settings probe"); + +// --- exceptions ---------------------------------------------------------------------------------------------------- + +static_assert(!Complete, "the partition-named raw-accessor exception is removed"); +static_assert(!Complete, "the partition-count agreement is removed"); +static_assert(Complete, "positive control: the raw-accessor exception"); +static_assert(std::is_base_of_v); +static_assert(Complete && Complete); + +} // namespace + +// The compile-time checks above are the test; this case keeps the file visible to the runner and reports the surface. +BOOST_AUTO_TEST_CASE(sharded_api_surface_has_no_partition_runtime) { + BOOST_TEST_MESSAGE("sharded OpenMP surface: no partition runtime, selector or replacement thread argument"); + BOOST_TEST(!Probe::kFacade<>); + BOOST_TEST(FacadeControl::kFacade<>); + BOOST_TEST((!kConstructible)); +} diff --git a/cpp/tests/sharded_construction_tests.cpp b/cpp/tests/sharded_construction_tests.cpp new file mode 100644 index 000000000..89d66713a --- /dev/null +++ b/cpp/tests/sharded_construction_tests.cpp @@ -0,0 +1,1911 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// First on purpose: the header must compile on its own. +#include "monoprop/detail/sharded/Construction.h" + +/* + * Direct-buffer construction and graph-free propagation over T shards of one process (detail/sharded/Construction.h). + * The team size T is the captured launch budget; cpp/tests/CMakeLists.txt reruns these cases in fresh processes at + * T = 1, 2 and 4. Cases that need a nonprimary worker are left out of T = 1. + * + * Oracles: + * - The rank-level root at the same launch (MPI_COMM_SELF, the launch's T), built through its public operations. Its + * shards come from the same seam, so it checks the root's wiring of the seam, never the seam's own arithmetic; so: + * - an independent insertion-order reference (plain sets and vectors, no engine code) for rows and IDs; + * - an independent coefficient-map propagator, and the frozen exact energy of tests/data/random_exact.msgpack. + * Bitwise equality with an independently implemented runtime at the same geometry, and the global retained maps + * across T, are checked from separate processes (tests/test_sharded_openmp.py); two runtimes never share a process. + * + * Workers never call BOOST_TEST. Observers write per-shard slots (visits, streams, kernel ranges) and assertions run + * after the team has joined. AllocationProbe.h injects real allocation failures on a chosen owner. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AllocationProbe.h" +#include "KernelTestSupport.h" +#include "PauliTestOracle.h" +#include "PropagatorTestAccess.h" +#include "TestUtilities.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/evolution/layer_build/QueryWire.h" +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/State.h" +#include "monoprop/detail/sharded/Team.h" + +namespace { + +using namespace monoprop; +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; +namespace allocation = test_utils::allocation; +using monoprop::detail::KernelRange; +using W = sharded::ConstructionWork; +using Key = std::vector; + +constexpr size_t kN = 8; +constexpr double kExactAtol = 1e-9; // fused_cos_sweep_tests' exactness tolerance against the frozen energy + +auto team_options() -> parallel::Options { + return parallel::capture_thread_budget(); +} + +auto team_size() -> size_t { + return static_cast(team_options().threads); +} + +auto has_nonprimary_worker(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto threads = parallel::capture_thread_budget().threads; + boost::test_tools::assertion_result result(threads >= 2); + result.message() << "the captured team budget is " << threads << "; this case needs a nonprimary worker"; + return result; +} + +auto has_allocation_probe(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + boost::test_tools::assertion_result result(allocation::available()); + result.message() << "this build's sanitizer runtime owns operator new/delete, so no allocation probe is linked"; + return result; +} + +template +auto key_of(const Monomial &m) -> Key { + Key key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + return key; +} + +auto bits(double v) -> uint64_t { + return std::bit_cast(v); +} + +auto bits(TermIndex v) -> uint64_t { + return v; +} + +// --- Fixtures ------------------------------------------------------------------------------------------------- + +struct Fixture { + std::string name; + OperatorDict op; + VecZ initial_state; + std::optional schrodinger_cutoff = std::nullopt; + Basis basis = Basis::Majorana; + CutoffType cutoff_type = CutoffType::Length; + unsigned int cutoff = 4; + size_t logical = kN; + std::optional lower_atol = std::nullopt; + std::optional upper_atol = std::nullopt; + std::optional> basis_change = std::nullopt; + bool opaque = false; // wrap the cutoff in an opaque closure +}; + +struct Circuit { + std::vector gates; + VecZ mapping; + VecD gen_coeffs; + VecD params; +}; + +struct Case { + Fixture f; + Circuit c; + std::optional k = std::nullopt; // only_rotate_len_k +}; + +auto add_term(OperatorDict &op, Basis basis, const VecZ &indices, double value) -> void { + op[indices] = algebra_decode_coeff(basis, std::complex(value, 0.0), indices_to_bitset(indices)); +} + +// Pairs, a ladder of quartics and long terms (which spill past the packed inline width), plus identity. +auto majorana_operator(size_t logical) -> OperatorDict { + OperatorDict op; + const size_t slots = 2 * logical; + auto value = 0.25; + for (size_t i = 0; i < slots; ++i) { + for (size_t j = i + 1; j < slots; j += 2) { + add_term(op, Basis::Majorana, {i, j}, value); + value += 0.125; + } + } + for (size_t i = 0; i + 3 < slots; ++i) { + add_term(op, Basis::Majorana, {i, i + 1, i + 2, i + 3}, -0.5 - (0.03125 * static_cast(i))); + } + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5}, 0.75); + add_term(op, Basis::Majorana, {1, 3, 5, 7, 9, 11}, -1.25); + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5, 6, 7}, 2.0); + add_term(op, Basis::Majorana, {}, 3.5); + return op; +} + +auto pauli_operator() -> OperatorDict { + OperatorDict op; + auto value = 0.5; + const auto add = [&](const std::string &pauli) { + add_term(op, Basis::Pauli, pauli_oracle::slots_of_string(pauli), value); + value += 0.25; + }; + for (size_t q = 0; q < kN; ++q) { + for (const char p : {'X', 'Z'}) { + std::string s(kN, 'I'); + s[q] = p; + add(s); + } + } + for (size_t q = 0; q + 1 < kN; ++q) { + std::string s(kN, 'I'); + s[q] = 'Z'; + s[q + 1] = 'Z'; + add(s); + } + add("XYZXYIII"); + add("ZZZZZZZZ"); + add_term(op, Basis::Pauli, {}, -1.75); + return op; +} + +// Two-, three- (odd), four- and six-slot generators, a repeated gate, a repeated parameter and an identity gate. +auto majorana_circuit(size_t logical) -> Circuit { + const size_t s = 2 * logical; + Circuit c; + c.gates = {{0, 1}, {2, 5}, {1, 3, 4, 6}, {0, s - 1}, {}, {3}, {2, 4, s - 3}, {5, 6, 7, s - 2}, {0, 1}, {1, 2}}; + c.mapping = {0, 1, 2, 3, 4, 5, 6, 2, 7, 8}; + for (size_t i = 0; i < c.gates.size(); ++i) { + c.gen_coeffs.push_back(i % 3 == 0 ? -1.0 : 0.5 + (0.25 * static_cast(i))); + } + for (size_t i = 0; i < 9; ++i) { + c.params.push_back(0.11 + (0.07 * static_cast(i))); + } + return c; +} + +auto pauli_circuit() -> Circuit { + Circuit c; + for (const std::string g : {"XXIIIIII", "YZXIIIII", "ZIIIIIII", "IIXYIIII", "IIIIIIII", "ZZZZIIII", "IYIIIXII"}) { + c.gates.push_back(pauli_oracle::slots_of_string(g)); + } + c.mapping.resize(c.gates.size()); + std::iota(c.mapping.begin(), c.mapping.end(), size_t{0}); + for (size_t i = 0; i < c.gates.size(); ++i) { + c.gen_coeffs.push_back(0.75 - (0.125 * static_cast(i))); + c.params.push_back(0.2 + (0.05 * static_cast(i))); + } + return c; +} + +auto data_circuit(const test_utils::CaseData &data) -> Circuit { + return {.gates = data.majoranas, + .mapping = data.param_inds, + .gen_coeffs = data.gen_coeffs, + .params = data.parameters}; +} + +// A basis change that permutes the slots pairwise, so the opaque closure really differs from the plain cutoff. +auto swap_basis(size_t logical) -> std::vector { + std::vector rows; + for (size_t i = 0; i < 2 * logical; ++i) { + rows.push_back({i ^ 1U}); + } + return rows; +} + +// Fixed-geometry comparisons: every basis and picture, cutoffs that trim, coefficient cutoffs, a length cap (two-pass +// fallback) and the basis-change closure. +auto fixture_cases() -> std::vector { + const auto data = test_utils::load_case_data("random_exact.msgpack"); + const Fixture maj{.name = "majorana-heisenberg-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .logical = 6}; + auto maj_s = maj; + maj_s.name = "majorana-schrodinger-logical-6"; + maj_s.schrodinger_cutoff = 5; + auto maj_atol = maj; + maj_atol.name = "majorana-heisenberg-atol"; + maj_atol.lower_atol = 0.02; + maj_atol.upper_atol = 0.3; + auto maj_s_atol = maj_s; + maj_s_atol.name = "majorana-schrodinger-atol"; + maj_s_atol.lower_atol = 0.001; + maj_s_atol.upper_atol = 0.05; + auto maj_bc = maj; + maj_bc.name = "majorana-heisenberg-basis-change"; + maj_bc.basis_change = swap_basis(6); + const Fixture rnd{.name = "majorana-heisenberg-random-exact", + .op = data.hamiltonian, + .initial_state = data.initial_state, + .cutoff = 6}; + auto rnd_s = rnd; + rnd_s.name = "majorana-schrodinger-random-exact"; + rnd_s.schrodinger_cutoff = 6; + const Fixture pauli{.name = "pauli-heisenberg", + .op = pauli_operator(), + .initial_state = {}, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 3}; + auto pauli_s = pauli; + pauli_s.name = "pauli-schrodinger"; + pauli_s.schrodinger_cutoff = 5; + return { + {.f = maj, .c = majorana_circuit(6)}, + {.f = maj_s, .c = majorana_circuit(6)}, + {.f = maj_atol, .c = majorana_circuit(6)}, + {.f = maj_s_atol, .c = majorana_circuit(6)}, + {.f = maj, .c = majorana_circuit(6), .k = 4}, + {.f = maj_s, .c = majorana_circuit(6), .k = 2}, + {.f = maj_bc, .c = majorana_circuit(6)}, + {.f = rnd, .c = data_circuit(data)}, + {.f = rnd_s, .c = data_circuit(data)}, + {.f = pauli, .c = pauli_circuit()}, + {.f = pauli_s, .c = pauli_circuit()}, + }; +} + +auto label(const Case &c) -> std::string { + return c.k ? std::format("{} k={}", c.f.name, *c.k) : c.f.name; +} + +// --- Sharded setup ---------------------------------------------------------------------------------------------- + +template +auto make_cutoff_fn(const Fixture &f) -> CutoffFn { + CutoffFn fn; + if (f.basis_change) { + MonomialList basis; + for (size_t i = 0; i < 2 * f.logical; ++i) { + basis.push_back(indices_to_bitset_checked((*f.basis_change)[i], 2 * f.logical)); + } + fn = monoprop::detail::cutoff_function_basis_change(f.cutoff_type, f.cutoff, basis, f.logical); + } + else { + fn = monoprop::detail::cutoff_function(f.cutoff_type, f.cutoff, f.logical); + } + if (f.opaque) { + return [inner = fn](const Monomial &m) { return inner(m); }; + } + return fn; +} + +// Shards seeded through the real S1 path, with the context that goes with them. +template +struct Sharded { + Fixture f; + CutoffFn cutoff_fn; + routing::Router router; + sharded::Shards shards; + sharded::PhysicalWorld world{}; // one process unless a multi-rank case seeds over MPI_COMM_WORLD + + [[nodiscard]] auto ctx() const -> sharded::ConstructionContext { + return {.cutoff_fn = cutoff_fn, + .router = router, + .lower_atol = f.lower_atol, + .upper_atol = f.upper_atol, + .basis = f.basis, + .schrodinger = f.schrodinger_cutoff.has_value(), + .world = world}; + } +}; + +template +auto seed(const Fixture &f, parallel::Options options, const sharded::PhysicalWorld &world = {}) -> Sharded { + const auto router = routing::make_router(world.ranks, static_cast(options.threads)); + const auto cutoff_fn = make_cutoff_fn(f); + std::optional paired; + if (f.schrodinger_cutoff) { + paired = sharded::paired_basis_bounds(*f.schrodinger_cutoff, f.logical, router.flat_world()); + } + // The inline width only selects the row packing (overflow spills losslessly), so the plain cutoff serves here. + const auto width_fn = monoprop::detail::cutoff_function(f.cutoff_type, f.cutoff, f.logical); + const auto seed_inputs = sharded::OperatorSeed{ + .initial_operator = f.op, + .initial_state = f.initial_state, + .router = router, + .basis = f.basis, + .logical_num_modes = f.logical, + .paired = paired, + .inline_width = sharded::packed_inline_width(paired.has_value(), f.basis_change ? cutoff_fn : width_fn)}; + return {.f = f, + .cutoff_fn = cutoff_fn, + .router = router, + .shards = sharded::seed_shards(options, seed_inputs, world.rank), + .world = world}; +} + +template +auto generators(const Fixture &f, const Circuit &c) -> std::vector> { + std::vector> out; + for (const auto &g : c.gates) { + out.push_back(indices_to_bitset_checked(g, 2 * f.logical)); + } + return out; +} + +template +auto run_graph(Sharded &s, const Circuit &c, std::optional k, const Observer &observer = {}) + -> sharded::ConstructionOutcome { + const auto gens = generators(s.f, c); + VecZ gate_indices(c.gates.size()); + std::iota(gate_indices.begin(), gate_indices.end(), size_t{0}); + const auto ctx = s.ctx(); + return sharded::build_graph(team_options(), + s.shards, + ctx, + {.generators = gens, + .parameter_mapping = c.mapping, + .gen_coeffs = c.gen_coeffs, + .gate_indices = gate_indices, + .only_rotate_len_k = k}, + observer); +} + +template +auto run_propagate(Sharded &s, const Circuit &c, std::optional k, const Observer &observer = {}) + -> sharded::ConstructionOutcome { + const auto gens = generators(s.f, c); + const auto mapped = map_params(c.params, c.mapping, c.gen_coeffs, 1.0); + const auto ctx = s.ctx(); + return sharded::propagate(team_options(), + s.shards, + ctx, + {.generators = gens, .mapped_params = mapped, .only_rotate_len_k = k}, + observer); +} + +auto require_success(const sharded::ConstructionOutcome &outcome) -> void { + if (outcome.error) { + try { + std::rethrow_exception(outcome.error); + } + catch (const std::exception &e) { + BOOST_FAIL("the construction failed: " << e.what()); + } + } +} + +// --- Exact shard digests ---------------------------------------------------------------------------------------- + +struct LayerDigest { + std::vector> occupied; // (slot, sin_send_count, in_count) + std::vector sin_send; + bool binary = false; + size_t phase_count = 0; + std::vector phase_words; + std::vector phase_values; + size_t world = 0; + size_t self_pos = 0; + size_t self_offset = 0; + std::vector generator; + uint64_t scaled_count = 0; + size_t param_index = 0; + uint64_t gen_coeff = 0; + size_t gate_index = 0; + auto operator==(const LayerDigest &) const -> bool = default; +}; + +struct ShardDigest { + std::vector rows; + std::vector op_coeffs; + std::vector state_coeffs; + std::vector state_rows; + std::vector state_vals; + std::vector> pending; + size_t index_rows = 0; + std::vector layers; + auto operator==(const ShardDigest &) const -> bool = default; +}; + +auto layer_digest(const LayerCore &core) -> LayerDigest { + LayerDigest d; + for (const auto &e : core.cross_rank.occupied) { + d.occupied.push_back({e.slot, e.sin_send_count, e.in_count}); + } + d.sin_send = core.cross_rank.sin_send_indices; + d.binary = core.cross_rank.sin_recv_phases.uses_binary_phases; + d.phase_count = core.cross_rank.sin_recv_phases.total_count; + d.phase_words = core.cross_rank.sin_recv_phases.phase_words; + d.phase_values = core.cross_rank.sin_recv_phases.phase_values; + d.world = core.cross_rank.world_size; + d.self_pos = core.cross_rank.self_pos; + d.self_offset = core.cross_rank.self_offset; + d.generator = core.generator_words; + d.scaled_count = core.scaled_count; + d.param_index = core.param_index; + d.gen_coeff = bits(core.gen_coeff); + d.gate_index = core.gate_index; + return d; +} + +template +auto digest(const sharded::ShardState &s) -> ShardDigest { + ShardDigest d; + for (size_t i = 0; i < s.op.store->size(); ++i) { + d.rows.push_back(key_of(s.op.store->row(i))); + } + for (const double v : s.op.op_coeffs) { + d.op_coeffs.push_back(bits(v)); + } + for (const double v : s.op.state_coeffs) { + d.state_coeffs.push_back(bits(v)); + } + d.state_rows = s.op.state_rows_; + for (const double v : s.op.state_vals_) { + d.state_vals.push_back(bits(v)); + } + for (const auto &[m, v] : s.op.init_op_map) { + d.pending.emplace_back(key_of(m), bits(v)); + } + std::ranges::sort(d.pending); + d.index_rows = s.op.inverted_index_ ? s.op.inverted_index_->rows() : 0; + for (size_t l = 0; l < s.graph.layers(); ++l) { + d.layers.push_back(layer_digest(s.graph.get_layer(l).core())); + } + return d; +} + +// The rank-level reference at geometry (1, T): the root over MPI_COMM_SELF, built through its public operations at the +// launch's T. Its shards come from the same seam, so a case that compares against it checks the root's wiring of the +// seam, and keeps its independent checks. +template +auto reference_root(const Fixture &f, size_t threads) -> MonomialPropagator { + auto p = MonomialPropagator(f.op, + f.cutoff, + f.initial_state, + f.schrodinger_cutoff, + MPI_COMM_SELF, + f.lower_atol, + f.upper_atol, + f.cutoff_type, + f.basis_change, + f.logical, + f.basis); + BOOST_TEST_REQUIRE(monoprop::detail::PropagatorTestAccess::shards(p).size() == threads); + return p; +} + +template +auto reference_digest(MonomialPropagator &p, size_t shard) -> ShardDigest { + return digest(*monoprop::detail::PropagatorTestAccess::shards(p).at(shard)); +} + +// --- Global retained maps --------------------------------------------------------------------------------------- + +using RetainedMap = std::map; + +template +auto picture_map(const sharded::ShardState &s, bool schrodinger, RetainedMap &out) -> void { + const VecD &c = schrodinger ? s.op.state_coeffs : s.op.op_coeffs; + BOOST_REQUIRE_EQUAL(c.size(), s.op.store->size()); + for (size_t i = 0; i < s.op.store->size(); ++i) { + const bool fresh = out.emplace(key_of(s.op.store->row(i)), c[i]).second; + BOOST_REQUIRE_MESSAGE(fresh, "a retained key is owned twice"); + } +} + +template +auto global_map(const sharded::Shards &shards, bool schrodinger) -> RetainedMap { + RetainedMap out; + for (const auto &s : shards) { + picture_map(*s, schrodinger, out); + } + return out; +} + +// The plan's general map tolerance: identical keys, abs(a - b) <= 1e-9 + 1e-7 * max(|a|, |b|). +auto check_maps_agree(const RetainedMap &got, const RetainedMap &want, const std::string &what) -> void { + BOOST_TEST_CONTEXT(what) { + BOOST_TEST(got.size() == want.size()); + size_t missing = 0; + size_t off = 0; + for (const auto &[key, value] : want) { + const auto it = got.find(key); + if (it == got.end()) { + ++missing; + continue; + } + const double scale = std::max(std::abs(value), std::abs(it->second)); + if (!(std::abs(value - it->second) <= 1e-9 + (1e-7 * scale))) { + ++off; + } + } + BOOST_TEST(missing == 0U); + BOOST_TEST(off == 0U); + } +} + +// --- Observation ------------------------------------------------------------------------------------------------ + +struct Visit { + W work; + size_t step; + int worker; + int team; + int level; + std::thread::id thread; +}; + +struct PassLog { + bool leaders = true; + std::vector> payload; + std::vector> sources; + std::vector> values; + std::vector> answers; + std::vector> received; + auto operator==(const PassLog &) const -> bool = default; +}; + +struct ShardLog { + std::vector visits; + std::vector passes; + kernel_test::PhaseLog ranges; +}; + +struct KernelFailure { + size_t shard = 0; + KernelRange kind = KernelRange::decode; + size_t call = 0; + size_t range = 0; +}; + +// The per-shard range observer: kernel ranges, and the streams of the exchange phases. +struct KernelProbe { + ShardLog *log = nullptr; + bool capture = true; + std::optional fail = std::nullopt; + std::optional fail_publication = std::nullopt; // throw once this pass's payload is built + + auto prepare(KernelRange kind, size_t ranges) const -> void { + kernel_test::AccumulatingObserver{&log->ranges}.prepare(kind, ranges); + } + // An armed failure fires in range `fail->range` of the `fail->call`-th call of its kind that has any range. + auto visit(KernelRange kind, size_t range) const -> void { + kernel_test::AccumulatingObserver{&log->ranges}.visit(kind, range); + if (fail && kind == fail->kind && range == fail->range) { + const auto calls = log->ranges.of(kind); + const auto nonempty = std::ranges::count_if(calls, [](const auto *call) { return !call->worker.empty(); }); + if (static_cast(nonempty) == fail->call + 1) { + throw std::runtime_error("injected kernel failure"); + } + } + } + template + static auto flatten(const mpi::WindowVec &blocks) -> std::vector> { + std::vector> out; + for (const auto &block : blocks) { + std::vector words; + for (const auto v : block) { + words.push_back(bits(v)); + } + out.push_back(std::move(words)); + } + return out; + } + static auto flatten_ids(const mpi::WindowVec &blocks) -> std::vector> { + std::vector> out; + for (const auto &block : blocks) { + out.emplace_back(block.begin(), block.end()); + } + return out; + } + auto exchange_prepared(bool leaders, + const mpi::WindowVec &payload, + const mpi::WindowVec> &sources, + const mpi::WindowVec> &values) const -> void { + if (capture) { + PassLog pass; + pass.leaders = leaders; + pass.payload = flatten_ids(payload); + pass.sources = flatten_ids(sources); + pass.values = flatten(values); + log->passes.push_back(std::move(pass)); + } + if (fail_publication == leaders) { + throw std::runtime_error("injected publication failure"); + } + } + template + auto exchange_resolved(bool /*leaders*/, const mpi::WindowVec> &answers) const -> void { + if (capture) { + log->passes.back().answers = flatten(answers); + } + } + template + auto exchange_consumed(bool /*leaders*/, const mpi::WindowVec> &received) const -> void { + if (capture) { + log->passes.back().received = flatten(received); + } + } +}; + +struct FailAt { + W work; + size_t step; + size_t shard; +}; + +struct AllocAt { + W work; + size_t step; + size_t shard; + size_t nth; +}; + +// The construction observer: per-shard visit logs, and optional injected failures. +struct Probe { + std::vector *logs = nullptr; + bool capture = true; + std::vector fail = {}; + std::optional alloc = std::nullopt; + std::optional kernel_fail = std::nullopt; + std::optional> publication_fail = std::nullopt; // (shard, leaders) + + auto visit(W work, size_t step, size_t shard) const -> void { + // A pending armed allocation failure never leaks into a later phase. + (void)allocation::disarm_failure(); + auto &log = (*logs)[shard]; + log.visits.push_back( + {work, step, omp_get_thread_num(), omp_get_num_threads(), omp_get_level(), std::this_thread::get_id()}); + for (const auto &f : fail) { + if (f.work == work && f.step == step && f.shard == shard) { + throw std::runtime_error( + std::format("injected failure: work {} step {} shard {}", static_cast(work), step, shard)); + } + } + if (alloc && alloc->work == work && alloc->step == step && alloc->shard == shard) { + allocation::arm_failure(alloc->nth); + } + } + [[nodiscard]] auto kernels(size_t shard) const -> KernelProbe { + KernelProbe probe{.log = &(*logs)[shard], .capture = capture}; + if (kernel_fail && kernel_fail->shard == shard) { + probe.fail = kernel_fail; + } + if (publication_fail && publication_fail->first == shard) { + probe.fail_publication = publication_fail->second; + } + return probe; + } +}; + +auto make_logs(size_t threads) -> std::vector { + std::vector logs(threads); + for (auto &log : logs) { + log.visits.reserve(4096); + } + return logs; +} + +// After an injected allocation failure, no worker thread keeps an armed failure. +auto disarm_team() -> void { + static_cast(sharded::run_team(team_options(), [](size_t, sharded::TeamFailure &) noexcept { + (void)allocation::disarm_failure(); + })); +} + +} // namespace + +// --- Independent references ------------------------------------------------------------------------------------- + +namespace { + +// The ordering contract, restated without engine code: per gate, each destination appends cross-owner leader +// misses (ascending source shard, source row order), then cross-owner follower misses (followers a leader matched +// are dropped), then its deferred same-shard leader misses, then its same-shard follower misses. +template +struct OrderReference { + std::vector>> rows; // per shard, in row order + std::map> where; // key -> (shard, row) + size_t cross_matches = 0; // leader hits that marked a row of another shard + size_t self_misses = 0; + size_t cross_misses = 0; + + auto append(size_t shard, const Monomial &m) -> void { + where.emplace(key_of(m), std::make_pair(shard, rows[shard].size())); + rows[shard].push_back(m); + } + + auto gate(const Monomial &gen, + const routing::Router &router, + Basis basis, + const std::function &)> &cutoff) -> void { + const size_t threads = rows.size(); + const auto [fold, odd] = with_algebra(basis, [&]() { + return std::pair{A::fold_generator(gen), A::fold_needs_odd_correction(gen)}; + }); + if (!gen.any()) { + return; + } + const size_t pivot = gen.find_first(); + struct Query { + size_t src; + Monomial partner; + size_t dest; + }; + // [source][dest] leaders and followers, in source row order. + std::vector>> lead(threads, std::vector>(threads)); + auto foll = lead; + for (size_t s = 0; s < threads; ++s) { + for (size_t i = 0; i < rows[s].size(); ++i) { + const auto &m = rows[s][i]; + const size_t overlap = (m & fold).count() + (odd ? m.count() : 0); + if (overlap % 2 == 0) { + continue; + } + const auto partner = m ^ gen; + if (!cutoff(partner)) { + continue; + } + const size_t d = router.dest(partner); + (m.test(pivot) ? foll : lead)[s][d].push_back({i, partner, d}); + } + } + const auto pre = rows; + const auto pre_hit = [&](size_t d, const Monomial &m) -> std::optional { + const auto it = where.find(key_of(m)); + if (it == where.end() || it->second.first != d || it->second.second >= pre[d].size()) { + return std::nullopt; + } + return it->second.second; + }; + std::vector> marked(threads); + for (size_t d = 0; d < threads; ++d) { + marked[d].assign(pre[d].size(), false); + } + std::vector>> self_lead_miss(threads); + std::vector>> self_foll_miss(threads); + // Leaders: same-shard resolution is frozen (deferred misses), cross-owner misses insert at once. + for (size_t d = 0; d < threads; ++d) { + for (const auto &q : lead[d][d]) { + if (const auto hit = pre_hit(d, q.partner)) { + marked[d][*hit] = true; + } + else { + self_lead_miss[d].push_back(q.partner); + } + } + } + for (size_t d = 0; d < threads; ++d) { + for (size_t s = 0; s < threads; ++s) { + if (s == d) { + continue; + } + for (const auto &q : lead[s][d]) { + if (const auto hit = pre_hit(d, q.partner)) { + marked[d][*hit] = true; + ++cross_matches; + } + else { + append(d, q.partner); + ++cross_misses; + } + } + } + } + // Followers: filtered by the source shard's completed marks. + for (size_t d = 0; d < threads; ++d) { + for (const auto &q : foll[d][d]) { + if (!marked[d][q.src] && !pre_hit(d, q.partner)) { + self_foll_miss[d].push_back(q.partner); + } + } + } + for (size_t d = 0; d < threads; ++d) { + for (size_t s = 0; s < threads; ++s) { + if (s == d) { + continue; + } + for (const auto &q : foll[s][d]) { + if (!marked[s][q.src] && !pre_hit(d, q.partner)) { + append(d, q.partner); + ++cross_misses; + } + } + } + } + for (size_t d = 0; d < threads; ++d) { + for (const auto &m : self_lead_miss[d]) { + append(d, m); + ++self_misses; + } + for (const auto &m : self_foll_miss[d]) { + append(d, m); + ++self_misses; + } + } + } +}; + +} // namespace + +// Rows and IDs per shard follow the ordering contract as an independent reference computes it, including followers +// matched by a leader of another shard (which must occur at T >= 2 on this fixture). +BOOST_AUTO_TEST_CASE(sharded_construction_rows_follow_the_ordering_contract) { + const auto options = team_options(); + const size_t threads = team_size(); + auto cases = fixture_cases(); + cases.resize(2); // Majorana, both pictures + cases.push_back(fixture_cases()[9]); // Pauli Heisenberg + size_t cross_matches = 0; + for (const auto &cs : cases) { + BOOST_TEST_CONTEXT(label(cs) << " T=" << threads) { + auto s = seed(cs.f, options); + OrderReference ref; + ref.rows.resize(threads); + for (size_t t = 0; t < threads; ++t) { + for (size_t i = 0; i < s.shards[t]->op.store->size(); ++i) { + ref.append(t, s.shards[t]->op.store->row(i)); + } + } + const auto gens = generators(cs.f, cs.c); + const auto plain = monoprop::detail::cutoff_function(cs.f.cutoff_type, cs.f.cutoff, cs.f.logical); + const bool schrodinger = cs.f.schrodinger_cutoff.has_value(); + for (size_t step = 0; step < gens.size(); ++step) { + ref.gate(gens[schrodinger ? step : gens.size() - 1 - step], s.router, cs.f.basis, plain); + } + require_success(run_graph(s, cs.c, cs.k)); + for (size_t t = 0; t < threads; ++t) { + std::vector got; + std::vector want; + for (size_t i = 0; i < s.shards[t]->op.store->size(); ++i) { + got.push_back(key_of(s.shards[t]->op.store->row(i))); + } + for (const auto &m : ref.rows[t]) { + want.push_back(key_of(m)); + } + BOOST_TEST((got == want), "shard " << t); + } + BOOST_TEST(ref.self_misses > 0U); + if (threads >= 2) { + BOOST_TEST(ref.cross_misses > 0U); + } + cross_matches += ref.cross_matches; + } + } + // A follower dropped because a leader of another shard matched it. + BOOST_TEST((threads < 2 || cross_matches > 0U)); +} + +namespace { + +// Independent propagation on a map: each anticommuting key M moves to cos(2a) M + sin(2a) phase(M) (M ^ G). An absent +// partner enters with its implicit value: 0 for an operator, and for a state its initial score. +template +auto reference_propagate(RetainedMap map, + const std::vector> &gens, + const VecD &mapped, + bool schrodinger, + Basis basis, + const VecZ &initial_state) -> RetainedMap { + const auto mask = initial_state_mask(initial_state); + const auto implicit = [&](const Monomial &m) -> double { + if (!schrodinger) { + return 0.0; + } + for (size_t mode = 0; mode < N; ++mode) { + if (m.test(2 * mode) != m.test((2 * mode) + 1)) { + return 0.0; + } + } + return algebra_state_phase(basis, m, mask); + }; + const auto mono_of = [](const Key &key) { + Monomial m; + for (size_t w = 0; w < key.size(); ++w) { + m.data()[w] = key[w]; + } + return m; + }; + const size_t n = gens.size(); + for (size_t step = 0; step < n; ++step) { + const size_t idx = schrodinger ? step : n - 1 - step; + const auto &gen = gens[idx]; + const double angle = schrodinger ? -mapped[idx] : mapped[idx]; + const double c = std::cos(2 * angle); + const double sn = std::sin(2 * angle); + with_algebra(basis, [&]() { + const auto fold = A::fold_generator(gen); + const bool odd = A::fold_needs_odd_correction(gen); + const auto ctx = A::make_gen_context(gen); + const auto anti = [&](const Monomial &m) { + return ((m & fold).count() + (odd ? m.count() : 0)) % 2 == 1; + }; + std::vector> partners; + for (const auto &[key, v] : map) { + const auto m = mono_of(key); + if (anti(m) && !map.contains(key_of(m ^ gen))) { + partners.push_back(m ^ gen); + } + } + for (const auto &m : partners) { + map.emplace(key_of(m), implicit(m)); + } + RetainedMap next; + for (const auto &[key, v] : map) { + const auto m = mono_of(key); + if (!anti(m)) { + next[key] += v; + continue; + } + const auto partner = m ^ gen; + const int phase = + A::emit_phase(A::rotation_sign(ctx, m, partner), m.count(), gen.count(), (m & gen).count()); + next[key] += c * v; + next[key_of(partner)] += sn * static_cast(phase) * v; + } + map = std::move(next); + }); + } + return map; +} + +// from shard coefficients: Heisenberg sums the evolved operator against the initial state's scores; +// Schrödinger sums the evolved state against the Hamiltonian's rows. The identity term is added once. +template +auto shard_energy(sharded::Shards &shards, bool schrodinger, double core) -> double { + auto energy = core; + for (auto &s : shards) { + auto op = s->op; // the dense oracles mutate caches + if (schrodinger) { + const auto &h = op.get_operator(); + for (size_t i = 0; i < op.size(); ++i) { + energy += op.state_coeffs[i] * h[i]; + } + } + else { + const auto state = op.materialize_state(); + for (size_t i = 0; i < op.size(); ++i) { + energy += op.op_coeffs[i] * state[i]; + } + } + } + return energy; +} + +} // namespace + +// Exact-cutoff propagation against the independent map propagator (both pictures, both bases, odd Majorana +// generators, native Pauli folding), and the frozen exact energy of random_exact in both pictures. +BOOST_AUTO_TEST_CASE(sharded_construction_propagation_matches_an_independent_reference) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto data = test_utils::load_case_data("random_exact.msgpack"); + std::vector> cases; + for (const auto sc : {std::optional{}, std::optional{16}}) { + cases.push_back({{.name = "random-exact", + .op = data.hamiltonian, + .initial_state = data.initial_state, + .schrodinger_cutoff = sc, + .cutoff = 2 * kN}, + data_circuit(data)}); + cases.push_back({{.name = "majorana-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .schrodinger_cutoff = sc ? std::optional{12} : std::nullopt, + .cutoff = 12, + .logical = 6}, + majorana_circuit(6)}); + cases.push_back({{.name = "pauli", + .op = pauli_operator(), + .initial_state = {}, + .schrodinger_cutoff = sc, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = kN}, + pauli_circuit()}); + } + for (const auto &[f, c] : cases) { + const bool schrodinger = f.schrodinger_cutoff.has_value(); + BOOST_TEST_CONTEXT(f.name << (schrodinger ? " schrodinger" : " heisenberg") << " T=" << threads) { + auto s = seed(f, options); + for (auto &state : s.shards) { + (void)state->op.current_picture(schrodinger); + } + const auto initial = global_map(s.shards, schrodinger); + require_success(run_propagate(s, c, std::nullopt)); + const auto mapped = map_params(c.params, c.mapping, c.gen_coeffs, 1.0); + const auto want = + reference_propagate(initial, generators(f, c), mapped, schrodinger, f.basis, f.initial_state); + check_maps_agree(global_map(s.shards, schrodinger), want, "reference map"); + if (f.name == "random-exact") { + const auto core = sharded::validate_initial_operator(f.op, f.basis, f.logical); + BOOST_CHECK_SMALL(shard_energy(s.shards, schrodinger, core) - data.actual_expval, kExactAtol); + } + } + } +} + +// --- Participation and the opaque path -------------------------------------------------------------------------- + +namespace { + +auto check_owner_participation(const std::vector &logs, size_t gates, bool opaque, const std::string &what) + -> void { + const size_t threads = logs.size(); + std::set threads_seen; + BOOST_TEST_CONTEXT(what) { + for (size_t t = 0; t < threads; ++t) { + const auto &visits = logs[t].visits; + BOOST_TEST_REQUIRE(!visits.empty()); + BOOST_TEST((visits.front().work == W::frame)); + BOOST_TEST((visits.back().work == W::caches)); + std::set steps; + std::set owner_threads; + bool owner_work = true; + bool one_team = true; + for (const auto &v : visits) { + // One region for a team; a one-worker team runs on the caller, without one. + one_team = one_team && v.team == static_cast(threads) && v.level == (threads > 1 ? 1 : 0); + if (v.step != sharded::no_step) { + steps.insert(v.step); + } + const int expected = (opaque && v.work == W::traverse) ? 0 : static_cast(t); + owner_work = owner_work && v.worker == expected; + if (v.worker == static_cast(t)) { + owner_threads.insert(v.thread); + } + } + BOOST_TEST(one_team, "shard " << t); + BOOST_TEST(owner_work, "shard " << t); + // Every gate of the circuit ran inside the same team, on the same owner thread. + BOOST_TEST(steps.size() == gates, "shard " << t); + BOOST_TEST(owner_threads.size() == 1U, "shard " << t); + threads_seen.insert(owner_threads.begin(), owner_threads.end()); + // The kernels of shard t ran serially on their owner (or, for an opaque traversal, the primary). + for (const auto &call : logs[t].ranges.calls) { + const int expected = (opaque && call.kind == KernelRange::scan) ? 0 : static_cast(t); + for (size_t r = 0; r < call.worker.size(); ++r) { + BOOST_TEST(call.worker[r] == expected, "shard " << t << " kind " << static_cast(call.kind)); + BOOST_TEST(call.level[r] == (threads > 1 ? 1 : 0)); // a one-worker team runs without a region + } + } + } + BOOST_TEST(threads_seen.size() == threads); + } +} + +auto count_visits(const std::vector &logs, W work) -> size_t { + size_t n = 0; + for (const auto &log : logs) { + n += static_cast(std::ranges::count_if(log.visits, [&](const Visit &v) { return v.work == work; })); + } + return n; +} + +auto count_ranges(const std::vector &logs, KernelRange kind) -> size_t { + size_t n = 0; + for (const auto &log : logs) { + for (const auto &call : log.ranges.of(kind)) { + n += call->worker.size(); + } + } + return n; +} + +} // namespace + +// Each owner performs its own traversal, resolution/publication, finalization and application, for every gate, inside +// one team of T workers at level 1, on one thread per owner; its kernels run serially on it. +BOOST_AUTO_TEST_CASE(sharded_construction_owners_do_their_own_work) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + for (const bool fused : {false, true}) { + auto s = seed(cs.f, options); + auto logs = make_logs(threads); + require_success(fused ? run_propagate(s, cs.c, cs.k, Probe{.logs = &logs}) + : run_graph(s, cs.c, cs.k, Probe{.logs = &logs})); + check_owner_participation(logs, cs.c.gates.size(), false, fused ? "propagate" : "graph"); + const size_t gates = cs.c.gates.size(); + BOOST_TEST(count_visits(logs, W::traverse) == threads * gates); + BOOST_TEST(count_visits(logs, W::finalize) == threads * gates); + BOOST_TEST(count_visits(logs, W::apply) == (fused ? threads * gates : 0U)); + // One identity gate: no exchange; with one shard, no cross-owner resolution at all. + const size_t exchanging = threads > 1 ? threads * (gates - 1) : 0U; + BOOST_TEST(count_visits(logs, W::resolve_leaders) == exchanging); + BOOST_TEST(count_visits(logs, W::consume_followers) == exchanging); + BOOST_TEST(count_ranges(logs, KernelRange::scan) > 0U); + BOOST_TEST(count_ranges(logs, KernelRange::self_probe) > 0U); + if (threads > 1) { + BOOST_TEST(count_ranges(logs, KernelRange::decode) > 0U); + BOOST_TEST(count_ranges(logs, KernelRange::scatter) > 0U); + } + if (fused) { + BOOST_TEST(count_ranges(logs, KernelRange::fused_apply) > 0U); + } + } +} + +// Opaque cutoffs (a closure over a typed one, and the basis-change closure) traverse every shard on the primary while +// the owners keep every other phase; the results equal the typed-cutoff run and the root. +BOOST_AUTO_TEST_CASE(sharded_construction_opaque_cutoff_traverses_on_the_primary) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cases = fixture_cases(); + for (const auto &base : {cases[0], cases[1], cases[6]}) { + auto opaque = base; + opaque.f.opaque = true; + for (const bool fused : {false, true}) { + BOOST_TEST_CONTEXT(label(base) << (fused ? " propagate" : " graph") << " T=" << threads) { + auto s = seed(opaque.f, options); + auto logs = make_logs(threads); + require_success(fused ? run_propagate(s, opaque.c, opaque.k, Probe{.logs = &logs}) + : run_graph(s, opaque.c, opaque.k, Probe{.logs = &logs})); + check_owner_participation(logs, opaque.c.gates.size(), true, "opaque"); + // Each traversal names its target shard, and the primary executed all of them. + for (size_t t = 0; t < threads; ++t) { + for (const auto &v : logs[t].visits) { + if (v.work == W::traverse) { + BOOST_TEST(v.worker == 0); + } + } + } + auto p = reference_root(base.f, threads); + if (fused) { + p.propagate(base.c.gates, base.c.mapping, base.c.gen_coeffs, base.c.params, base.k); + } + else { + p.build_graph(base.c.gates, base.c.mapping, base.c.gen_coeffs, std::nullopt, std::nullopt, base.k); + } + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == reference_digest(p, t)), "shard " << t); + } + } + } + } +} + +// --- Degenerate inputs ------------------------------------------------------------------------------------------ + +// An empty circuit opens no team and changes nothing; an empty operator and identity-only circuits leave every shard +// (including empty owners) consistent with the root; empty owners still reach every checkpoint. +BOOST_AUTO_TEST_CASE(sharded_construction_empty_operators_circuits_and_identity) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + { + auto s = seed(cs.f, options); + std::vector before; + for (const auto &state : s.shards) { + before.push_back(digest(*state)); + } + auto logs = make_logs(threads); + const auto outcome = run_graph(s, Circuit{}, std::nullopt, Probe{.logs = &logs}); + BOOST_TEST(!outcome.error); + BOOST_TEST(!outcome.mutation_started); + const auto outcome2 = run_propagate(s, Circuit{}, std::nullopt, Probe{.logs = &logs}); + BOOST_TEST(!outcome2.mutation_started); + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == before[t])); + BOOST_TEST(logs[t].visits.empty()); + } + } + Fixture empty = cs.f; + empty.name = "empty-operator"; + empty.op = {}; + Fixture two = cs.f; + two.name = "two-terms"; + two.op = {}; + add_term(two.op, Basis::Majorana, {0, 1}, 0.5); + add_term(two.op, Basis::Majorana, {2, 3}, -0.25); + const Circuit identity{.gates = {{}, {}}, .mapping = {0, 1}, .gen_coeffs = {1.0, 1.0}, .params = {0.3, 0.4}}; + for (const auto &f : {empty, two, cs.f}) { + for (const auto &c : {identity, cs.c}) { + for (const bool fused : {false, true}) { + BOOST_TEST_CONTEXT(f.name << " gates=" << c.gates.size() << (fused ? " propagate" : " graph")) { + auto s = seed(f, options); + auto logs = make_logs(threads); + require_success(fused ? run_propagate(s, c, std::nullopt, Probe{.logs = &logs}) + : run_graph(s, c, std::nullopt, Probe{.logs = &logs})); + auto p = reference_root(f, threads); + if (fused) { + p.propagate(c.gates, c.mapping, c.gen_coeffs, c.params); + } + else { + p.build_graph(c.gates, c.mapping, c.gen_coeffs); + } + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == reference_digest(p, t)), "shard " << t); + // Empty owners reach every checkpoint: the frame, every gate's finish, the caches. + BOOST_TEST(static_cast( + std::ranges::count_if(logs[t].visits, + [](const Visit &v) { return v.work == W::finalize; })) + == c.gates.size()); + } + } + } + } + } +} + +// --- The shared span-helper seam -------------------------------------------------------------------------------- + +namespace { + +template +auto make_store(const std::vector> &terms) -> monoprop::detail::MPOperator { + monoprop::detail::MPOperator op; + if (!terms.empty()) { + monoprop::detail::insert_absent_terms( + op, + terms.size(), + [&](size_t k) -> const Monomial & { return terms[k]; }, + [&](size_t k, size_t base) { assign_row(*op.store, base + k, terms[k]); }); + } + return op; +} + +// A Majorana term with a coefficient its basis can encode (real or imaginary by its length). +template +auto add_random_term(OperatorDict &op, const Monomial &m, double value) -> void { + VecZ idx; + for (size_t b = m.find_first(); b < m.size(); b = m.find_next(b)) { + idx.push_back(b); + } + op[idx] = algebra_decode_coeff(Basis::Majorana, std::complex(value, 0.0), m); +} + +template +auto random_term(std::mt19937_64 &rng, size_t k) -> Monomial { + Monomial m; + std::uniform_int_distribution slot(0, (2 * N) - 1); + while (m.count() < k) { + m.set(slot(rng)); + } + return m; +} + +template +auto encode(const std::vector> &keys, bool fused, size_t salt) -> VecZ { + using QW = monoprop::detail::QueryWire; + VecZ buf; + for (size_t q = 0; q < keys.size(); ++q) { + std::vector pos; + for (size_t b = keys[q].find_first(); b < keys[q].size(); b = keys[q].find_next(b)) { + pos.push_back(static_cast(b)); + } + QW::push(buf, pos, static_cast((q + salt) % 3) - 1); + if (fused) { + QW::push_value(buf, 0.25 + static_cast(q + salt)); + } + } + return buf; +} + +// One resolver owner's state after resolving `incoming` through the owning-VecZ path or the published-span path. +template +auto resolve_both(const std::vector> &seeds, + const mpi::WindowVec &incoming, + size_t owner, + size_t world) -> std::pair>, std::vector>> { + using Sink = std::conditional_t, monoprop::detail::GraphSink>; + const auto run = [&](bool spans) { + auto op = make_store(seeds); + monoprop::detail::MatchedEpochSet matched; + monoprop::detail::FusedContract fc; + VecD coeffs(seeds.size(), 0.5); + const auto window = incoming.window(); + auto engine = [&] { + if constexpr (Fused) { + return monoprop::detail::LayerBuildEngine(op, + world, + owner, + matched, + op.size(), + Sink{.R = world, + .my_rank = owner, + .fc = fc, + .op_coeffs = coeffs, + .fused_scale = false, + .inv_cos = 1.0, + .schrodinger = false, + .basis = Basis::Majorana}, + window); + } + else { + return monoprop::detail::LayerBuildEngine(op, + world, + owner, + matched, + op.size(), + Sink(world, owner), + window); + } + }(); + mpi::WindowVec> answers; + if (spans) { + answers = engine.resolve_published(mpi::views_of(incoming), true); + } + else { + answers = monoprop::detail::resolve_incoming(incoming, op, true, matched, op.size(), engine.sink); + } + std::vector> out; + for (const auto &block : answers) { + std::vector words; + for (const auto v : block) { + words.push_back(bits(v)); + } + out.push_back(std::move(words)); + } + for (size_t i = 0; i < op.size(); ++i) { + out.push_back(key_of(op.store->row(i))); + } + for (const auto &h : fc.cross_half) { + out.push_back({h.local_idx, bits(h.v_partner), static_cast(h.phase_signed), h.is_insert}); + } + for (size_t i = 0; i < op.size() && i < matched.epoch_.size(); ++i) { + out.push_back({static_cast(i < seeds.size() && matched.is_marked(i))}); + } + return out; + }; + return {run(false), run(true)}; +} + +} // namespace + +// The published-view helpers keep a nonzero window base, give empty senders and absent publishers empty views, and +// alias rather than copy. +BOOST_AUTO_TEST_CASE(sharded_construction_published_views_cover_the_window) { + const mpi::SlotWindow window{.base = 2, .count = 3}; + mpi::WindowVec blocks(window); + blocks.at_slot(2) = {7, 8}; + blocks.at_slot(4) = {9}; + const auto views = mpi::views_of(blocks); + BOOST_TEST((views.window() == window)); + BOOST_TEST(views.at_slot(2).data() == blocks.at_slot(2).data()); + BOOST_TEST(views.at_slot(3).empty()); + BOOST_TEST(views.at_slot(4).size() == 1U); + + // Owners 2..4 of a process whose first flat slot is 2 publish buffers over windows of their own. + mpi::WindowVec from2(mpi::SlotWindow{.base = 2, .count = 3}); + from2.at_slot(3) = {1, 2, 3}; + mpi::WindowVec from4(mpi::SlotWindow{.base = 4, .count = 1}); // does not reach owner 3 + from4.at_slot(4) = {5}; + const std::vector *> published{&from2, nullptr, &from4}; + const auto gathered = sharded::gather_published(mpi::SlotWindow{.base = 1, .count = 5}, + 3, + 2, + std::span *const>(published)); + BOOST_TEST((gathered.window() == mpi::SlotWindow{.base = 1, .count = 5})); + BOOST_TEST(gathered.at_slot(1).empty()); // not a local owner + BOOST_TEST(gathered.at_slot(2).data() == from2.at_slot(3).data()); + BOOST_TEST(gathered.at_slot(3).empty()); // published nothing + BOOST_TEST(gathered.at_slot(4).empty()); // its window does not reach owner 3 + BOOST_TEST(gathered.at_slot(5).empty()); // beyond the local owners +} + +// resolve_published over read-only views equals resolve_incoming over owning blocks, exactly, for Plain and Fused +// records, at 255/256/257 queries per sender, with an empty sender and a nonzero window base, for narrow (8-mode) and +// wide, escaped, multiword (250-mode) positions; and a malformed Plain stream is rejected before any mutation. +BOOST_AUTO_TEST_CASE(sharded_construction_published_resolution_matches_owning_blocks) { + const auto check = [&](size_t min_k, size_t max_k) { + std::mt19937_64 rng(N + max_k); + std::vector> seeds; + std::set used; + const auto fresh = [&](size_t k) { + for (;;) { + const auto m = random_term(rng, k); + if (used.insert(key_of(m)).second) { + return m; + } + } + }; + const auto width = [&](size_t i) { return min_k + (i % (max_k - min_k + 1)); }; + for (size_t i = 0; i < 600; ++i) { + seeds.push_back(fresh(width(i))); + } + // Mixed, all-hit and all-miss senders. + // Clang cannot capture a structured binding under OpenMP, so the pair is unpacked by name. + for (const auto &shape : {std::pair{size_t{255}, 0}, + std::pair{size_t{256}, 0}, + std::pair{size_t{257}, 0}, + std::pair{size_t{256}, 1}, + std::pair{size_t{257}, 2}}) { + const size_t count = shape.first; + const int pattern = shape.second; + const mpi::SlotWindow window{.base = 3, .count = 3}; + std::vector>> per_sender(3); + const auto hit = [&](size_t q, size_t modulus) { + return pattern == 1 || (pattern == 0 && q % modulus == 0); + }; + for (size_t q = 0; q < count; ++q) { + per_sender[0].push_back(hit(q, 2) ? seeds[q] : fresh(width(q))); + per_sender[2].push_back(hit(q, 3) ? seeds[300 + (q % 290)] : fresh(width(q * 7))); + } + // Distinct queries overall, as XOR with a fixed generator guarantees in production. + std::set distinct; + for (auto &sender : per_sender) { + std::erase_if(sender, [&](const Monomial &m) { return !distinct.insert(key_of(m)).second; }); + } + for (const bool fused : {false, true}) { + mpi::WindowVec incoming(window); + for (size_t k = 0; k < 3; ++k) { + incoming[mpi::WindowIndex{k}] = encode(per_sender[k], fused, k); + } + // Owner 4 resolves; slot 4 (its own) is the empty sender. + const auto [owning, viewed] = fused ? resolve_both(seeds, incoming, 4, 6) + : resolve_both(seeds, incoming, 4, 6); + BOOST_TEST((owning == viewed), + "N=" << N << " count=" << count << " pattern=" << pattern << " fused=" << fused); + } + } + }; + check.template operator()(3, 6); + check.template operator()<250>(1, 40); // positions above 255, escaped counts (k >= 31), multiword keys + + // The retained malformed Plain QueryWire<128> fixture: a header past its one-word stream. + using Engine = monoprop::detail::LayerBuildEngine<128, monoprop::detail::GraphSink<128>>; + std::mt19937_64 rng128(3); + auto op = make_store<128>({random_term<128>(rng128, 4)}); + monoprop::detail::MatchedEpochSet matched; + const mpi::SlotWindow window{.base = 0, .count = 2}; + Engine engine(op, 2, 0, matched, op.size(), monoprop::detail::GraphSink<128>(2, 0), window); + mpi::WindowVec malformed(window); + malformed.at_slot(1) = VecZ{size_t{0x8037e}}; + BOOST_CHECK_THROW((void)engine.resolve_published(mpi::views_of(malformed), true), + monoprop::detail::MalformedQueryStream); + BOOST_TEST(op.size() == 1U); + // A view over the wrong window is refused before anything is read. + mpi::WindowVec shifted(mpi::SlotWindow{.base = 1, .count = 2}); + BOOST_CHECK_THROW((void)engine.resolve_published(mpi::views_of(shifted), true), std::invalid_argument); +} + +// --- Large streams and wide terms through the orchestration ----------------------------------------------------- + +// Same-shard streams beyond one 4096-query self window, fresh growth and reindexing, through the whole team: the +// result equals the root's exactly, and at T = 1 the self stream exceeds 4096 queries. +BOOST_AUTO_TEST_CASE(sharded_construction_large_self_streams_match_the_root) { + constexpr size_t N = 16; + const auto options = team_options(); + const size_t threads = team_size(); + std::mt19937_64 rng(17); + Fixture f{.name = "large", .op = {}, .initial_state = {0, 1}, .cutoff = 2 * N, .logical = N}; + std::set used; + while (f.op.size() < 20000) { + const auto m = random_term(rng, 2 + (2 * (rng() % 4))); + if (used.insert(key_of(m)).second) { + add_random_term(f.op, m, 0.001 * static_cast(f.op.size() % 97)); + } + } + const Circuit c{.gates = {{0, 1}, {3, 17}, {2, 5, 9, 30}}, + .mapping = {0, 1, 2}, + .gen_coeffs = {1.0, 0.5, -1.0}, + .params = {0.1, 0.2, 0.3}}; + for (const bool fused : {false, true}) { + auto s = seed(f, options); + auto logs = make_logs(threads); + require_success(fused ? run_propagate(s, c, std::nullopt, Probe{.logs = &logs, .capture = true}) + : run_graph(s, c, std::nullopt, Probe{.logs = &logs, .capture = true})); + auto p = reference_root(f, threads); + if (fused) { + p.propagate(c.gates, c.mapping, c.gen_coeffs, c.params); + } + else { + p.build_graph(c.gates, c.mapping, c.gen_coeffs); + } + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == reference_digest(p, t)), "shard " << t); + } + // A full self window probes 4096 queries in 16 blocks; a stream reaching one exceeds a single window. + size_t full_windows = 0; + for (const auto &log : logs) { + for (const auto *call : log.ranges.of(KernelRange::self_probe)) { + full_windows += static_cast(call->worker.size() == 16); + } + } + if (threads == 1) { + BOOST_TEST(full_windows > 0U); + } + } +} + +// Wide positions (250 modes: positions above 255, more than 31 positions per record, multiword keys) through graph +// construction and propagation, against the root. +BOOST_AUTO_TEST_CASE(sharded_construction_wide_terms_match_the_root) { + constexpr size_t N = 250; + const auto options = team_options(); + const size_t threads = team_size(); + std::mt19937_64 rng(250); + Fixture f{.name = "wide", .op = {}, .initial_state = {0, 1, 2}, .cutoff = 2 * N, .logical = N}; + std::set used; + while (f.op.size() < 300) { + const auto m = random_term(rng, 2 * (1 + (rng() % 20))); + if (used.insert(key_of(m)).second) { + add_random_term(f.op, m, 0.01 * static_cast(1 + (f.op.size() % 13))); + } + } + const Circuit c{.gates = {{1, 300}, {7, 260, 261, 499}, {2}, {40, 41}}, + .mapping = {0, 1, 2, 3}, + .gen_coeffs = {1.0, -0.5, 0.25, 1.0}, + .params = {0.3, 0.2, 0.1, 0.4}}; + for (const bool fused : {false, true}) { + auto s = seed(f, options); + require_success(fused ? run_propagate(s, c, std::nullopt) : run_graph(s, c, std::nullopt)); + auto p = reference_root(f, threads); + if (fused) { + p.propagate(c.gates, c.mapping, c.gen_coeffs, c.params); + } + else { + p.build_graph(c.gates, c.mapping, c.gen_coeffs); + } + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == reference_digest(p, t)), "shard " << t); + } + } +} + +// --- Failures ----------------------------------------------------------------------------------------------------- + +namespace { + +// The checkpoint group a work item belongs to, so "later work" is ordered across owners. +auto group_of(W work) -> int { + switch (work) { + case W::frame: + return -2; + case W::picture: + return -1; + case W::traverse: + case W::prepare_leaders: + return 0; + case W::resolve_leaders: + return 1; + case W::consume_leaders: + case W::prepare_followers: + return 2; + case W::resolve_followers: + return 3; + case W::consume_followers: + case W::finalize: + case W::apply: + return 4; + case W::caches: + return 5; + case W::seed: + case W::replay: + return 6; // informed construction only; never reported by these cases + } + return 7; +} + +auto order_of(size_t step, W work) -> std::pair { + const long long s = step == sharded::no_step ? (work == W::caches ? 1LL << 40 : -1) : static_cast(step); + return {s, group_of(work)}; +} + +// Nothing ran after the failing checkpoint: every visit is at or before the failing (step, group). +auto no_later_work(const std::vector &logs, size_t step, W work) -> bool { + const auto limit = order_of(step, work); + for (const auto &log : logs) { + for (const auto &v : log.visits) { + if (order_of(v.step, v.work) > limit) { + return false; + } + } + } + return true; +} + +auto message_of(const std::exception_ptr &error) -> std::string { + try { + std::rethrow_exception(error); + } + catch (const std::exception &e) { + return e.what(); + } + catch (...) { + return "non-standard exception"; + } +} + +} // namespace + +// A failure in any phase, on the primary or the last owner, joins the team, suppresses every later phase body on every +// worker, returns the original exception, reports whether mutation started, and leaves an earlier copy intact. +BOOST_AUTO_TEST_CASE(sharded_construction_failure_in_each_phase_suppresses_later_work) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + const auto cross_works = {W::resolve_leaders, W::consume_leaders, W::resolve_followers, W::consume_followers}; + for (const bool fused : {false, true}) { + std::vector works = + {W::frame, W::traverse, W::prepare_leaders, W::prepare_followers, W::finalize, W::caches}; + if (fused) { + works.push_back(W::picture); + works.push_back(W::apply); + } + if (threads > 1) { + works.insert(works.end(), cross_works.begin(), cross_works.end()); + } + for (const W work : works) { + for (const size_t shard : {size_t{0}, threads - 1}) { + const bool loop = work != W::frame && work != W::picture && work != W::caches; + const size_t step = loop ? 1 : sharded::no_step; + BOOST_TEST_CONTEXT((fused ? "propagate" : "graph") + << " work " << static_cast(work) << " shard " << shard) { + auto s = seed(cs.f, options); + const auto copy = sharded::copy_shards(options, s.shards); + std::vector copy_before; + for (const auto &state : copy) { + copy_before.push_back(digest(*state)); + } + auto logs = make_logs(threads); + const Probe probe{.logs = &logs, .capture = false, .fail = {{work, step, shard}}}; + const auto outcome = + fused ? run_propagate(s, cs.c, cs.k, probe) : run_graph(s, cs.c, cs.k, probe); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) + == std::format("injected failure: work {} step {} shard {}", + static_cast(work), + step, + shard)); + BOOST_TEST(outcome.mutation_started == (work != W::frame)); + BOOST_TEST(no_later_work(logs, step, work)); + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*copy[t]) == copy_before[t])); + } + if (work == W::frame) { + // Nothing mutated: the shards still equal the untouched copy. + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST((digest(*s.shards[t]) == copy_before[t])); + } + } + } + } + } + } +} + +// Real failures inside the kernels and phases: a decode worker of a destination, the fused apply of the last owner, +// and a publication after the payload was built; each joins and suppresses later work. +BOOST_AUTO_TEST_CASE(sharded_construction_kernel_decode_and_publication_failures, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + struct Scenario { + const char *name; + bool fused; + std::optional kernel; + std::optional> publication; + const char *message; + }; + const std::vector scenarios = { + {"decode", + false, + KernelFailure{threads - 1, KernelRange::decode, 1, 0}, + std::nullopt, + "injected kernel failure"}, + {"scatter", true, KernelFailure{1, KernelRange::scatter, 0, 0}, std::nullopt, "injected kernel failure"}, + {"fused apply", + true, + KernelFailure{threads - 1, KernelRange::fused_apply, 2, 0}, + std::nullopt, + "injected kernel failure"}, + {"leader publication", false, std::nullopt, std::pair{threads - 1, true}, "injected publication failure"}, + {"follower publication", true, std::nullopt, std::pair{size_t{0}, false}, "injected publication failure"}, + }; + for (const auto &sc : scenarios) { + BOOST_TEST_CONTEXT(sc.name) { + auto s = seed(cs.f, options); + const auto copy = sharded::copy_shards(options, s.shards); + const auto copy_digest = digest(*copy[0]); + auto logs = make_logs(threads); + const Probe probe{.logs = &logs, + .capture = false, + .kernel_fail = sc.kernel, + .publication_fail = sc.publication}; + const auto outcome = + sc.fused ? run_propagate(s, cs.c, cs.k, probe) : run_graph(s, cs.c, cs.k, probe); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) == sc.message); + BOOST_TEST(outcome.mutation_started); + BOOST_TEST((digest(*copy[0]) == copy_digest)); + // No shard started the caches phase after the failure. + BOOST_TEST(count_visits(logs, W::caches) == 0U); + } + } +} + +// Several owners failing in the same phase: the lowest-numbered owner's exception is returned. +BOOST_AUTO_TEST_CASE(sharded_construction_concurrent_failures_select_the_lowest_owner, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + auto s = seed(cs.f, options); + auto logs = make_logs(threads); + const Probe probe{.logs = &logs, + .capture = false, + .fail = {{W::resolve_followers, 2, threads - 1}, {W::resolve_followers, 2, 1}}}; + const auto outcome = run_graph(s, cs.c, cs.k, probe); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) + == std::format("injected failure: work {} step 2 shard 1", static_cast(W::resolve_followers))); + BOOST_TEST(no_later_work(logs, 2, W::resolve_followers)); +} + +// Real allocation failures at every allocation of one owner's frame, traversal, resolution and finish: each is a +// catchable std::bad_alloc after the join, with no leak; a frame failure mutates nothing; earlier copies survive. +BOOST_AUTO_TEST_CASE(sharded_construction_allocation_failures_are_contained, + *boost::unit_test::precondition(has_allocation_probe)) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[1]; // Schrödinger: fresh-insert scoring in the apply + struct Site { + W work; + size_t step; + size_t shard; + }; + std::vector sites = {{W::frame, sharded::no_step, threads - 1}, + {W::traverse, 2, 0}, + {W::finalize, 3, threads - 1}}; + sites.push_back({W::apply, 2, 0}); // propagation only; graph construction has no apply + if (threads > 1) { + sites.push_back({W::resolve_leaders, 1, threads - 1}); + sites.push_back({W::consume_followers, 1, 0}); + } + for (const bool fused : {false, true}) { + // Warm every worker's retained thread-local scan scratch, which later runs reuse and never free. + { + auto warm = seed(cs.f, options); + require_success(fused ? run_propagate(warm, cs.c, cs.k) : run_graph(warm, cs.c, cs.k)); + } + for (const auto &site : sites) { + if (site.work == W::apply && !fused) { + continue; + } + size_t injected = 0; + for (size_t nth = 1; nth < 2000; ++nth) { + bool failed = false; + { + const allocation::LiveBytes live; + { + auto s = seed(cs.f, options); + const auto copy = sharded::copy_shards(options, s.shards); + const auto copy_digest = digest(*copy[threads - 1]); + auto logs = make_logs(threads); + const Probe probe{.logs = &logs, + .capture = false, + .alloc = AllocAt{site.work, site.step, site.shard, nth}}; + const auto outcome = + fused ? run_propagate(s, cs.c, cs.k, probe) : run_graph(s, cs.c, cs.k, probe); + disarm_team(); + failed = static_cast(outcome.error); + if (failed) { + ++injected; + bool bad_alloc = false; + try { + std::rethrow_exception(outcome.error); + } + catch (const std::bad_alloc &) { + bad_alloc = true; + } + catch (...) { + } + BOOST_TEST(bad_alloc); + BOOST_TEST(outcome.mutation_started == (site.work != W::frame)); + BOOST_TEST((digest(*copy[threads - 1]) == copy_digest)); + } + } + BOOST_TEST(live.net() == 0); + } + if (!failed) { + break; + } + } + BOOST_TEST_MESSAGE("allocation site work " << static_cast(site.work) << " shard " << site.shard + << (fused ? " propagate" : " graph") << ": " << injected + << " injected failures"); + BOOST_TEST(injected > 0U); + } + } +} + +// Arguments are checked before the team, with nothing mutated. +BOOST_AUTO_TEST_CASE(sharded_construction_rejects_invalid_arguments_before_the_team) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + auto s = seed(cs.f, options); + const auto before = digest(*s.shards[0]); + auto logs = make_logs(threads); + const auto gens = generators(cs.f, cs.c); + VecZ gate_indices(gens.size()); + const auto good = s.ctx(); + const auto graph = + [&](const sharded::ConstructionContext &ctx, std::span coeffs, std::optional k) { + return sharded::build_graph(options, + s.shards, + ctx, + {.generators = gens, + .parameter_mapping = cs.c.mapping, + .gen_coeffs = coeffs, + .gate_indices = gate_indices, + .only_rotate_len_k = k}, + Probe{.logs = &logs}); + }; + BOOST_CHECK_THROW((void)graph(good, std::span(cs.c.gen_coeffs).first(2), std::nullopt), + std::invalid_argument); + BOOST_CHECK_THROW((void)graph(good, cs.c.gen_coeffs, 2 * kN + 1), std::runtime_error); + auto two_ranks = good; + two_ranks.router = routing::Router::splitmix(2 * threads); + BOOST_CHECK_THROW((void)graph(two_ranks, cs.c.gen_coeffs, std::nullopt), std::invalid_argument); + auto wrong_rank = good; + wrong_rank.world.rank = 1; + BOOST_CHECK_THROW((void)graph(wrong_rank, cs.c.gen_coeffs, std::nullopt), std::invalid_argument); + sharded::Shards short_shards; + BOOST_CHECK_THROW( + (void)sharded::propagate(options, + short_shards, + good, + {.generators = gens, .mapped_params = {}, .only_rotate_len_k = std::nullopt}), + std::invalid_argument); + BOOST_TEST((digest(*s.shards[0]) == before)); + for (const auto &log : logs) { + BOOST_TEST(log.visits.empty()); + } +} diff --git a/cpp/tests/sharded_evaluation_tests.cpp b/cpp/tests/sharded_evaluation_tests.cpp new file mode 100644 index 000000000..0490cb15a --- /dev/null +++ b/cpp/tests/sharded_evaluation_tests.cpp @@ -0,0 +1,2044 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// First on purpose: the header must compile on its own. +#include "monoprop/detail/sharded/Evaluation.h" + +/* + * Snapshot-safe replay, energy, gradients, retained/pared functionals and coefficient-informed construction over T + * shards of one process (detail/sharded/Evaluation.h, Construction.h). The team size T is the captured launch budget; + * cpp/tests/CMakeLists.txt reruns these cases in fresh processes at T = 1, 2 and 4. Cases that need a nonprimary worker + * are left out of T = 1. + * + * Oracles: + * - The rank-level root at the same launch (MPI_COMM_SELF, the launch's T), through its public operations: per-shard + * evolved coefficients, local terms, energies and every gradient component, bitwise. It drives the same seams, so it + * checks the root's wiring of them, never their arithmetic; so: + * - an independent coefficient-map propagator (algebra primitives only) for exact-cutoff energies, the frozen exact + * energy of tests/data/random_exact.msgpack, closed-form single-gate fits and central finite differences. + * Bitwise equality with an independently implemented runtime at the same geometry, and the global retained maps + * across T, are checked from separate processes (tests/test_sharded_openmp.py); two runtimes never share a process. + * + * Workers never call BOOST_TEST. Observers write per-shard slots and assertions run after the team has joined. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AllocationProbe.h" +#include "PauliTestOracle.h" +#include "PropagatorTestAccess.h" +#include "TestUtilities.h" +#include "monoprop/MPFunctions.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/Validation.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/evolution/CosineRecompute.h" +#include "monoprop/detail/evolution/LayerReplay.h" +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/pare/PareGraph.h" +#include "monoprop/detail/sharded/Construction.h" +#include "monoprop/detail/sharded/State.h" +#include "monoprop/detail/sharded/Team.h" + +namespace { + +using namespace monoprop; +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; +namespace replay = monoprop::detail::replay; +namespace allocation = test_utils::allocation; +using E = sharded::EvaluationWork; +using W = sharded::ConstructionWork; +using Key = std::vector; + +constexpr size_t kN = 8; +constexpr double kExactAtol = 1e-9; // the frozen exact-energy tolerance of the construction and fused-sweep tests + +auto team_options() -> parallel::Options { + return parallel::capture_thread_budget(); +} + +auto team_size() -> size_t { + return static_cast(team_options().threads); +} + +auto has_nonprimary_worker(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto threads = parallel::capture_thread_budget().threads; + boost::test_tools::assertion_result result(threads >= 2); + result.message() << "the captured team budget is " << threads << "; this case needs a nonprimary worker"; + return result; +} + +auto has_allocation_probe(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + boost::test_tools::assertion_result result(allocation::available()); + result.message() << "this build's sanitizer runtime owns operator new/delete, so no allocation probe is linked"; + return result; +} + +template +auto key_of(const Monomial &m) -> Key { + Key key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + return key; +} + +auto bits(double v) -> uint64_t { + return std::bit_cast(v); +} + +auto bits_of(const VecD &v) -> std::vector { + std::vector out; + out.reserve(v.size()); + for (const double x : v) { + out.push_back(bits(x)); + } + return out; +} + +auto message_of(const std::exception_ptr &error) -> std::string { + try { + std::rethrow_exception(error); + } + catch (const std::exception &e) { + return e.what(); + } + catch (...) { + return "non-standard exception"; + } +} + +// --- Fixtures ------------------------------------------------------------------------------------------------- + +struct Fixture { + std::string name; + OperatorDict op; + VecZ initial_state; + std::optional schrodinger_cutoff = std::nullopt; + Basis basis = Basis::Majorana; + CutoffType cutoff_type = CutoffType::Length; + unsigned int cutoff = 4; + size_t logical = kN; + std::optional lower_atol = std::nullopt; + std::optional upper_atol = std::nullopt; + std::optional> basis_change = std::nullopt; +}; + +struct Circuit { + std::vector gates; + VecZ mapping; + VecD gen_coeffs; + VecD params; +}; + +struct Case { + Fixture f; + Circuit c; + std::optional k = std::nullopt; // only_rotate_len_k +}; + +auto add_term(OperatorDict &op, Basis basis, const VecZ &indices, double value) -> void { + op[indices] = algebra_decode_coeff(basis, std::complex(value, 0.0), indices_to_bitset(indices)); +} + +// Pairs, a ladder of quartics and long terms (which spill past the packed inline width), plus identity. +auto majorana_operator(size_t logical) -> OperatorDict { + OperatorDict op; + const size_t slots = 2 * logical; + auto value = 0.25; + for (size_t i = 0; i < slots; ++i) { + for (size_t j = i + 1; j < slots; j += 2) { + add_term(op, Basis::Majorana, {i, j}, value); + value += 0.125; + } + } + for (size_t i = 0; i + 3 < slots; ++i) { + add_term(op, Basis::Majorana, {i, i + 1, i + 2, i + 3}, -0.5 - (0.03125 * static_cast(i))); + } + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5}, 0.75); + add_term(op, Basis::Majorana, {1, 3, 5, 7, 9, 11}, -1.25); + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5, 6, 7}, 2.0); + add_term(op, Basis::Majorana, {}, 3.5); + return op; +} + +auto pauli_operator() -> OperatorDict { + OperatorDict op; + auto value = 0.5; + const auto add = [&](const std::string &pauli) { + add_term(op, Basis::Pauli, pauli_oracle::slots_of_string(pauli), value); + value += 0.25; + }; + for (size_t q = 0; q < kN; ++q) { + for (const char p : {'X', 'Z'}) { + std::string s(kN, 'I'); + s[q] = p; + add(s); + } + } + for (size_t q = 0; q + 1 < kN; ++q) { + std::string s(kN, 'I'); + s[q] = 'Z'; + s[q + 1] = 'Z'; + add(s); + } + add("XYZXYIII"); + add("ZZZZZZZZ"); + add_term(op, Basis::Pauli, {}, -1.75); + return op; +} + +// Two-, three- (odd), four- and six-slot generators, a repeated gate, a repeated parameter and an identity gate. +auto majorana_circuit(size_t logical) -> Circuit { + const size_t s = 2 * logical; + Circuit c; + c.gates = {{0, 1}, {2, 5}, {1, 3, 4, 6}, {0, s - 1}, {}, {3}, {2, 4, s - 3}, {5, 6, 7, s - 2}, {0, 1}, {1, 2}}; + c.mapping = {0, 1, 2, 3, 4, 5, 6, 2, 7, 8}; + for (size_t i = 0; i < c.gates.size(); ++i) { + c.gen_coeffs.push_back(i % 3 == 0 ? -1.0 : 0.5 + (0.25 * static_cast(i))); + } + for (size_t i = 0; i < 9; ++i) { + c.params.push_back(0.11 + (0.07 * static_cast(i))); + } + return c; +} + +auto pauli_circuit() -> Circuit { + Circuit c; + for (const std::string g : {"XXIIIIII", "YZXIIIII", "ZIIIIIII", "IIXYIIII", "IIIIIIII", "ZZZZIIII", "IYIIIXII"}) { + c.gates.push_back(pauli_oracle::slots_of_string(g)); + } + c.mapping.resize(c.gates.size()); + std::iota(c.mapping.begin(), c.mapping.end(), size_t{0}); + for (size_t i = 0; i < c.gates.size(); ++i) { + c.gen_coeffs.push_back(0.75 - (0.125 * static_cast(i))); + c.params.push_back(0.2 + (0.05 * static_cast(i))); + } + return c; +} + +auto data_circuit(const test_utils::CaseData &data) -> Circuit { + return {.gates = data.majoranas, + .mapping = data.param_inds, + .gen_coeffs = data.gen_coeffs, + .params = data.parameters}; +} + +// A basis change that permutes the slots pairwise. +auto swap_basis(size_t logical) -> std::vector { + std::vector rows; + for (size_t i = 0; i < 2 * logical; ++i) { + rows.push_back({i ^ 1U}); + } + return rows; +} + +// Every basis and picture, trimming cutoffs, coefficient cutoffs, length caps (two-pass fallback), a basis change, +// odd Majorana generators and native Pauli folding. +auto fixture_cases() -> std::vector { + const auto data = test_utils::load_case_data("random_exact.msgpack"); + const Fixture maj{.name = "majorana-heisenberg-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .logical = 6}; + auto maj_s = maj; + maj_s.name = "majorana-schrodinger-logical-6"; + maj_s.schrodinger_cutoff = 5; + auto maj_atol = maj; + maj_atol.name = "majorana-heisenberg-atol"; + maj_atol.lower_atol = 0.02; + maj_atol.upper_atol = 0.3; + auto maj_s_atol = maj_s; + maj_s_atol.name = "majorana-schrodinger-atol"; + maj_s_atol.lower_atol = 0.001; + maj_s_atol.upper_atol = 0.05; + auto maj_bc = maj; + maj_bc.name = "majorana-heisenberg-basis-change"; + maj_bc.basis_change = swap_basis(6); + const Fixture rnd{.name = "majorana-heisenberg-random-exact", + .op = data.hamiltonian, + .initial_state = data.initial_state, + .cutoff = 6}; + auto rnd_s = rnd; + rnd_s.name = "majorana-schrodinger-random-exact"; + rnd_s.schrodinger_cutoff = 6; + const Fixture pauli{.name = "pauli-heisenberg", + .op = pauli_operator(), + .initial_state = {}, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 3}; + auto pauli_s = pauli; + pauli_s.name = "pauli-schrodinger"; + pauli_s.schrodinger_cutoff = 5; + return { + {.f = maj, .c = majorana_circuit(6)}, + {.f = maj_s, .c = majorana_circuit(6)}, + {.f = maj_atol, .c = majorana_circuit(6)}, + {.f = maj_s_atol, .c = majorana_circuit(6)}, + {.f = maj, .c = majorana_circuit(6), .k = 4}, + {.f = maj_s, .c = majorana_circuit(6), .k = 2}, + {.f = maj_bc, .c = majorana_circuit(6)}, + {.f = rnd, .c = data_circuit(data)}, + {.f = rnd_s, .c = data_circuit(data)}, + {.f = pauli, .c = pauli_circuit()}, + {.f = pauli_s, .c = pauli_circuit()}, + }; +} + +auto label(const Case &c) -> std::string { + return c.k ? std::format("{} k={}", c.f.name, *c.k) : c.f.name; +} + +// Parameter vectors on the case's axis: the case's own, a deep-amplification set (every layer records what the layer +// below rotates), and one with vanishing cosines on two consecutive layers (duplicate record indices). +auto parameter_sets(const Circuit &c) -> std::vector> { + std::vector> sets = {{"case", c.params}}; + VecD deep(c.params.size()); + for (size_t i = 0; i < deep.size(); ++i) { + deep[i] = 0.7 + (0.013 * static_cast(i)); + } + sets.emplace_back("deep", deep); + if (c.gen_coeffs.size() >= 2) { + // The last two gates' layers are the first two replayed in Heisenberg and the last two in Schrödinger; both + // are driven to cos(2 * gen * param) ~ 6e-17. + VecD vanish = c.params; + const size_t n = c.gates.size(); + for (const size_t gate : {n - 1, n - 2}) { + const double g = c.gen_coeffs[gate]; + if (g != 0.0) { + vanish[c.mapping[gate]] = std::numbers::pi / (4.0 * g); + } + } + sets.emplace_back("vanishing", vanish); + } + return sets; +} + +// --- The rank-level reference --------------------------------------------------------------------------------- + +template +using Access = monoprop::detail::PropagatorTestAccess; + +// The rank-level reference at geometry (1, T): the root over MPI_COMM_SELF at the launch's T, through its public +// operations. It drives the same seams, so comparing against it checks the root's wiring of them, while each case +// keeps its independent references. +template +auto reference_root(const Fixture &f, size_t threads) -> MonomialPropagator { + auto p = MonomialPropagator(f.op, + f.cutoff, + f.initial_state, + f.schrodinger_cutoff, + MPI_COMM_SELF, + f.lower_atol, + f.upper_atol, + f.cutoff_type, + f.basis_change, + f.logical, + f.basis); + BOOST_TEST_REQUIRE(Access::shards(p).size() == threads); + return p; +} + +// --- Sharded setup ---------------------------------------------------------------------------------------------- + +template +auto make_cutoff_fn(const Fixture &f) -> CutoffFn { + if (f.basis_change) { + MonomialList basis; + for (size_t i = 0; i < 2 * f.logical; ++i) { + basis.push_back(indices_to_bitset_checked((*f.basis_change)[i], 2 * f.logical)); + } + return monoprop::detail::cutoff_function_basis_change(f.cutoff_type, f.cutoff, basis, f.logical); + } + return monoprop::detail::cutoff_function(f.cutoff_type, f.cutoff, f.logical); +} + +template +struct Sharded { + Fixture f; + CutoffFn cutoff_fn; + routing::Router router; + sharded::Shards shards; + sharded::PhysicalWorld world{}; // one process unless a multi-rank case seeds over MPI_COMM_WORLD + + [[nodiscard]] auto ctx() const -> sharded::ConstructionContext { + return {.cutoff_fn = cutoff_fn, + .router = router, + .lower_atol = f.lower_atol, + .upper_atol = f.upper_atol, + .basis = f.basis, + .schrodinger = f.schrodinger_cutoff.has_value(), + .world = world}; + } + [[nodiscard]] auto schrodinger() const -> bool { return f.schrodinger_cutoff.has_value(); } + [[nodiscard]] auto core() const -> double { + return sharded::validate_initial_operator(f.op, f.basis, f.logical); + } +}; + +template +auto seed(const Fixture &f, parallel::Options options, const sharded::PhysicalWorld &world = {}) -> Sharded { + const auto router = routing::make_router(world.ranks, static_cast(options.threads)); + const auto cutoff_fn = make_cutoff_fn(f); + std::optional paired; + if (f.schrodinger_cutoff) { + paired = sharded::paired_basis_bounds(*f.schrodinger_cutoff, f.logical, router.flat_world()); + } + const auto width_fn = monoprop::detail::cutoff_function(f.cutoff_type, f.cutoff, f.logical); + const auto seed_inputs = sharded::OperatorSeed{ + .initial_operator = f.op, + .initial_state = f.initial_state, + .router = router, + .basis = f.basis, + .logical_num_modes = f.logical, + .paired = paired, + .inline_width = sharded::packed_inline_width(paired.has_value(), f.basis_change ? cutoff_fn : width_fn)}; + return {.f = f, + .cutoff_fn = cutoff_fn, + .router = router, + .shards = sharded::seed_shards(options, seed_inputs, world.rank), + .world = world}; +} + +template +auto generators(const Fixture &f, const Circuit &c) -> std::vector> { + std::vector> out; + for (const auto &g : c.gates) { + out.push_back(indices_to_bitset_checked(g, 2 * f.logical)); + } + return out; +} + +auto require_success(const std::exception_ptr &error, const char *what) -> void { + if (error) { + BOOST_FAIL(what << " failed: " << message_of(error)); + } +} + +template +auto run_graph(Sharded &s, const Circuit &c, std::optional k, size_t gate_offset = 0) -> void { + const auto gens = generators(s.f, c); + VecZ gate_indices(c.gates.size()); + std::iota(gate_indices.begin(), gate_indices.end(), gate_offset); + const auto ctx = s.ctx(); + const auto outcome = sharded::build_graph(team_options(), + s.shards, + ctx, + {.generators = gens, + .parameter_mapping = c.mapping, + .gen_coeffs = c.gen_coeffs, + .gate_indices = gate_indices, + .only_rotate_len_k = k}); + require_success(outcome.error, "build_graph"); +} + +// Optimizer-order gate arrays of a shard graph, as MonomialPropagator::graph_gate_arrays_() reads them. +auto gate_arrays(const MPGraph &graph) -> std::pair { + const size_t count = graph.layers(); + VecZ mapping(count); + VecD gen(count); + for (size_t layer = 0; layer < count; ++layer) { + const auto t = graph.get_layer_traversal(layer); + mapping[count - 1 - layer] = t.param_index(); + gen[count - 1 - layer] = t.gen_coeff(); + } + return {std::move(mapping), std::move(gen)}; +} + +// contract_partially()'s replay angles for a picture over a graph's layers. +auto contraction_angles(const MPGraph &graph, bool schrodinger, const VecD ¶ms) -> VecD { + const auto [mapping, gen] = gate_arrays(graph); + return schrodinger ? map_params(params, mapping, gen, -1.0) : map_params(params, mapping, gen, 1.0, true); +} + +// Coefficient-informed construction on the shards, with the angles the root's build_graph derives. +template +auto run_informed(Sharded &s, + const Circuit &c, + const VecD ¶meters, + std::optional k, + size_t gate_offset = 0, + const Observer &observer = {}) -> sharded::ConstructionOutcome { + const auto gens = generators(s.f, c); + VecZ gate_indices(c.gates.size()); + std::iota(gate_indices.begin(), gate_indices.end(), gate_offset); + const auto mapped = map_params(parameters, c.mapping, c.gen_coeffs, 1.0); + std::optional seed_angles; + if (s.shards.front()->graph.layers() > 0) { + const auto [existing, _] = gate_arrays(s.shards.front()->graph); + const size_t m = expected_num_params(existing); + const VecD prefix(parameters.begin(), parameters.begin() + static_cast(m)); + seed_angles = contraction_angles(s.shards.front()->graph, s.schrodinger(), prefix); + } + const auto ctx = s.ctx(); + return sharded::build_graph_informed( + team_options(), + s.shards, + ctx, + {.graph = {.generators = gens, + .parameter_mapping = c.mapping, + .gen_coeffs = c.gen_coeffs, + .gate_indices = gate_indices, + .only_rotate_len_k = k}, + .mapped_params = mapped, + .seed_params = seed_angles ? std::optional>(*seed_angles) : std::nullopt}, + observer); +} + +template +auto retained_context(const Sharded &s, std::optional pare) -> sharded::RetainedContext { + auto [mapping, gen] = gate_arrays(s.shards.front()->graph); + return {.core_term = s.core(), + .parameter_mapping = std::move(mapping), + .gen_coeffs = std::move(gen), + .pare_threshold = pare, + .basis = s.f.basis, + .schrodinger = s.schrodinger(), + .rank = s.world.rank}; +} + +template +auto retain(Sharded &s, + std::optional pare = std::nullopt, + const sharded::EvaluationObserver *observer = nullptr) -> sharded::RetainedEvaluation { + auto outcome = sharded::prepare_retained(team_options(), s.shards, retained_context(s, pare), observer); + require_success(outcome.error, "prepare_retained"); + BOOST_TEST_REQUIRE(outcome.retained.has_value()); + return std::move(*outcome.retained); +} + +auto evaluate(const sharded::RetainedEvaluation &r, + const VecD ¶ms, + bool gradient, + const sharded::EvaluationObserver *observer = nullptr, + const sharded::PhysicalWorld &world = {}) -> sharded::EvaluationOutcome { + const auto requests = r.requests(params); + return sharded::evaluate_shards(requests, r.callbacks, team_options(), gradient, observer, world); +} + +auto energy(const sharded::RetainedEvaluation &r, const VecD ¶ms, const sharded::PhysicalWorld &world = {}) + -> double { + const auto requests = r.requests(params); + return sharded::ev_sharded(requests, r.callbacks, team_options(), world.comm); +} + +auto energy_and_gradient(const sharded::RetainedEvaluation &r, + const VecD ¶ms, + const sharded::PhysicalWorld &world = {}) -> std::pair { + const auto requests = r.requests(params); + return sharded::ev_and_grad_sharded(requests, r.callbacks, team_options(), world.comm); +} + +// --- Observation ------------------------------------------------------------------------------------------------ + +struct Visit { + E work; + size_t step; + int worker; + int team; + int level; + std::thread::id thread; +}; + +struct FailAt { + E work; + size_t step; + size_t shard; +}; + +struct AllocAt { + E work; + size_t step; + size_t shard; + size_t nth; +}; + +// Per-shard visit logs and optional injected failures. Slot s is written by the thread performing shard s's work: +// its owner, or the primary inside an exclusive phase; never two threads between the same checkpoints. +class Recorder final : public sharded::EvaluationObserver { +public: + explicit Recorder(size_t threads, std::vector fail = {}, std::optional alloc = std::nullopt) + : logs_(threads), + fail_(std::move(fail)), + alloc_(alloc) { + for (auto &log : logs_) { + log.reserve(8192); + } + } + + auto visit(E work, size_t step, size_t shard) const -> void override { + (void)allocation::disarm_failure(); + logs_[shard].push_back( + {work, step, omp_get_thread_num(), omp_get_num_threads(), omp_get_level(), std::this_thread::get_id()}); + for (const auto &f : fail_) { + if (f.work == work && f.step == step && f.shard == shard) { + throw std::runtime_error( + std::format("injected failure: work {} step {} shard {}", static_cast(work), step, shard)); + } + } + if (alloc_ && alloc_->work == work && alloc_->step == step && alloc_->shard == shard) { + allocation::arm_failure(alloc_->nth); + } + } + + [[nodiscard]] auto logs() const -> const std::vector> & { return logs_; } + + [[nodiscard]] auto count(E work) const -> size_t { + size_t n = 0; + for (const auto &log : logs_) { + n += static_cast(std::ranges::count_if(log, [&](const Visit &v) { return v.work == work; })); + } + return n; + } + +private: + mutable std::vector> logs_; + std::vector fail_; + std::optional alloc_; +}; + +// After an injected allocation failure, no worker thread keeps an armed failure. +auto disarm_team() -> void { + static_cast(sharded::run_team(team_options(), [](size_t, sharded::TeamFailure &) noexcept { + (void)allocation::disarm_failure(); + })); +} + +// --- Independent references ------------------------------------------------------------------------------------- + +using RetainedMap = std::map; + +template +auto mono_of(const Key &key) -> Monomial { + Monomial m; + for (size_t w = 0; w < key.size(); ++w) { + m.data()[w] = key[w]; + } + return m; +} + +// Independent propagation on a map: each anticommuting key M moves to cos(2a) M + sin(2a) phase(M) (M ^ G). Absent +// partners enter with 0 (an operator has no implicit terms). +template +auto reference_operator(RetainedMap map, const std::vector> &gens, const VecD &mapped, Basis basis) + -> RetainedMap { + const size_t n = gens.size(); + for (size_t step = 0; step < n; ++step) { + const size_t idx = n - 1 - step; + const auto &gen = gens[idx]; + const double c = std::cos(2 * mapped[idx]); + const double sn = std::sin(2 * mapped[idx]); + with_algebra(basis, [&]() { + const auto fold = A::fold_generator(gen); + const bool odd = A::fold_needs_odd_correction(gen); + const auto ctx = A::make_gen_context(gen); + const auto anti = [&](const Monomial &m) { + return ((m & fold).count() + (odd ? m.count() : 0)) % 2 == 1; + }; + RetainedMap next; + for (const auto &[key, v] : map) { + const auto m = mono_of(key); + if (!anti(m)) { + next[key] += v; + continue; + } + const auto partner = m ^ gen; + const int phase = + A::emit_phase(A::rotation_sign(ctx, m, partner), m.count(), gen.count(), (m & gen).count()); + next[key] += c * v; + next[key_of(partner)] += sn * static_cast(phase) * v; + } + map = std::move(next); + }); + } + return map; +} + +// from the reference Heisenberg operator: the identity once, plus each term's state score. +template +auto reference_energy(const Fixture &f, const Circuit &c, const VecD ¶ms) -> double { + RetainedMap map; + double core = 0.0; + for (const auto &[indices, coefficient] : f.op) { + const auto mono = indices_to_bitset_checked(indices, 2 * f.logical); + const double encoded = algebra_encode_coeff(f.basis, coefficient, mono); + if (indices.empty()) { + core = encoded; + continue; + } + map[key_of(mono)] = encoded; + } + const auto mapped = map_params(params, c.mapping, c.gen_coeffs, 1.0); + const auto evolved = reference_operator(map, generators(f, c), mapped, f.basis); + const auto mask = initial_state_mask(f.initial_state); + double total = core; + for (const auto &[key, v] : evolved) { + const auto m = mono_of(key); + bool diagonal = true; + for (size_t mode = 0; mode < N; ++mode) { + if (m.test(2 * mode) != m.test((2 * mode) + 1)) { + diagonal = false; + break; + } + } + if (diagonal) { + total += v * algebra_state_phase(f.basis, m, mask); + } + } + return total; +} + +template +auto central_differences(Fn &&fn, const VecD ¶ms, double eps = 1e-5) -> VecD { + VecD grad(params.size()); + for (size_t i = 0; i < params.size(); ++i) { + auto plus = params; + auto minus = params; + plus[i] += eps; + minus[i] -= eps; + grad[i] = (fn(plus) - fn(minus)) / (2 * eps); + } + return grad; +} + +auto close(double a, double b, double atol, double rtol = 0.0) -> bool { + return std::abs(a - b) <= atol + (rtol * std::max(std::abs(a), std::abs(b))); +} + +// Layout census of a set of shard graphs: what the fixtures exercise. +struct Census { + size_t self_pairs = 0; // layers with a nonempty self slot + size_t cross_slots = 0; // occupied slots of another local shard + size_t multi_peer = 0; // layers with two or more partner slots + size_t empty_layers = 0; // layers with no occupied slot at all + size_t empty_owners = 0; // shards without rows +}; + +template +auto census(const sharded::Shards &shards) -> Census { + Census c; + for (size_t t = 0; t < shards.size(); ++t) { + const auto &state = *shards[t]; + c.empty_owners += state.op.size() == 0 ? 1 : 0; + for (size_t l = 0; l < state.graph.layers(); ++l) { + const auto layer = state.graph.get_layer_traversal(l); + size_t partners = 0; + size_t occupied = 0; + layer.for_each_occupied_slot([&](size_t slot, const monoprop::detail::CrossRankSlotView &view) { + ++occupied; + if (slot != t && view.sin_send_count > 0) { + ++partners; + } + }); + c.self_pairs += layer.cross_rank_self_slot().sin_send_count >= 2 ? 1 : 0; + c.cross_slots += partners; + c.multi_peer += partners >= 2 ? 1 : 0; + c.empty_layers += occupied == 0 ? 1 : 0; + } + } + return c; +} + +} // namespace + +// A small cross-shard fixture: every shard both publishes and reads partner endpoints in the same layer, so a finish +// that read a partner's live (already cos-scaled) coefficients instead of its snapshot would be off by a factor cos. +// Checked in both directions against the independent map propagator and its central differences. +BOOST_AUTO_TEST_CASE(sharded_evaluation_peers_read_published_snapshots, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + Fixture f{.name = "cross-shard", .op = {}, .initial_state = {0, 2}, .cutoff = 2 * kN, .logical = kN}; + for (size_t i = 0; i + 1 < 2 * kN; ++i) { + add_term(f.op, Basis::Majorana, {i, i + 1}, 0.3 + (0.05 * static_cast(i))); + } + add_term(f.op, Basis::Majorana, {0, 1, 2, 3}, -0.4); + add_term(f.op, Basis::Majorana, {}, 0.25); + const Circuit c{.gates = {{1, 2}, {3, 4, 5, 6}, {0, 3}, {2, 7}}, + .mapping = {0, 1, 2, 3}, + .gen_coeffs = {0.5, -0.75, 1.0, 0.25}, + .params = {0.4, 0.9, -0.3, 1.1}}; + auto s = seed(f, options); + run_graph(s, c, std::nullopt); + const auto layout = census(s.shards); + BOOST_TEST_REQUIRE(layout.cross_slots > 0U); + const auto r = retain(s); + const auto reference = [&](const VecD &p) { return reference_energy(f, c, p); }; + const auto [e, grad] = energy_and_gradient(r, c.params); + BOOST_TEST(close(energy(r, c.params), reference(c.params), kExactAtol)); + BOOST_TEST(close(e, reference(c.params), kExactAtol)); + const auto fd = central_differences(reference, c.params); + for (size_t i = 0; i < grad.size(); ++i) { + BOOST_TEST(close(grad[i], fd[i], 1e-6), "component " << i << ": " << grad[i] << " vs " << fd[i]); + } +} + +// Whole self pairs, other-local-shard and multi-peer layouts, empty owners and no-work layers are all exercised at +// T >= 2, and evaluation over them matches the root. +BOOST_AUTO_TEST_CASE(sharded_evaluation_covers_every_layout, *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const size_t threads = team_size(); + Census total; + const auto accumulate = [&](const Census &c) { + total.self_pairs += c.self_pairs; + total.cross_slots += c.cross_slots; + total.multi_peer += c.multi_peer; + total.empty_layers += c.empty_layers; + total.empty_owners += c.empty_owners; + }; + Fixture two{.name = "two-terms", .op = {}, .initial_state = {0}, .cutoff = 4, .logical = 6}; + add_term(two.op, Basis::Majorana, {0, 1}, 0.5); + add_term(two.op, Basis::Majorana, {}, 1.5); + const Circuit few{.gates = {{}, {1, 2}}, .mapping = {0, 1}, .gen_coeffs = {1.0, 0.5}, .params = {0.3, 0.4}}; + std::vector cases = fixture_cases(); + cases.push_back({.f = two, .c = few}); + for (const auto &cs : cases) { + BOOST_TEST_CONTEXT(label(cs) << " T=" << threads) { + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + accumulate(census(s.shards)); + auto old = reference_root(cs.f, threads); + old.build_graph(cs.c.gates, cs.c.mapping, cs.c.gen_coeffs, std::nullopt, std::nullopt, cs.k); + const auto r = retain(s); + const auto [ge, grad] = old.expectation_value_and_gradient(cs.c.params); + const auto [se, sgrad] = energy_and_gradient(r, cs.c.params); + BOOST_TEST(bits(se) == bits(ge)); + BOOST_TEST((bits_of(sgrad) == bits_of(grad))); + } + } + BOOST_TEST(total.self_pairs > 0U); + BOOST_TEST(total.cross_slots > 0U); + BOOST_TEST(total.empty_layers > 0U); + if (threads >= 4) { + // Two partner slots need at least three shards. + BOOST_TEST(total.multi_peer > 0U); + BOOST_TEST(total.empty_owners > 0U); + } +} + +// The replicated identity is added once, never once per shard; a Schrödinger identity basis row is an ordinary routed +// row with a zero operator coefficient. Empty parameters contract the un-evolved operator and need no callbacks. +BOOST_AUTO_TEST_CASE(sharded_evaluation_identity_once_and_empty_parameters) { + const auto options = team_options(); + const size_t threads = team_size(); + for (const auto &cs : fixture_cases()) { + BOOST_TEST_CONTEXT(label(cs) << " T=" << threads) { + auto s = seed(cs.f, options); + auto r = retain(s); + BOOST_TEST(r.num_params == 0U); + // No callbacks at all: empty parameters must not need them. + r.callbacks.assign(threads, monoprop::detail::CosCallbacks{}); + const VecD none; + const auto outcome = evaluate(r, none, true); + require_success(outcome.error, "evaluate"); + double expected = r.core_term; + double folded = 0.0; + for (size_t t = 0; t < threads; ++t) { + const double local = r.shards[t].state.dot(r.shards[t].op); + BOOST_TEST(bits(outcome.contributions[t]) == bits(local)); + BOOST_TEST(outcome.gradients[t].empty()); + folded += local; + } + expected += folded; + BOOST_TEST(bits(energy(r, none)) == bits(expected)); + const auto [e, g] = energy_and_gradient(r, none); + BOOST_TEST(bits(e) == bits(expected)); + BOOST_TEST(g.empty()); + auto old = reference_root(cs.f, threads); + BOOST_TEST(bits(old.expectation_value(none)) == bits(expected)); + // The identity is not a shard row in Heisenberg; in Schrödinger its basis row carries no operator weight. + for (const auto &state : s.shards) { + if (const auto row = state->op.store->find(Monomial{})) { + BOOST_TEST(cs.f.schrodinger_cutoff.has_value()); + BOOST_TEST(state->op.get_operator()[*row] == 0.0); + } + } + } + } +} + +// Missing callbacks and mismatched arguments are rejected before the team: no observer visit, nothing mutated. +BOOST_AUTO_TEST_CASE(sharded_evaluation_rejects_invalid_arguments_before_the_team) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s); + const auto requests = r.requests(cs.c.params); + const Recorder recorder(threads); + const auto run = [&](std::span req, + std::span cos, + bool g) { return sharded::evaluate_shards(req, cos, options, g, &recorder); }; + const auto accepted = [&](std::span req, + std::span cos, + bool g) { return sharded::evaluate_shards(req, cos, options, g); }; + auto no_scale = r.callbacks; + no_scale.back().scale = nullptr; + BOOST_CHECK_THROW((void)run(requests, no_scale, false), MissingLayerCallback); + auto no_acc = r.callbacks; + no_acc.front().accumulate = nullptr; + BOOST_CHECK_NO_THROW(require_success(accepted(requests, no_acc, false).error, "energy without accumulate")); + BOOST_CHECK_THROW((void)run(requests, no_acc, true), MissingLayerCallback); + // Indices are needed only where a cosine vanishes. + auto no_indices = r.callbacks; + no_indices.back().indices = nullptr; + BOOST_CHECK_NO_THROW( + require_success(accepted(requests, no_indices, true).error, "gradient without vanishing cosine")); + const auto vanishing = parameter_sets(cs.c).back().second; + const auto vanish_requests = r.requests(vanishing); + BOOST_CHECK_THROW((void)run(vanish_requests, no_indices, true), MissingLayerCallback); + // Counts and replicated inputs. + BOOST_CHECK_THROW((void)run(std::span(requests).first(threads - 1), r.callbacks, false), std::invalid_argument); + BOOST_CHECK_THROW((void)run(requests, std::span(r.callbacks).first(threads - 1), false), std::invalid_argument); + if (threads > 1) { + auto other = cs.c.params; + other[0] += 1.0; + auto mixed = r.requests(cs.c.params); + const auto odd = r.requests(other); + std::vector combined; + combined.push_back(mixed[0]); + for (size_t t = 1; t < threads; ++t) { + combined.push_back(odd[t]); + } + BOOST_CHECK_THROW((void)run(combined, r.callbacks, false), std::invalid_argument); + } + const VecD too_short(1, 0.1); + BOOST_CHECK_THROW((void)run(r.requests(too_short), r.callbacks, false), std::invalid_argument); + // A state longer than its operator. + const VecD short_op(1, 1.0); + std::vector longer; + for (size_t t = 0; t < threads; ++t) { + longer.push_back(EvalRequest{.e_core = r.core_term, + .state = r.shards[t].state, + .op = r.shards[t].state.length() > 1 ? short_op : r.shards[t].op, + .parameter_mapping = r.parameter_mapping, + .gen_coeffs = r.gen_coeffs, + .graph = r.shards[t].graph->replay_view(), + .params = cs.c.params}); + } + BOOST_CHECK_THROW((void)run(longer, r.callbacks, false), EvalStateArgumentError); + BOOST_CHECK_THROW((void)sharded::evaluate_shards(requests, r.callbacks, parallel::Options{.threads = 0}, false), + std::invalid_argument); + // Replay: window lengths and a missing scale. + std::vector replays; + for (const auto &shard : r.shards) { + replays.push_back({.coeffs = shard.op, .graph = shard.graph->replay_view()}); + } + const VecD one_angle(1, 0.2); + BOOST_CHECK_THROW((void)sharded::replay_shards(replays, one_angle, r.callbacks, options), std::invalid_argument); + const auto mapped = map_params(cs.c.params, r.parameter_mapping, r.gen_coeffs, 1.0, true); + BOOST_CHECK_THROW((void)sharded::replay_shards(replays, mapped, no_scale, options), MissingLayerCallback); + for (const auto &log : recorder.logs()) { + BOOST_TEST(log.empty()); + } +} + +// Exact-cutoff graphs against the independent map propagator: energies (and the frozen exact energy of +// random_exact), gradients against central differences of the reference, and single-gate closed-form fits. +BOOST_AUTO_TEST_CASE(sharded_evaluation_matches_independent_references) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto data = test_utils::load_case_data("random_exact.msgpack"); + std::vector> cases; + for (const auto sc : {std::optional{}, std::optional{16}}) { + cases.push_back({{.name = "random-exact", + .op = data.hamiltonian, + .initial_state = data.initial_state, + .schrodinger_cutoff = sc, + .cutoff = 2 * kN}, + data_circuit(data)}); + cases.push_back({{.name = "majorana-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .schrodinger_cutoff = sc ? std::optional{12} : std::nullopt, + .cutoff = 12, + .logical = 6}, + majorana_circuit(6)}); + cases.push_back({{.name = "pauli", + .op = pauli_operator(), + .initial_state = {}, + .schrodinger_cutoff = sc, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = kN}, + pauli_circuit()}); + } + // Named elements, not structured bindings: Clang cannot capture a binding in a lambda under OpenMP. + for (const auto &entry : cases) { + const Fixture &f = entry.first; + const Circuit &c = entry.second; + const bool schrodinger = f.schrodinger_cutoff.has_value(); + BOOST_TEST_CONTEXT(f.name << (schrodinger ? " schrodinger" : " heisenberg") << " T=" << threads) { + auto s = seed(f, options); + run_graph(s, c, std::nullopt); + const auto r = retain(s); + const auto reference = [&](const VecD &p) { return reference_energy(f, c, p); }; + const double e = energy(r, c.params); + BOOST_TEST(close(e, reference(c.params), kExactAtol, 1e-12)); + if (f.name == "random-exact") { + BOOST_CHECK_SMALL(e - data.actual_expval, kExactAtol); + } + const auto [ge, grad] = energy_and_gradient(r, c.params); + BOOST_TEST(close(ge, e, 1e-12, 1e-12)); + const auto fd = central_differences(reference, c.params); + for (size_t i = 0; i < grad.size(); ++i) { + BOOST_TEST(close(grad[i], fd[i], 1e-6, 1e-6), "component " << i << ": " << grad[i] << " vs " << fd[i]); + } + } + } + // Closed form: with one gate, E(theta) = a + b cos(2 g theta) + c sin(2 g theta); fit a, b, c from three angles and + // check two further angles and the gradient, for an odd Majorana generator and a Pauli generator. + const std::vector> single = { + {{.name = "odd-majorana", .op = majorana_operator(6), .initial_state = {0, 3}, .cutoff = 12, .logical = 6}, + {1, 4, 6}}, + {{.name = "pauli-xy", + .op = pauli_operator(), + .initial_state = {}, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = kN}, + pauli_oracle::slots_of_string("YIIIIIII")}, + }; + for (const auto &[f, gate] : single) { + BOOST_TEST_CONTEXT(f.name << " closed form T=" << threads) { + const double g = 0.8; + const Circuit c{.gates = {gate}, .mapping = {0}, .gen_coeffs = {g}, .params = {0.0}}; + auto s = seed(f, options); + run_graph(s, c, std::nullopt); + const auto r = retain(s); + const auto at = [&](double theta) { return energy(r, VecD{theta}); }; + const double e0 = at(0.0); + const double eq = at(std::numbers::pi / (4 * g)); // cos = 0, sin = 1 + const double eh = at(std::numbers::pi / (2 * g)); // cos = -1, sin = 0 + const double a = 0.5 * (e0 + eh); + const double b = 0.5 * (e0 - eh); + const double cc = eq - a; + for (const double theta : {0.17, -0.61}) { + const double want = a + (b * std::cos(2 * g * theta)) + (cc * std::sin(2 * g * theta)); + BOOST_TEST(close(at(theta), want, 1e-12, 1e-12)); + const auto [e, grad] = energy_and_gradient(r, VecD{theta}); + const double dwant = 2 * g * ((-b * std::sin(2 * g * theta)) + (cc * std::cos(2 * g * theta))); + BOOST_TEST(close(grad[0], dwant, 1e-12, 1e-10), grad[0] << " vs " << dwant); + } + // The gate really moves the energy. (An odd Majorana generator never maps an even term onto a diagonal + // partner, so its sine coefficient vanishes and only the cosine term carries the angle.) + BOOST_TEST(std::abs(b) + std::abs(cc) > 1e-6); + } + } +} + +// The substantive cases of tests/test_deep_circuit_gradient.py through the sharded engine: a ZZ observable over eight +// qubits propagated through layers of every ZZ term and an X on every qubit, with a pruning cutoff. Depth, vanishing +// cosines (theta = pi/4 at unit generator coefficient) and angles that record nothing; each against central +// differences of the sharded energy and bitwise against the root. +BOOST_AUTO_TEST_CASE(sharded_evaluation_deep_circuit_gradients) { + const auto options = team_options(); + const size_t threads = team_size(); + Fixture f{.name = "deep-zz", + .op = {}, + .initial_state = {}, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 4}; + std::vector zz; + for (size_t i = 0; i < kN; ++i) { + for (size_t j = i + 1; j < kN; ++j) { + std::string p(kN, 'I'); + p[i] = 'Z'; + p[j] = 'Z'; + zz.push_back(p); + add_term(f.op, Basis::Pauli, pauli_oracle::slots_of_string(p), 0.5); + } + } + const auto circuit = [&](size_t layers) { + Circuit c; + for (size_t k = 0; k < layers; ++k) { + for (const auto &p : zz) { + c.gates.push_back(pauli_oracle::slots_of_string(p)); + c.mapping.push_back(k); + c.gen_coeffs.push_back(0.5); + } + for (size_t q = 0; q < kN; ++q) { + std::string p(kN, 'I'); + p[q] = 'X'; + c.gates.push_back(pauli_oracle::slots_of_string(p)); + c.mapping.push_back(layers + k); + c.gen_coeffs.push_back(-1.0); + } + } + return c; + }; + struct Run { + size_t layers; + double theta; + uint8_t flags; // the record branches the angles must reach + }; + const std::vector runs = {{4, 1.0, replay::kRecordRotationsBelow}, + {8, 1.0, replay::kRecordRotationsBelow}, + {1, std::numbers::pi / 4, replay::kRecordCosineSet}, + {2, std::numbers::pi / 4, replay::kRecordCosineSet}, + {4, std::numbers::pi / 4, replay::kRecordCosineSet}, + {4, 0.05, 0}, + {8, 0.05, 0}}; + for (const auto &run : runs) { + BOOST_TEST_CONTEXT("layers " << run.layers << " theta " << run.theta << " T=" << threads) { + auto c = circuit(run.layers); + c.params.assign(2 * run.layers, run.theta); + auto s = seed(f, options); + run_graph(s, c, std::nullopt); + const auto r = retain(s); + // The cutoff prunes, or the checks would be vacuous. + size_t cos_only = 0; + for (const auto &state : s.shards) { + for (size_t l = 0; l < state->graph.layers(); ++l) { + const auto t = state->graph.get_layer_traversal(l); + const auto mask = monoprop::detail::full_cos_mask(state->op.inverted_index(), t, f.basis); + cos_only += mask.total_count > t.total_rotation_endpoints() ? 1 : 0; + } + } + BOOST_TEST_REQUIRE(cos_only > 0U); + std::vector wanted; + (void)replay::plan_cos_records(map_params(c.params, r.parameter_mapping, r.gen_coeffs, 1.0, true), wanted); + uint8_t reached = 0; + for (const uint8_t w : wanted) { + reached |= w; + } + BOOST_TEST((reached & run.flags) == run.flags); + if (run.flags == 0) { + BOOST_TEST(reached == 0); + } + const auto [e, grad] = energy_and_gradient(r, c.params); + BOOST_TEST(std::ranges::all_of(grad, [](double x) { return std::isfinite(x); })); + const auto fd = central_differences([&](const VecD &p) { return energy(r, p); }, c.params); + for (size_t i = 0; i < grad.size(); ++i) { + BOOST_TEST(close(grad[i], fd[i], 1e-6), "component " << i << ": " << grad[i] << " vs " << fd[i]); + } + auto old = reference_root(f, threads); + old.build_graph(c.gates, c.mapping, c.gen_coeffs, std::nullopt, std::nullopt, std::nullopt); + const auto [le, lgrad] = old.expectation_value_and_gradient(c.params); + BOOST_TEST(bits(e) == bits(le)); + BOOST_TEST((bits_of(grad) == bits_of(lgrad))); + } + } +} + +// Repeated parameters accumulate per shard in the reverse order, and two consecutive vanishing layers record the same +// indices twice (rotations below and the cosine set): restoration is idempotent and the gradient stays exact. +BOOST_AUTO_TEST_CASE(sharded_evaluation_repeated_parameters_and_duplicate_records) { + const auto options = team_options(); + const size_t threads = team_size(); + for (const auto &cs : {fixture_cases()[0], fixture_cases()[1], fixture_cases()[9]}) { + BOOST_TEST_CONTEXT(label(cs) << " T=" << threads) { + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s); + // Drive every other replayed layer to a vanishing cosine: consecutive vanishing layers pile both flags up. + VecD params = cs.c.params; + for (size_t gate = 0; gate + 1 < cs.c.gates.size(); gate += 2) { + if (cs.c.gen_coeffs[gate] != 0.0) { + params[cs.c.mapping[gate]] = std::numbers::pi / (4.0 * cs.c.gen_coeffs[gate]); + } + } + std::vector wanted; + (void)replay::plan_cos_records(map_params(params, r.parameter_mapping, r.gen_coeffs, 1.0, true), wanted); + const bool both = std::ranges::any_of(wanted, [](uint8_t w) { + return w == (replay::kRecordRotationsBelow | replay::kRecordCosineSet); + }); + BOOST_TEST(both); + auto old = reference_root(cs.f, threads); + old.build_graph(cs.c.gates, cs.c.mapping, cs.c.gen_coeffs, std::nullopt, std::nullopt, cs.k); + const auto [le, lgrad] = old.expectation_value_and_gradient(params); + const auto [e, grad] = energy_and_gradient(r, params); + BOOST_TEST(bits(e) == bits(le)); + BOOST_TEST((bits_of(grad) == bits_of(lgrad))); + // A repeated parameter's component is the sum over its layers: perturbing the parameter moves the energy + // by the whole component. + const auto fd = central_differences([&](const VecD &p) { return energy(r, p); }, params, 1e-6); + for (size_t i = 0; i < grad.size(); ++i) { + if (std::ranges::count(cs.c.mapping, i) > 1) { + BOOST_TEST(close(grad[i], fd[i], 1e-5, 1e-5), + "repeated " << i << ": " << grad[i] << " vs " << fd[i]); + } + } + } + } +} + +// Pared functionals equal the root's (bitwise, energy and gradient) for positive, zero and negative +// thresholds in both pictures; cross-owner lists are the unchanged shared cores; some layers really store a pruned +// mask; and the owner seam uses the flat owner, not a communicator rank. +BOOST_AUTO_TEST_CASE(sharded_evaluation_pared_functionals_match_the_root) { + const auto options = team_options(); + const size_t threads = team_size(); + size_t stored = 0; + size_t empty_stored = 0; + for (const auto &cs : {fixture_cases()[0], fixture_cases()[1], fixture_cases()[7], fixture_cases()[9]}) { + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + auto old = reference_root(cs.f, threads); + old.build_graph(cs.c.gates, cs.c.mapping, cs.c.gen_coeffs, std::nullopt, std::nullopt, cs.k); + for (const double threshold : {1e-3, 0.05, 0.0, -1.0}) { + BOOST_TEST_CONTEXT(label(cs) << " threshold " << threshold << " T=" << threads) { + const auto r = retain(s, threshold); + const auto ev = old.expectation_value_functional(threshold); + const auto evg = old.expectation_value_and_gradient_functional(threshold); + for (const auto &[name, params] : parameter_sets(cs.c)) { + BOOST_TEST_CONTEXT(name) { + BOOST_TEST(bits(energy(r, params)) == bits(ev(params))); + const auto [le, lgrad] = evg(params); + const auto [e, grad] = energy_and_gradient(r, params); + BOOST_TEST(bits(e) == bits(le)); + BOOST_TEST((bits_of(grad) == bits_of(lgrad))); + } + } + for (size_t t = 0; t < threads; ++t) { + const MPGraph &pared = *r.shards[t].graph; + const MPGraph &live = s.shards[t]->graph; + BOOST_TEST_REQUIRE(pared.layers() == live.layers()); + for (size_t l = 0; l < pared.layers(); ++l) { + BOOST_TEST(pared.get_layer(l).shared_core() == live.get_layer(l).shared_core()); + stored += pared.get_layer(l).pruned_cos() != nullptr ? 1 : 0; + } + // The owner seam at the flat slot reproduces the retained pared graph exactly. + const auto &state = *s.shards[t]; + const auto &index = state.op.inverted_index(); + const auto full = [&](size_t i, CosMask &out) { + out = monoprop::detail::full_cos_mask(index, live.get_layer_traversal(i), cs.f.basis); + }; + const bool pic = cs.f.schrodinger_cutoff.has_value(); + const auto keep = + pic ? indices_above(r.shards[t].op, threshold) : r.shards[t].state.indices_above(threshold); + const size_t count = pic ? r.shards[t].op.size() : r.shards[t].state.length(); + const auto again = monoprop::detail::pare_graph_owner(live, keep, count, pic, t, full); + for (size_t l = 0; l < pared.layers(); ++l) { + const CosMask *a = pared.get_layer(l).pruned_cos(); + const CosMask *b = again.get_layer(l).pruned_cos(); + const bool same = + (a == nullptr && b == nullptr) || (a != nullptr && b != nullptr && a->blocks == b->blocks); + BOOST_TEST(same, "shard " << t << " layer " << l); + empty_stored += (a != nullptr && a->blocks.empty()) ? 1 : 0; + } + } + } + } + } + BOOST_TEST(stored > 0U); + BOOST_TEST_MESSAGE("pared layers storing a mask: " << stored << ", of which empty: " << empty_stored); +} + +// A step's staging holds what the step reads when that fits one run, else one run: a small layer stages exactly as +// before (all of it in one run) without reserving a whole default run per owner, and a block larger than the run is +// read where its partner published it instead of widening every owner's staging to it. +BOOST_AUTO_TEST_CASE(sharded_evaluation_staging_capacity_is_sized_by_need) { + BOOST_TEST(sharded::staging_capacity(4096, 300) == 300U); // a small step: exactly its values + BOOST_TEST(sharded::staging_capacity(4096, 10000) == 4096U); // a large step: one default run + BOOST_TEST(sharded::staging_capacity(4096, 9000) == 4096U); // never above the run, whatever its largest block + BOOST_TEST(sharded::staging_capacity(1, 300) == 1U); // test seam: one value per run + BOOST_TEST(sharded::staging_capacity(4096, 0) == 1U); // never empty +} + +// Per-step buffers whose size falls and rises reallocate only past their capacity, and then at least double it; a plain +// resize() grows from the current size, by little more than the step's increase. +BOOST_AUTO_TEST_CASE(sharded_evaluation_step_buffers_grow_by_capacity) { + std::vector v; + sharded::grow_to(v, 1000); + for (size_t i = 0; i < v.size(); ++i) { + v[i] = static_cast(i); + } + const size_t capacity = v.capacity(); + sharded::grow_to(v, 100); // a smaller step keeps the block + const double *block = v.data(); + sharded::grow_to(v, 900); + BOOST_TEST(v.data() == block); // within the capacity: no reallocation + BOOST_TEST(v.capacity() == capacity); + sharded::grow_to(v, capacity + 1); // past it: at least doubled + BOOST_TEST(v.capacity() >= 2 * capacity); + BOOST_TEST(v[99] == 99.0); // the values below the size are kept + BOOST_TEST(v.size() == capacity + 1); +} + +// Runs of `values` values: blocks up to that size are staged, several to a run where they fit; larger ones are read in +// place. One value per run reads nearly every block in place; two mixes the two kinds within a step. +class SmallRuns final : public sharded::EvaluationObserver { +public: + explicit SmallRuns(size_t values) : values_(values) {} + auto visit(E /*work*/, size_t /*step*/, size_t /*shard*/) const -> void override {} + [[nodiscard]] auto staging_run() const noexcept -> size_t override { return values_; } + +private: + size_t values_; +}; + +/* + * Partner blocks staged in small runs, or read in place when larger than a run, give bitwise the results of the default + * runs, which hold a whole small layer: the staging changes where the finishes read the partners' values, never which + * values or in which order. Covers forward and reverse finishes (energy, gradient) and the in-team forward replay + * (replay_shards()). + */ +BOOST_AUTO_TEST_CASE(sharded_evaluation_staging_runs_do_not_change_results) { + const auto options = team_options(); + const size_t threads = team_size(); + const SmallRuns one_value_runs(1); + const SmallRuns two_value_runs(2); + size_t multi_partner_layers = 0; + for (const auto &cs : {fixture_cases()[0], fixture_cases()[1], fixture_cases()[7], fixture_cases()[9]}) { + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s); + for (size_t t = 0; t < threads; ++t) { + const MPGraph &graph = *r.shards[t].graph; + for (size_t l = 0; l < graph.layers(); ++l) { + size_t partners = 0; + graph.get_layer_traversal(l).for_each_occupied_slot( + [&](size_t rank, const auto & /*slot*/) { partners += rank != t ? 1 : 0; }); + multi_partner_layers += partners > 1 ? 1 : 0; + } + } + for (const auto &[name, params] : parameter_sets(cs.c)) { + for (const SmallRuns *small : {&one_value_runs, &two_value_runs}) { + BOOST_TEST_CONTEXT(label(cs) << " " << name << " T=" << threads << " run=" << small->staging_run()) { + const auto requests = r.requests(params); + for (const bool gradient : {false, true}) { + const auto plain = sharded::evaluate_shards(requests, r.callbacks, options, gradient); + const auto runs = sharded::evaluate_shards(requests, r.callbacks, options, gradient, small); + require_success(plain.error, "evaluate_shards"); + require_success(runs.error, "evaluate_shards, small runs"); + BOOST_TEST((bits_of(runs.contributions) == bits_of(plain.contributions))); + BOOST_TEST_REQUIRE(runs.gradients.size() == plain.gradients.size()); + for (size_t t = 0; t < plain.gradients.size(); ++t) { + BOOST_TEST((bits_of(runs.gradients[t]) == bits_of(plain.gradients[t])), "shard " << t); + } + } + std::vector replays; + for (const auto &shard : r.shards) { + replays.push_back({.coeffs = shard.op, .graph = shard.graph->replay_view()}); + } + const auto mapped = map_params(params, r.parameter_mapping, r.gen_coeffs, 1.0, true); + const auto plain = sharded::replay_shards(replays, mapped, r.callbacks, options); + const auto runs = sharded::replay_shards(replays, mapped, r.callbacks, options, small); + require_success(plain.error, "replay_shards"); + require_success(runs.error, "replay_shards, small runs"); + BOOST_TEST_REQUIRE(runs.coeffs.size() == plain.coeffs.size()); + for (size_t t = 0; t < plain.coeffs.size(); ++t) { + BOOST_TEST((bits_of(runs.coeffs[t]) == bits_of(plain.coeffs[t])), "shard " << t); + } + } + } + } + } + // With three or more shards some owner has several partners in a layer, so small runs mean several runs. + if (threads >= 3) { + BOOST_TEST(multi_partner_layers > 0U); + } +} + +/* + * CosCallbacks::count reports exactly what CosCallbacks::indices appends, for recomputed folds, pared stored masks and + * empty stored masks, so replay::reserve_records() reserves the records once: recording every layer with both flags + * fills the reservation exactly and never reallocates. + */ +BOOST_AUTO_TEST_CASE(sharded_evaluation_cosine_counts_reserve_records_exactly) { + const auto options = team_options(); + size_t nonempty = 0; + for (const auto &cs : {fixture_cases()[0], fixture_cases()[1], fixture_cases()[7], fixture_cases()[9]}) { + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s, 0.05); + for (size_t t = 0; t < s.shards.size(); ++t) { + const auto &state = *s.shards[t]; + const auto &index = state.op.inverted_index(); + std::vector empty_layers; + for (size_t l = 0; l < state.graph.layers(); ++l) { + empty_layers.emplace_back(state.graph.get_layer(l).shared_core(), CosMask{}); + } + const MPGraph empty_masks(false, std::move(empty_layers)); + for (const auto &[name, view] : {std::pair{"recomputed", state.graph.replay_view()}, + std::pair{"pared", r.shards[t].graph->replay_view()}, + std::pair{"empty stored", empty_masks.replay_view()}}) { + BOOST_TEST_CONTEXT(label(cs) << " shard " << t << " " << name) { + const auto cos = monoprop::detail::make_cos_callbacks(index, view, cs.f.basis); + BOOST_TEST_REQUIRE(static_cast(cos.count)); + std::vector wanted(view.layers()); + for (size_t l = 0; l < view.layers(); ++l) { + std::vector indices; + cos.indices(l, indices); + BOOST_TEST(cos.count(l) == indices.size(), "layer " << l); + nonempty += indices.empty() ? 0 : 1; + wanted[l] = replay::kRecordCosineSet | (l > 0 ? replay::kRecordRotationsBelow : 0); + } + replay::CosRecords records; + replay::reset_records(records, view.layers()); + replay::reserve_records(records, wanted, view, cos.count); + const size_t reserved = records.values.capacity(); + const auto *const storage = records.values.data(); + const VecD op(state.op.size(), 1.0); + for (size_t l = 0; l < view.layers(); ++l) { + replay::record_pre_layer(records, wanted, view, cos.indices, l, op); + } + BOOST_TEST(records.values.size() == reserved); + BOOST_TEST(records.indices.size() == records.indices.capacity()); + BOOST_TEST(records.values.data() == storage); + } + } + } + } + BOOST_TEST(nonempty > 0U); +} + +// A stored empty pruned cosine mask replays nothing: it is not a request to recompute the full mask. +BOOST_AUTO_TEST_CASE(sharded_evaluation_empty_stored_mask_applies_nothing) { + const auto options = team_options(); + auto s = seed(fixture_cases()[0].f, options); + run_graph(s, fixture_cases()[0].c, std::nullopt); + const auto &state = *s.shards[0]; + std::vector layers; + for (size_t l = 0; l < state.graph.layers(); ++l) { + layers.emplace_back(state.graph.get_layer(l).shared_core(), CosMask{}); + } + const MPGraph empty_masks(false, std::move(layers)); + const auto view = empty_masks.replay_view(); + const auto &index = state.op.inverted_index(); + const auto cos = monoprop::detail::make_cos_callbacks(index, view, Basis::Majorana); + const auto recompute = monoprop::detail::make_cos_callbacks(index, state.graph.replay_view(), Basis::Majorana); + size_t changed_by_recompute = 0; + for (size_t l = 0; l < view.layers(); ++l) { + VecD coeffs(state.op.size()); + std::iota(coeffs.begin(), coeffs.end(), 1.0); + const VecD before = coeffs; + cos.scale(l, coeffs.data(), 0.5); + BOOST_TEST((coeffs == before), "layer " << l); + std::vector indices; + cos.indices(l, indices); + BOOST_TEST(indices.empty()); + VecD st = before; + VecD op = before; + BOOST_TEST(cos.accumulate(l, st.data(), op.data(), 0.5, 2.0) == 0.0); + recompute.scale(l, coeffs.data(), 0.5); + changed_by_recompute += coeffs != before ? 1 : 0; + } + BOOST_TEST(changed_by_recompute > 0U); +} + +// Two differently sized instances evaluated interleaved and repeatedly give bitwise-stable results through the +// retained thread-local scratch; a copy mutated afterwards leaves the original's retained evaluation unchanged; and +// retained callbacks stay exact after the owning index grows, since parity words are fetched at use time. +BOOST_AUTO_TEST_CASE(sharded_evaluation_repeated_instances_copies_and_index_growth) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto small_case = fixture_cases()[9]; + const auto large_case = fixture_cases()[7]; + auto small = seed(small_case.f, options); + run_graph(small, small_case.c, std::nullopt); + auto large = seed(large_case.f, options); + run_graph(large, large_case.c, std::nullopt); + const auto rs = retain(small); + const auto rl = retain(large); + const auto first_small = energy_and_gradient(rs, small_case.c.params); + const auto first_large = energy_and_gradient(rl, large_case.c.params); + for (size_t round = 0; round < 3; ++round) { + const auto a = energy_and_gradient(rl, large_case.c.params); + const auto b = energy_and_gradient(rs, small_case.c.params); + const auto c = energy(rl, large_case.c.params); + BOOST_TEST(bits(a.first) == bits(first_large.first)); + BOOST_TEST((bits_of(a.second) == bits_of(first_large.second))); + BOOST_TEST(bits(b.first) == bits(first_small.first)); + BOOST_TEST((bits_of(b.second) == bits_of(first_small.second))); + BOOST_TEST(bits(c) == bits(first_large.first)); + } + + // Copies are independent: extend the copy's graph, the original's retained evaluation is unchanged. + Sharded copy{.f = large.f, + .cutoff_fn = large.cutoff_fn, + .router = large.router, + .shards = sharded::copy_shards(options, large.shards)}; + run_graph(copy, pauli_circuit(), std::nullopt, large_case.c.gates.size()); + const auto again = energy_and_gradient(rl, large_case.c.params); + BOOST_TEST(bits(again.first) == bits(first_large.first)); + BOOST_TEST((bits_of(again.second) == bits_of(first_large.second))); + + // Index growth under retained callbacks: an owned graph (negative threshold keeps every recomputing layer) keeps + // its layer count while the shards' store grows. + const auto owned = retain(large, -1.0); + const auto before = energy_and_gradient(owned, large_case.c.params); + std::vector rows_before; + for (const auto &state : large.shards) { + rows_before.push_back(state->op.size()); + } + run_graph(large, majorana_circuit(kN), std::nullopt, large_case.c.gates.size()); + size_t grown = 0; + for (size_t t = 0; t < threads; ++t) { + grown += large.shards[t]->op.size() > rows_before[t] ? 1 : 0; + } + BOOST_TEST_REQUIRE(grown > 0U); + BOOST_CHECK_THROW(rl.validate_call(large_case.c.params), std::runtime_error); // the aliasing graph grew + BOOST_CHECK_NO_THROW(owned.validate_call(large_case.c.params)); + const auto after = energy_and_gradient(owned, large_case.c.params); + BOOST_TEST(bits(after.first) == bits(before.first)); + BOOST_TEST((bits_of(after.second) == bits_of(before.second))); +} + +namespace { + +// Captures each shard's evolving coefficients after every new-layer replay of an informed build. +struct ReplayCapture { + std::vector *last; + std::vector *calls; + auto visit(W /*work*/, size_t /*step*/, size_t /*shard*/) const -> void {} + [[nodiscard]] auto kernels(size_t) const -> monoprop::detail::NoRangeObserver { return {}; } + auto replayed(size_t shard, size_t /*step*/, const VecD &coeffs) const -> void { + (*last)[shard] = coeffs; + ++(*calls)[shard]; + } +}; + +// The evolving coefficients an informed build ends with equal an independent replay of the finished graph at the +// picture's contraction angles (Schrödinger negated), from the live picture: rows created by a later layer lie past +// every earlier layer's scaled_count, so earlier layers never touch them. +template +auto check_replayed_coefficients(Sharded &s, const std::vector &last, const VecD ¶ms) -> void { + const bool schrodinger = s.schrodinger(); + std::vector start; + std::vector views; + std::vector callbacks; + for (auto &state : s.shards) { + start.push_back(state->op.current_picture(schrodinger)); + views.push_back(state->graph.slice_view(state->graph.layers())); + } + std::vector requests; + for (size_t t = 0; t < s.shards.size(); ++t) { + callbacks.push_back( + monoprop::detail::make_cos_callbacks(s.shards[t]->op.inverted_index(), views[t], s.f.basis)); + requests.push_back({.coeffs = start[t], .graph = views[t]}); + } + const auto angles = contraction_angles(s.shards.front()->graph, schrodinger, params); + const auto outcome = sharded::replay_shards(requests, angles, callbacks, team_options()); + require_success(outcome.error, "replay_shards"); + for (size_t t = 0; t < s.shards.size(); ++t) { + BOOST_TEST((bits_of(last[t]) == bits_of(outcome.coeffs[t])), "shard " << t); + } +} + +} // namespace + +// Actual participation, asserted after the join: every evaluation, replay, retained-preparation and informed seed or +// replay visit of shard t ran on worker t at nesting level 1 in a team of T, and every step is present. +BOOST_AUTO_TEST_CASE(sharded_evaluation_owners_do_their_own_work) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[1]; // Schrödinger, cross-owner traffic at T >= 2 + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const Recorder retain_log(threads); + const auto r = retain(s, 1e-3, &retain_log); + const size_t layers = r.expected_layers; + const auto check = [&](const Recorder &rec, const std::string &what) { + BOOST_TEST_CONTEXT(what) { + std::set owners; + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST_REQUIRE(!rec.logs()[t].empty()); + for (const auto &v : rec.logs()[t]) { + BOOST_TEST(v.worker == static_cast(t)); + BOOST_TEST(v.team == static_cast(threads)); + BOOST_TEST(v.level == (threads > 1 ? 1 : 0)); // a one-worker team runs without a region + } + owners.insert(rec.logs()[t].front().thread); + } + BOOST_TEST(owners.size() == threads); + } + }; + check(retain_log, "retain"); + BOOST_TEST(retain_log.count(E::retain) == threads); + + const Recorder energy_log(threads); + require_success(evaluate(r, cs.c.params, false, &energy_log).error, "evaluate"); + check(energy_log, "energy"); + for (const E work : {E::publish, E::cosine, E::finish}) { + BOOST_TEST(energy_log.count(work) == threads * layers); + } + BOOST_TEST(energy_log.count(E::contribution) == threads); + BOOST_TEST(energy_log.count(E::reverse_publish) == 0U); + + const Recorder grad_log(threads); + require_success(evaluate(r, cs.c.params, true, &grad_log).error, "evaluate gradient"); + check(grad_log, "gradient"); + for (const E work : + {E::record, E::publish, E::cosine, E::finish, E::reverse_publish, E::reverse_accumulate, E::reverse_finish}) { + BOOST_TEST(grad_log.count(work) == threads * layers, "work " << static_cast(work)); + } + // Every step of every shard, in phase order. + for (size_t t = 0; t < threads; ++t) { + std::vector finishes; + for (const auto &v : grad_log.logs()[t]) { + if (v.work == E::finish || v.work == E::reverse_finish) { + finishes.push_back(v.step); + } + } + std::vector want(2 * layers); + std::iota(want.begin(), want.end(), size_t{0}); + BOOST_TEST((finishes == want)); + } + + // Informed construction's seed replay and new-layer replays. + struct ConstructionLog { + std::vector> *logs; + auto visit(W work, size_t step, size_t shard) const -> void { + if (work == W::seed || work == W::replay) { + (*logs)[shard].push_back({E::frame, + step, + omp_get_thread_num(), + omp_get_num_threads(), + omp_get_level(), + std::this_thread::get_id()}); + } + } + [[nodiscard]] auto kernels(size_t) const -> monoprop::detail::NoRangeObserver { return {}; } + }; + std::vector> clogs(threads); + auto inf = seed(cs.f, options); + require_success(run_informed(inf, cs.c, cs.c.params, cs.k).error, "first informed"); + const size_t first_layers = inf.shards.front()->graph.layers(); + require_success(run_informed( + inf, + cs.c, + [&] { + VecD p = cs.c.params; + p.insert(p.end(), cs.c.params.begin(), cs.c.params.end()); + return p; + }(), + cs.k, + first_layers, + ConstructionLog{&clogs}) + .error, + "incremental informed"); + for (size_t t = 0; t < threads; ++t) { + // One seed-preparation visit, three per seed replay step, and two replay visits per gate. + BOOST_TEST(clogs[t].size() == 1 + (3 * first_layers) + (2 * cs.c.gates.size())); + for (const auto &v : clogs[t]) { + BOOST_TEST(v.worker == static_cast(t)); + BOOST_TEST(v.level == (threads > 1 ? 1 : 0)); // a one-worker team runs without a region + } + } +} + +// Opaque callback sets run every callback call on the primary, one shard at a time, while snapshots, publication and +// finishes stay on the owners; the results are bitwise those of the owner-parallel run. +BOOST_AUTO_TEST_CASE(sharded_evaluation_opaque_callbacks_run_on_the_primary) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + auto r = retain(s); + const auto params = parameter_sets(cs.c).back().second; // vanishing: the indices callback runs too + const auto want = energy_and_gradient(r, params); + // Per-shard call logs: every call is made by the primary, so each vector has one writer. + std::vector> calls(threads); + std::vector opaque(threads); + for (size_t t = 0; t < threads; ++t) { + const auto inner = r.callbacks[t]; + auto *log = &calls[t]; + opaque[t].scale = [inner, log](size_t l, double *c, double v) { + log->push_back(omp_get_thread_num()); + inner.scale(l, c, v); + }; + opaque[t].accumulate = [inner, log](size_t l, double *st, double *h, double v, double sec) { + log->push_back(omp_get_thread_num()); + return inner.accumulate(l, st, h, v, sec); + }; + opaque[t].indices = [inner, log](size_t l, std::vector &out) { + log->push_back(omp_get_thread_num()); + inner.indices(l, out); + }; + } + r.callbacks = opaque; + const Recorder rec(threads); + const auto requests = r.requests(params); + const auto outcome = sharded::evaluate_shards(requests, r.callbacks, options, true, &rec); + require_success(outcome.error, "opaque evaluation"); + const auto got = energy_and_gradient(r, params); + BOOST_TEST(bits(got.first) == bits(want.first)); + BOOST_TEST((bits_of(got.second) == bits_of(want.second))); + for (size_t t = 0; t < threads; ++t) { + BOOST_TEST(!calls[t].empty()); + BOOST_TEST(std::ranges::all_of(calls[t], [](int worker) { return worker == 0; })); + for (const auto &v : rec.logs()[t]) { + const bool callback_work = v.work == E::record || v.work == E::cosine || v.work == E::reverse_accumulate; + BOOST_TEST(v.worker == (callback_work ? 0 : static_cast(t)), "work " << static_cast(v.work)); + } + } +} + +// The ascending-shard fold is the reference association, visible with order-sensitive values. +BOOST_AUTO_TEST_CASE(sharded_evaluation_combination_folds_in_shard_order) { + const std::vector values = {1e16, 1.0, -1e16, 1.0}; + BOOST_TEST(sharded::combine_contributions(values) == ((((0.0 + 1e16) + 1.0) + -1e16) + 1.0)); + BOOST_TEST(sharded::combine_contributions(values) != 2.0); + const std::vector gradients = {{1e16, 1.0}, {1.0, 1e16}, {-1e16, 1.0}, {1.0, -1e16}}; + const auto folded = sharded::combine_gradients(gradients); + BOOST_TEST(folded[0] == ((((0.0 + 1e16) + 1.0) + -1e16) + 1.0)); + BOOST_TEST(folded[1] == ((((0.0 + 1.0) + 1e16) + 1.0) + -1e16)); + BOOST_TEST(sharded::combine_contributions(std::vector{-0.0}) == 0.0); + BOOST_TEST(!std::signbit(sharded::combine_contributions(std::vector{-0.0}))); + BOOST_CHECK_THROW((void)sharded::combine_gradients(std::vector{{1.0}, {1.0, 2.0}}), std::invalid_argument); + // The evaluator adds the identity to the fold of its per-shard terms, once. + const auto options = team_options(); + const auto cs = fixture_cases()[7]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s); + const auto outcome = evaluate(r, cs.c.params, false); + require_success(outcome.error, "evaluate"); + BOOST_TEST(bits(energy(r, cs.c.params)) + == bits(r.core_term + sharded::combine_contributions(outcome.contributions))); +} + +// --- Failures ----------------------------------------------------------------------------------------------------- + +namespace { + +// The phase a visit belongs to: frame (-1), then phase p for step work (finish of p - 1 belongs to phase p), and the +// opaque/exclusive work has no separate phase in the owner-parallel runs used here. +auto phase_of(const Visit &v, size_t layers) -> long long { + switch (v.work) { + case E::frame: + case E::retain: + return -1; + case E::finish: + case E::reverse_finish: + return static_cast(v.step) + 1; + case E::contribution: + return static_cast(layers); + case E::record: + case E::publish: + case E::cosine: + case E::reverse_publish: + case E::reverse_accumulate: + return static_cast(v.step); + } + return 1LL << 40; +} + +auto no_later_work(const Recorder &rec, long long limit, size_t layers) -> bool { + for (const auto &log : rec.logs()) { + for (const auto &v : log) { + if (phase_of(v, layers) > limit) { + return false; + } + } + } + return true; +} + +} // namespace + +// A failure in any evaluation phase, on the primary or the last owner, joins the team, suppresses every later phase, +// returns the original exception from evaluate_shards(), and is rethrown unchanged by ev_sharded() and +// ev_and_grad_sharded(); independent copies survive. +BOOST_AUTO_TEST_CASE(sharded_evaluation_failure_in_each_phase_suppresses_later_work) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[1]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto copy = sharded::copy_shards(options, s.shards); + const auto r = retain(s); + const size_t layers = r.expected_layers; + const auto params = parameter_sets(cs.c)[1].second; + const auto reference = energy_and_gradient(r, params); + struct Site { + E work; + size_t step; + bool gradient; + }; + const std::vector sites = {{E::frame, sharded::no_step, false}, + {E::publish, 0, false}, + {E::cosine, 2, false}, + {E::finish, 1, false}, + {E::contribution, sharded::no_step, false}, + {E::record, 3, true}, + {E::reverse_publish, layers + 1, true}, + {E::reverse_accumulate, layers + 2, true}, + {E::reverse_finish, layers, true}, + {E::finish, layers - 1, true}}; + for (const auto &site : sites) { + for (const size_t shard : {size_t{0}, threads - 1}) { + BOOST_TEST_CONTEXT("work " << static_cast(site.work) << " step " << site.step << " shard " << shard) { + const std::string message = std::format("injected failure: work {} step {} shard {}", + static_cast(site.work), + site.step, + shard); + const Recorder rec(threads, {{site.work, site.step, shard}}); + const auto outcome = evaluate(r, params, site.gradient, &rec); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) == message); + const Visit probe{site.work, site.step, 0, 0, 0, {}}; + BOOST_TEST(no_later_work(rec, phase_of(probe, layers), layers)); + const Recorder again(threads, {{site.work, site.step, shard}}); + const auto requests = r.requests(params); + try { + if (site.gradient) { + (void)sharded::ev_and_grad_sharded(requests, r.callbacks, options, MPI_COMM_SELF, &again); + } + else { + (void)sharded::ev_sharded(requests, r.callbacks, options, MPI_COMM_SELF, &again); + } + BOOST_FAIL("the failure was not rethrown"); + } + catch (const std::runtime_error &e) { + BOOST_TEST(std::string(e.what()) == message); + } + } + } + } + // After every failure: an independent copy made earlier evaluates identically, and so does the original. + Sharded c{.f = s.f, + .cutoff_fn = s.cutoff_fn, + .router = s.router, + .shards = sharded::copy_shards(options, copy)}; + const auto rc = retain(c); + const auto from_copy = energy_and_gradient(rc, params); + BOOST_TEST(bits(from_copy.first) == bits(reference.first)); + BOOST_TEST((bits_of(from_copy.second) == bits_of(reference.second))); + const auto after = energy_and_gradient(r, params); + BOOST_TEST(bits(after.first) == bits(reference.first)); + BOOST_TEST((bits_of(after.second) == bits_of(reference.second))); +} + +// Real failures inside callbacks (forward cosine pass, reverse accumulation, record indices), inside the retained +// preparation (paring) and inside informed construction (seed replay, new-layer replay): each joins, suppresses later +// work and returns the original exception, with mutation reported where it began. +BOOST_AUTO_TEST_CASE(sharded_evaluation_callback_paring_and_construction_failures) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[0]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + auto r = retain(s); + const auto params = parameter_sets(cs.c).back().second; + std::vector wanted; + (void)replay::plan_cos_records(map_params(params, r.parameter_mapping, r.gen_coeffs, 1.0, true), wanted); + const auto vanishing = std::ranges::find_if(wanted, [](uint8_t w) { return (w & replay::kRecordCosineSet) != 0; }); + BOOST_TEST_REQUIRE((vanishing != wanted.end())); + const auto vanishing_layer = static_cast(vanishing - wanted.begin()); + struct Throwing { + size_t shard; + int which; // 0 scale, 1 accumulate, 2 indices + size_t layer; + }; + for (const Throwing site : + {Throwing{threads - 1, 0, 3}, Throwing{0, 1, 2}, Throwing{threads - 1, 2, vanishing_layer}}) { + BOOST_TEST_CONTEXT("callback " << site.which << " shard " << site.shard) { + auto callbacks = r.callbacks; + auto &target = callbacks[site.shard]; + const auto inner = target; + // Still owner-parallel: this wrapper only adds a throw to the built-in closures. + if (site.which == 0) { + target.scale = [inner, site](size_t l, double *c, double v) { + if (l == site.layer) { + throw std::runtime_error("injected callback failure"); + } + inner.scale(l, c, v); + }; + } + else if (site.which == 1) { + target.accumulate = [inner, site](size_t l, double *st, double *h, double v, double sec) { + if (l == site.layer) { + throw std::runtime_error("injected callback failure"); + } + return inner.accumulate(l, st, h, v, sec); + }; + } + else { + target.indices = [inner, site](size_t l, std::vector &out) { + if (l == site.layer) { + throw std::runtime_error("injected callback failure"); + } + inner.indices(l, out); + }; + } + const Recorder rec(threads); + const auto requests = r.requests(params); + const auto outcome = sharded::evaluate_shards(requests, callbacks, options, true, &rec); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) == "injected callback failure"); + BOOST_TEST(rec.count(E::contribution) == (site.which == 1 ? threads : 0U)); + } + } + + // Paring: the last owner fails inside its retained preparation; mutation had started. + { + const Recorder rec(threads, {{E::retain, sharded::no_step, threads - 1}}); + const auto outcome = sharded::prepare_retained(options, s.shards, retained_context(s, 1e-3), &rec); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(outcome.mutation_started); + BOOST_TEST(!outcome.retained.has_value()); + BOOST_TEST(message_of(outcome.error) + == std::format("injected failure: work {} step {} shard {}", + static_cast(E::retain), + sharded::no_step, + threads - 1)); + } + + // Informed construction: a seed-replay step and a new-layer replay fail; nothing after them runs, and an earlier + // copy survives unchanged. + struct ConstructionFail { + std::vector>> *logs; + W work; + size_t step; + size_t shard; + size_t *seen; + auto visit(W w, size_t st, size_t sh) const -> void { + (*logs)[sh].emplace_back(w, st); + if (w == work && st == step && sh == shard) { + ++*seen; + if (*seen == 1) { + throw std::runtime_error("injected construction failure"); + } + } + } + [[nodiscard]] auto kernels(size_t) const -> monoprop::detail::NoRangeObserver { return {}; } + }; + for (const auto &[work, step] : {std::pair{W::seed, size_t{2}}, std::pair{W::replay, size_t{1}}}) { + BOOST_TEST_CONTEXT("construction work " << static_cast(work)) { + auto inf = seed(cs.f, options); + require_success(run_informed(inf, cs.c, cs.c.params, cs.k).error, "first informed"); + const auto copy = sharded::copy_shards(options, inf.shards); + const size_t copy_rows = sharded::total_size(copy); + std::vector>> logs(threads); + size_t seen = 0; + VecD p = cs.c.params; + p.insert(p.end(), cs.c.params.begin(), cs.c.params.end()); + const auto outcome = run_informed(inf, + cs.c, + p, + cs.k, + inf.shards.front()->graph.layers(), + ConstructionFail{&logs, work, step, threads - 1, &seen}); + BOOST_TEST_REQUIRE(static_cast(outcome.error)); + BOOST_TEST(message_of(outcome.error) == "injected construction failure"); + BOOST_TEST(outcome.mutation_started); + for (const auto &log : logs) { + BOOST_TEST(std::ranges::none_of(log, [](const auto &v) { return v.first == W::caches; })); + } + BOOST_TEST(sharded::total_size(copy) == copy_rows); + } + } +} + +// Real allocation failures at every allocation of one owner's frame, snapshot publication and reverse work: each is a +// catchable std::bad_alloc after the join, and the retained scratch left behind does not change later results. +BOOST_AUTO_TEST_CASE(sharded_evaluation_allocation_failures_are_contained, + *boost::unit_test::precondition(has_allocation_probe)) { + const auto options = team_options(); + const size_t threads = team_size(); + const auto cs = fixture_cases()[1]; + auto s = seed(cs.f, options); + run_graph(s, cs.c, cs.k); + const auto r = retain(s); + const size_t layers = r.expected_layers; + // Vanishing cosines: the records hold a cosine-set layer's indices. The frame phase reserves them exactly + // (replay::reserve_records), so the frame allocates and the record of that layer does not. + const auto params = parameter_sets(cs.c).back().second; + const auto reference = energy_and_gradient(r, params); + std::vector wanted; + (void)replay::plan_cos_records(map_params(params, r.parameter_mapping, r.gen_coeffs, 1.0, true), wanted); + const auto vanishing = std::ranges::find_if(wanted, [](uint8_t w) { return (w & replay::kRecordCosineSet) != 0; }); + BOOST_TEST_REQUIRE((vanishing != wanted.end())); + // The scratch sites run on shard 0, the fresh host thread, whose thread-local scratch is always cold. libomp may + // hand a new root thread pooled workers whose scratch is already warm; the frame site's per-call gradient still + // allocates there, so it keeps a nonprimary owner covered. + struct Site { + AllocAt at; + bool allocates; + }; + const std::vector sites = {{{E::frame, sharded::no_step, threads - 1, 0}, true}, + {{E::frame, sharded::no_step, 0, 0}, true}, + {{E::publish, 1, 0, 0}, true}, + {{E::reverse_publish, layers + 1, 0, 0}, true}, + {{E::record, static_cast(vanishing - wanted.begin()), 0, 0}, false}}; + for (const Site &entry : sites) { + const AllocAt &site = entry.at; + const bool allocates = entry.allocates; + size_t injected = 0; + for (size_t nth = 1; nth < 200; ++nth) { + bool failed = false; + bool bad_alloc = false; + // A fresh host thread gets a fresh OpenMP team, whose workers start with cold thread-local scratch. It + // makes no Boost.Test call; the main thread asserts after the join. + std::thread host([&] { + const Recorder rec(threads, {}, AllocAt{site.work, site.step, site.shard, nth}); + const auto requests = r.requests(params); + const auto outcome = sharded::evaluate_shards(requests, r.callbacks, options, true, &rec); + disarm_team(); + failed = static_cast(outcome.error); + if (failed) { + try { + std::rethrow_exception(outcome.error); + } + catch (const std::bad_alloc &) { + bad_alloc = true; + } + catch (...) { + } + } + }); + host.join(); + if (failed) { + ++injected; + BOOST_TEST(bad_alloc); + } + if (!failed) { + break; + } + } + BOOST_TEST_MESSAGE("allocation site work " << static_cast(site.work) << " shard " << site.shard << ": " + << injected << " injected failures"); + BOOST_TEST((injected > 0U) == allocates); + } + const auto after = energy_and_gradient(r, params); + BOOST_TEST(bits(after.first) == bits(reference.first)); + BOOST_TEST((bits_of(after.second) == bits_of(reference.second))); +} diff --git a/cpp/tests/sharded_exchange_tests.cpp b/cpp/tests/sharded_exchange_tests.cpp new file mode 100644 index 000000000..7aac4e22a --- /dev/null +++ b/cpp/tests/sharded_exchange_tests.cpp @@ -0,0 +1,734 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// First on purpose: the header must compile on its own. +#include "monoprop/detail/sharded/Exchange.h" + +/* + * The physical-MPI round of the sharded runtime (detail/sharded/Exchange.h), driven the way the orchestration drives + * it: owners fill their count rows and pack their own slices inside protected phases, the primary plans, posts and + * waits in primary-only phases, and destination owners read the received blocks after the checkpoint that follows + * the wait. cpp/tests/CMakeLists.txt launches the multi-rank cases under mpiexec with 2 and 3 processes at + * T = 1, 2 and 4; at world size 1 they skip, and only the single-process contract cases run. + * + * Oracle: every block's size and contents are a closed-form function of (source rank, source shard, destination + * rank, destination shard, element index, round), so each receiver checks exactly what it must have received + * without trusting the sender's layout. Workers never call BOOST_TEST; they record mismatches in per-shard slots. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/mpi/Comm.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/Team.h" + +namespace { + +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; +using sharded::ExchangeElement; +using sharded::ExchangeTransport; +using sharded::PhysicalExchange; +using sharded::PhysicalWorld; + +constexpr int kTag = 0x5E01; + +auto team_options() -> parallel::Options { + return parallel::capture_thread_budget(); +} + +auto world() -> PhysicalWorld { + return PhysicalWorld::of(monoprop::mpi::Comm(MPI_COMM_WORLD)); +} + +// Multi-rank cases run only in their dedicated launches, which name the world size (cpp/tests/CMakeLists.txt). +auto dedicated_launch() -> bool { + const char *text = std::getenv("monoprop_TEST_SHARDED_RANKS"); + return text != nullptr && std::stoul(text) == world().ranks; +} + +auto has_two_ranks(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto ranks = world().ranks; + boost::test_tools::assertion_result result(ranks >= 2 && dedicated_launch()); + result.message() << "the launch has " << ranks << " MPI rank(s); this case needs a dedicated launch of at least 2"; + return result; +} + +auto has_even_ranks(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto ranks = world().ranks; + boost::test_tools::assertion_result result(ranks >= 2 && ranks % 2 == 0 && dedicated_launch()); + result.message() << "the launch has " << ranks << " MPI rank(s); the XOR peer needs a dedicated launch of an even " + << "count of at least 2"; + return result; +} + +auto has_nonprimary_worker(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto threads = team_options().threads; + boost::test_tools::assertion_result result(threads >= 2); + result.message() << "the captured team budget is " << threads << "; this case needs a nonprimary worker"; + return result; +} + +// Block size from source (rank a, shard u) to destination (rank b, shard t) in round `round`: 0 .. 5 elements, so +// empty legs, empty shards and whole empty peers all occur, and a scaled round grows the staging. +auto block_size(size_t a, size_t u, size_t b, size_t t, size_t round, size_t scale) -> size_t { + return ((a * 7) + (u * 3) + (b * 5) + (t * 11) + (round * 13)) % 6 * scale; +} + +template +auto encode(size_t a, size_t u, size_t b, size_t t, size_t i) -> T { + // Unique per (source, destination, index) and exact in every element type. + const auto v = static_cast((((((a * 16) + u) * 16 + b) * 16 + t) << 12) + i); + if constexpr (std::is_same_v) { + return static_cast(v) + 0.25; + } + else { + return static_cast(v); + } +} + +template +constexpr auto element_of() -> ExchangeElement { + if constexpr (std::is_same_v) { + return ExchangeElement::u32; + } + else if constexpr (std::is_same_v) { + return ExchangeElement::u64; + } + else { + return ExchangeElement::f64; + } +} + +auto all_other_ranks(const PhysicalWorld &w) -> std::vector { + std::vector peers; + for (size_t b = 0; b < w.ranks; ++b) { + if (b != w.rank) { + peers.push_back(b); + } + } + return peers; +} + +struct RoundSpec { + std::vector peers; + ExchangeTransport transport = ExchangeTransport::pairwise; + bool counts_known = false; // receivers fill their receive rows; otherwise the count round supplies them + size_t round = 0; + size_t scale = 1; +}; + +// One round through the phase protocol; returns per-shard mismatch descriptions and the join error. +template +auto run_round(PhysicalExchange &ex, const PhysicalWorld &w, const RoundSpec &spec) + -> std::pair> { + const auto options = team_options(); + const auto threads = static_cast(options.threads); + const size_t me = w.rank; + std::vector> problems(threads); + const auto error = sharded::run_team(options, [&](size_t s, sharded::TeamFailure &failure) noexcept { + // Owner s fills its rows: what it sends to every remote shard, and (when known) what it receives. + if (!sharded::phase(failure, s, [&] { + ex.reset_rows(s); + for (size_t k = 0; k < spec.peers.size(); ++k) { + const size_t b = spec.peers[k]; + for (size_t t = 0; t < threads; ++t) { + ex.set_send_count(s, k, t, block_size(me, s, b, t, spec.round, spec.scale)); + if (spec.counts_known) { + ex.set_recv_count(s, k, t, block_size(b, t, me, s, spec.round, spec.scale)); + } + } + } + })) { + return; + } + if (!sharded::phase(failure, s, [&] { + if (s == 0) { + ex.plan_send(spec.peers, spec.transport); + if (!spec.counts_known) { + ex.post_counts(); + } + } + })) { + return; + } + // Each owner packs its own disjoint slices. + if (!sharded::phase(failure, s, [&] { + for (size_t k = 0; k < spec.peers.size(); ++k) { + const size_t b = spec.peers[k]; + for (size_t t = 0; t < threads; ++t) { + auto block = ex.send_block(s, k, t); + if (block.size() != block_size(me, s, b, t, spec.round, spec.scale)) { + throw std::logic_error("send slice has the wrong size"); + } + for (size_t i = 0; i < block.size(); ++i) { + block[i] = encode(me, s, b, t, i); + } + } + } + })) { + return; + } + if (!sharded::phase(failure, s, [&] { + if (s == 0) { + if (!spec.counts_known) { + ex.wait_counts(); + } + ex.plan_recv(); + ex.post(); + ex.wait(); + } + })) { + return; + } + // Destination owner s checks every block it received, against the closed form. + static_cast(sharded::phase(failure, s, [&] { + for (size_t k = 0; k < spec.peers.size(); ++k) { + const size_t a = spec.peers[k]; + for (size_t u = 0; u < threads; ++u) { + const size_t expected = block_size(a, u, me, s, spec.round, spec.scale); + if (ex.recv_count(s, k, u) != expected) { + problems[s].push_back(std::format("count from ({}, {}) is {}, expected {}", + a, + u, + ex.recv_count(s, k, u), + expected)); + } + const auto block = ex.recv_block(s, k, u); + if (block.size() != expected) { + problems[s].push_back(std::format("block from ({}, {}) has {} values, expected {}", + a, + u, + block.size(), + expected)); + continue; + } + for (size_t i = 0; i < block.size(); ++i) { + if (block[i] != encode(a, u, me, s, i)) { + problems[s].push_back(std::format("block from ({}, {}) differs at {}", a, u, i)); + break; + } + } + } + } + })); + }); + std::vector flat; + for (auto &p : problems) { + flat.insert(flat.end(), p.begin(), p.end()); + } + return {error, flat}; +} + +auto describe(const std::exception_ptr &error) -> std::string { + if (!error) { + return "none"; + } + try { + std::rethrow_exception(error); + } + catch (const std::exception &e) { + return e.what(); + } + catch (...) { + return "unknown"; + } +} + +template +auto check_rounds(const std::vector &specs) -> void { + const auto w = world(); + PhysicalExchange ex(w, static_cast(team_options().threads), element_of(), kTag); + for (const auto &spec : specs) { + BOOST_TEST_CONTEXT("rank " << w.rank << " of " << w.ranks << ", T " << team_options().threads << ", round " + << spec.round << " scale " << spec.scale << ", " + << (spec.transport == ExchangeTransport::pairwise ? "pairwise" : "collective") + << ", counts " << (spec.counts_known ? "known" : "exchanged") << ", " + << spec.peers.size() << " peer(s)") { + const auto [error, problems] = run_round(ex, w, spec); + BOOST_TEST(describe(error) == "none"); + BOOST_TEST(problems.empty(), (problems.empty() ? std::string{} : problems.front())); + BOOST_TEST(ex.live() == 0); + } + } +} + +auto every_variant(const std::vector &peers) -> std::vector { + std::vector specs; + size_t round = 0; + // Grow, shrink and grow again on the same object: staging is reused, never assumed fresh. + for (const size_t scale : {size_t{1}, size_t{300}, size_t{1}, size_t{2}}) { + for (const auto transport : {ExchangeTransport::pairwise, ExchangeTransport::collective}) { + for (const bool known : {false, true}) { + specs.push_back( + {.peers = peers, .transport = transport, .counts_known = known, .round = round++, .scale = scale}); + } + } + } + return specs; +} + +} // namespace + +// --- Single-process contract -------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(sharded_exchange_world_reads_the_communicator) { + const auto w = world(); + BOOST_TEST(w.ranks == static_cast(monoprop::mpi::size(monoprop::mpi::Comm(MPI_COMM_WORLD)))); + BOOST_TEST(w.rank == static_cast(monoprop::mpi::rank(monoprop::mpi::Comm(MPI_COMM_WORLD)))); + const auto self = PhysicalWorld::of(monoprop::mpi::Comm(MPI_COMM_SELF)); + BOOST_TEST(self.ranks == 1U); + BOOST_TEST(self.rank == 0U); + BOOST_TEST(!self.replay_pairwise); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_rejects_invalid_layouts_before_posting) { + const auto w = world(); + const size_t threads = 2; + PhysicalExchange ex(w, threads, ExchangeElement::f64, kTag); + // A peer set must name other ranks, ascending, each once. + BOOST_CHECK_THROW(ex.plan_send(std::vector{w.rank}, ExchangeTransport::pairwise), std::invalid_argument); + BOOST_CHECK_THROW(ex.plan_send(std::vector{w.ranks}, ExchangeTransport::pairwise), std::invalid_argument); + if (w.ranks >= 3) { + const auto peers = all_other_ranks(w); + BOOST_CHECK_THROW(ex.plan_send(std::vector{peers[1], peers[0]}, ExchangeTransport::pairwise), + std::invalid_argument); + BOOST_CHECK_THROW(ex.plan_send(std::vector{peers[0], peers[0]}, ExchangeTransport::pairwise), + std::invalid_argument); + } + // Counts must fit an MPI int, and indices must address the table (a one-rank world has no peer to address). + ex.reset_rows(0); + if (w.ranks >= 2) { + BOOST_CHECK_THROW(ex.set_send_count(0, 0, 0, size_t{std::numeric_limits::max()} + 1), std::length_error); + BOOST_CHECK_THROW(ex.set_recv_count(0, 0, 0, size_t{std::numeric_limits::max()} + 1), std::length_error); + } + else { + BOOST_CHECK_THROW(ex.set_send_count(0, 0, 0, 1), std::out_of_range); + } + BOOST_CHECK_THROW(ex.set_send_count(threads, 0, 0, 1), std::out_of_range); + BOOST_CHECK_THROW(ex.set_send_count(0, w.ranks, 0, 1), std::out_of_range); + BOOST_CHECK_THROW(ex.set_send_count(0, 0, threads, 1), std::out_of_range); + // The element type is fixed per exchange. + ex.plan_send({}, ExchangeTransport::pairwise); + BOOST_CHECK_THROW(static_cast(ex.send_block(0, 0, 0)), std::logic_error); + // Nothing was posted by any of the failures. + BOOST_TEST(ex.live() == 0); +} + +// Owner-parallel planning (close_rows, plan_totals, plan_column, place_column) lays every block out exactly where +// plan_send + plan_recv do: same slice address and size for every (shard, peer, other shard), same staging sizes. +// No MPI call is made, so a world of several ranks can be described without launching them; every rank position is +// covered, sparse and dense count tables, empty rows and columns. +BOOST_AUTO_TEST_CASE(sharded_exchange_owner_parallel_planning_matches_the_primary_layout) { + uint64_t state = 0x9E3779B97F4A7C15ULL; + const auto next = [&state](uint64_t bound) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state % bound; + }; + for (const size_t ranks : {size_t{2}, size_t{3}, size_t{4}}) { + for (const size_t threads : {size_t{1}, size_t{2}, size_t{5}}) { + for (size_t rank = 0; rank < ranks; ++rank) { + for (const uint64_t density : {uint64_t{0}, uint64_t{3}, uint64_t{10}}) { + BOOST_TEST_CONTEXT("ranks " << ranks << " threads " << threads << " rank " << rank << " density " + << density) { + const PhysicalWorld w{.comm = monoprop::mpi::Comm(MPI_COMM_SELF), .rank = rank, .ranks = ranks}; + PhysicalExchange primary(w, threads, ExchangeElement::f64, kTag); + PhysicalExchange owners(w, threads, ExchangeElement::f64, kTag); + std::vector peers; + for (size_t b = 0; b < ranks; ++b) { + if (b != rank) { + peers.push_back(b); + } + } + for (size_t t = 0; t < threads; ++t) { + primary.reset_rows(t); + owners.reset_rows(t); + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t o = 0; o < threads; ++o) { + // density 10: every entry nonzero; 3: about a third; 0: all empty. + const size_t send = density == 0 || next(10) >= density ? 0 : 1 + next(50); + const size_t recv = density == 0 || next(10) >= density ? 0 : 1 + next(50); + primary.set_send_count(t, k, o, send); + primary.set_recv_count(t, k, o, recv); + owners.set_send_count(t, k, o, send); + owners.set_recv_count(t, k, o, recv); + } + } + owners.close_rows(t); + } + primary.plan_send(peers, ExchangeTransport::pairwise); + primary.plan_recv(); + owners.plan_totals(peers, ExchangeTransport::pairwise); + for (size_t t = 0; t < threads; ++t) { + owners.plan_column(t); + } + for (size_t t = threads; t-- > 0;) { // any order: each owner writes only its own column + owners.place_column(t); + } + // Offsets relative to each staging's start, since the two exchanges own different buffers. + const auto *send0 = primary.send_block(0, 0, 0).data(); + const auto *send1 = owners.send_block(0, 0, 0).data(); + const auto *recv0 = primary.recv_block(0, 0, 0).data(); + const auto *recv1 = owners.recv_block(0, 0, 0).data(); + size_t mismatches = 0; + for (size_t t = 0; t < threads; ++t) { + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t o = 0; o < threads; ++o) { + const auto a = primary.send_block(t, k, o); + const auto b = owners.send_block(t, k, o); + const auto c = primary.recv_block(t, k, o); + const auto d = owners.recv_block(t, k, o); + mismatches += static_cast( + a.size() != b.size() || a.data() - send0 != b.data() - send1 + || c.size() != d.size() || c.data() - recv0 != d.data() - recv1); + } + } + } + BOOST_TEST(mismatches == 0U); + BOOST_TEST(owners.live() == 0); + } + } + } + } + } + // Placing a column before the round is sized is refused. + const PhysicalWorld w{.comm = monoprop::mpi::Comm(MPI_COMM_SELF), .rank = 0, .ranks = 2}; + PhysicalExchange unplanned(w, 2, ExchangeElement::f64, kTag); + BOOST_CHECK_THROW(unplanned.place_column(0), std::logic_error); + BOOST_CHECK_THROW(unplanned.plan_column(2), std::out_of_range); + BOOST_CHECK_THROW(unplanned.close_rows(2), std::out_of_range); + // A per-peer total past INT_MAX is refused before anything is placed or posted, as by plan_send(). + PhysicalExchange big(w, 2, ExchangeElement::f64, kTag); + for (size_t t = 0; t < 2; ++t) { + big.reset_rows(t); + big.set_send_count(t, 0, 0, static_cast(std::numeric_limits::max())); + big.close_rows(t); + } + BOOST_CHECK_THROW(big.plan_totals(std::vector{1}, ExchangeTransport::pairwise), std::length_error); + BOOST_CHECK_THROW(big.place_column(0), std::logic_error); + BOOST_TEST(big.live() == 0); +} + +// A replay step's bulk path (write_symmetric_rows, the all-other-ranks plan_totals, send_column) produces the same +// tables, staging layout and slices as the entry-wise path, and send_column's view of column t is exactly the +// send_block slices (u, peer, t) of every local source u. Layouts shorter than the world fill the missing slots with +// zero counts; the owner's own rank is never a peer. +BOOST_AUTO_TEST_CASE(sharded_exchange_symmetric_rows_and_column_views_match_the_entrywise_path) { + uint64_t state = 0xD1B54A32D192ED03ULL; + const auto next = [&state](uint64_t bound) { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + return state % bound; + }; + for (const size_t ranks : {size_t{2}, size_t{3}, size_t{4}}) { + for (const size_t threads : {size_t{1}, size_t{2}, size_t{5}}) { + for (size_t rank = 0; rank < ranks; ++rank) { + for (const bool short_layouts : {false, true}) { + BOOST_TEST_CONTEXT("ranks " << ranks << " threads " << threads << " rank " << rank << " short " + << short_layouts) { + const PhysicalWorld w{.comm = monoprop::mpi::Comm(MPI_COMM_SELF), .rank = rank, .ranks = ranks}; + PhysicalExchange entrywise(w, threads, ExchangeElement::f64, kTag); + PhysicalExchange bulk(w, threads, ExchangeElement::f64, kTag); + std::vector peers; + for (size_t b = 0; b < ranks; ++b) { + if (b != rank) { + peers.push_back(b); + } + } + for (size_t t = 0; t < threads; ++t) { + // One publication layout per owner, indexed by flat slot; about a third of the slots + // empty, the own rank's slots arbitrary (never read), and sometimes cut short. + const size_t slots = short_layouts ? next(ranks * threads + 1) : ranks * threads; + std::vector counts(slots); + for (int &count : counts) { + count = next(3) == 0 ? 0 : static_cast(1 + next(40)); + } + entrywise.reset_rows(t); + for (size_t k = 0; k < peers.size(); ++k) { + for (size_t o = 0; o < threads; ++o) { + const size_t slot = (peers[k] * threads) + o; + const size_t count = slot < counts.size() ? static_cast(counts[slot]) : 0; + entrywise.set_send_count(t, k, o, count); + entrywise.set_recv_count(t, k, o, count); + } + } + entrywise.close_rows(t); + bulk.write_symmetric_rows(t, counts); + } + entrywise.plan_totals(peers, ExchangeTransport::collective); + bulk.plan_totals(ExchangeTransport::collective); + BOOST_TEST(std::ranges::equal(bulk.peers(), peers)); + for (size_t t = 0; t < threads; ++t) { + entrywise.plan_column(t); + bulk.plan_column(t); + } + for (size_t t = threads; t-- > 0;) { + entrywise.place_column(t); + bulk.place_column(t); + } + const auto *send0 = entrywise.send_block(0, 0, 0).data(); + const auto *send1 = bulk.send_block(0, 0, 0).data(); + const auto *recv0 = entrywise.recv_block(0, 0, 0).data(); + const auto *recv1 = bulk.recv_block(0, 0, 0).data(); + size_t mismatches = 0; + for (size_t t = 0; t < threads; ++t) { + for (size_t k = 0; k < peers.size(); ++k) { + const auto column = bulk.send_column(t, k); + mismatches += static_cast(column.offsets.size() != threads + || column.counts.size() != threads); + for (size_t o = 0; o < threads; ++o) { + mismatches += static_cast( + entrywise.send_count(t, k, o) != bulk.send_count(t, k, o) + || entrywise.recv_count(t, k, o) != bulk.recv_count(t, k, o)); + const auto a = entrywise.send_block(t, k, o); + const auto b = bulk.send_block(t, k, o); + const auto c = entrywise.recv_block(t, k, o); + const auto d = bulk.recv_block(t, k, o); + mismatches += static_cast( + a.size() != b.size() || a.data() - send0 != b.data() - send1 + || c.size() != d.size() || c.data() - recv0 != d.data() - recv1); + // Column t's entry for source o is send_block(o, k, t). + const auto slice = bulk.send_block(o, k, t); + const auto block = column.block(o); + mismatches += static_cast(block.data() != slice.data() + || block.size() != slice.size()); + } + } + } + BOOST_TEST(mismatches == 0U); + } + } + } + } + } + const PhysicalWorld w{.comm = monoprop::mpi::Comm(MPI_COMM_SELF), .rank = 1, .ranks = 3}; + PhysicalExchange round(w, 2, ExchangeElement::f64, kTag); + BOOST_CHECK_THROW(round.write_symmetric_rows(2, std::vector{}), std::out_of_range); + BOOST_CHECK_THROW(round.write_symmetric_rows(0, std::vector{0, 0, 0, 0, -1, 0}), std::length_error); + // Columns are readable only once the round is planned, with the round's element type, inside the table. + BOOST_CHECK_THROW(static_cast(round.send_column(0, 0)), std::logic_error); + round.write_symmetric_rows(0, std::vector{1, 1, 1, 1, 1, 1}); + round.write_symmetric_rows(1, std::vector{1, 1, 1, 1, 1, 1}); + round.plan_totals(ExchangeTransport::pairwise); + round.plan_column(0); + round.plan_column(1); + round.place_column(0); + BOOST_CHECK_THROW(static_cast(round.send_column(0, 0)), std::logic_error); + BOOST_CHECK_THROW(static_cast(round.send_column(2, 0)), std::out_of_range); + BOOST_CHECK_THROW(static_cast(round.send_column(0, 2)), std::out_of_range); + BOOST_TEST(round.send_column(0, 1).block(1).size() == 1U); + BOOST_TEST(round.live() == 0); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_owner_rounds_persist_by_kind) { + using Kind = sharded::PhysicalRounds::Kind; + const PhysicalWorld w{.comm = monoprop::mpi::Comm(MPI_COMM_SELF), .rank = 0, .ranks = 2}; + sharded::PhysicalRounds rounds; + // Nothing exists before first use. + for (const auto kind : {Kind::queries, Kind::graph_answers, Kind::fused_answers, Kind::replay}) { + BOOST_TEST(rounds.peek(kind).threads() == 0U); + } + PhysicalExchange &queries = rounds.get(Kind::queries, w, 2); + // The same round on every later call with the same geometry: its staging persists. + BOOST_TEST(&rounds.get(Kind::queries, w, 2) == &queries); + BOOST_TEST(&rounds.peek(Kind::queries) == &queries); + BOOST_TEST(queries.threads() == 2U); + BOOST_TEST(rounds.peek(Kind::graph_answers).threads() == 0U); // kinds are separate rounds + // Each kind carries its payload type. + const std::vector peer{1}; + for (const auto kind : {Kind::queries, Kind::graph_answers, Kind::fused_answers, Kind::replay}) { + PhysicalExchange &round = rounds.get(kind, w, 2); + round.reset_rows(0); + round.reset_rows(1); + round.set_send_count(0, 0, 0, 1); + round.plan_send(peer, ExchangeTransport::pairwise); + const bool u64 = kind == Kind::queries; + const bool u32 = kind == Kind::graph_answers; + const auto accepts = [&](auto element) { + try { + return round.send_block(0, 0, 0).size() == 1U; + } + catch (const std::logic_error &) { + return false; + } + }; + BOOST_TEST(accepts(uint64_t{}) == u64); + BOOST_TEST(accepts(uint32_t{}) == u32); + BOOST_TEST(accepts(double{}) == (!u64 && !u32)); + } + // A different geometry gets a fresh round. + BOOST_TEST(rounds.get(Kind::queries, w, 3).threads() == 3U); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_payload_totals_must_fit_mpi_counts) { + // Per-peer totals are prefix sums of int counts; a sum past INT_MAX is refused by plan_send, before posting. + const auto w = world(); + if (w.ranks < 2 || !dedicated_launch()) { + // One process has no peer to address; the per-count bound is covered above. + BOOST_TEST_MESSAGE("world size 1: no peer to overflow"); + return; + } + const size_t threads = 2; + PhysicalExchange ex(w, threads, ExchangeElement::f64, kTag); + ex.reset_rows(0); + ex.reset_rows(1); + const int big = std::numeric_limits::max(); + ex.set_send_count(0, 0, 0, static_cast(big)); + ex.set_send_count(1, 0, 0, static_cast(big)); + BOOST_CHECK_THROW(ex.plan_send(std::vector{all_other_ranks(w).front()}, ExchangeTransport::pairwise), + std::length_error); + BOOST_TEST(ex.live() == 0); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_mpi_calls_run_only_on_the_primary, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto w = world(); + const auto options = team_options(); + PhysicalExchange ex(w, static_cast(options.threads), ExchangeElement::u64, kTag); + // No peers: a primary post is a no-op, a worker's post is refused before it reaches MPI. + std::vector refused(static_cast(options.threads), 0); + const auto error = sharded::run_team(options, [&](size_t s, sharded::TeamFailure &failure) noexcept { + if (!sharded::phase(failure, s, [&] { + if (s == 0) { + ex.plan_send({}, ExchangeTransport::pairwise); + } + })) { + return; + } + static_cast(sharded::phase(failure, s, [&] { + if (s == 1) { + for (const auto &call : {+[](PhysicalExchange &e) { e.post_counts(); }, + +[](PhysicalExchange &e) { e.wait_counts(); }, + +[](PhysicalExchange &e) { e.post(); }, + +[](PhysicalExchange &e) { e.wait(); }}) { + try { + call(ex); + } + catch (const std::logic_error &) { + ++refused[s]; + } + } + } + })); + }); + BOOST_TEST(describe(error) == "none"); + BOOST_TEST(refused[1] == 4); + BOOST_TEST(ex.live() == 0); +} + +// --- Multi-rank rounds -------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(sharded_exchange_moves_every_block_to_its_owner, *boost::unit_test::precondition(has_two_ranks)) { + const auto peers = all_other_ranks(world()); + check_rounds(every_variant(peers)); + check_rounds(every_variant(peers)); + check_rounds(every_variant(peers)); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_count_blocks_of_every_width_arrive, + *boost::unit_test::precondition(has_two_ranks)) { + // Counts of at most 5, 300 and 100000 travel as 1-, 2- and 4-byte counts; a later light round after a heavy one + // shrinks the block again. The receiver must read each block's width from its size. + const auto peers = all_other_ranks(world()); + std::vector specs; + size_t round = 0; + for (const size_t scale : {size_t{1}, size_t{60}, size_t{20000}, size_t{1}, size_t{60}}) { + for (const auto transport : {ExchangeTransport::pairwise, ExchangeTransport::collective}) { + specs.push_back( + {.peers = peers, .transport = transport, .counts_known = false, .round = round++, .scale = scale}); + } + } + check_rounds(specs); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_sparse_peer_round, *boost::unit_test::precondition(has_even_ranks)) { + const auto w = world(); + check_rounds(every_variant({w.rank ^ 1U})); +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_empty_rounds_complete, *boost::unit_test::precondition(has_two_ranks)) { + // A round whose every count is zero still completes on every rank, under both transports. + const auto w = world(); + PhysicalExchange ex(w, static_cast(team_options().threads), ExchangeElement::f64, kTag); + for (const auto transport : {ExchangeTransport::pairwise, ExchangeTransport::collective}) { + for (const bool known : {false, true}) { + const RoundSpec spec{.peers = all_other_ranks(w), + .transport = transport, + .counts_known = known, + .round = 0, + .scale = 0}; + const auto [error, problems] = run_round(ex, w, spec); + BOOST_TEST(describe(error) == "none"); + BOOST_TEST(problems.empty()); + BOOST_TEST(ex.live() == 0); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_exchange_moves_keep_and_drain_requests, *boost::unit_test::precondition(has_two_ranks)) { + // A posted round moved into a new owner keeps its requests; assigning over a live owner drains it first. + const auto w = world(); + const auto peers = all_other_ranks(w); + PhysicalExchange ex(w, 1, ExchangeElement::f64, kTag); + const auto post_one = [&](PhysicalExchange &e, double value) { + e.reset_rows(0); + for (size_t k = 0; k < peers.size(); ++k) { + e.set_send_count(0, k, 0, 1); + e.set_recv_count(0, k, 0, 1); + } + e.plan_send(peers, ExchangeTransport::pairwise); + for (size_t k = 0; k < peers.size(); ++k) { + e.send_block(0, k, 0)[0] = value + static_cast(w.rank); + } + e.plan_recv(); + e.post(); + }; + post_one(ex, 10.0); + BOOST_TEST(ex.live() > 0); + PhysicalExchange moved(std::move(ex)); + BOOST_TEST(moved.live() > 0); + moved.wait(); + for (size_t k = 0; k < peers.size(); ++k) { + BOOST_TEST(moved.recv_block(0, k, 0)[0] == 10.0 + static_cast(peers[k])); + } + PhysicalExchange target(w, 1, ExchangeElement::f64, kTag + 2); + post_one(target, 20.0); + BOOST_TEST(target.live() > 0); + target = std::move(moved); // drains the round target had posted + BOOST_TEST(target.live() == 0); +#ifdef monoprop_ENABLE_MPI + // The barrier keeps the second round's tag from meeting the next case's first round. + MPI_Barrier(MPI_COMM_WORLD); +#endif +} diff --git a/cpp/tests/sharded_root_tests.cpp b/cpp/tests/sharded_root_tests.cpp new file mode 100644 index 000000000..efe6a5382 --- /dev/null +++ b/cpp/tests/sharded_root_tests.cpp @@ -0,0 +1,1441 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * The sharded OpenMP root: MonomialPropagator over the T shard states of one rank, through its public operations. The + * team size T is the captured launch budget; cpp/tests/CMakeLists.txt reruns every case in fresh processes at T = 1, 2 + * and 4 (sharded_root_env_t*), and the serial per-case registrations run at T = 1. Cases that need a nonprimary worker + * return early at T = 1. + * + * Private inspection, phase observation and failure injection go through PropagatorTestAccess (shards(), the (1, T) + * router, and the test-only RootObserver every seam reports to). Observers record the executing worker inside the + * protected phase; assertions run after the team has joined. Numerical parity against an independently built runtime at + * the same (1, T) geometry, and across T, is checked from separate processes by tests/test_sharded_openmp.py. + */ + +#include + +#include +#include + +#include "TestUtilities.h" +#include "monoprop/MonomialPropagator.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AllocationProbe.h" +#include "PropagatorTestAccess.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/mpi/MPICompat.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/RootObserver.h" + +namespace { + +using namespace monoprop; +using namespace test_utils; +namespace sharded = monoprop::detail::sharded; + +constexpr size_t kN = 12; +using MP = MonomialPropagator; +using Access = monoprop::detail::PropagatorTestAccess; + +auto launch_team() -> size_t { + const char *text = std::getenv("monoprop_TEST_EXPECT_TEAM"); + return text == nullptr ? 1 : static_cast(std::stoul(text)); +} + +auto lih() -> const CaseData & { + static const auto data = load_case_data("lih_fermionic_spin_exact.msgpack"); + return data; +} + +struct Injected : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +enum class Seam : std::uint8_t { shard, construction, evaluation, root }; + +struct Visit { + Seam seam; + int work; + size_t step; + size_t shard; + int worker; + int team; + int level; + std::thread::id thread; +}; + +// Records every observed visit with its executing worker, and optionally throws at one armed visit. +class Recorder final : public sharded::RootObserver { +public: + struct Arm { + Seam seam; + int work; + size_t shard; + std::optional step; // any step when empty + }; + + auto arm(Arm target) -> void { + arm_ = target; + fired_ = false; + } + auto disarm() -> void { arm_.reset(); } + [[nodiscard]] auto fired() const -> bool { return fired_; } + auto clear() -> void { + const std::lock_guard lock(mutex_); + visits_.clear(); + } + [[nodiscard]] auto visits() const -> std::vector { + const std::lock_guard lock(mutex_); + return visits_; + } + + auto visit(sharded::EvaluationWork work, size_t step, size_t shard) const -> void override { + note(Seam::evaluation, static_cast(work), step, shard); + } + auto shard_work(sharded::ShardWork work, bool end, size_t shard) const -> void override { + note(Seam::shard, (2 * static_cast(work)) + (end ? 1 : 0), sharded::no_step, shard); + } + auto construction(sharded::ConstructionWork work, size_t step, size_t shard) const -> void override { + note(Seam::construction, static_cast(work), step, shard); + } + auto root(sharded::RootWork work, size_t shard) const -> void override { + note(Seam::root, static_cast(work), sharded::no_step, shard); + } + +private: + auto note(Seam seam, int work, size_t step, size_t shard) const -> void { + bool fire = false; + { + const std::lock_guard lock(mutex_); + visits_.push_back(Visit{seam, + work, + step, + shard, + omp_get_thread_num(), + omp_get_num_threads(), + omp_get_level(), + std::this_thread::get_id()}); + if (arm_ && !fired_ && arm_->seam == seam && arm_->work == work && arm_->shard == shard + && (!arm_->step || *arm_->step == step)) { + fired_ = true; + fire = true; + } + } + if (fire) { + throw Injected("injected root-seam failure"); + } + } + + mutable std::mutex mutex_; + mutable std::vector visits_; + std::optional arm_; + mutable bool fired_ = false; +}; + +struct Config { + bool schrodinger = false; + std::optional lower_atol = std::nullopt; + std::optional> basis_change = std::nullopt; + Basis basis = Basis::Majorana; +}; + +auto make(const Config &config = {}, const sharded::RootObserver *observer = nullptr) -> std::unique_ptr { + const auto &data = lih(); + return Access::construct_observed(observer, + data.hamiltonian, + 4U, + data.initial_state, + config.schrodinger ? std::optional{6U} : std::nullopt, + mpi::Comm(MPI_COMM_SELF), + config.lower_atol, + std::optional{}, + CutoffType::Length, + config.basis_change, + kN, + config.basis); +} + +auto build(MP &sim) -> void { + const auto &data = lih(); + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); +} + +auto params() -> const VecD & { + return lih().parameters; +} + +// The root's own caller-side work, outside any team. +auto caller_work(const Visit &v) -> bool { + return v.seam == Seam::root + && (v.work == static_cast(sharded::RootWork::combine) + || v.work == static_cast(sharded::RootWork::export_block)); +} + +// Every owner visit of shard t ran on worker t of a level-1 team of T workers (a one-worker team runs on the caller, +// at level 0, without a region); caller work ran outside any team. +auto check_owner_visits(const std::vector &visits, size_t team, const char *what) -> void { + BOOST_TEST_CONTEXT(what) { + BOOST_TEST(!visits.empty()); + std::set threads; + for (const auto &v : visits) { + if (caller_work(v)) { + BOOST_TEST(v.level == 0); + continue; + } + BOOST_TEST(static_cast(v.worker) == v.shard); + BOOST_TEST(static_cast(v.team) == team); + BOOST_TEST(v.level == (team > 1 ? 1 : 0)); + threads.insert(v.thread); + } + BOOST_TEST(threads.size() == team); + } +} + +auto count(const std::vector &visits, Seam seam, int work, size_t shard) -> size_t { + return static_cast(std::ranges::count_if(visits, [&](const Visit &v) { + return v.seam == seam && v.work == work && v.shard == shard; + })); +} + +auto energy_of(MP &sim) -> double { + return sim.expectation_value(params()); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sharded_root_raw_accessors_follow_the_launch_team) { + const auto data = load_case_data<8>("random_exact.msgpack"); + MonomialPropagator<8> sim(data.hamiltonian, 16, data.initial_state, std::nullopt, MPI_COMM_SELF); + const auto &shards = monoprop::detail::PropagatorTestAccess<8>::shards(sim); + BOOST_TEST(shards.size() == launch_team()); + if (launch_team() == 1) { + // The actual sole shard, not a copy or a merge. + BOOST_TEST(&sim.mp_op() == &shards.front()->op); + BOOST_TEST(&sim.graph() == &shards.front()->graph); + BOOST_TEST(&sim.indexing() == shards.front()->op.store.get()); + BOOST_TEST(sim.graph_data().size() == sim.graph_layers()); + return; + } + const auto remedy = [](const MultiShardUnsupported &e) { + return std::string(e.what()).find("monoprop_NUM_THREADS=1") != std::string::npos + && std::string(e.what()).find("partitions=1") == std::string::npos; + }; + BOOST_CHECK_EXCEPTION(sim.mp_op(), MultiShardUnsupported, remedy); + BOOST_CHECK_EXCEPTION(std::as_const(sim).mp_op(), MultiShardUnsupported, remedy); + BOOST_CHECK_EXCEPTION(sim.graph(), MultiShardUnsupported, remedy); + BOOST_CHECK_EXCEPTION(sim.indexing(), MultiShardUnsupported, remedy); + BOOST_CHECK_EXCEPTION(sim.graph_data(), MultiShardUnsupported, remedy); + // The rejection mutates nothing: the object stays valid and keeps answering. + BOOST_TEST(!monoprop::detail::PropagatorTestAccess<8>::is_invalid(sim)); + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + BOOST_CHECK_EXCEPTION(sim.graph(), MultiShardUnsupported, remedy); + BOOST_TEST(std::isfinite(sim.expectation_value(data.parameters))); + BOOST_TEST(!monoprop::detail::PropagatorTestAccess<8>::is_invalid(sim)); +} + +// At T = 1 on one rank the root evaluates its sole shard with the low-level serial evaluator; with an observer attached +// it runs the sharded phases. Both give bitwise the same energies and gradients through every entry point, both +// pictures, pared and unpared, and a bad parameter vector fails the same way on both, leaving the object valid. +BOOST_AUTO_TEST_CASE(sharded_root_sole_shard_serial_evaluation_matches_the_phases) { + if (launch_team() != 1) { + return; // the serial path is only for a sole shard + } + using Access = monoprop::detail::PropagatorTestAccess<8>; + const auto data = load_case_data<8>("random_exact.msgpack"); + const sharded::RootObserver phases; // observes nothing; its presence keeps the sharded phases + const auto bits = [](double v) { return std::bit_cast(v); }; + const auto same = [&](const std::pair &a, const std::pair &b) { + return bits(a.first) == bits(b.first) && a.second.size() == b.second.size() + && std::ranges::equal(a.second, b.second, [&](double x, double y) { return bits(x) == bits(y); }); + }; + for (const std::optional schrodinger : + {std::optional{}, std::optional{16}}) { + for (const std::optional pare : {std::optional{}, std::optional{1e-3}}) { + BOOST_TEST_CONTEXT("schrodinger " << schrodinger.has_value() << " pare " << pare.has_value()) { + MonomialPropagator<8> sim(data.hamiltonian, 16, data.initial_state, schrodinger, MPI_COMM_SELF); + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const auto energy = sim.expectation_value_functional(pare); + const auto gradient = sim.expectation_value_and_gradient_functional(pare); + const std::pair serial_e{energy(data.parameters), {}}; + const auto serial_g = gradient(data.parameters); + const auto serial_direct = sim.expectation_value_and_gradient(data.parameters); + Access::observe(sim, &phases); + const std::pair phased_e{energy(data.parameters), {}}; + const auto phased_g = gradient(data.parameters); + const auto phased_direct = sim.expectation_value_and_gradient(data.parameters); + Access::observe(sim, nullptr); + BOOST_TEST(same(serial_e, phased_e)); + BOOST_TEST(same(serial_g, phased_g)); + BOOST_TEST(same(serial_direct, phased_direct)); + BOOST_TEST(!serial_g.second.empty()); + // Too few parameters for the mapping: the same diagnostic on both paths, nothing mutated. + const VecD short_params(1, 0.1); + std::string serial_message; + std::string phased_message; + try { + static_cast(gradient(short_params)); + } + catch (const std::exception &e) { + serial_message = e.what(); + } + Access::observe(sim, &phases); + try { + static_cast(gradient(short_params)); + } + catch (const std::exception &e) { + phased_message = e.what(); + } + Access::observe(sim, nullptr); + BOOST_TEST(!serial_message.empty()); + BOOST_TEST(serial_message == phased_message); + BOOST_TEST(!Access::is_invalid(sim)); + BOOST_TEST(same(gradient(data.parameters), serial_g)); + } + } + } +} + +// An invalid owner fails its validity guard before the shard-count check, at every T. +BOOST_AUTO_TEST_CASE(sharded_root_raw_accessors_check_validity_first) { + auto sim = make(); + build(*sim); + Access::set_cutoff_fn(*sim, [](const Monomial &) -> bool { throw Injected("injected cutoff failure"); }); + BOOST_CHECK_THROW(build(*sim), Injected); + BOOST_TEST_REQUIRE(Access::is_invalid(*sim)); + BOOST_CHECK_THROW(sim->mp_op(), InvalidPropagatorError); + BOOST_CHECK_THROW(std::as_const(*sim).mp_op(), InvalidPropagatorError); + BOOST_CHECK_THROW(sim->indexing(), InvalidPropagatorError); + BOOST_CHECK_THROW(std::as_const(*sim).indexing(), InvalidPropagatorError); + BOOST_CHECK_THROW(sim->graph(), InvalidPropagatorError); + BOOST_CHECK_THROW(sim->graph_data(), InvalidPropagatorError); +} + +namespace { + +// Sets one environment variable for a scope (or unsets it, for nullopt), then restores the previous state. +class ScopedEnv { +public: + ScopedEnv(const char *name, std::optional value) : name_(name) { + if (const char *old = std::getenv(name)) { + old_ = std::string(old); + } + if (value) { + ::setenv(name, value->c_str(), 1); + } + else { + ::unsetenv(name); + } + } + ScopedEnv(const ScopedEnv &) = delete; + auto operator=(const ScopedEnv &) -> ScopedEnv & = delete; + ~ScopedEnv() { + if (old_) { + ::setenv(name_, old_->c_str(), 1); + } + else { + ::unsetenv(name_); + } + } + +private: + const char *name_; + std::optional old_; +}; + +} // namespace + +// The obsolete partition selector is not read at all: whatever it holds while a root is constructed and used, the +// root has the launch's T shards and answers bitwise as with the variable unset. Only the captured budget decides. +BOOST_AUTO_TEST_CASE(sharded_root_ignores_the_obsolete_partition_variable) { + double energy = 0.0; + std::vector>> terms; + { + const ScopedEnv unset("monoprop_PARTITIONS", std::nullopt); + auto sim = make(); + build(*sim); + energy = energy_of(*sim); + terms = sim->evolved_operator_terms(params(), 0.0); + } + for (const char *value : {"off", "auto", "1", "2", "4", "0", "", "-3", "not-a-count"}) { + BOOST_TEST_CONTEXT("monoprop_PARTITIONS=\"" << value << "\"") { + const ScopedEnv set("monoprop_PARTITIONS", std::string(value)); + std::unique_ptr sim; + BOOST_CHECK_NO_THROW(sim = make()); + BOOST_TEST_REQUIRE(sim != nullptr); + BOOST_TEST(Access::shards(*sim).size() == launch_team()); + BOOST_TEST(Access::options(*sim).threads == static_cast(launch_team())); + build(*sim); + BOOST_TEST(energy_of(*sim) == energy); + BOOST_TEST((sim->evolved_operator_terms(params(), 0.0) == terms)); + const MP copy(*sim); + BOOST_TEST(Access::shards(copy).size() == launch_team()); + } + } +} + +// Registered as sharded_root_env_openmp_default_t3: monoprop_NUM_THREADS unset and OMP_NUM_THREADS=3. Neither the test +// runner nor the library may supply the variable, so the root takes omp_get_max_threads() and runs that many owners. +BOOST_AUTO_TEST_CASE(sharded_root_unset_budget_takes_the_openmp_default) { + const char *expected = std::getenv("monoprop_TEST_EXPECT_UNSET_BUDGET"); + if (expected == nullptr) { + BOOST_TEST_MESSAGE("not the unset-budget launch"); + return; + } + const auto team = static_cast(std::stoul(expected)); + BOOST_TEST(std::getenv("monoprop_NUM_THREADS") == nullptr); + BOOST_TEST(static_cast(omp_get_max_threads()) == team); + Recorder recorder; + auto sim = make({}, &recorder); + BOOST_TEST(static_cast(Access::options(*sim).threads) == team); + BOOST_TEST(Access::shards(*sim).size() == team); + build(*sim); + static_cast(energy_of(*sim)); + check_owner_visits(recorder.visits(), team, "seeding, construction and evaluation at the OpenMP default"); +} + +BOOST_AUTO_TEST_CASE(sharded_root_owns_t_routed_shards) { + for (const bool schrodinger : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger) { + auto sim = make({.schrodinger = schrodinger}); + build(*sim); + const size_t team = launch_team(); + const auto &shards = Access::shards(*sim); + const auto &router = Access::router(*sim); + BOOST_TEST(static_cast(Access::options(*sim).threads) == team); + BOOST_TEST(shards.size() == team); + BOOST_TEST(router.ranks() == 1U); + BOOST_TEST(router.shards_per_rank() == team); + std::set keys; + size_t rows = 0; + for (size_t t = 0; t < team; ++t) { + BOOST_TEST(shards[t]->graph.layers() == sim->graph_layers()); + shards[t]->op.store->for_each([&](const auto &mono, size_t) { + BOOST_TEST(router.dest(mono) == t); // flat owner rank * T + t at rank 0 + BOOST_TEST(keys.insert(bitset_to_indices(mono)).second); // no key on two shards + ++rows; + }); + } + BOOST_TEST(rows == sim->size()); + // Heisenberg keeps the identity out of every shard; it is rank-level metadata, counted once. + if (!schrodinger) { + BOOST_TEST(!keys.contains(VecZ{})); + BOOST_TEST(sim->core_term() == Access::core_term(*sim)); + } + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_root_aggregates_cover_every_shard_once) { + auto sim = make(); + build(*sim); + const auto &shards = Access::shards(*sim); + size_t rows = 0; + std::pair graph_size{0, 0}; + auto op_memory = monoprop::detail::MPOperatorMemoryBreakdown{}; + auto graph_memory = GraphMemoryBreakdown{}; + for (const auto &state : shards) { + rows += state->size(); + op_memory += state->operator_memory_usage(); + graph_memory += state->graph_memory_usage(); + graph_size.second += state->graph.total_cycles(); + BOOST_TEST(state->operator_memory_usage().matched_scratch_bytes == state->matched.memory_bytes()); + } + BOOST_TEST(sim->size() == rows); + BOOST_TEST(sim->operator_memory_usage().total_bytes() == op_memory.total_bytes()); + BOOST_TEST(sim->operator_memory_usage().matched_scratch_bytes == op_memory.matched_scratch_bytes); + BOOST_TEST(sim->operator_memory_usage().inverted_index_bytes == op_memory.inverted_index_bytes); + BOOST_TEST(sim->graph_memory_usage().total_bytes() == graph_memory.total_bytes()); + BOOST_TEST(sim->graph_memory_usage().cross_rank_bytes == graph_memory.cross_rank_bytes); + BOOST_TEST(sim->graph_size().second == graph_size.second); + // Shared metadata is read once: layers and gates are not multiplied by T. + BOOST_TEST(sim->graph_layers() == lih().majoranas.size()); + BOOST_TEST(sim->n_gates() == lih().majoranas.size()); + BOOST_TEST(sim->parameter_mapping().size() == lih().majoranas.size()); +} + +BOOST_AUTO_TEST_CASE(sharded_root_owners_do_the_work_of_every_operation) { + const size_t team = launch_team(); + if (team == 1) { + return; // needs a nonprimary worker + } + Recorder recorder; + auto sim = make({}, &recorder); + auto visits = recorder.visits(); + check_owner_visits(visits, team, "seeding"); + for (size_t t = 0; t < team; ++t) { + BOOST_TEST(count(visits, Seam::shard, 2 * static_cast(sharded::ShardWork::seed), t) == 1U); + BOOST_TEST(count(visits, Seam::shard, 2 * static_cast(sharded::ShardWork::seed) + 1, t) == 1U); + } + + const size_t gates = lih().majoranas.size(); + const auto expect = [&](const char *what, Seam seam, int work, size_t per_shard) { + visits = recorder.visits(); + check_owner_visits(visits, team, what); + for (size_t t = 0; t < team; ++t) { + BOOST_TEST_CONTEXT(what << " shard " << t) { + BOOST_TEST(count(visits, seam, work, t) == per_shard); + } + } + recorder.clear(); + }; + recorder.clear(); + build(*sim); + expect("build_graph", Seam::construction, static_cast(sharded::ConstructionWork::traverse), gates); + + const size_t layers = sim->graph_layers(); + static_cast(sim->expectation_value_and_gradient(params())); + expect("gradient: retained preparation", Seam::evaluation, static_cast(sharded::EvaluationWork::retain), 1); + static_cast(sim->expectation_value_and_gradient(params())); + expect("gradient: reverse derivatives", + Seam::evaluation, + static_cast(sharded::EvaluationWork::reverse_finish), + layers); + static_cast(sim->expectation_value(params())); + expect("energy: forward publication", Seam::evaluation, static_cast(sharded::EvaluationWork::publish), layers); + const auto functional = sim->expectation_value_and_gradient_functional(1e-3); + recorder.clear(); + static_cast(functional(params())); + expect("pared functional: reverse accumulation", + Seam::evaluation, + static_cast(sharded::EvaluationWork::reverse_accumulate), + layers); + static_cast(sim->contract_partially(params(), false)); + expect("contraction: replay finish", Seam::evaluation, static_cast(sharded::EvaluationWork::finish), layers); + + auto informed = make({.lower_atol = 1e-4}, &recorder); + recorder.clear(); + informed->build_graph(lih().majoranas, lih().param_inds, lih().gen_coeffs, std::nullopt, params()); + expect("informed build: new-layer replay", + Seam::construction, + static_cast(sharded::ConstructionWork::replay), + 2 * gates); + // Extending a built graph with coefficients replays the existing layers as the seed, in the same team. + VecD doubled = params(); + doubled.insert(doubled.end(), params().begin(), params().end()); + VecZ shifted = lih().param_inds; + for (auto &p : shifted) { + p += params().size(); + } + informed->build_graph(lih().majoranas, shifted, lih().gen_coeffs, std::nullopt, doubled); + visits = recorder.visits(); + check_owner_visits(visits, team, "informed seed replay"); + for (size_t t = 0; t < team; ++t) { + BOOST_TEST(count(visits, Seam::construction, static_cast(sharded::ConstructionWork::seed), t) + == 1 + (3 * gates)); // the seed phase, then publish, cosine and finish per existing layer + } + recorder.clear(); + + auto direct = make({}, &recorder); + recorder.clear(); + direct->propagate(lih().majoranas, lih().param_inds, lih().gen_coeffs, params()); + expect("propagate: fused apply", Seam::construction, static_cast(sharded::ConstructionWork::apply), gates); + + sim->update_initial_operator(OperatorDict{{lih().hamiltonian.begin()->first, lih().hamiltonian.begin()->second}}); + expect("initial-operator update", Seam::root, static_cast(sharded::RootWork::initial_operator), 1); + sim->set_parameter_mapping(VecZ(layers, 0)); + expect("remap", Seam::root, static_cast(sharded::RootWork::remap), 1); + + const MP copy(*sim); + expect("copy", Seam::shard, 2 * static_cast(sharded::ShardWork::copy), 1); +} + +BOOST_AUTO_TEST_CASE(sharded_root_opaque_cutoff_traverses_on_the_primary) { + const size_t team = launch_team(); + if (team == 1) { + return; + } + // A basis-change cutoff is an opaque closure: never called from two threads at once. + std::vector identity_basis; + for (size_t i = 0; i < 2 * kN; ++i) { + identity_basis.push_back(VecZ{i}); + } + Recorder recorder; + auto sim = make({.basis_change = identity_basis}, &recorder); + recorder.clear(); + build(*sim); + for (const auto &v : recorder.visits()) { + if (v.seam == Seam::construction && v.work == static_cast(sharded::ConstructionWork::traverse)) { + BOOST_TEST(v.worker == 0); + } + else { + BOOST_TEST(static_cast(v.worker) == v.shard); + } + } + auto reference = make(); + build(*reference); + BOOST_TEST(energy_of(*sim) == energy_of(*reference)); +} + +BOOST_AUTO_TEST_CASE(sharded_root_export_pairs_each_shard_with_its_block) { + for (const bool schrodinger : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger) { + auto sim = make({.schrodinger = schrodinger}); + build(*sim); + const auto contracted = sim->contract_partially(params(), false); + const auto terms = sim->evolved_operator_terms(params(), 0.0); + const auto &shards = Access::shards(*sim); + // Blocks are concatenated in shard order, each indexed by its own shard's rows. + std::vector>> expected; + size_t offset = 0; + for (const auto &state : shards) { + state->op.store->for_each([&](const auto &mono, size_t idx) { + const auto decoded = algebra_decode_coeff(Basis::Majorana, contracted[offset + idx], mono); + expected.emplace_back(bitset_to_indices(mono), + std::complex(std::round(decoded.real() * 1e12) / 1e12, + std::round(decoded.imag() * 1e12) / 1e12)); + }); + offset += state->size(); + } + BOOST_TEST(offset == contracted.size()); + BOOST_TEST(terms.size() == expected.size()); + BOOST_TEST((terms == expected)); + // atol = 0 keeps every stored row, zeros included, and no key twice. + std::set keys; + for (const auto &[key, coeff] : terms) { + BOOST_TEST(keys.insert(key).second); + } + BOOST_TEST(keys.size() == sim->size()); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_root_contraction_updates_the_right_picture) { + for (const bool schrodinger : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger) { + auto sim = make({.schrodinger = schrodinger}); + build(*sim); + const auto &shards = Access::shards(*sim); + const auto picture = [&](size_t t) -> const VecD & { + return schrodinger ? shards[t]->op.state_coeffs : shards[t]->op.op_coeffs; + }; + std::vector before; + for (size_t t = 0; t < shards.size(); ++t) { + before.push_back(picture(t)); + } + const auto layers = sim->graph_layers(); + const auto evolved = sim->contract_partially(params(), false); + // Non-inplace consumes nothing. + BOOST_TEST(sim->graph_layers() == layers); + for (size_t t = 0; t < shards.size(); ++t) { + BOOST_TEST((picture(t) == before[t])); + } + const auto consumed = sim->contract_partially(params(), true); + BOOST_TEST((consumed == evolved)); + BOOST_TEST(sim->graph_layers() == 0U); + size_t offset = 0; + for (size_t t = 0; t < shards.size(); ++t) { + BOOST_TEST(shards[t]->graph.layers() == 0U); + const VecD block(evolved.begin() + static_cast(offset), + evolved.begin() + static_cast(offset + picture(t).size())); + BOOST_TEST((picture(t) == block)); + offset += picture(t).size(); + } + // A further build starts from the contracted picture. + build(*sim); + BOOST_TEST(std::isfinite(energy_of(*sim))); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_root_initial_operator_updates_route_to_owners) { + auto sim = make(); + build(*sim); + const auto epoch = Access::epoch(*sim); + OperatorDict scaled; + for (const auto &[key, value] : lih().hamiltonian) { + scaled[key] = value * 2.0; + } + const auto stale = sim->expectation_value_functional(); + sim->update_initial_operator(scaled); + BOOST_TEST(Access::epoch(*sim) == epoch + 1); // once, not once per shard + BOOST_CHECK_THROW(stale(params()), std::exception); + BOOST_TEST(!Access::is_invalid(*sim)); + const auto &router = Access::router(*sim); + const auto &shards = Access::shards(*sim); + for (size_t t = 0; t < shards.size(); ++t) { + for (const auto &[mono, coeff] : shards[t]->op.init_op_map) { + BOOST_TEST(router.dest(mono) == t); + } + } + auto reference = make(); + build(*reference); + // Linear in the operator: every coefficient doubled, the identity included. + BOOST_TEST(energy_of(*sim) == 2.0 * energy_of(*reference), boost::test_tools::tolerance(1e-12)); +} + +BOOST_AUTO_TEST_CASE(sharded_root_remap_replaces_immutable_cores) { + auto sim = make(); + build(*sim); + const MP before(*sim); + const auto &shards = Access::shards(*sim); + const auto &old_shards = Access::shards(before); + // The copy shares the source's immutable cores. + BOOST_TEST(&shards.front()->graph.get_layer(0).core() == &old_shards.front()->graph.get_layer(0).core()); + const size_t layers = sim->graph_layers(); + sim->set_parameter_mapping(VecZ(layers, 0)); + for (size_t t = 0; t < shards.size(); ++t) { + for (size_t l = 0; l < layers; ++l) { + BOOST_TEST(&shards[t]->graph.get_layer(l).core() != &old_shards[t]->graph.get_layer(l).core()); + BOOST_TEST(shards[t]->graph.get_layer_traversal(l).param_index() == 0U); + } + } + BOOST_TEST((before.parameter_mapping() != sim->parameter_mapping())); +} + +BOOST_AUTO_TEST_CASE(sharded_root_copies_are_owner_allocated_and_independent) { + auto sim = make(); + build(*sim); + const double energy = energy_of(*sim); + auto copy = std::make_unique(*sim); + const auto clone = Access::clone(*sim); + BOOST_TEST(Access::options(*copy).threads == Access::options(*sim).threads); + for (size_t t = 0; t < Access::shards(*sim).size(); ++t) { + BOOST_TEST(Access::shards(*copy)[t].get() != Access::shards(*sim)[t].get()); + BOOST_TEST(Access::shards(*copy)[t]->op.store.get() != Access::shards(*sim)[t]->op.store.get()); + } + copy->update_lower_atol(1e-3); + build(*copy); + BOOST_TEST(energy_of(*sim) == energy); + BOOST_TEST(clone->expectation_value(params()) == energy); + copy.reset(); + BOOST_TEST(energy_of(*sim) == energy); +} + +namespace { + +// An ordinary extension: the virtual clone and initial-operator hooks, without the obsolete partition factory. +class Derived final : public MP { +public: + using MP::MP; + Derived(const Derived &) = default; + + auto update_initial_operator(const OperatorDict &op_dict) -> void override { + ++calls; + std::tie(last_terms, last_coeffs) = apply_initial_operator_(op_dict); + } + + size_t calls = 0; + MonomialList last_terms; + VecD last_coeffs; + +protected: + auto clone_() const -> std::unique_ptr override { return std::make_unique(*this); } +}; + +} // namespace + +BOOST_AUTO_TEST_CASE(sharded_root_virtual_hooks_keep_working) { + const auto &data = lih(); + Derived sim(data.hamiltonian, 4U, data.initial_state, std::nullopt, MPI_COMM_SELF); + build(sim); + const double energy = sim.expectation_value(params()); + // clone_() keeps the derived type, with its own shards and the source's T. + const auto clone = Access::clone(sim); + BOOST_TEST(dynamic_cast(clone.get()) != nullptr); + BOOST_TEST(Access::options(*clone).threads == Access::options(sim).threads); + BOOST_TEST(Access::shards(*clone).size() == launch_team()); + for (size_t t = 0; t < Access::shards(sim).size(); ++t) { + BOOST_TEST(Access::shards(*clone)[t]->op.store.get() != Access::shards(sim)[t]->op.store.get()); + } + BOOST_TEST(clone->expectation_value(params()) == energy); + Derived copy(sim); + + // Through the base interface: the override runs, and the protected hook returns this rank's routed share, every + // shard's block in shard order (the identity is rank-level metadata and is not among the terms). + OperatorDict scaled; + for (const auto &[key, value] : data.hamiltonian) { + scaled[key] = value * 0.5; + } + MP &base = sim; + base.update_initial_operator(scaled); + BOOST_TEST(sim.calls == 1U); + const size_t routed = data.hamiltonian.size() - (data.hamiltonian.contains(VecZ{}) ? 1 : 0); + BOOST_TEST(sim.last_terms.size() == routed); + BOOST_TEST(sim.last_coeffs.size() == routed); + // The returned share is each key once, routed to one of this rank's shards, with its encoded new coefficient. + const auto &router = Access::router(sim); + std::set returned; + for (size_t i = 0; i < sim.last_terms.size(); ++i) { + const auto &mono = sim.last_terms[i]; + const auto key = bitset_to_indices(mono); + BOOST_TEST(returned.insert(key).second); + BOOST_TEST(router.dest(mono) < launch_team()); + BOOST_TEST_REQUIRE(scaled.contains(key)); + BOOST_TEST(sim.last_coeffs[i] == algebra_encode_coeff(Basis::Majorana, scaled.at(key), mono)); + } + // The update reached the source only: the copy and the clone answer as before. + const double updated = sim.expectation_value(params()); + BOOST_TEST(updated != energy); + BOOST_TEST(clone->expectation_value(params()) == energy); + BOOST_TEST(copy.expectation_value(params()) == energy); + BOOST_TEST(copy.calls == 0U); + // And the other way round: updating the copy leaves the source alone. + copy.update_initial_operator(data.hamiltonian); + BOOST_TEST(copy.calls == 1U); + BOOST_TEST(copy.expectation_value(params()) == energy); + BOOST_TEST(sim.expectation_value(params()) == updated); +} + +BOOST_AUTO_TEST_CASE(sharded_root_rejections_before_mutation_leave_it_usable) { + auto sim = make(); + build(*sim); + const double energy = energy_of(*sim); + const auto &data = lih(); + BOOST_CHECK_THROW(sim->build_graph(data.majoranas, VecZ{0}, data.gen_coeffs), std::exception); + BOOST_CHECK_THROW(sim->build_graph({VecZ{0, 2 * kN}}, VecZ{0}, VecD{1.0}), std::exception); + BOOST_CHECK_THROW(sim->propagate(data.majoranas, data.param_inds, data.gen_coeffs, params()), std::exception); + BOOST_CHECK_THROW(sim->expectation_value(VecD{0.1}), std::exception); + BOOST_CHECK_THROW(sim->contract_partially(VecD{}, false), std::exception); + BOOST_CHECK_THROW(sim->update_initial_operator(OperatorDict{{VecZ{0, 2 * kN}, 1.0}}), std::exception); + BOOST_CHECK_THROW(sim->set_parameter_mapping(VecZ{0}), std::exception); + BOOST_CHECK_THROW(sim->update_basis_change(std::vector{VecZ{0}}), std::exception); + BOOST_CHECK_THROW(sim->build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, VecD{0.1}), + std::exception); + BOOST_TEST(!Access::is_invalid(*sim)); + BOOST_TEST(energy_of(*sim) == energy); + BOOST_TEST(sim->graph_layers() == data.majoranas.size()); +} + +namespace { + +// Arms one seam visit on the last shard (a nonprimary owner at T > 1), runs `op`, and checks the original error +// arrives, the object is invalid with every dependent operation refused, and an earlier copy keeps working. +template +auto check_post_mutation_failure(const char *what, Recorder::Arm target, Prepare &&prepare, Op &&op) -> void { + BOOST_TEST_CONTEXT(what) { + Recorder recorder; + auto sim = make({}, &recorder); + prepare(*sim); + MP earlier(*sim); + Access::observe(earlier, nullptr); + const VecD at = earlier.graph_layers() > 0 ? params() : VecD{}; + const double earlier_energy = earlier.expectation_value(at); + recorder.arm(target); + BOOST_CHECK_THROW(op(*sim), Injected); + BOOST_TEST(recorder.fired()); + recorder.disarm(); + BOOST_TEST(Access::is_invalid(*sim)); + BOOST_CHECK_THROW(sim->size(), InvalidPropagatorError); + BOOST_CHECK_THROW(energy_of(*sim), InvalidPropagatorError); + BOOST_CHECK_THROW(build(*sim), InvalidPropagatorError); + BOOST_CHECK_THROW(MP{*sim}, InvalidPropagatorError); + BOOST_TEST(!Access::is_invalid(earlier)); + BOOST_TEST(earlier.expectation_value(at) == earlier_energy); + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sharded_root_failures_after_mutation_invalidate_the_root) { + const size_t last = launch_team() - 1; + const auto none = [](MP &) {}; + const auto built = [](MP &sim) { build(sim); }; + using CW = sharded::ConstructionWork; + using EW = sharded::EvaluationWork; + using RW = sharded::RootWork; + check_post_mutation_failure("construction traversal", + {Seam::construction, static_cast(CW::traverse), last, 1}, + none, + [](MP &sim) { build(sim); }); + check_post_mutation_failure("construction publication", + {Seam::construction, static_cast(CW::finalize), last, 2}, + none, + [](MP &sim) { build(sim); }); + check_post_mutation_failure( + "propagation apply", + {Seam::construction, static_cast(CW::apply), last, 0}, + none, + [](MP &sim) { sim.propagate(lih().majoranas, lih().param_inds, lih().gen_coeffs, params()); }); + check_post_mutation_failure("informed seed replay", + {Seam::construction, static_cast(CW::seed), last, 1}, + built, + [](MP &sim) { + VecD doubled = params(); + doubled.insert(doubled.end(), params().begin(), params().end()); + VecZ shifted = lih().param_inds; + for (auto &p : shifted) { + p += params().size(); + } + sim.build_graph(lih().majoranas, shifted, lih().gen_coeffs, std::nullopt, doubled); + }); + check_post_mutation_failure("retained preparation", + {Seam::evaluation, static_cast(EW::retain), last, std::nullopt}, + built, + [](MP &sim) { static_cast(sim.expectation_value_functional(1e-3)); }); + check_post_mutation_failure("forward replay of a direct energy", + {Seam::evaluation, static_cast(EW::finish), last, 1}, + built, + [](MP &sim) { static_cast(sim.expectation_value(params())); }); + check_post_mutation_failure("reverse derivative of a direct gradient", + {Seam::evaluation, static_cast(EW::reverse_finish), last, std::nullopt}, + built, + [](MP &sim) { static_cast(sim.expectation_value_and_gradient(params())); }); + check_post_mutation_failure("result combination after the team", + {Seam::root, static_cast(RW::combine), 0, std::nullopt}, + built, + [](MP &sim) { static_cast(sim.expectation_value_and_gradient(params())); }); + check_post_mutation_failure("partial-contraction replay", + {Seam::evaluation, static_cast(EW::finish), last, std::nullopt}, + built, + [](MP &sim) { static_cast(sim.contract_partially(params(), true)); }); + check_post_mutation_failure("decoded export", + {Seam::root, static_cast(RW::export_block), last, std::nullopt}, + built, + [](MP &sim) { static_cast(sim.evolved_operator_terms(params(), 0.0)); }); + check_post_mutation_failure("initial-operator owner phase", + {Seam::root, static_cast(RW::initial_operator), last, std::nullopt}, + built, + [](MP &sim) { sim.update_initial_operator(lih().hamiltonian); }); + check_post_mutation_failure("remap owner phase", + {Seam::root, static_cast(RW::remap), last, std::nullopt}, + built, + [](MP &sim) { sim.set_parameter_mapping(VecZ(sim.graph_layers(), 0)); }); +} + +BOOST_AUTO_TEST_CASE(sharded_root_functional_failures_invalidate_their_owner) { + const size_t last = launch_team() - 1; + Recorder recorder; + auto sim = make({}, &recorder); + build(*sim); + const auto functional = sim->expectation_value_and_gradient_functional(); + const auto value = functional(params()); + // Argument errors are checked before the team: the owner stays usable. + BOOST_CHECK_THROW(functional(VecD{0.1}), std::exception); + BOOST_TEST(!Access::is_invalid(*sim)); + recorder.arm({Seam::evaluation, static_cast(sharded::EvaluationWork::reverse_accumulate), last, std::nullopt}); + BOOST_CHECK_THROW(functional(params()), Injected); + BOOST_TEST(Access::is_invalid(*sim)); + BOOST_CHECK_THROW(functional(params()), InvalidPropagatorError); + static_cast(value); +} + +BOOST_AUTO_TEST_CASE(sharded_root_functional_epoch_and_graph_guards) { + auto sim = make(); + build(*sim); + const auto energy = sim->expectation_value_functional(); + const auto pared = sim->expectation_value_and_gradient_functional(1e-3); + const double before = energy(params()); + sim->update_initial_operator(lih().hamiltonian); + BOOST_CHECK_THROW(energy(params()), std::exception); + BOOST_CHECK_THROW(pared(params()), std::exception); + const auto fresh = sim->expectation_value_functional(); + BOOST_TEST(fresh(params()) == before); + // An unpared functional aliases the live graphs, so appending layers is detected. + build(*sim); + BOOST_CHECK_THROW(fresh(params()), std::exception); + BOOST_TEST(!Access::is_invalid(*sim)); +} + +BOOST_AUTO_TEST_CASE(sharded_root_copy_and_seed_failures_are_contained) { + const size_t last = launch_team() - 1; + Recorder recorder; + recorder.arm({Seam::shard, 2 * static_cast(sharded::ShardWork::seed), last, std::nullopt}); + BOOST_CHECK_THROW(make({}, &recorder), Injected); + recorder.disarm(); + auto sim = make({}, &recorder); + build(*sim); + const double energy = energy_of(*sim); + recorder.arm({Seam::shard, 2 * static_cast(sharded::ShardWork::copy) + 1, last, std::nullopt}); + BOOST_CHECK_THROW(MP{*sim}, Injected); + recorder.disarm(); + // A failed copy leaves its source intact and valid. + BOOST_TEST(!Access::is_invalid(*sim)); + BOOST_TEST(energy_of(*sim) == energy); +} + +BOOST_AUTO_TEST_CASE(sharded_root_allocation_failures_are_catchable) { + if (!allocation::available()) { + return; // the probe is compiled out under this sanitizer + } + // Every operator new on the calling thread (the caller and, inside teams, worker 0) fails in turn. Each failure + // must surface as std::bad_alloc: before mutation the object is unchanged, after it the object is invalid. + const auto data = load_case_data<8>("random_exact.msgpack"); + using Small = MonomialPropagator<8>; + using SmallAccess = monoprop::detail::PropagatorTestAccess<8>; + const auto make_small = [&] { + return Small(data.hamiltonian, 16, data.initial_state, std::nullopt, MPI_COMM_SELF); + }; + auto reference = make_small(); + reference.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + const double expected = reference.expectation_value(data.parameters); + size_t failures = 0; + for (size_t nth = 1; nth < 20000; ++nth) { + auto sim = make_small(); + allocation::arm_failure(nth); + bool threw = false; + try { + sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + static_cast(sim.expectation_value_and_gradient(data.parameters)); + } + catch (const std::bad_alloc &) { + threw = true; + } + const bool pending = allocation::disarm_failure(); + if (!threw) { + if (pending) { + break; // the armed allocation never happened: the sweep is complete + } + // A nothrow allocation (a standard algorithm's temporary buffer) failed and its caller fell back: the + // results must be unaffected. + BOOST_TEST_CONTEXT("nth=" << nth) { + BOOST_TEST(!SmallAccess::is_invalid(sim)); + BOOST_TEST(sim.expectation_value(data.parameters) == expected); + } + continue; + } + ++failures; + if (!SmallAccess::is_invalid(sim)) { + if (sim.graph_layers() == data.majoranas.size()) { + BOOST_TEST(sim.expectation_value(data.parameters) == expected); + } + else { + BOOST_TEST(sim.graph_layers() == 0U); + } + } + else { + BOOST_CHECK_THROW(sim.size(), InvalidPropagatorError); + } + } + BOOST_TEST(failures > 0U); +} + +// --- Multi-rank roots (P ranks x T threads; cpp/tests/CMakeLists.txt launches these under mpiexec) ------------------- + +namespace { + +auto world_size() -> size_t { + return static_cast(mpi::size(mpi::Comm(MPI_COMM_WORLD))); +} + +auto world_rank() -> size_t { + return static_cast(mpi::rank(mpi::Comm(MPI_COMM_WORLD))); +} + +// A linear router needs a power-of-two rank count; splitmix routes any. +auto routable_world() -> bool { + return !routing::linear_requested() || std::has_single_bit(world_size()); +} + +// Multi-rank cases run only in their dedicated launches, which name the world size (cpp/tests/CMakeLists.txt). +auto dedicated_launch() -> bool { + const char *text = std::getenv("monoprop_TEST_SHARDED_RANKS"); + return text != nullptr && std::stoul(text) == world_size(); +} + +auto has_routable_ranks(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + boost::test_tools::assertion_result result(world_size() >= 2 && dedicated_launch() && routable_world()); + result.message() << "the launch has " << world_size() << " rank(s)" + << (routing::linear_requested() ? " under linear routing" : "") + << "; this case needs at least 2 ranks the router accepts"; + return result; +} + +auto make_on(mpi::Comm comm, const Config &config = {}, const sharded::RootObserver *observer = nullptr) + -> std::unique_ptr { + const auto &data = lih(); + return Access::construct_observed(observer, + data.hamiltonian, + 4U, + data.initial_state, + config.schrodinger ? std::optional{6U} : std::nullopt, + comm, + config.lower_atol, + std::optional{}, + CutoffType::Length, + config.basis_change, + kN, + config.basis); +} + +// The plan's general tolerance for cross-geometry comparisons. +auto close(double a, double b) -> bool { + return std::abs(a - b) <= 1e-9 + (1e-7 * std::max(std::abs(a), std::abs(b))); +} + +auto key_hash(const VecZ &key) -> uint64_t { + uint64_t h = 0x9E37'79B9'7F4A'7C15ULL; + for (const size_t i : key) { + h = (h ^ i) * 0x100'0000'01B3ULL; + h ^= h >> 29; + } + return h; +} + +/* + * The rank-local terms of `world`, gathered implicitly: every local key must be in `reference` (one process's whole + * map) with a close value, and over all ranks the key count and the wrapping sum of key hashes must equal the + * reference's, so no key is missing or owned twice. + */ +auto check_global_terms(const std::vector>> &local, + const std::vector>> &reference, + const char *what) -> void { + BOOST_TEST_CONTEXT(what << " on rank " << world_rank() << " of " << world_size()) { + std::map> want(reference.begin(), reference.end()); + BOOST_TEST(want.size() == reference.size()); + size_t missing = 0; + size_t off = 0; + uint64_t hashes = 0; + for (const auto &[key, value] : local) { + hashes += key_hash(key); + const auto it = want.find(key); + if (it == want.end()) { + ++missing; + continue; + } + if (!close(value.real(), it->second.real()) || !close(value.imag(), it->second.imag())) { + ++off; + } + } + BOOST_TEST(missing == 0U); + BOOST_TEST(off == 0U); + uint64_t want_hashes = 0; + for (const auto &[key, value] : reference) { + want_hashes += key_hash(key); + } + const auto comm = mpi::Comm(MPI_COMM_WORLD); + BOOST_TEST(mpi::allreduce_sum(local.size(), comm) == reference.size()); + BOOST_TEST(mpi::allreduce_sum(hashes, comm) == want_hashes); + } +} + +auto check_close_vectors(const VecD &got, const VecD &want, const char *what) -> void { + BOOST_TEST_CONTEXT(what) { + BOOST_TEST_REQUIRE(got.size() == want.size()); + for (size_t i = 0; i < got.size(); ++i) { + BOOST_TEST(close(got[i], want[i]), "component " << i << ": " << got[i] << " vs " << want[i]); + } + } +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_rejects_an_unroutable_geometry) { + if (world_size() < 2 || !dedicated_launch()) { + BOOST_TEST_MESSAGE("not a dedicated multi-rank launch"); + return; + } + if (routable_world()) { + BOOST_CHECK_NO_THROW(make_on(mpi::Comm(MPI_COMM_WORLD))); + return; + } + // Every rank derives the same verdict from the agreed mode and the shared size, so all of them throw together. + BOOST_CHECK_THROW(make_on(mpi::Comm(MPI_COMM_WORLD)), routing::UnroutableGeometry); +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_owns_its_flat_owners, *boost::unit_test::precondition(has_routable_ranks)) { + const size_t team = launch_team(); + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + auto self = make_on(mpi::Comm(MPI_COMM_SELF)); + const auto &router = Access::router(*world); + BOOST_TEST(router.ranks() == world_size()); + BOOST_TEST(router.shards_per_rank() == team); + const auto &shards = Access::shards(*world); + BOOST_TEST_REQUIRE(shards.size() == team); + size_t misrouted = 0; + for (size_t t = 0; t < team; ++t) { + shards[t]->op.store->for_each( + [&](const auto &mono, size_t) { misrouted += router.dest(mono) == (world_rank() * team) + t ? 0 : 1; }); + } + BOOST_TEST(misrouted == 0U); + // The seeded rows tile the one-process operator, and so do the built rows. + BOOST_TEST(mpi::allreduce_sum(world->size(), mpi::Comm(MPI_COMM_WORLD)) == self->size()); + build(*world); + build(*self); + BOOST_TEST(mpi::allreduce_sum(world->size(), mpi::Comm(MPI_COMM_WORLD)) == self->size()); + BOOST_TEST(world->graph_layers() == self->graph_layers()); +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_operations_reuse_the_owners_rounds, + *boost::unit_test::precondition(has_routable_ranks)) { + using Kind = sharded::PhysicalRounds::Kind; + const auto &data = lih(); + const size_t team = launch_team(); + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + const sharded::PhysicalRounds *rounds = Access::rounds(*world); + BOOST_TEST_REQUIRE(rounds != nullptr); + BOOST_TEST(rounds->peek(Kind::queries).threads() == 0U); + // Graph construction creates the owner's query and answer rounds; evaluation, its replay round; later calls reuse + // them: the same objects, not rounds of their own. + build(*world); + BOOST_TEST(rounds->peek(Kind::queries).threads() == team); + BOOST_TEST(rounds->peek(Kind::graph_answers).threads() == team); + const sharded::PhysicalExchange *queries = &rounds->peek(Kind::queries); + static_cast(world->expectation_value(data.parameters)); + BOOST_TEST(rounds->peek(Kind::replay).threads() == team); + BOOST_TEST(rounds->peek(Kind::queries).live() == 0); + BOOST_TEST(&rounds->peek(Kind::queries) == queries); + // A copy never shares them. + auto copy = std::make_unique(*world); + BOOST_TEST(Access::rounds(*copy) != rounds); + BOOST_TEST(Access::rounds(*copy)->peek(Kind::queries).threads() == 0U); +} + +/* + * Construction's rounds keep no staging between operations: an evaluation reuses none of it, and holding it would add + * the largest construction round to every later operation's memory. The replay round keeps its staging, which every + * evaluation reuses. The rounds themselves stay the owner's, and a later construction gives the same results. + */ +BOOST_AUTO_TEST_CASE(sharded_root_multirank_construction_returns_its_staging, + *boost::unit_test::precondition(has_routable_ranks)) { + using Kind = sharded::PhysicalRounds::Kind; + const auto &data = lih(); + const size_t team = launch_team(); + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + auto self = make_on(mpi::Comm(MPI_COMM_SELF)); + const sharded::PhysicalRounds *rounds = Access::rounds(*world); + BOOST_TEST_REQUIRE(rounds != nullptr); + build(*world); + build(*self); + BOOST_TEST(rounds->peek(Kind::queries).threads() == team); + BOOST_TEST(rounds->peek(Kind::graph_answers).threads() == team); + BOOST_TEST(rounds->peek(Kind::queries).staging_bytes() == 0U); + BOOST_TEST(rounds->peek(Kind::graph_answers).staging_bytes() == 0U); + BOOST_TEST(close(world->expectation_value(data.parameters), self->expectation_value(data.parameters))); + BOOST_TEST(rounds->peek(Kind::replay).staging_bytes() > 0U); + // A second construction plans and maps its staging again. + build(*world); + build(*self); + BOOST_TEST(rounds->peek(Kind::queries).staging_bytes() == 0U); + BOOST_TEST(mpi::allreduce_sum(world->size(), mpi::Comm(MPI_COMM_WORLD)) == self->size()); + BOOST_TEST(close(world->expectation_value(data.parameters), self->expectation_value(data.parameters))); +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_matches_one_process, *boost::unit_test::precondition(has_routable_ranks)) { + const auto &data = lih(); + for (const Config config : {Config{}, + Config{.schrodinger = true}, + Config{.lower_atol = 1e-3}, + Config{.schrodinger = true, .lower_atol = 1e-4}}) { + BOOST_TEST_CONTEXT("schrodinger=" << config.schrodinger << " lower_atol=" << config.lower_atol.has_value()) { + auto world = make_on(mpi::Comm(MPI_COMM_WORLD), config); + auto self = make_on(mpi::Comm(MPI_COMM_SELF), config); + // Structural construction, or coefficient-informed when an atol is set. + if (config.lower_atol) { + world->build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, data.parameters); + self->build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, data.parameters); + } + else { + build(*world); + build(*self); + } + const auto &p = params(); + BOOST_TEST(close(world->expectation_value(p), self->expectation_value(p))); + const auto [wv, wg] = world->expectation_value_and_gradient(p); + const auto [sv, sg] = self->expectation_value_and_gradient(p); + BOOST_TEST(close(wv, sv)); + check_close_vectors(wg, sg, "gradient"); + const auto [pv, pg] = world->expectation_value_and_gradient_functional(1e-3)(p); + const auto [qv, qg] = self->expectation_value_and_gradient_functional(1e-3)(p); + BOOST_TEST(close(pv, qv)); + check_close_vectors(pg, qg, "pared gradient"); + check_global_terms(world->evolved_operator_terms(p, 0.0), self->evolved_operator_terms(p, 0.0), "export"); + // A copy evaluates alike and mutates independently. + auto copy = std::make_unique(*world); + BOOST_TEST(close(copy->expectation_value(p), wv)); + // Contract in place, then extend the contracted picture. + BOOST_TEST(mpi::allreduce_sum(world->contract_partially(p, true).size(), mpi::Comm(MPI_COMM_WORLD)) + == self->contract_partially(p, true).size()); + BOOST_TEST(close(world->expectation_value({}), self->expectation_value({}))); + build(*world); + build(*self); + BOOST_TEST(close(world->expectation_value(p), self->expectation_value(p))); + BOOST_TEST(close(copy->expectation_value(p), wv)); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_propagation_matches_one_process, + *boost::unit_test::precondition(has_routable_ranks)) { + const auto &data = lih(); + for (const bool schrodinger : {false, true}) { + BOOST_TEST_CONTEXT("schrodinger=" << schrodinger) { + auto world = make_on(mpi::Comm(MPI_COMM_WORLD), Config{.schrodinger = schrodinger}); + auto self = make_on(mpi::Comm(MPI_COMM_SELF), Config{.schrodinger = schrodinger}); + world->propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); + self->propagate(data.majoranas, data.param_inds, data.gen_coeffs, data.parameters); + BOOST_TEST(close(world->expectation_value({}), self->expectation_value({}))); + check_global_terms(world->evolved_operator_terms({}, 0.0), self->evolved_operator_terms({}, 0.0), "map"); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_updates_route_to_their_owners, + *boost::unit_test::precondition(has_routable_ranks)) { + const auto &data = lih(); + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + auto self = make_on(mpi::Comm(MPI_COMM_SELF)); + build(*world); + build(*self); + OperatorDict scaled; + for (const auto &[key, value] : data.hamiltonian) { + scaled[key] = value * 0.5; + } + world->update_initial_operator(scaled); + self->update_initial_operator(scaled); + BOOST_TEST(close(world->expectation_value(params()), self->expectation_value(params()))); +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_mpi_runs_on_the_primary_and_owners_on_their_workers, + *boost::unit_test::precondition(has_routable_ranks)) { + const size_t team = launch_team(); + Recorder recorder; + auto world = make_on(mpi::Comm(MPI_COMM_WORLD), Config{.lower_atol = 1e-3}, &recorder); + const auto &data = lih(); + world->build_graph(data.majoranas, data.param_inds, data.gen_coeffs, std::nullopt, data.parameters); + static_cast(world->expectation_value_and_gradient(params())); + const auto visits = recorder.visits(); + size_t exchanges = 0; + for (const auto &v : visits) { + if (v.seam == Seam::construction && v.work == static_cast(sharded::ConstructionWork::exchange)) { + ++exchanges; + // A physical round's MPI calls run on the team's primary, the caller's thread. + BOOST_TEST(v.worker == 0); + BOOST_TEST(v.thread == std::this_thread::get_id()); + } + } + BOOST_TEST(exchanges > 0U); + std::vector owners; + std::ranges::copy_if(visits, std::back_inserter(owners), [](const Visit &v) { + return !(v.seam == Seam::construction && v.work == static_cast(sharded::ConstructionWork::exchange)); + }); + check_owner_visits(owners, team, "multi-rank construction and evaluation"); +} + +// Raw access depends on T alone: at T = 1 every rank hands out its actual sole shard, whatever P; above it every rank +// rejects. The ordinary extension hooks work over P ranks, each rank applying and returning its own share only. +BOOST_AUTO_TEST_CASE(sharded_root_multirank_raw_accessors_and_hooks, + *boost::unit_test::precondition(has_routable_ranks)) { + const size_t team = launch_team(); + const auto &data = lih(); + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + build(*world); + const auto &shards = Access::shards(*world); + if (team == 1) { + BOOST_TEST(&world->mp_op() == &shards.front()->op); + BOOST_TEST(&std::as_const(*world).mp_op() == &shards.front()->op); + BOOST_TEST(&world->indexing() == shards.front()->op.store.get()); + BOOST_TEST(&world->graph() == &shards.front()->graph); + BOOST_TEST(world->graph_data().size() == world->graph_layers()); + BOOST_TEST(world->mp_op().size() == world->size()); + } + else { + BOOST_CHECK_THROW(world->mp_op(), MultiShardUnsupported); + BOOST_CHECK_THROW(world->indexing(), MultiShardUnsupported); + BOOST_CHECK_THROW(world->graph(), MultiShardUnsupported); + BOOST_CHECK_THROW(world->graph_data(), MultiShardUnsupported); + } + BOOST_TEST(!Access::is_invalid(*world)); + BOOST_TEST(std::isfinite(world->expectation_value(params()))); + + Derived derived(data.hamiltonian, 4U, data.initial_state, std::nullopt, MPI_COMM_WORLD); + build(derived); + const double energy = derived.expectation_value(params()); + const auto clone = Access::clone(derived); + BOOST_TEST(dynamic_cast(clone.get()) != nullptr); + MP &base = derived; + base.update_initial_operator(data.hamiltonian); + BOOST_TEST(derived.calls == 1U); + const auto &router = Access::router(derived); + size_t foreign = 0; + for (const auto &mono : derived.last_terms) { + const size_t owner = router.dest(mono); + foreign += owner / team == world_rank() ? 0 : 1; + } + BOOST_TEST(foreign == 0U); + const size_t routed = data.hamiltonian.size() - (data.hamiltonian.contains(VecZ{}) ? 1 : 0); + BOOST_TEST(mpi::allreduce_sum(derived.last_terms.size(), mpi::Comm(MPI_COMM_WORLD)) == routed); + BOOST_TEST(close(derived.expectation_value(params()), energy)); + BOOST_TEST(clone->expectation_value(params()) == energy); +} + +BOOST_AUTO_TEST_CASE(sharded_root_multirank_rejections_before_mutation_leave_it_usable, + *boost::unit_test::precondition(has_routable_ranks)) { + // Replicated arguments fail the same validation on every rank, before any team or collective. + auto world = make_on(mpi::Comm(MPI_COMM_WORLD)); + build(*world); + const double before = world->expectation_value(params()); + BOOST_CHECK_THROW(world->expectation_value(VecD{0.1}), std::exception); + BOOST_CHECK_THROW(world->propagate({VecZ{0, 1}}, VecZ{0}, VecD{1.0}, VecD{0.1}), GraphStateConflict); + BOOST_TEST(!Access::is_invalid(*world)); + BOOST_TEST(world->expectation_value(params()) == before); +} + +BOOST_AUTO_TEST_CASE(sharded_root_empty_operators_and_identity_generators) { + auto empty = Access::construct_observed(nullptr, + OperatorDict{}, + 4U, + VecZ{}, + std::optional{}, + mpi::Comm(MPI_COMM_SELF), + std::optional{}, + std::optional{}, + CutoffType::Length, + std::optional>{}, + kN, + Basis::Majorana); + BOOST_TEST(empty->size() == 0U); + empty->build_graph({VecZ{0, 1}, VecZ{}}, VecZ{0, 1}, VecD{1.0, 1.0}); + BOOST_TEST(empty->graph_layers() == 2U); + BOOST_TEST(empty->expectation_value(VecD{0.3, 0.2}) == 0.0); + const auto [value, gradient] = empty->expectation_value_and_gradient(VecD{0.3, 0.2}); + BOOST_TEST(value == 0.0); + BOOST_TEST(gradient.size() == 2U); + BOOST_TEST(empty->evolved_operator_terms(VecD{0.3, 0.2}, 0.0).empty()); + BOOST_TEST(empty->contract_partially(VecD{0.3, 0.2}, true).empty()); + + // The replicated identity is rank-level metadata: added once to every result, at every T, never exported by + // a shard. + auto core_only = Access::construct_observed(nullptr, + OperatorDict{{VecZ{}, std::complex{2.5, 0.0}}}, + 4U, + VecZ{}, + std::optional{}, + mpi::Comm(MPI_COMM_SELF), + std::optional{}, + std::optional{}, + CutoffType::Length, + std::optional>{}, + kN, + Basis::Majorana); + core_only->build_graph({VecZ{0, 1}}, VecZ{0}, VecD{1.0}); + BOOST_TEST(core_only->core_term() == 2.5); + BOOST_TEST(core_only->expectation_value(VecD{0.3}) == 2.5); + BOOST_TEST(core_only->expectation_value_and_gradient(VecD{0.3}).first == 2.5); + BOOST_TEST(core_only->expectation_value_functional(1e-3)(VecD{0.3}) == 2.5); + BOOST_TEST(core_only->evolved_operator_terms(VecD{0.3}, 0.0).empty()); +} diff --git a/cpp/tests/sharded_state_tests.cpp b/cpp/tests/sharded_state_tests.cpp new file mode 100644 index 000000000..f3c6cba23 --- /dev/null +++ b/cpp/tests/sharded_state_tests.cpp @@ -0,0 +1,1388 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// First on purpose: the header must compile on its own. +#include "monoprop/detail/sharded/State.h" + +/* + * Owner-initialized shard state (detail/sharded/State.h). The team size T is the captured launch budget; + * cpp/tests/CMakeLists.txt reruns these cases in fresh processes at T = 1, 2 and 4. Cases that need a nonprimary + * worker skip below T = 2. + * + * Oracles: an independent reference for seeding, derived in the test from the routing, enumeration and store + * primitives, and the rank-level root (the same seam, driven through the public constructor at the launch's T, + * constructed on the test thread outside any sharded team) wherever a built source or an aggregate reference is needed. + * + * Workers never call BOOST_TEST. Initializers and the library's test observer write per-owner slots; assertions run + * after the team has joined. AllocationProbe.h supplies per-thread allocation counts and real allocation failures at + * the n-th operator new of a chosen owner. Owner-thread observations show where construction executes, not physical + * memory placement. + */ + +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "AllocationProbe.h" +#include "PauliTestOracle.h" +#include "PropagatorTestAccess.h" +#include "TestUtilities.h" +#include "monoprop/MonomialPropagator.h" +#include "monoprop/algebra/Algebra.h" +#include "monoprop/detail/monomial_propagator/MonomialPropagatorCommon.h" +#include "monoprop/detail/mpi/Routing.h" +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" + +namespace { + +using namespace monoprop; +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; +namespace allocation = test_utils::allocation; +using namespace std::chrono_literals; + +constexpr size_t kN = 8; +using State = sharded::ShardState; +using Shards = sharded::Shards; +using Operator = monoprop::detail::MPOperator; +using Propagator = MonomialPropagator; +using Access = monoprop::detail::PropagatorTestAccess; +using Mono = Monomial; + +// Monomials have no ordering of their own; any strict order serves the key-set comparisons. +struct MonoLess { + auto operator()(const Mono &a, const Mono &b) const -> bool { + return std::lexicographical_compare(a.data(), + a.data() + Mono::num_words(), + b.data(), + b.data() + Mono::num_words()); + } +}; + +auto team_options() -> parallel::Options { + return parallel::capture_thread_budget(); +} + +auto team_size() -> size_t { + return static_cast(team_options().threads); +} + +auto has_nonprimary_worker(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto threads = parallel::capture_thread_budget().threads; + boost::test_tools::assertion_result result(threads >= 2); + result.message() << "the captured team budget is " << threads << "; this case needs a nonprimary worker"; + return result; +} + +auto has_allocation_probe(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + boost::test_tools::assertion_result result(allocation::available()); + result.message() << "this build's sanitizer runtime owns operator new/delete, so no allocation probe is linked"; + return result; +} + +// --- Fixtures ------------------------------------------------------------------------------------------------- + +struct Fixture { + std::string name; + OperatorDict op; + VecZ initial_state; + std::optional schrodinger_cutoff = std::nullopt; + Basis basis = Basis::Majorana; + CutoffType cutoff_type = CutoffType::Length; + unsigned int cutoff = 4; + size_t logical = kN; +}; + +// Stores the coefficient whose encoding is `value`, so every fixture is encodable in its basis. +auto add_term(OperatorDict &op, Basis basis, const VecZ &indices, double value) -> void { + op[indices] = algebra_decode_coeff(basis, std::complex(value, 0.0), indices_to_bitset(indices)); +} + +// Pairs, a ladder of quartics and a few six- and eight-slot terms over the first 2 * logical slots, plus identity. +// Under a length-4 cutoff the long terms exceed the packed inline width and spill to the overflow map. +auto majorana_operator(size_t logical) -> OperatorDict { + OperatorDict op; + const size_t slots = 2 * logical; + auto value = 0.25; + for (size_t i = 0; i < slots; ++i) { + for (size_t j = i + 1; j < slots; ++j) { + add_term(op, Basis::Majorana, {i, j}, value); + value += 0.125; + } + } + for (size_t i = 0; i + 3 < slots; ++i) { + add_term(op, Basis::Majorana, {i, i + 1, i + 2, i + 3}, -0.5 - (0.03125 * static_cast(i))); + } + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5}, 0.75); + add_term(op, Basis::Majorana, {1, 3, 5, 7, 9, 11}, -1.25); + add_term(op, Basis::Majorana, {0, 2, 4, 6, 8, 10}, 1.5); + add_term(op, Basis::Majorana, {0, 1, 2, 3, 4, 5, 6, 7}, 2.0); + add_term(op, Basis::Majorana, {}, 3.5); + return op; +} + +auto pauli_operator() -> OperatorDict { + OperatorDict op; + auto value = 0.5; + const auto add = [&](const std::string &pauli) { + add_term(op, Basis::Pauli, pauli_oracle::slots_of_string(pauli), value); + value += 0.25; + }; + for (size_t q = 0; q < kN; ++q) { + for (const char p : {'X', 'Y', 'Z'}) { + std::string s(kN, 'I'); + s[q] = p; + add(s); + } + } + for (size_t q = 0; q + 1 < kN; ++q) { + std::string s(kN, 'I'); + s[q] = 'Z'; + s[q + 1] = 'Z'; + add(s); + } + add("XYZXYIII"); + add("IYYYYYII"); + add("ZZZZZZZZ"); + add_term(op, Basis::Pauli, {}, -1.75); + return op; +} + +auto nontrivial_fixtures() -> std::vector { + const auto data = test_utils::load_case_data("random_exact.msgpack"); + return { + {.name = "majorana-heisenberg-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .schrodinger_cutoff = std::nullopt, + .logical = 6}, + // Odd: rounds up to three pairs, so the paired basis has sum C(6, 0..3) = 42 rows including identity. The + // unpaired Hamiltonian terms stay pending in init_op_map. + {.name = "majorana-schrodinger-logical-6", + .op = majorana_operator(6), + .initial_state = {0, 3}, + .schrodinger_cutoff = 5, + .logical = 6}, + {.name = "majorana-heisenberg-random-exact", .op = data.hamiltonian, .initial_state = data.initial_state}, + {.name = "majorana-schrodinger-random-exact", + .op = data.hamiltonian, + .initial_state = data.initial_state, + .schrodinger_cutoff = 6}, + {.name = "pauli-heisenberg", + .op = pauli_operator(), + .initial_state = {}, + .schrodinger_cutoff = std::nullopt, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 3}, + {.name = "pauli-schrodinger", + .op = pauli_operator(), + .initial_state = {}, + .schrodinger_cutoff = 5, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 3}, + }; +} + +auto degenerate_fixtures() -> std::vector { + OperatorDict identity; + add_term(identity, Basis::Majorana, {}, 2.0); + OperatorDict pauli_identity; + add_term(pauli_identity, Basis::Pauli, {}, -2.0); + return { + {.name = "majorana-heisenberg-empty", .op = {}, .initial_state = {0, 1}}, + {.name = "majorana-heisenberg-identity-only", .op = identity, .initial_state = {0, 1}}, + {.name = "majorana-schrodinger-identity-only", + .op = identity, + .initial_state = {0, 1}, + .schrodinger_cutoff = 3, + .logical = 4}, + {.name = "pauli-heisenberg-identity-only", + .op = pauli_identity, + .initial_state = {}, + .schrodinger_cutoff = std::nullopt, + .basis = Basis::Pauli, + .cutoff_type = CutoffType::Support, + .cutoff = 3}, + }; +} + +auto all_fixtures() -> std::vector { + auto fixtures = nontrivial_fixtures(); + for (auto &fixture : degenerate_fixtures()) { + fixtures.push_back(std::move(fixture)); + } + return fixtures; +} + +// The root at the launch's T over MPI_COMM_SELF: its shards are seeded by the seam under test. +auto root(const Fixture &f) -> Propagator { + return Propagator(f.op, + f.cutoff, + f.initial_state, + f.schrodinger_cutoff, + MPI_COMM_SELF, + std::nullopt, + std::nullopt, + f.cutoff_type, + std::nullopt, + f.logical, + f.basis); +} + +// The rank-level reference for aggregates: the root, which must run at the launch's T. +auto oracle_of(const Fixture &f, size_t threads) -> Propagator { + auto sim = root(f); + BOOST_TEST_REQUIRE(Access::shards(sim).size() == threads); + return sim; +} + +// Caller-side preparation, as the rank-level constructor performs it. +auto make_seed(const Fixture &f, routing::Router router) -> sharded::OperatorSeed { + const auto cutoff_fn = monoprop::detail::cutoff_function(f.cutoff_type, f.cutoff, f.logical); + std::optional paired; + if (f.schrodinger_cutoff) { + paired = sharded::paired_basis_bounds(*f.schrodinger_cutoff, f.logical, router.flat_world()); + } + return {.initial_operator = f.op, + .initial_state = f.initial_state, + .router = router, + .basis = f.basis, + .logical_num_modes = f.logical, + .paired = paired, + .inline_width = sharded::packed_inline_width(f.schrodinger_cutoff.has_value(), cutoff_fn)}; +} + +auto local_router(size_t threads) -> routing::Router { + return routing::make_router(1, threads); +} + +// --- Owner observation ---------------------------------------------------------------------------------------- + +// What one owner saw inside the library's own seed or copy body, written only by that owner. +struct alignas(64) OwnerSlot { + std::atomic begins{0}; + std::atomic ends{0}; + int worker = -1; + int team = 0; + std::thread::id thread; + sharded::ShardWork work = sharded::ShardWork::seed; + size_t bytes_at_begin = 0; + size_t bytes = 0; //!< Allocated on this thread between begin() and end(). + bool armed = false; + bool unfired = false; //!< An armed failure was still pending at end(). +}; + +// Test observer for seed_shards/copy_shards. Optionally arms a real allocation failure on one owner. +struct OwnerProbe { + std::vector *slots = nullptr; + std::optional fail_shard = std::nullopt; + size_t fail_nth = 0; + + auto begin(sharded::ShardWork work, size_t shard) const -> void { + auto &slot = slots->at(shard); + slot.begins.fetch_add(1); + slot.worker = omp_get_thread_num(); + slot.team = omp_get_num_threads(); + slot.thread = std::this_thread::get_id(); + slot.work = work; + slot.bytes_at_begin = allocation::thread_bytes(); + if (fail_shard == shard) { + slot.armed = true; + allocation::arm_failure(fail_nth); + } + } + + auto end(sharded::ShardWork /*work*/, size_t shard) const -> void { + auto &slot = slots->at(shard); + slot.bytes = allocation::thread_bytes() - slot.bytes_at_begin; + if (slot.armed) { + slot.unfired = allocation::disarm_failure(); + } + slot.ends.fetch_add(1); + } +}; + +// Every owner ran exactly one begin/end of `work` on its own worker, one distinct thread each, worker 0 the caller. +auto check_owner_bodies(const std::vector &slots, sharded::ShardWork work, const std::string &what) -> void { + const auto threads = slots.size(); + std::set distinct; + for (size_t shard = 0; shard < threads; ++shard) { + const auto &slot = slots[shard]; + BOOST_TEST_CONTEXT(what << " shard " << shard) { + BOOST_TEST(slot.begins.load() == 1); + BOOST_TEST(slot.ends.load() == 1); + BOOST_TEST(slot.worker == static_cast(shard)); + BOOST_TEST(slot.team == static_cast(threads)); + BOOST_TEST((slot.work == work)); + if (allocation::available()) { + BOOST_TEST(slot.bytes > 0U, "the owner body allocated its own state"); + } + } + distinct.insert(slot.thread); + } + BOOST_TEST(distinct.size() == threads); + BOOST_TEST((slots.at(0).thread == std::this_thread::get_id())); +} + +// --- State comparison --------------------------------------------------------------------------------------- + +auto describe(const Mono &mono) -> std::string { + std::string out = "{"; + for (size_t b = mono.find_first(); b < mono.size(); b = mono.find_next(b)) { + out += std::format("{} ", b); + } + return out + "}"; +} + +// First difference between two operators, or empty. Rows, keys, lookups, overflow placement, coefficients, pending +// entries, sparse/dense state, the inverted index and every memory-accounting field (so capacities and reserves too). +auto operator_difference(const Operator &a, const Operator &b) -> std::string { + if (a.size() != b.size()) { + return std::format("size {} != {}", a.size(), b.size()); + } + if (a.store->overflow_size() != b.store->overflow_size()) { + return std::format("overflow rows {} != {}", a.store->overflow_size(), b.store->overflow_size()); + } + for (size_t i = 0; i < a.size(); ++i) { + const auto row_a = a.store->row(i); + if (row_a != b.store->row(i)) { + return std::format("row {}: {} != {}", i, describe(row_a), describe(b.store->row(i))); + } + if (a.store->row_positions(i).inlined() != b.store->row_positions(i).inlined()) { + return std::format("row {} inline placement differs", i); + } + if (a.store->find(row_a) != std::optional(i) || b.store->find(row_a) != std::optional(i)) { + return std::format("row {} lookup does not return its own index", i); + } + } + if (a.op_coeffs != b.op_coeffs) { + return "op_coeffs differ"; + } + if (a.state_rows_ != b.state_rows_ || a.state_vals_ != b.state_vals_ + || a.state_scored_rows_ != b.state_scored_rows_) { + return "sparse state differs"; + } + if (a.state_coeffs != b.state_coeffs) { + return "dense state differs"; + } + if (a.init_op_map.size() != b.init_op_map.size()) { + return std::format("pending entries {} != {}", a.init_op_map.size(), b.init_op_map.size()); + } + for (const auto &[key, value] : a.init_op_map) { + const auto found = b.init_op_map.find(key); + if (found == b.init_op_map.end() || found->second != value) { + return std::format("pending entry {} differs", describe(key)); + } + } + if (a.initial_state != b.initial_state || a.basis != b.basis) { + return "initial state or basis differs"; + } + if (a.inverted_index_.has_value() != b.inverted_index_.has_value()) { + return "inverted index presence differs"; + } + if (a.inverted_index_) { + const auto &ia = *a.inverted_index_; + const auto &ib = *b.inverted_index_; + if (ia.row_count != ib.row_count || ia.row_parity_ != ib.row_parity_) { + return "inverted index rows differ"; + } + for (size_t c = 0; c < ia.cols.size(); ++c) { + if (ia.cols[c].is_dense != ib.cols[c].is_dense || ia.cols[c].words != ib.cols[c].words + || ia.cols[c].set_rows != ib.cols[c].set_rows) { + return std::format("inverted index column {} differs", c); + } + } + } + const auto ma = monoprop::detail::estimate_memory_usage(a); + const auto mb = monoprop::detail::estimate_memory_usage(b); + const auto fields_a = std::vector{ma.operator_terms_bytes, + ma.op_coeffs_bytes, + ma.state_coeffs_bytes, + ma.indexing_bytes, + ma.init_operator_bytes, + ma.initial_state_bytes, + ma.inverted_index_bytes, + ma.operator_terms_slack_bytes, + ma.init_operator_entries, + ma.state_coeffs_nonzero}; + const auto fields_b = std::vector{mb.operator_terms_bytes, + mb.op_coeffs_bytes, + mb.state_coeffs_bytes, + mb.indexing_bytes, + mb.init_operator_bytes, + mb.initial_state_bytes, + mb.inverted_index_bytes, + mb.operator_terms_slack_bytes, + mb.init_operator_entries, + mb.state_coeffs_nonzero}; + for (size_t k = 0; k < fields_a.size(); ++k) { + if (fields_a[k] != fields_b[k]) { + return std::format("memory accounting field {}: {} != {}", k, fields_a[k], fields_b[k]); + } + } + return {}; +} + +// Graph layers compare by shared core identity: a copy shares its immutable cores with its source. +auto graph_difference(const MPGraph &a, const MPGraph &b) -> std::string { + if (a.is_schrodinger() != b.is_schrodinger() || a.layers() != b.layers()) { + return std::format("graph shape differs ({} vs {} layers)", a.layers(), b.layers()); + } + for (size_t i = 0; i < a.layers(); ++i) { + const auto &la = a.get_layer(i); + const auto &lb = b.get_layer(i); + if (la.shared_core() != lb.shared_core()) { + return std::format("layer {} core differs", i); + } + if ((la.pruned_cos() == nullptr) != (lb.pruned_cos() == nullptr)) { + return std::format("layer {} cosine form differs", i); + } + } + return {}; +} + +auto state_difference(const State &a, const State &b) -> std::string { + if (auto d = operator_difference(a.op, b.op); !d.empty()) { + return d; + } + if (auto d = graph_difference(a.graph, b.graph); !d.empty()) { + return d; + } + if (a.matched.epoch_ != b.matched.epoch_ || a.matched.cur_ != b.matched.cur_) { + return "matched scratch differs"; + } + return {}; +} + +auto shards_difference(const Shards &a, const Shards &b) -> std::string { + if (a.size() != b.size()) { + return std::format("shard count {} != {}", a.size(), b.size()); + } + for (size_t shard = 0; shard < a.size(); ++shard) { + if (!a[shard] || !b[shard]) { + return std::format("shard {} is null", shard); + } + if (auto d = state_difference(*a[shard], *b[shard]); !d.empty()) { + return std::format("shard {}: {}", shard, d); + } + } + return {}; +} + +// A caller-side deep snapshot through the plain copy constructor, as an independent reference. +auto snapshot(const Shards &shards) -> Shards { + Shards out; + for (const auto &state : shards) { + out.push_back(std::make_unique(*state)); + } + return out; +} + +// Shards bridged from a built root: a caller-side deep snapshot of its own T shards. +auto bridged_shards(const Propagator &p, size_t threads) -> Shards { + BOOST_TEST_REQUIRE(Access::shards(p).size() == threads); + return snapshot(Access::shards(p)); +} + +// A root at T with a built graph: nontrivial grown stores, layers, caches and +// matched scratch. The Hamiltonian's long terms exceed the inline width, so the stores have overflow rows too. +struct BuiltSource { + test_utils::CaseData data; + Fixture fixture; + Propagator propagator; +}; + +auto built_source(size_t threads) -> BuiltSource { + auto data = test_utils::load_case_data("random_exact.msgpack"); + auto fixture = Fixture{.name = "built", .op = majorana_operator(kN), .initial_state = data.initial_state}; + auto propagator = oracle_of(fixture, threads); + propagator.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); + return {std::move(data), std::move(fixture), std::move(propagator)}; +} + +class ScopedEnv { +public: + ScopedEnv(const char *name, const char *value) : name_(name) { + if (const char *old = std::getenv(name)) { + old_ = old; + } + setenv(name, value, 1); + } + ScopedEnv(const ScopedEnv &) = delete; + auto operator=(const ScopedEnv &) -> ScopedEnv & = delete; + ~ScopedEnv() { + if (old_) { + setenv(name_, old_->c_str(), 1); + } + else { + unsetenv(name_); + } + } + +private: + const char *name_; + std::optional old_; +}; + +struct WorkerError { + size_t shard; +}; + +class InjectedFailure : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +// An initializer body for make_shards that builds a small real state on the owner. +auto small_state(bool schrodinger) -> std::unique_ptr { + auto op = Operator{}; + op.store->push_back(indices_to_bitset({0, 1})); + op.store->emplace(indices_to_bitset({0, 1}), 0); + op.op_coeffs.assign(1, 1.0); + return std::make_unique(std::move(op), schrodinger); +} + +// --- Worker stack use ------------------------------------------------------------------------------------------- + +// Instrumented builds lay out frames differently (redzones, fake stacks), so the page count below means nothing there. +auto has_plain_stack_frames(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) + constexpr bool instrumented = true; +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) || __has_feature(thread_sanitizer) + constexpr bool instrumented = true; +#else + constexpr bool instrumented = false; +#endif +#else + constexpr bool instrumented = false; +#endif + boost::test_tools::assertion_result result(!instrumented); + result.message() << "a sanitizer build lays out stack frames differently"; + return result; +} + +// mincore's residency vector is `unsigned char *` on Linux, `char *` on macOS. +#ifdef __APPLE__ +using ResidencyByte = char; +#else +using ResidencyByte = unsigned char; +#endif + +/* + * The bytes of a fresh, never-touched thread stack that `body` makes resident, `body` run on that thread. The stack is + * an anonymous mapping without huge pages (macOS has no MADV_NOHUGEPAGE, nor transparent huge pages to opt out of), so + * residency is per page and counts every page a frame touched (GCC's stack-clash probes touch each page of a large + * frame). Pages the thread start itself touched are not counted. + */ +template +auto stack_bytes_touched(Body &&body) -> size_t { + constexpr size_t stack_bytes = size_t{8} << 20; + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + void *stack = + ::mmap(nullptr, stack_bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + BOOST_TEST_REQUIRE(stack != MAP_FAILED); +#ifdef MADV_NOHUGEPAGE + (void)::madvise(stack, stack_bytes, MADV_NOHUGEPAGE); +#endif + struct Run { + Body *body; + void *stack; + size_t bytes; + size_t page; + size_t before = 0; + size_t after = 0; + std::exception_ptr error{}; + } run{.body = &body, .stack = stack, .bytes = stack_bytes, .page = page}; + static constexpr auto trampoline = [](void *arg) -> void * { + auto &r = *static_cast(arg); + try { + const auto resident_pages = [&r] { + auto pages = std::vector(r.bytes / r.page); + if (::mincore(r.stack, r.bytes, pages.data()) != 0) { + throw std::runtime_error("mincore failed"); + } + return static_cast(std::ranges::count_if(pages, [](ResidencyByte p) { return (p & 1U) != 0; })); + }; + r.before = resident_pages(); + (*r.body)(); + r.after = resident_pages(); + } + catch (...) { + r.error = std::current_exception(); + } + return nullptr; + }; + pthread_attr_t attr; + ::pthread_attr_init(&attr); + ::pthread_attr_setstack(&attr, stack, stack_bytes); + pthread_t thread; + const int created = ::pthread_create(&thread, &attr, +trampoline, &run); + ::pthread_attr_destroy(&attr); + if (created == 0) { + ::pthread_join(thread, nullptr); + } + ::munmap(stack, stack_bytes); + BOOST_TEST_REQUIRE(created == 0); + if (run.error) { + std::rethrow_exception(run.error); + } + return (run.after - run.before) * page; +} + +} // namespace + +// --- Ownership and initialization ----------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(sharded_state_make_shards_initializes_each_owner_once) { + const auto options = team_options(); + const auto threads = team_size(); + auto slots = std::vector(threads); + auto shards = sharded::make_shards(options, [&](size_t shard) { + auto &slot = slots.at(shard); + slot.begins.fetch_add(1); + slot.worker = omp_get_thread_num(); + slot.team = omp_get_num_threads(); + slot.thread = std::this_thread::get_id(); + slot.bytes_at_begin = allocation::thread_bytes(); + auto state = small_state(false); + slot.bytes = allocation::thread_bytes() - slot.bytes_at_begin; + slot.ends.fetch_add(1); + return state; + }); + check_owner_bodies(slots, sharded::ShardWork::seed, "make_shards"); + + BOOST_REQUIRE(shards.size() == threads); + std::set stores; + std::vector addresses; + for (const auto &state : shards) { + BOOST_REQUIRE(state != nullptr); + stores.insert(state->op.store.get()); + addresses.push_back(state.get()); + BOOST_TEST(state->size() == 1U); + } + BOOST_TEST(stores.size() == threads); + // Moving the owning container moves pointer metadata only. + const auto moved = std::move(shards); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(moved[shard].get() == addresses[shard]); + } +} + +// An oracle independent of the extracted seed: rows, coefficients, pending +// entries, inline width and reservation derived here from the routing, enumeration and store primitives alone. +BOOST_AUTO_TEST_CASE(sharded_state_seed_matches_an_independent_reference) { + const auto options = team_options(); + const auto threads = team_size(); + for (const size_t ranks : {size_t{1}, size_t{2}}) { + const auto router = routing::make_router(ranks, threads); + for (const auto &fixture : all_fixtures()) { + BOOST_TEST_CONTEXT(fixture.name << " ranks " << ranks) { + const auto seed = make_seed(fixture, router); + // The width rule by hand: Schrödinger takes the default; a length cutoff bounds positions, a support + // cutoff bounds qubits (two slots each). + const size_t width = fixture.schrodinger_cutoff ? size_t{11} + : fixture.cutoff_type == CutoffType::Length + ? std::min(fixture.cutoff, 32) + : std::min(2 * fixture.cutoff, 32); + BOOST_TEST(seed.inline_width == width); + std::vector paired_basis; + size_t paired_reserve = 0; + if (fixture.schrodinger_cutoff) { + const auto effective = std::min(*fixture.schrodinger_cutoff, 2 * fixture.logical); + const auto pairs = (effective + 1) / 2; + paired_basis = generate_paired_op(pairs, fixture.logical); + paired_reserve = std::max(1, paired_basis.size() / (ranks * threads)); + } + const size_t rank = ranks - 1; + const auto shards = sharded::seed_shards(options, seed, rank); + for (size_t shard = 0; shard < threads; ++shard) { + const auto flat = (rank * threads) + shard; + const auto owned = [&](const Mono &mono) { return router.dest(mono) == flat; }; + std::vector rows; + std::map coefficients; + for (const auto &[indices, coefficient] : fixture.op) { + const auto mono = indices_to_bitset(indices); + if (indices.empty() || !owned(mono)) { + continue; + } + coefficients[mono] = algebra_encode_coeff(fixture.basis, coefficient, mono); + if (!fixture.schrodinger_cutoff) { + rows.push_back(mono); + } + } + for (const auto &mono : paired_basis) { + if (owned(mono)) { + rows.push_back(mono); + } + } + auto store = monoprop::detail::OperatorIndex(width); + store.reserve(fixture.schrodinger_cutoff ? paired_reserve : std::max(1, rows.size())); + for (size_t i = 0; i < rows.size(); ++i) { + store.push_back(rows[i]); + store.emplace(rows[i], i); + } + + const auto &op = shards[shard]->op; + BOOST_TEST_CONTEXT("shard " << shard) { + BOOST_REQUIRE(op.size() == rows.size()); + size_t wrong_rows = 0; + size_t wrong_coefficients = 0; + auto pending = coefficients; + for (size_t i = 0; i < rows.size(); ++i) { + wrong_rows += op.store->row(i) != rows[i] ? 1 : 0; + const auto found = pending.find(rows[i]); + const auto expected = found == pending.end() ? 0.0 : found->second; + wrong_coefficients += op.op_coeffs.at(i) != expected ? 1 : 0; + if (found != pending.end()) { + pending.erase(found); + } + } + BOOST_TEST(wrong_rows == 0U); + BOOST_TEST(wrong_coefficients == 0U); + BOOST_TEST(op.init_op_map.size() == pending.size()); + for (const auto &[key, value] : pending) { + BOOST_TEST(op.init_op_map.count(key) == 1U); + BOOST_TEST((op.init_op_map.count(key) == 1 && op.init_op_map.at(key) == value)); + } + BOOST_TEST(op.store->overflow_size() == store.overflow_size()); + BOOST_TEST(op.store->memory_bytes() == store.memory_bytes()); + BOOST_TEST(op.store->index_estimated_memory_bytes() == store.index_estimated_memory_bytes()); + BOOST_TEST(op.initial_state == fixture.initial_state); + } + } + } + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_state_routed_rows_belong_to_their_flat_owner) { + const auto options = team_options(); + const auto threads = team_size(); + struct Geometry { + size_t ranks; + bool linear; + }; + for (const auto geometry : {Geometry{1, true}, Geometry{2, true}, Geometry{4, true}, Geometry{3, false}}) { + const auto router = routing::Router::for_modes(geometry.ranks, threads, geometry.linear); + for (const auto &fixture : all_fixtures()) { + BOOST_TEST_CONTEXT(fixture.name << " ranks " << geometry.ranks << " linear " << geometry.linear) { + const auto seed = make_seed(fixture, router); + std::set expected; + if (seed.paired) { + for (const auto &mono : generate_paired_op(seed.paired->max_pairs, fixture.logical)) { + expected.insert(mono); + } + } + else { + for (const auto &[indices, coefficient] : fixture.op) { + if (!indices.empty()) { + expected.insert(indices_to_bitset(indices)); + } + } + } + std::map owners; + size_t rows = 0; + size_t misrouted = 0; + size_t identity_pending = 0; + size_t identity_coefficient_nonzero = 0; + for (size_t rank = 0; rank < geometry.ranks; ++rank) { + // One rank's shards at a time, as each rank of the launch would hold them. + const auto shards = sharded::seed_shards(options, seed, rank); + for (size_t shard = 0; shard < threads; ++shard) { + const auto flat = (rank * threads) + shard; + const auto &op = shards[shard]->op; + for (size_t i = 0; i < op.size(); ++i) { + const auto key = op.store->row(i); + misrouted += router.dest(key) != flat ? 1 : 0; + owners.emplace(key, flat); + ++rows; + if (key.none() && op.op_coeffs.at(i) != 0.0) { + ++identity_coefficient_nonzero; + } + } + identity_pending += op.init_op_map.count(Mono{}); + } + } + BOOST_TEST(misrouted == 0U); + BOOST_TEST(rows == owners.size(), "no key is held by two owners"); + BOOST_TEST(owners.size() == expected.size()); + BOOST_TEST(std::ranges::all_of(expected, [&](const Mono &key) { return owners.contains(key); })); + // The identity core is never seeded into a shard. Schrödinger's identity basis row is ordinary. + BOOST_TEST(identity_pending == 0U); + BOOST_TEST(identity_coefficient_nonzero == 0U); + BOOST_TEST(owners.contains(Mono{}) == seed.paired.has_value()); + if (seed.paired) { + BOOST_TEST(owners.at(Mono{}) == router.dest(Mono{})); + } + } + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_state_empty_owners_participate, *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const auto threads = team_size(); + OperatorDict op; + add_term(op, Basis::Majorana, {}, 1.0); + add_term(op, Basis::Majorana, {0, 1}, 0.5); + const auto fixture = Fixture{.name = "one-term", .op = op, .initial_state = {0, 1}}; + const auto seed = make_seed(fixture, local_router(threads)); + auto slots = std::vector(threads); + const auto shards = sharded::seed_shards(options, seed, 0, OwnerProbe{.slots = &slots}); + check_owner_bodies(slots, sharded::ShardWork::seed, "one-term"); + size_t empty = 0; + for (const auto &state : shards) { + BOOST_REQUIRE(state != nullptr); + empty += state->size() == 0 ? 1 : 0; + BOOST_TEST(state->op.inverted_index_.has_value()); + } + BOOST_TEST(empty == threads - 1); + BOOST_TEST(sharded::total_size(shards) == 1U); +} + +BOOST_AUTO_TEST_CASE(sharded_state_keeps_sparse_heisenberg_and_dense_schrodinger_caches) { + const auto options = team_options(); + const auto threads = team_size(); + size_t pending = 0; + for (const auto &fixture : nontrivial_fixtures()) { + BOOST_TEST_CONTEXT(fixture.name) { + const auto shards = sharded::seed_shards(options, make_seed(fixture, local_router(threads)), 0); + for (const auto &state : shards) { + const auto &op = state->op; + BOOST_TEST(op.state_scored_rows_ == op.size()); + BOOST_TEST(op.op_coeffs.size() == op.size()); + BOOST_TEST(op.inverted_index_.has_value()); + BOOST_TEST(op.inverted_index_->rows() == op.size()); + if (fixture.schrodinger_cutoff) { + BOOST_TEST(op.state_coeffs.size() == op.size(), "the live Schrödinger vector is dense"); + pending += op.init_op_map.size(); + } + else { + BOOST_TEST(op.state_coeffs.empty(), "Heisenberg keeps the sparse state only"); + BOOST_TEST(op.init_op_map.empty()); + } + } + } + } + // Unpaired Hamiltonian terms have no Schrödinger row yet and stay pending. + BOOST_TEST(pending > 0U); +} + +BOOST_AUTO_TEST_CASE(sharded_state_counts_and_accounting_sum_over_shards) { + const auto options = team_options(); + const auto threads = team_size(); + for (const auto &fixture : all_fixtures()) { + BOOST_TEST_CONTEXT(fixture.name) { + const auto oracle = oracle_of(fixture, threads); + const auto shards = sharded::seed_shards(options, make_seed(fixture, local_router(threads)), 0); + BOOST_TEST(sharded::total_size(shards) == oracle.size()); + const auto ours = sharded::operator_memory_usage(shards); + const auto theirs = oracle.operator_memory_usage(); + BOOST_TEST(ours.total_bytes() == theirs.total_bytes()); + BOOST_TEST(ours.operator_terms_bytes == theirs.operator_terms_bytes); + BOOST_TEST(ours.indexing_bytes == theirs.indexing_bytes); + BOOST_TEST(ours.inverted_index_bytes == theirs.inverted_index_bytes); + BOOST_TEST(ours.state_coeffs_bytes == theirs.state_coeffs_bytes); + BOOST_TEST(ours.init_operator_entries == theirs.init_operator_entries); + BOOST_TEST(ours.matched_scratch_bytes == theirs.matched_scratch_bytes); + BOOST_TEST(sharded::graph_memory_usage(shards).total_bytes() == oracle.graph_memory_usage().total_bytes()); + } + } + // A built state reports its matched scratch per shard. + auto source = built_source(threads); + const auto shards = bridged_shards(source.propagator, threads); + const auto ours = sharded::operator_memory_usage(shards); + const auto theirs = source.propagator.operator_memory_usage(); + BOOST_TEST(ours.total_bytes() == theirs.total_bytes()); + BOOST_TEST(ours.matched_scratch_bytes == theirs.matched_scratch_bytes); + BOOST_TEST(ours.matched_scratch_bytes > 0U); + BOOST_TEST(sharded::graph_memory_usage(shards).total_bytes() + == source.propagator.graph_memory_usage().total_bytes()); + BOOST_TEST(sharded::total_size(shards) == source.propagator.size()); +} + +BOOST_AUTO_TEST_CASE(sharded_state_rejects_invalid_arguments_before_the_team) { + const auto options = team_options(); + const auto threads = team_size(); + auto calls = std::atomic{0}; + const auto counting = [&](size_t) { + calls.fetch_add(1); + return small_state(false); + }; + BOOST_CHECK_THROW((void)sharded::make_shards(parallel::Options{.threads = 0}, counting), std::invalid_argument); + BOOST_CHECK_THROW((void)sharded::make_shards(parallel::Options{.threads = -3}, counting), + std::invalid_argument); + BOOST_TEST(calls.load() == 0); + + // An initializer that publishes nothing is a failure, reported after the join. + BOOST_CHECK_THROW((void)sharded::make_shards(options, [](size_t) { return std::unique_ptr{}; }), + std::logic_error); + + const auto fixture = nontrivial_fixtures().front(); + const auto mismatched = make_seed(fixture, routing::make_router(1, threads + 1)); + BOOST_CHECK_THROW((void)sharded::seed_shards(options, mismatched, 0), std::invalid_argument); + const auto seed = make_seed(fixture, local_router(threads)); + BOOST_CHECK_THROW((void)sharded::seed_shards(options, seed, 1), std::invalid_argument); + + auto source = sharded::seed_shards(options, seed, 0); + BOOST_CHECK_THROW((void)sharded::copy_shards(parallel::Options{.threads = options.threads + 1}, source), + std::invalid_argument); + source.back().reset(); + BOOST_CHECK_THROW((void)sharded::copy_shards(options, source), std::invalid_argument); + + // Caller-side validation of the logical-mode bound and the coefficient encoding. + OperatorDict out_of_range; + add_term(out_of_range, Basis::Majorana, {0, 12}, 1.0); + BOOST_CHECK_THROW((void)sharded::validate_initial_operator(out_of_range, Basis::Majorana, 6), std::exception); + BOOST_CHECK_NO_THROW((void)sharded::validate_initial_operator(out_of_range, Basis::Majorana, kN)); + OperatorDict non_hermitian; + non_hermitian[VecZ{0, 1}] = std::complex(1.0, 0.0); + BOOST_CHECK_THROW((void)sharded::validate_initial_operator(non_hermitian, Basis::Majorana, kN), std::exception); +} + +// --- Copying and lifecycle ------------------------------------------------------------------------------------ + +BOOST_AUTO_TEST_CASE(sharded_state_copy_is_exact_and_owner_allocated) { + const auto options = team_options(); + const auto threads = team_size(); + auto source = built_source(threads); + const auto shards = bridged_shards(source.propagator, threads); + size_t overflow = 0; + size_t layers = 0; + for (const auto &state : shards) { + overflow += state->op.store->overflow_size(); + layers += state->graph.layers(); + } + BOOST_TEST_MESSAGE("copy source: rows " << sharded::total_size(shards) << ", overflow rows " << overflow + << ", layers " << layers); + BOOST_REQUIRE(overflow > 0U); + BOOST_REQUIRE(layers > 0U); + + auto slots = std::vector(threads); + const auto copy = sharded::copy_shards(options, shards, OwnerProbe{.slots = &slots}); + check_owner_bodies(slots, sharded::ShardWork::copy, "copy"); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST_MESSAGE("copy owner " << shard << ": worker " << slots[shard].worker << ", bytes " + << slots[shard].bytes); + } + const auto difference = shards_difference(copy, shards); + BOOST_TEST(difference.empty(), difference); + for (size_t shard = 0; shard < threads; ++shard) { + const auto &a = *shards[shard]; + const auto &b = *copy[shard]; + BOOST_TEST_CONTEXT("shard " << shard) { + BOOST_TEST(&a != &b); + BOOST_TEST(a.op.store.get() != b.op.store.get()); + BOOST_TEST((a.op.op_coeffs.empty() || a.op.op_coeffs.data() != b.op.op_coeffs.data())); + BOOST_TEST((a.matched.epoch_.empty() || a.matched.epoch_.data() != b.matched.epoch_.data())); + // The copy's bytes were requested inside the owner's copy body: at least its packed rows. + if (allocation::available()) { + BOOST_TEST(slots[shard].bytes >= a.op.store->size()); + } + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_state_copies_mutate_independently) { + const auto options = team_options(); + const auto threads = team_size(); + auto source_propagator = built_source(threads); + auto source = bridged_shards(source_propagator.propagator, threads); + const auto reference = snapshot(source); + auto copy = sharded::copy_shards(options, source); + + const auto mutate = [](Shards &shards) { + for (auto &state : shards) { + auto &op = state->op; + // Touch the inverted index first so later growth has a materialized cache to diverge from. + (void)op.inverted_index(); + for (size_t i = 0; i < op.size(); ++i) { + if (!op.store->row_positions(i).inlined()) { + op.store->set(i, indices_to_bitset({0, 1, 2, 3, 4, 5, 6, 7, 8, 9})); + break; + } + } + if (op.size() > 0) { + op.store->set(0, indices_to_bitset({15})); + op.op_coeffs.at(0) += 1.0; + } + op.store->push_back(indices_to_bitset({14, 15})); + op.reindex_after_growth(op.size() - 1, 1); + op.op_coeffs.push_back(2.0); + op.state_coeffs.assign(op.size(), 0.5); + op.init_op_map[indices_to_bitset({13})] = 4.0; + state->matched.begin_gate(op.size()); + state->matched.mark(0); + if (state->graph.layers() > 0) { + (void)state->graph.slice_graph(1, /*contract=*/true); + } + } + }; + + mutate(copy); + auto difference = shards_difference(source, reference); + BOOST_CHECK_MESSAGE(difference.empty(), "source after copy mutation: " << difference); + + auto second = sharded::copy_shards(options, source); + mutate(source); + difference = shards_difference(second, reference); + BOOST_CHECK_MESSAGE(difference.empty(), "copy after source mutation: " << difference); + BOOST_TEST(!shards_difference(source, reference).empty(), "the mutation is visible where it was applied"); +} + +BOOST_AUTO_TEST_CASE(sharded_state_graph_cores_are_shared_and_lists_independent) { + const auto options = team_options(); + const auto threads = team_size(); + auto built = std::make_unique(built_source(threads)); + auto source = bridged_shards(built->propagator, threads); + auto copy = sharded::copy_shards(options, source); + std::vector> cores; + std::vector layer_counts; + for (size_t shard = 0; shard < threads; ++shard) { + const auto &a = source[shard]->graph; + const auto &b = copy[shard]->graph; + BOOST_REQUIRE(a.layers() == b.layers()); + layer_counts.push_back(b.layers()); + for (size_t i = 0; i < a.layers(); ++i) { + BOOST_TEST(a.get_layer(i).shared_core() == b.get_layer(i).shared_core()); + cores.push_back(b.get_layer(i).shared_core()); + } + // A layer-list change in the copy leaves the source's list alone. + if (b.layers() > 0) { + (void)copy[shard]->graph.slice_graph(1, /*contract=*/true); + BOOST_TEST(source[shard]->graph.layers() == layer_counts.back()); + BOOST_TEST(copy[shard]->graph.layers() == layer_counts.back() - 1); + } + } + const auto use_counts_before = [&] { + std::vector counts; + for (const auto &core : cores) { + counts.push_back(core.use_count()); + } + return counts; + }(); + // Destroying the source releases only its references; the copy's remaining layers stay readable. + source.clear(); + built.reset(); + size_t index = 0; + for (size_t shard = 0; shard < threads; ++shard) { + const auto &graph = copy[shard]->graph; + for (size_t i = 0; i < graph.layers(); ++i) { + const auto traversal = graph.get_layer_traversal(i); + BOOST_TEST(traversal.scaled_count() <= copy[shard]->op.size()); + } + for (size_t i = 0; i < layer_counts[shard]; ++i, ++index) { + BOOST_TEST(cores[index].use_count() < use_counts_before[index]); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_state_copy_keeps_the_captured_budget) { + const auto options = team_options(); + const auto threads = team_size(); + const auto fixture = nontrivial_fixtures().front(); + const auto source = sharded::seed_shards(options, make_seed(fixture, local_router(threads)), 0); + auto root_source = oracle_of(fixture, threads); + const auto root_budget = Access::options(root_source).threads; + + // A reread of the environment would now fail, so neither copy may capture a fresh budget. + const ScopedEnv poisoned("monoprop_NUM_THREADS", "0"); + BOOST_CHECK_THROW((void)parallel::capture_thread_budget(), std::invalid_argument); + auto slots = std::vector(threads); + const auto copy = sharded::copy_shards(options, source, OwnerProbe{.slots = &slots}); + check_owner_bodies(slots, sharded::ShardWork::copy, "copy under a changed environment"); + BOOST_TEST(copy.size() == threads); + const auto difference = shards_difference(copy, source); + BOOST_TEST(difference.empty(), difference); + // Preservation: the rank-level copy keeps the source's captured budget. + const Propagator root_copy(root_source); + BOOST_TEST(Access::options(root_copy).threads == root_budget); +} + +// Preservation evidence for the rank-level owner guard (MonomialPropagator::require_valid_), which S1 reuses rather +// than duplicating per shard: an invalid owner's state is rejected before anything is copied, while copies made +// before the failure stay exact and usable. + +// --- Failure paths -------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(sharded_state_initializer_failure_on_the_primary_is_returned) { + const auto options = team_options(); + const auto threads = team_size(); + long long net = -1; + auto caught = false; + { + const allocation::LiveBytes live; + try { + (void)sharded::make_shards(options, [](size_t shard) { + auto state = small_state(false); // unwinds inside the failing initializer + if (shard == 0) { + throw InjectedFailure("primary initializer failure"); + } + return state; + }); + } + catch (const InjectedFailure &e) { + // Compared in place: a copied message would itself be a tracked allocation. + caught = std::string_view(e.what()) == "primary initializer failure"; + } + net = live.net(); + } + BOOST_TEST(caught); + if (allocation::available()) { + BOOST_TEST(net == 0, "published sibling shards are released after the join"); + } + // A fresh construction afterwards succeeds. + const auto shards = sharded::make_shards(options, [](size_t) { return small_state(true); }); + BOOST_TEST(shards.size() == threads); +} + +BOOST_AUTO_TEST_CASE(sharded_state_initializer_failure_on_a_nonprimary_owner, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const auto threads = team_size(); + for (const bool after_publication : {false, true}) { + BOOST_TEST_CONTEXT("after publication " << after_publication) { + auto published = std::atomic{0}; + auto waited_out = std::atomic{false}; + long long net = -1; + auto caught = std::optional{}; + { + const allocation::LiveBytes live; + try { + (void)sharded::make_shards(options, [&](size_t shard) { + if (shard + 1 == threads) { + if (after_publication) { + // Bounded wait for every sibling to finish its state; nobody waits on this owner. + const auto deadline = std::chrono::steady_clock::now() + 10s; + while (published.load() + 1 < threads) { + if (std::chrono::steady_clock::now() > deadline) { + waited_out.store(true); + break; + } + std::this_thread::yield(); + } + } + throw WorkerError{shard}; + } + auto state = small_state(false); + published.fetch_add(1); + return state; + }); + } + catch (const WorkerError &e) { + caught = e.shard; + } + net = live.net(); + } + BOOST_TEST(caught.value_or(0) == threads - 1); + BOOST_TEST(!waited_out.load()); + if (allocation::available()) { + BOOST_TEST(net == 0); + } + } + } + const auto shards = sharded::make_shards(options, [](size_t) { return small_state(false); }); + BOOST_TEST(sharded::total_size(shards) == threads); +} + +BOOST_AUTO_TEST_CASE(sharded_state_concurrent_failures_select_the_lowest_owner, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const auto threads = team_size(); + for (const size_t stride : {size_t{1}, size_t{2}}) { + auto caught = std::optional{}; + try { + (void)sharded::make_shards(options, [&](size_t shard) { + auto state = small_state(false); + if (shard % stride == stride - 1) { + throw WorkerError{shard}; + } + return state; + }); + } + catch (const WorkerError &e) { + caught = e.shard; + } + BOOST_CHECK_MESSAGE(caught.value_or(threads) == stride - 1, "stride " << stride); + } +} + +// Real allocation failures at every operator new of one owner's seed body. Each must reach the caller as the +// original std::bad_alloc after the join (an allocating noexcept constructor would terminate instead), with every +// published sibling released. The sweep ends at the first count the body does not reach, which must seed exactly. +BOOST_AUTO_TEST_CASE(sharded_state_seed_allocation_failures_are_catchable, + *boost::unit_test::precondition(has_allocation_probe)) { + const auto options = team_options(); + const auto threads = team_size(); + const auto fixture = nontrivial_fixtures().at(1); + const auto seed = make_seed(fixture, local_router(threads)); + const auto reference = sharded::seed_shards(options, seed, 0); + for (const size_t failing : std::set{0, threads - 1}) { + BOOST_TEST_CONTEXT("failing owner " << failing) { + size_t failures = 0; + size_t leaks = 0; + size_t other_errors = 0; + bool completed = false; + for (size_t nth = 1; nth <= 100000 && !completed; ++nth) { + auto slots = std::vector(threads); + const allocation::LiveBytes live; + try { + auto shards = + sharded::seed_shards(options, + seed, + 0, + OwnerProbe{.slots = &slots, .fail_shard = failing, .fail_nth = nth}); + completed = slots[failing].unfired; + if (completed) { + const auto difference = shards_difference(shards, reference); + BOOST_TEST(difference.empty(), difference); + } + } + catch (const std::bad_alloc &) { + ++failures; + } + catch (...) { + ++other_errors; + } + leaks += live.net() != 0 ? 1 : 0; + } + BOOST_TEST_MESSAGE("allocation failures injected and caught: " << failures); + BOOST_TEST(completed); + BOOST_TEST(failures > 0U); + BOOST_TEST(other_errors == 0U); + BOOST_TEST(leaks == 0U); + } + } +} + +// The same sweep over one owner's copy body: every failure leaves the source and an earlier independent copy exact. +BOOST_AUTO_TEST_CASE(sharded_state_copy_allocation_failures_leave_sources_intact, + *boost::unit_test::precondition(has_allocation_probe)) { + const auto options = team_options(); + const auto threads = team_size(); + auto built = built_source(threads); + const auto source = bridged_shards(built.propagator, threads); + const auto reference = snapshot(source); + const auto earlier = sharded::copy_shards(options, source); + for (const size_t failing : std::set{0, threads - 1}) { + BOOST_TEST_CONTEXT("failing owner " << failing) { + size_t failures = 0; + size_t leaks = 0; + size_t other_errors = 0; + size_t damaged = 0; + bool completed = false; + for (size_t nth = 1; nth <= 100000 && !completed; ++nth) { + auto slots = std::vector(threads); + { + const allocation::LiveBytes live; + try { + auto copy = + sharded::copy_shards(options, + source, + OwnerProbe{.slots = &slots, .fail_shard = failing, .fail_nth = nth}); + completed = slots[failing].unfired; + if (completed) { + const auto difference = shards_difference(copy, reference); + BOOST_TEST(difference.empty(), difference); + } + } + catch (const std::bad_alloc &) { + ++failures; + } + catch (...) { + ++other_errors; + } + leaks += live.net() != 0 ? 1 : 0; + } + damaged += + (!shards_difference(source, reference).empty() || !shards_difference(earlier, reference).empty()) + ? 1 + : 0; + } + BOOST_TEST_MESSAGE("allocation failures injected and caught: " << failures); + BOOST_TEST(completed); + BOOST_TEST(failures > 0U); + BOOST_TEST(other_errors == 0U); + BOOST_TEST(leaks == 0U); + BOOST_TEST(damaged == 0U); + } + } +} + +// --- Worker stack use ------------------------------------------------------------------------------------------- + +/* + * Seeding runs on every worker, and the OpenMP pool keeps each worker's stack pages resident for the process's life. + * An MPOperator is about 112 N bytes (its inverted index holds 2N columns inline), so an operator in a seeding + * frame would cost every worker that much stack: the state is built in place on the heap instead. Measured on the + * calling thread, which runs a team of one, at a large NumModes where one operator spans many pages. + */ +BOOST_AUTO_TEST_CASE(sharded_state_seeding_keeps_operators_off_the_worker_stack, + *boost::unit_test::precondition(has_plain_stack_frames)) { + constexpr size_t modes = 512; + constexpr size_t logical = 8; + OperatorDict op; + for (const auto &[indices, value] : + std::vector>{{{0, 1}, 0.5}, {{2, 3}, -0.25}, {{0, 1, 2, 3}, 0.125}, {{}, 1.0}}) { + op[indices] = algebra_decode_coeff(Basis::Majorana, + std::complex(value, 0.0), + indices_to_bitset(indices)); + } + const VecZ initial_state = {0, 1}; + const auto cutoff_fn = monoprop::detail::cutoff_function(CutoffType::Length, 4, logical); + const auto seed = + sharded::OperatorSeed{.initial_operator = op, + .initial_state = initial_state, + .router = routing::make_router(1, 1), + .basis = Basis::Majorana, + .logical_num_modes = logical, + .paired = std::nullopt, + .inline_width = sharded::packed_inline_width(false, cutoff_fn)}; + auto shards = sharded::Shards{}; + const auto touched = + stack_bytes_touched([&] { shards = sharded::seed_shards(parallel::Options{.threads = 1}, seed, 0); }); + BOOST_TEST_REQUIRE(shards.size() == 1U); + BOOST_TEST(shards.front()->size() == 3U); + BOOST_TEST(shards.front()->op.inverted_index_.has_value()); + BOOST_TEST_MESSAGE("seeding touched " << touched << " stack bytes; one operator is " + << sizeof(monoprop::detail::MPOperator)); + BOOST_TEST(touched < sizeof(monoprop::detail::MPOperator)); +} diff --git a/cpp/tests/sharded_team_tests.cpp b/cpp/tests/sharded_team_tests.cpp new file mode 100644 index 000000000..57134c929 --- /dev/null +++ b/cpp/tests/sharded_team_tests.cpp @@ -0,0 +1,830 @@ +// Copyright 2026 Algorithmiq +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* + * The fixed-team phase primitive (detail/sharded/Team.h). The team size T is the captured launch budget + * (monoprop_NUM_THREADS, else omp_get_max_threads()); cpp/tests/CMakeLists.txt reruns these cases in fresh + * processes at T = 1, 2 and 4. Cases that need a nonprimary worker skip below T = 2; the T = 2 and T = 4 + * launches run them. + * + * Workers never call BOOST_TEST. Each writes only its own preallocated observation slot (or an atomic, so a + * mutated helper that repeats an owner cannot race), and every assertion runs on the caller after run_team() + * has joined the team. Observing the actual team is test-side evidence, not a library check. + * + * The sleeps only shape interleavings that make a broken checkpoint visible: waiting workers park in the + * barrier while a delayed worker is still inside its phase. Correctness never depends on them. + */ + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "monoprop/detail/parallel/Options.h" +#include "monoprop/detail/parallel/ThreadBudget.h" +#include "monoprop/detail/sharded/Team.h" + +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; +using namespace std::chrono_literals; + +namespace { + +// Orchestration bodies must be noexcept; potentially throwing work belongs inside phase(). +static_assert(sharded::TeamBody); +static_assert(!sharded::TeamBody); +static_assert(!sharded::TeamBody); + +auto team_options() -> parallel::Options { + return parallel::capture_thread_budget(); +} + +auto team_size(parallel::Options options) -> size_t { + return static_cast(options.threads); +} + +// The team a fixed-T launch in cpp/tests/CMakeLists.txt promises; absent in ordinary discovered runs. +auto expected_team() -> std::optional { + const auto *value = std::getenv("monoprop_TEST_EXPECT_TEAM"); + if (value == nullptr) { + return std::nullopt; + } + return std::stoi(value); +} + +auto has_nonprimary_worker(boost::unit_test::test_unit_id) -> boost::test_tools::assertion_result { + const auto threads = parallel::capture_thread_budget().threads; + boost::test_tools::assertion_result result(threads >= 2); + result.message() << "the captured team budget is " << threads << "; this case needs a nonprimary worker"; + return result; +} + +// What one worker saw of its own team, written only by that worker. +struct alignas(64) Observation { + std::atomic calls{0}; //!< Orchestration-body invocations for this owner. + int worker = -1; //!< omp_get_thread_num() inside the body. + int team = 0; //!< omp_get_num_threads() inside the body. + int level = -1; //!< omp_get_level() inside the body. + int active_level = -1; //!< omp_get_active_level() inside the body. + std::thread::id thread; //!< The OS thread running the body. +}; + +auto observe(Observation &slot) -> void { + slot.calls.fetch_add(1); + slot.worker = omp_get_thread_num(); + slot.team = omp_get_num_threads(); + slot.level = omp_get_level(); + slot.active_level = omp_get_active_level(); + slot.thread = std::this_thread::get_id(); +} + +/* + * Test-only orchestration for multi-phase failure cases: run `count` phases, `body(p)` in phase p, and record each + * decision in `decisions` until the first false one. After a false decision the worker runs no further phase body, + * but still reaches the remaining checkpoints. With the correct helper every worker stops at the same checkpoint, so + * those extra checkpoints are uniform and change nothing. With a broken checkpoint, workers that disagree still + * reach the same number of barriers, so the divergence is recorded and asserted after the join instead of + * deadlocking the team. + */ +template +auto run_phases(sharded::TeamFailure &failure, size_t shard, size_t count, std::vector &decisions, Body &&body) + -> void { + auto proceeding = true; + for (size_t p = 0; p < count; ++p) { + if (!proceeding) { + failure.checkpoint(); + continue; + } + proceeding = sharded::phase(failure, shard, [&] { body(p); }); + decisions.push_back(proceeding ? 1 : 0); + } +} + +// A non-std exception type that also identifies the thrower. +struct WorkerError { + size_t worker; +}; + +// The OpenMP runtime settings (ICVs) that a library helper must never change. +struct RuntimeSettings { + int max_threads = 0; + int dynamic = 0; + int max_active_levels = 0; + int thread_limit = 0; + omp_sched_t schedule = omp_sched_static; + int chunk = 0; + omp_proc_bind_t proc_bind = omp_proc_bind_false; + int level = 0; + + auto operator==(const RuntimeSettings &) const -> bool = default; +}; + +auto runtime_settings() -> RuntimeSettings { + auto settings = RuntimeSettings{}; + settings.max_threads = omp_get_max_threads(); + settings.dynamic = omp_get_dynamic(); + settings.max_active_levels = omp_get_max_active_levels(); + settings.thread_limit = omp_get_thread_limit(); + omp_get_schedule(&settings.schedule, &settings.chunk); + settings.proc_bind = omp_get_proc_bind(); + settings.level = omp_get_level(); + return settings; +} + +} // namespace + +BOOST_AUTO_TEST_CASE(sharded_team_visits_each_owner) { + const auto options = team_options(); + const auto threads = team_size(options); + auto visits = std::vector>(threads); + auto observations = std::vector(threads); + auto decisions = std::vector(threads, -1); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + observe(observations[shard]); + decisions[shard] = sharded::phase(failure, shard, [&] { visits[shard].fetch_add(1); }) ? 1 : 0; + }); + BOOST_TEST(!error); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(visits[shard].load() == 1, "shard " << shard); + BOOST_TEST(observations[shard].calls.load() == 1, "shard " << shard); + BOOST_TEST(observations[shard].team == options.threads, "shard " << shard); + BOOST_TEST(decisions[shard] == 1, "shard " << shard); + } +} + +// At most `limit` workers are inside the section at once, every worker enters exactly once, and the section's +// value is returned. The sleep keeps holders inside long enough for the others to queue behind the limit. +BOOST_AUTO_TEST_CASE(sharded_team_section_limit_bounds_concurrent_holders) { + const auto options = team_options(); + const auto threads = team_size(options); + for (const size_t limit : {size_t{1}, size_t{2}, threads}) { + auto gate = sharded::SectionLimit(limit); + auto inside = std::atomic(0); + auto most = std::atomic(0); + auto entered = std::vector(threads, 0); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + static_cast(sharded::phase(failure, shard, [&] { + entered[shard] = gate.run([&] { + const size_t now = inside.fetch_add(1) + 1; + for (size_t seen = most.load(); seen < now && !most.compare_exchange_weak(seen, now);) { + } + std::this_thread::sleep_for(2ms); + inside.fetch_sub(1); + return shard + 1; + }); + })); + }); + BOOST_TEST(!error); + BOOST_TEST(most.load() <= limit, "limit " << limit); + BOOST_TEST(most.load() >= 1U, "limit " << limit); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(entered[shard] == shard + 1, "limit " << limit << " shard " << shard); + } + } +} + +// A section that throws releases its slot: with one slot, every worker still gets in, and the exception reaches +// the phase. +BOOST_AUTO_TEST_CASE(sharded_team_section_limit_releases_on_exception) { + const auto options = team_options(); + const auto threads = team_size(options); + auto gate = sharded::SectionLimit(1); + auto entered = std::atomic(0); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + static_cast(sharded::phase(failure, shard, [&] { + gate.run([&] { + entered.fetch_add(1); + throw std::runtime_error("section failed"); + }); + })); + }); + BOOST_TEST(entered.load() == threads); + BOOST_REQUIRE(error); + BOOST_CHECK_THROW(std::rethrow_exception(error), std::runtime_error); +} + +BOOST_AUTO_TEST_CASE(sharded_team_section_limit_admits_at_least_one) { + auto gate = sharded::SectionLimit(0); + BOOST_TEST(gate.run([] { return 7; }) == 7); +} + +BOOST_AUTO_TEST_CASE(sharded_team_observes_worker_identity) { + const auto options = team_options(); + const auto threads = team_size(options); + const auto caller = std::this_thread::get_id(); + const auto caller_level = omp_get_level(); + auto in_body = std::vector(threads); + auto in_phase = std::vector(threads); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + observe(in_body[shard]); + sharded::phase(failure, shard, [&] { observe(in_phase[shard]); }); + }); + BOOST_TEST(!error); + auto distinct = std::set{}; + for (size_t shard = 0; shard < threads; ++shard) { + for (const auto *slot : {&in_body[shard], &in_phase[shard]}) { + BOOST_TEST(slot->calls.load() == 1, "shard " << shard); + BOOST_TEST(slot->worker == static_cast(shard), "shard " << shard); + BOOST_TEST(slot->team == options.threads, "shard " << shard); + // One structured region for a team; a one-worker team runs on the caller, without a region. + BOOST_TEST(slot->level == caller_level + (threads > 1 ? 1 : 0), "shard " << shard); + BOOST_TEST(slot->active_level == (threads > 1 ? 1 : 0), "shard " << shard); + BOOST_TEST((slot->thread == in_body[shard].thread), "shard " << shard); + } + distinct.insert(in_body[shard].thread); + } + BOOST_TEST(distinct.size() == threads); + BOOST_TEST((in_body[0].thread == caller)); // the entering caller is the OpenMP primary thread + BOOST_TEST_MESSAGE("sharded_team_observes_worker_identity: " << distinct.size() + << " distinct threads for T=" << threads); +} + +BOOST_AUTO_TEST_CASE(sharded_team_keeps_owner_across_phases) { + constexpr size_t phases = 6; + const auto options = team_options(); + const auto threads = team_size(options); + auto workers = std::vector>(threads, std::vector(phases, -1)); + auto identities = std::vector>(threads, std::vector(phases)); + auto decisions = std::vector>(threads, std::vector(phases, -1)); + auto regions = std::vector>(threads, std::vector(phases, -1)); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + for (size_t p = 0; p < phases; ++p) { + const auto proceed = sharded::phase(failure, shard, [&] { + workers[shard][p] = omp_get_thread_num(); + identities[shard][p] = std::this_thread::get_id(); + // Distinguishes one team from a region per phase. + regions[shard][p] = omp_get_level(); + }); + decisions[shard][p] = proceed ? 1 : 0; + if (!proceed) { + return; + } + } + }); + BOOST_TEST(!error); + for (size_t shard = 0; shard < threads; ++shard) { + for (size_t p = 0; p < phases; ++p) { + BOOST_TEST(workers[shard][p] == static_cast(shard), "shard " << shard << " phase " << p); + BOOST_TEST((identities[shard][p] == identities[shard][0]), "shard " << shard << " phase " << p); + BOOST_TEST(decisions[shard][p] == 1, "shard " << shard << " phase " << p); + BOOST_TEST(regions[shard][p] == regions[0][0], "shard " << shard << " phase " << p); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_primary_only_phase_runs_on_the_caller) { + const auto options = team_options(); + const auto threads = team_size(options); + const auto caller = std::this_thread::get_id(); + auto primary_runs = std::atomic{0}; + auto primary_thread = std::thread::id{}; + auto published = 0; + auto seen = std::vector(threads, -1); + auto decisions = std::vector>(threads); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + auto proceed = sharded::phase(failure, shard, [&] { + if (shard == 0) { + // The other owners have no work here but still wait for this write. + std::this_thread::sleep_for(10ms); + primary_runs.fetch_add(1); + primary_thread = std::this_thread::get_id(); + published = 42; + } + }); + decisions[shard].push_back(proceed ? 1 : 0); + if (!proceed) { + return; + } + proceed = sharded::phase(failure, shard, [&] { seen[shard] = published; }); + decisions[shard].push_back(proceed ? 1 : 0); + }); + BOOST_TEST(!error); + BOOST_TEST(primary_runs.load() == 1); + BOOST_TEST((primary_thread == caller)); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(seen[shard] == 42, "shard " << shard); + BOOST_TEST((decisions[shard] == std::vector{1, 1}), "shard " << shard); + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_empty_owners_reach_the_checkpoint) { + const auto options = team_options(); + const auto threads = team_size(options); + const auto last = threads - 1; + auto value = 0; + auto seen = std::vector(threads, -1); + auto empty_arrivals = std::vector>(threads); + auto decisions = std::vector>(threads); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + // Only the last owner has work; every other owner's phase body is empty. + auto proceed = sharded::phase(failure, shard, [&] { + if (shard == last) { + std::this_thread::sleep_for(10ms); + value = 7; + } + }); + if (shard != last) { + empty_arrivals[shard].fetch_add(1); + } + decisions[shard].push_back(proceed ? 1 : 0); + if (!proceed) { + return; + } + proceed = sharded::phase(failure, shard, [&] { seen[shard] = value; }); + decisions[shard].push_back(proceed ? 1 : 0); + }); + BOOST_TEST(!error); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(seen[shard] == 7, "shard " << shard); + BOOST_TEST((decisions[shard] == std::vector{1, 1}), "shard " << shard); + BOOST_TEST(empty_arrivals[shard].load() == (shard == last ? 0 : 1), "shard " << shard); + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_publishes_writes_to_the_next_phase) { + constexpr size_t block = 4096; + constexpr size_t rounds = 3; + const auto options = team_options(); + const auto threads = team_size(options); + auto buffers = std::vector>(threads, std::vector(block, 0)); + auto sums = std::vector>(threads, std::vector(rounds, 0)); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + for (size_t round = 0; round < rounds; ++round) { + // Producers finish at staggered times; a consumer must still see its peer's whole block. + if (!sharded::phase(failure, shard, [&] { + std::this_thread::sleep_for(std::chrono::milliseconds(2 * (shard + 1))); + for (size_t i = 0; i < block; ++i) { + buffers[shard][i] = (round + 1) * 1000003 + shard * block + i; + } + })) { + return; + } + const auto peer = (shard + 1) % threads; + if (!sharded::phase(failure, shard, [&] { + sums[shard][round] = std::accumulate(buffers[peer].begin(), buffers[peer].end(), size_t{0}); + })) { + return; + } + } + }); + BOOST_TEST(!error); + for (size_t shard = 0; shard < threads; ++shard) { + const auto peer = (shard + 1) % threads; + for (size_t round = 0; round < rounds; ++round) { + auto expected = size_t{0}; + for (size_t i = 0; i < block; ++i) { + expected += (round + 1) * 1000003 + peer * block + i; + } + BOOST_TEST(sums[shard][round] == expected, "shard " << shard << " round " << round); + } + } +} + +// Explicit T = 1, independent of the launch budget: the failure is returned after the (inactive) region +// ends, never rethrown by run_team(). +BOOST_AUTO_TEST_CASE(sharded_team_single_worker_failure_is_returned_after_join) { + const auto caller = std::this_thread::get_id(); + const auto caller_level = omp_get_level(); + auto observation = Observation{}; + auto later_phase = 0; + auto finished = 0; + auto decision = -1; + auto error = std::exception_ptr{}; + try { + error = sharded::run_team( + parallel::Options{.threads = 1}, + [&](size_t shard, sharded::TeamFailure &failure) noexcept { + observe(observation); + const auto proceed = sharded::phase(failure, shard, [] { throw std::runtime_error("single worker"); }); + decision = proceed ? 1 : 0; + if (proceed) { + sharded::phase(failure, shard, [&] { ++later_phase; }); + } + finished = 1; + }); + } + catch (...) { + BOOST_FAIL("run_team rethrew a phase exception instead of returning it"); + } + BOOST_TEST(observation.calls.load() == 1); + BOOST_TEST(observation.worker == 0); + BOOST_TEST(observation.team == 1); + BOOST_TEST(observation.level == caller_level); // no region for a one-worker team + BOOST_TEST((observation.thread == caller)); + BOOST_TEST(decision == 0); + BOOST_TEST(later_phase == 0); + BOOST_TEST(finished == 1); + BOOST_REQUIRE(error); + BOOST_CHECK_EXCEPTION(std::rethrow_exception(error), std::runtime_error, [](const std::runtime_error &e) { + return std::string(e.what()) == "single worker"; + }); +} + +BOOST_AUTO_TEST_CASE(sharded_team_failure_is_returned_not_rethrown) { + const auto options = team_options(); + const auto threads = team_size(options); + const auto failing = threads - 1; // the primary at T = 1 + auto finished = std::atomic{0}; + auto error = std::exception_ptr{}; + try { + error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { + if (shard == failing) { + throw std::runtime_error("returned"); + } + }); + finished.fetch_add(1); + }); + } + catch (...) { + BOOST_FAIL("run_team rethrew a phase exception instead of returning it"); + } + BOOST_TEST(finished.load() == options.threads); + BOOST_REQUIRE(error); + BOOST_CHECK_EXCEPTION(std::rethrow_exception(error), std::runtime_error, [](const std::runtime_error &e) { + return std::string(e.what()) == "returned"; + }); +} + +BOOST_AUTO_TEST_CASE(sharded_team_nonprimary_failure_joins_and_skips_later_phases, + *boost::unit_test::precondition(has_nonprimary_worker)) { + constexpr size_t phases = 4; + constexpr size_t failing_phase = 1; + constexpr int iterations = 10; + const auto options = team_options(); + const auto threads = team_size(options); + const auto failing = threads - 1; + for (int iteration = 0; iteration < iterations; ++iteration) { + auto executed = std::vector>(threads, std::vector(phases, 0)); + auto decisions = std::vector>(threads); + auto in_flight = std::atomic{0}; + auto finished = std::atomic{0}; + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + in_flight.fetch_add(1); + run_phases(failure, shard, phases, decisions[shard], [&](size_t p) { + if (p == failing_phase && shard == failing) { + // Every peer is already waiting in the checkpoint when this worker fails. + std::this_thread::sleep_for(5ms); + throw std::runtime_error("worker " + std::to_string(shard)); + } + executed[shard][p] = 1; + }); + finished.fetch_add(1); + in_flight.fetch_sub(1); + }); + const auto joined = in_flight.load(); + BOOST_TEST(joined == 0, "iteration " << iteration); + BOOST_TEST(finished.load() == options.threads, "iteration " << iteration); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST((decisions[shard] == std::vector{1, 0}), "iteration " << iteration << " shard " << shard); + BOOST_TEST(executed[shard][0] == 1, "iteration " << iteration << " shard " << shard); + BOOST_TEST(executed[shard][failing_phase] == (shard == failing ? 0 : 1), + "iteration " << iteration << " shard " << shard); + for (size_t p = failing_phase + 1; p < phases; ++p) { + BOOST_TEST(executed[shard][p] == 0, "iteration " << iteration << " shard " << shard << " phase " << p); + } + } + BOOST_REQUIRE(error); + BOOST_CHECK_EXCEPTION(std::rethrow_exception(error), std::runtime_error, [&](const std::runtime_error &e) { + return std::string(e.what()) == "worker " + std::to_string(failing); + }); + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_simultaneous_failures_select_the_lowest_worker, + *boost::unit_test::precondition(has_nonprimary_worker)) { + constexpr int iterations = 5; + const auto options = team_options(); + const auto threads = team_size(options); + // Higher workers fail first in time, so selection cannot follow arrival order. + for (const size_t lowest_failing : {size_t{0}, size_t{1}}) { + for (int iteration = 0; iteration < iterations; ++iteration) { + auto later = std::atomic{0}; + auto decisions = std::vector>(threads); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + run_phases(failure, shard, 2, decisions[shard], [&](size_t p) { + if (p == 1) { + later.fetch_add(1); + } + else if (shard >= lowest_failing) { + std::this_thread::sleep_for(std::chrono::milliseconds(2 * (threads - shard))); + throw WorkerError{shard}; + } + }); + }); + BOOST_TEST(later.load() == 0); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST((decisions[shard] == std::vector{0}), "iteration " << iteration << " shard " << shard); + } + BOOST_REQUIRE(error); + try { + std::rethrow_exception(error); + } + catch (const WorkerError &e) { + BOOST_TEST(e.worker == lowest_failing, "iteration " << iteration); + } + catch (...) { + BOOST_FAIL("the selected exception changed type"); + } + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_failure_selects_the_lowest_recorded_slot) { + auto failure = sharded::TeamFailure(4); + BOOST_TEST(!failure.first_error()); + // Outside a parallel region the checkpoint binds to a one-thread team. + BOOST_TEST(failure.checkpoint()); + const auto third = std::make_exception_ptr(WorkerError{3}); + const auto second = std::make_exception_ptr(WorkerError{2}); + const auto later = std::make_exception_ptr(WorkerError{99}); + failure.record(3, third); + failure.record(2, second); + failure.record(2, later); // a slot keeps its first error + failure.record(1, std::exception_ptr{}); + BOOST_TEST((failure.first_error() == second)); + BOOST_TEST(!failure.checkpoint()); +} + +BOOST_AUTO_TEST_CASE(sharded_team_preserves_the_original_exception) { + struct PayloadError : std::runtime_error { + explicit PayloadError(int value) : std::runtime_error("payload"), payload(value) {} + int payload; + }; + struct Opaque { + int code; + std::string text; + }; + const auto options = team_options(); + const auto failing = team_size(options) - 1; // a nonprimary worker when T > 1 + auto fail_with = [&](auto thrower) { + return sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { + if (shard == failing) { + thrower(); + } + }); + }); + }; + + const auto payload = fail_with([] { throw PayloadError(17); }); + BOOST_REQUIRE(payload); + try { + std::rethrow_exception(payload); + } + catch (const PayloadError &e) { + BOOST_TEST(e.payload == 17); + BOOST_TEST(std::string(e.what()) == "payload"); + } + catch (...) { + BOOST_FAIL("PayloadError changed type"); + } + + const auto opaque = fail_with([] { throw Opaque{5, "not a std::exception"}; }); + BOOST_REQUIRE(opaque); + try { + std::rethrow_exception(opaque); + } + catch (const Opaque &e) { + BOOST_TEST(e.code == 5); + BOOST_TEST(e.text == "not a std::exception"); + } + catch (...) { + BOOST_FAIL("Opaque changed type"); + } + + const auto integer = fail_with([] { throw 23; }); + BOOST_REQUIRE(integer); + try { + std::rethrow_exception(integer); + } + catch (const int value) { + BOOST_TEST(value == 23); + } + catch (...) { + BOOST_FAIL("int changed type"); + } +} + +// Checkpoint N's decision must stay stable while slow workers consume it: a fast worker that passes +// checkpoint N and fails in phase N+1 must not turn N into a failure for the others. +BOOST_AUTO_TEST_CASE(sharded_team_checkpoint_generations_are_stable, + *boost::unit_test::precondition(has_nonprimary_worker)) { + constexpr int iterations = 20; + const auto options = team_options(); + const auto threads = team_size(options); + for (const size_t fast : {size_t{0}, threads - 1}) { + for (int iteration = 0; iteration < iterations; ++iteration) { + auto decisions = std::vector>(threads); + auto executed_second = std::vector(threads, 0); + auto executed_third = std::vector(threads, 0); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + run_phases(failure, shard, 3, decisions[shard], [&](size_t p) { + if (p == 0 && shard == fast) { + // The slow workers park in checkpoint N before the fast worker arrives last. + std::this_thread::sleep_for(5ms); + } + else if (p == 1) { + if (shard == fast) { + throw WorkerError{shard}; // at once, while the others may still be leaving checkpoint N + } + executed_second[shard] = 1; + } + else if (p == 2) { + executed_third[shard] = 1; + } + }); + }); + for (size_t shard = 0; shard < threads; ++shard) { + const auto context = [&] { + return "fast " + std::to_string(fast) + " iteration " + std::to_string(iteration) + " shard " + + std::to_string(shard); + }; + BOOST_TEST((decisions[shard] == std::vector{1, 0}), context()); + BOOST_TEST(executed_second[shard] == (shard == fast ? 0 : 1), context()); + BOOST_TEST(executed_third[shard] == 0, context()); + } + BOOST_REQUIRE(error); + try { + std::rethrow_exception(error); + } + catch (const WorkerError &e) { + BOOST_TEST(e.worker == fast); + } + catch (...) { + BOOST_FAIL("the selected exception changed type"); + } + } + } +} + +/* + * The same property with the interleaving forced: every other worker reads checkpoint 0's decision only after the + * fast worker has left it and recorded its failure in phase 1. A decision that is not generation-stamped (any recorded + * failure fails every checkpoint still being read) would fail checkpoint 0 for the slow workers. + */ +BOOST_AUTO_TEST_CASE(sharded_team_checkpoint_ignores_a_later_generation_failure, + *boost::unit_test::precondition(has_nonprimary_worker)) { + const auto options = team_options(); + const auto threads = team_size(options); + for (const size_t fast : {size_t{0}, threads - 1}) { + auto decisions = std::vector>(threads); + auto fast_recorded = std::atomic(false); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + const auto hold_until_recorded = [&](size_t generation) noexcept { + if (generation == 0 && shard != fast) { + while (!fast_recorded.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + }; + auto proceeding = true; + for (size_t p = 0; p < 3; ++p) { + if (proceeding) { + try { + if (p == 1 && shard == fast) { + failure.record(shard, std::make_exception_ptr(WorkerError{shard})); + fast_recorded.store(true, std::memory_order_release); + throw WorkerError{shard}; + } + } + catch (...) { + failure.record(shard, std::current_exception()); + } + } + const auto decision = failure.checkpoint(hold_until_recorded); + if (proceeding) { + decisions[shard].push_back(decision ? 1 : 0); + } + proceeding = proceeding && decision; + } + }); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST((decisions[shard] == std::vector{1, 0}), + "fast " + std::to_string(fast) + " shard " + std::to_string(shard)); + } + BOOST_REQUIRE(error); + try { + std::rethrow_exception(error); + } + catch (const WorkerError &e) { + BOOST_TEST(e.worker == fast); + } + catch (...) { + BOOST_FAIL("the selected exception changed type"); + } + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_fresh_invocation_has_no_stale_failure) { + const auto options = team_options(); + const auto threads = team_size(options); + const auto failed = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { + if (shard == threads - 1) { + throw std::runtime_error("first invocation"); + } + }); + }); + BOOST_REQUIRE(failed); + + auto executed = std::vector>(threads, std::vector(3, 0)); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + for (size_t p = 0; p < 3; ++p) { + if (!sharded::phase(failure, shard, [&] { executed[shard][p] = 1; })) { + return; + } + } + }); + BOOST_TEST(!error); + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST((executed[shard] == std::vector{1, 1, 1}), "shard " << shard); + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_rejects_a_nonpositive_budget_before_the_region) { + for (const int threads : {0, -1}) { + auto calls = std::atomic{0}; + BOOST_CHECK_THROW(sharded::run_team(parallel::Options{.threads = threads}, + [&](size_t, sharded::TeamFailure &) noexcept { calls.fetch_add(1); }), + std::invalid_argument); + BOOST_TEST(calls.load() == 0); + } +} + +BOOST_AUTO_TEST_CASE(sharded_team_leaves_runtime_settings_unchanged) { + const auto options = team_options(); + const auto before = runtime_settings(); + auto sink = std::vector(team_size(options), 0); + const auto ok = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { sink[shard] = 1; }); + }); + const auto failed = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [] { throw std::runtime_error("fail"); }); + }); + const auto single = + sharded::run_team(parallel::Options{.threads = 1}, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { sink[shard] = 2; }); + }); + const auto after = runtime_settings(); + BOOST_TEST(!ok); + BOOST_TEST(static_cast(failed)); + BOOST_TEST(!single); + BOOST_TEST((before == after)); + BOOST_TEST(after.max_threads == before.max_threads); + BOOST_TEST(after.dynamic == before.dynamic); + BOOST_TEST(after.thread_limit == before.thread_limit); + BOOST_TEST(after.level == before.level); +} + +// Only meaningful under the fixed-T launches, which set monoprop_TEST_EXPECT_TEAM; it checks that the +// launch really supplied the team the other cases are rerun with. +BOOST_AUTO_TEST_CASE(sharded_team_launch_supplies_the_expected_team) { + const auto expected = expected_team(); + if (!expected) { + BOOST_TEST_MESSAGE("monoprop_TEST_EXPECT_TEAM is unset; not a fixed-team launch"); + return; + } + const auto options = team_options(); + BOOST_TEST(options.threads == *expected); + BOOST_TEST(omp_get_dynamic() == 0); + const auto threads = team_size(options); + auto observations = std::vector(threads); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { observe(observations[shard]); }); + }); + BOOST_TEST(!error); + auto distinct = std::set{}; + for (size_t shard = 0; shard < threads; ++shard) { + BOOST_TEST(observations[shard].calls.load() == 1, "shard " << shard); + BOOST_TEST(observations[shard].team == *expected, "shard " << shard); + distinct.insert(observations[shard].thread); + } + BOOST_TEST(distinct.size() == static_cast(*expected)); + BOOST_TEST_MESSAGE("sharded_team_launch_supplies_the_expected_team: observed " + << distinct.size() << " workers, expected " << *expected); +} diff --git a/cpp/tests/shm_comm_tests.cpp b/cpp/tests/shm_comm_tests.cpp deleted file mode 100644 index 7470181cb..000000000 --- a/cpp/tests/shm_comm_tests.cpp +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2026 Algorithmiq -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include -#include -#include -#include -#include -#include - -#include "SlotBlocks.h" -#include "ThreadHarness.h" -#include "monoprop/detail/mpi/Comm.h" -#include "monoprop/detail/mpi/Exchange.h" -#include "monoprop/detail/mpi/MPICompat.h" -#include "monoprop/detail/mpi/ShmComm.h" - -using monoprop::mpi::Comm; -using monoprop::mpi::ShmComm; -using monoprop::mpi::ShmCommPoisoned; - -namespace { - -template -auto run_shm(int s, Body body) -> std::vector { - ShmComm sh(s); - return test_utils::run_comm_threads(sh, s, body); -} - -} // namespace - -// alltoall_counts is a transpose: recv[s] on rank r == what s declared it sends to r. -BOOST_AUTO_TEST_CASE(shm_comm_alltoall_counts_transpose) { - for (const int S : {2, 4, 8}) { - std::vector> recv(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - std::vector send(static_cast(S)); - for (int t = 0; t < S; ++t) { - send[static_cast(t)] = r * 100 + t; - } - std::vector got(static_cast(S)); - sh.alltoall_counts(r, send.data(), got.data()); - recv[static_cast(r)] = got; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int r = 0; r < S; ++r) { - for (int s = 0; s < S; ++s) { - BOOST_CHECK_EQUAL(recv[static_cast(r)][static_cast(s)], s * 100 + r); - } - } - } -} - -// begin_alltoallv (the vector-of-vectors facade) must deliver each source's block contiguously in -// ascending source order — what Resolve.h's positional pairing depends on. Rank r sends (r+1) tagged elts. -BOOST_AUTO_TEST_CASE(shm_comm_begin_alltoallv_source_order_and_tags) { - for (const int S : {2, 4, 8}) { - std::vector>> recv(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - Comm c = Comm::make_shm(&sh, r); - std::vector> send(static_cast(S)); - for (int t = 0; t < S; ++t) { - for (int j = 0; j <= r; ++j) { - send[static_cast(t)].push_back(r * 1000 + j); - } - } - const auto got = test_utils::exchange_blocks(send, c); - recv[static_cast(r)] = got; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int r = 0; r < S; ++r) { - const auto &got = recv[static_cast(r)]; - BOOST_REQUIRE_EQUAL(static_cast(got.size()), S); - for (int s = 0; s < S; ++s) { - const auto &blk = got[static_cast(s)]; - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), s + 1); // source s sent (s+1) - for (int j = 0; j <= s; ++j) { - BOOST_CHECK_EQUAL(blk[static_cast(j)], s * 1000 + j); - } - } - } - } -} - -// skip_self: the self slot is neither sent nor received; every other source arrives intact. -BOOST_AUTO_TEST_CASE(shm_comm_begin_alltoallv_skip_self) { - const int S = 4; - std::vector>> recv(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - Comm c = Comm::make_shm(&sh, r); - std::vector> send(static_cast(S)); - for (int t = 0; t < S; ++t) { - send[static_cast(t)] = {r * 10 + 1, r * 10 + 2}; - } - const auto got = test_utils::exchange_blocks(send, c, /*skip_self=*/true); - recv[static_cast(r)] = got; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int r = 0; r < S; ++r) { - for (int s = 0; s < S; ++s) { - const auto &blk = recv[static_cast(r)][static_cast(s)]; - if (s == r) { - BOOST_CHECK(blk.empty()); - } - else { - BOOST_REQUIRE_EQUAL(static_cast(blk.size()), 2); - BOOST_CHECK_EQUAL(blk[0], s * 10 + 1); - } - } - } -} - -// allreduce_sum is bit-identical on every rank and equals the fixed-order reference. -BOOST_AUTO_TEST_CASE(shm_comm_allreduce_sum_bit_identical) { - for (const int S : {2, 4, 8}) { - std::vector int_res(static_cast(S)); - std::vector dbl_res(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - int_res[static_cast(r)] = sh.allreduce_sum(r, static_cast(r) + 1); - dbl_res[static_cast(r)] = sh.allreduce_sum(r, static_cast(r) + 0.5); - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - const size_t expect_int = static_cast(S) * (static_cast(S) + 1) / 2; // sum 1..S - const double expect_dbl = static_cast(S) * static_cast(S) / 2.0; // sum (r+0.5) - for (int r = 0; r < S; ++r) { - BOOST_CHECK_EQUAL(int_res[static_cast(r)], expect_int); - BOOST_CHECK_EQUAL(dbl_res[static_cast(r)], dbl_res[0]); - BOOST_CHECK_CLOSE(dbl_res[static_cast(r)], expect_dbl, 1e-12); - } - } -} - -// allreduce_sum_inplace sums element-wise; every rank ends bit-identical to the ascending-rank-order -// reference. Lengths straddle the slice edges: shorter than S (empty slices), partial lines, many lines. -BOOST_AUTO_TEST_CASE(shm_comm_allreduce_sum_inplace_vector) { - for (const int S : {2, 4, 8}) { - for (const size_t N : - {size_t{1}, size_t{5}, size_t{8 * 2 + 3}, size_t{8} * static_cast(S) + 7, size_t{257}}) { - // Materialize every rank's input once and reduce those exact stored doubles at both sites: - // aarch64 gcc-14 -ffp-contract=fast fuses `r*0.3 + k*1.7` at one site and not the other (1 ulp). - std::vector> inputs(static_cast(S), std::vector(N)); - for (int r = 0; r < S; ++r) { - for (size_t k = 0; k < N; ++k) { - inputs[static_cast(r)][k] = static_cast(r + 1) * 0.3 + static_cast(k) * 1.7; - } - } - std::vector> res(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - std::vector v = inputs[static_cast(r)]; - sh.allreduce_sum_inplace(r, v.data(), N); - res[static_cast(r)] = v; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - std::vector ref(N); // ascending-rank-order reference over the identical stored inputs - for (size_t k = 0; k < N; ++k) { - double acc = 0.0; - for (int r = 0; r < S; ++r) { - acc += inputs[static_cast(r)][k]; - } - ref[k] = acc; - } - for (int r = 0; r < S; ++r) { - BOOST_REQUIRE_EQUAL(res[static_cast(r)].size(), N); - for (size_t k = 0; k < N; ++k) { - BOOST_CHECK_EQUAL(res[static_cast(r)][k], ref[k]); - } - } - } - } -} - -// post_flat_alltoallv over caller-owned flat buffers (the Evolution/Pare replay path): each rank sends -// its own id, so every target must receive [0,1,..,S-1] in source order. -BOOST_AUTO_TEST_CASE(shm_comm_post_flat_alltoallv_flat_buffers) { - const int S = 4; - std::vector> recv(static_cast(S)); - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - Comm c = Comm::make_shm(&sh, r); - std::vector send(static_cast(S), r); - std::vector sc(static_cast(S), 1), sd(static_cast(S)); - for (int i = 0; i < S; ++i) { - sd[static_cast(i)] = i; - } - std::vector rc(static_cast(S), 1), rd(static_cast(S)); - for (int i = 0; i < S; ++i) { - rd[static_cast(i)] = i; - } - std::vector out(static_cast(S), -1); - auto ticket = monoprop::mpi::post_flat_alltoallv({.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = out.data(), - .recv_counts = rc.data(), - .recv_displs = rd.data()}, - S, - c); - ticket.wait(); - recv[static_cast(r)] = out; - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - for (int r = 0; r < S; ++r) { - for (int s = 0; s < S; ++s) { - BOOST_CHECK_EQUAL(recv[static_cast(r)][static_cast(s)], s); - } - } -} - -// alltoallv_resolve resolves recv_counts (the transpose) AND moves the payload in one 2-sync round, -// sizing recv itself — what begin_alltoallv takes for unknown-layout Shm rounds. -BOOST_AUTO_TEST_CASE(shm_comm_alltoallv_resolve_fused) { - for (const int S : {2, 4, 8}) { - const int rounds = 25; - std::atomic failures{0}; - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - std::vector recv; // reused across rounds (HWM) - std::vector rc(static_cast(S)), rd(static_cast(S)); - for (int round = 0; round < rounds; ++round) { - // Every 5th round is big: pushes the recv HWM up so later rounds run over a larger buffer. - const auto len_of = [&](int src) { - return (round % 5 == 4) ? (src % 3 + 1) * 11 : (src + round) % 4; // includes 0 - }; - const int my_len = len_of(r); - std::vector send; - std::vector sc(static_cast(S)), sd(static_cast(S)); - int off = 0; - for (int t = 0; t < S; ++t) { - sc[static_cast(t)] = my_len; - sd[static_cast(t)] = off; - for (int j = 0; j < my_len; ++j) { - send.push_back(r * 100000 + round * 100 + j); - } - off += my_len; - } - sh.alltoallv_resolve(r, - {.send = send.data(), - .send_counts = sc.data(), - .send_displs = sd.data(), - .recv = recv, - .recv_counts = rc.data(), - .recv_displs = rd.data()}); - int expected_total = 0; - for (int s = 0; s < S; ++s) { - const int len = len_of(s); // s sends me exactly len(s) (same block to every target) - if (rc[static_cast(s)] != len) { - failures.fetch_add(1); - } - if (rd[static_cast(s)] != expected_total) { - failures.fetch_add(1); - } - for (int j = 0; j < len; ++j) { - if (recv[static_cast(expected_total + j)] != s * 100000 + round * 100 + j) { - failures.fetch_add(1); - } - } - expected_total += len; - } - if (static_cast(recv.size()) != expected_total) { - failures.fetch_add(1); - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_EQUAL(failures.load(), 0); - } -} - -// Repeated collectives reuse one ShmComm across many rounds without drift (barrier generation reuse). -BOOST_AUTO_TEST_CASE(shm_comm_repeated_collectives) { - const int S = 8; - std::atomic failures{0}; - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - for (int round = 0; round < 200; ++round) { - const size_t got = sh.allreduce_sum(r, static_cast(round)); - if (got != static_cast(round) * static_cast(S)) { - failures.fetch_add(1); - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_EQUAL(failures.load(), 0); -} - -// Oversubscribed: the barrier's bounded spin must fall back to yielding or spinners starve the completer -// of a core. The test completing at all proves liveness. -BOOST_AUTO_TEST_CASE(shm_comm_oversubscribed_repeated_collectives) { - const unsigned hw = std::max(1u, std::thread::hardware_concurrency()); - const int S = static_cast(std::min(64u, std::max(8u, 2 * hw))); - std::atomic failures{0}; - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - for (int round = 0; round < 50; ++round) { - const size_t got = sh.allreduce_sum(r, static_cast(round)); - if (got != static_cast(round) * static_cast(S)) { - failures.fetch_add(1); - } - } - }); - for (const auto &e : errs) { - BOOST_CHECK(e == nullptr); - } - BOOST_CHECK_EQUAL(failures.load(), 0); -} - -// Poison: if one participant unwinds instead of arriving, peers waiting in a barrier must throw -// ShmCommPoisoned rather than hang. The test completing at all proves no deadlock. -BOOST_AUTO_TEST_CASE(shm_comm_poison_releases_waiters) { - for (const int S : {2, 4, 8}) { - auto errs = run_shm(S, [&](ShmComm &sh, int r) { - if (r == 0) { - sh.poison(); // simulate an engine exception on rank 0 before it reaches the collective - return; - } - // Peers enter a collective; only S-1 arrive, so they must observe poison and throw. - std::vector send(static_cast(S), 1), got(static_cast(S)); - sh.alltoall_counts(r, send.data(), got.data()); - }); - BOOST_CHECK(errs[0] == nullptr); - for (int r = 1; r < S; ++r) { - BOOST_REQUIRE(errs[static_cast(r)] != nullptr); - BOOST_CHECK_THROW(std::rethrow_exception(errs[static_cast(r)]), ShmCommPoisoned); - } - } -} diff --git a/cpp/tests/sparse_fold_tests.cpp b/cpp/tests/sparse_fold_tests.cpp index cd84fe042..a8773a303 100644 --- a/cpp/tests/sparse_fold_tests.cpp +++ b/cpp/tests/sparse_fold_tests.cpp @@ -342,3 +342,14 @@ BOOST_AUTO_TEST_CASE(sparse_fold_replay_kernels_match_oracle) { } } } + +// The word sweep and the posting fold use one block per thread (every owner folds at T = 96, so a second block is +// resident on each worker): the posting fold gets it all-zero even straight after a word sweep left it dirty. +BOOST_AUTO_TEST_CASE(sparse_fold_word_and_posting_kernels_share_one_block) { + std::vector &swept = column_block_scratch(); + std::ranges::fill(swept, ~uint64_t{0}); // what a word sweep leaves behind + const uint64_t *const storage = swept.data(); + const ZeroBlockScratch &zs = fold_zero_scratch(); + BOOST_TEST(zs.words.data() == storage); + BOOST_TEST(zero_scratch_is_clean(zs)); +} diff --git a/cpp/tests/sparse_query_tests.cpp b/cpp/tests/sparse_query_tests.cpp index 0843d2e56..0a1821ef9 100644 --- a/cpp/tests/sparse_query_tests.cpp +++ b/cpp/tests/sparse_query_tests.cpp @@ -17,13 +17,19 @@ #include #include +#include +#include #include #include +#include +#include #include +#include #include #include "monoprop/detail/evolution/layer_build/Common.h" #include "monoprop/detail/evolution/layer_build/QueryWire.h" +#include "monoprop/detail/evolution/layer_build/Scan.h" #include "dense_query_reference.h" @@ -571,3 +577,337 @@ BOOST_AUTO_TEST_CASE(sparse_record_round_trips_every_reachable_k_and_gap_width) BOOST_TEST(differential<128>(pos, 1) == 1U); } } + +// --- ordered merge of per-range scan results ------------------------------------------------------------------ + +namespace { + +constexpr size_t kMergeModes = 32; +using MergeResult = FusedScanResult; + +auto reset_streams(MergeResult &r, mpi::SlotWindow w, bool capture) -> void { + r.leader_queries.reset(w); + r.leader_src.reset(w); + r.follower_queries.reset(w); + r.follower_src.reset(w); + if (capture) { + r.leader_val.reset(w); + r.follower_val.reset(w); + } +} + +auto same_stage(const SelfQueryStage &a, const SelfQueryStage &b) -> bool { + if (a.size() != b.size() || a.positions() != b.positions()) { + return false; + } + for (size_t q = 0; q < a.size(); ++q) { + if (a.pos_off[q] != b.pos_off[q] || a.k_of[q] != b.k_of[q] || a.phase_of[q] != b.phase_of[q]) { + return false; + } + } + return std::equal(a.pos_flat.begin(), + a.pos_flat.begin() + static_cast(a.positions()), + b.pos_flat.begin()); +} + +auto bits_equal(const std::vector &a, const std::vector &b) -> bool { + return a.size() == b.size() && (a.empty() || std::memcmp(a.data(), b.data(), a.size() * sizeof(double)) == 0); +} + +} // namespace + +// Hand-built per-range pieces with variable-length wire records (escaped k included), a window at a +// non-zero base, self-owned queries on both stages, an empty middle piece and value capture on and off. +// The merge must reproduce the single stream a serial scan would have produced: each destination's +// records, sources and values in range order, each self stage rebased in range order, cosine sets in +// range order, and every piece's buffers released afterwards. +BOOST_AUTO_TEST_CASE(openmp_scan_merge_keeps_source_order_windows_and_self_stages) { + using QW = QueryWire; + const mpi::SlotWindow window{.base = 2, .count = 3}; + const size_t me = 3; + for (const bool capture : {false, true}) { + std::mt19937_64 rng(0x3E26EULL); + std::vector pieces(4); + MergeResult expect; + reset_streams(expect, window, capture); + size_t src = 0; + for (size_t p = 0; p < pieces.size(); ++p) { + auto &piece = pieces[p]; + reset_streams(piece, window, capture); + CosineWordBuilder cos; + if (p == 1) { + piece.cos_blocks.push_back(cos.finish()); // an empty range still reports its (empty) set + continue; + } + for (size_t q = 0; q < 40 + (p * 7); ++q) { + const size_t dest = window.slot(mpi::WindowIndex{rng() % window.count}); + const bool follower = (rng() & 1U) != 0; + const std::array widths{0, 1, 5, 31, 40, 64}; + const size_t k = widths[rng() % widths.size()]; + std::vector pos(64); + std::iota(pos.begin(), pos.end(), uint16_t{0}); + std::shuffle(pos.begin(), pos.end(), rng); + pos.resize(k); + std::ranges::sort(pos); + const int phase = static_cast(rng() % 3) - 1; + const double v = std::ldexp(static_cast(rng() % 1000) - 500.0, -7); + for (auto *r : {&piece, &expect}) { + if (dest == me) { + (follower ? r->follower_self : r->leader_self).push(pos, phase); + } + else { + QW::push((follower ? r->follower_queries : r->leader_queries).at_slot(dest), pos, phase); + } + (follower ? r->follower_src : r->leader_src).at_slot(dest).push_back(src); + if (capture) { + (follower ? r->follower_val : r->leader_val).at_slot(dest).push_back(v); + } + } + cos.push_index((p * 1024 * 64) + (q * 3)); + ++src; + } + piece.cos_blocks.push_back(cos.finish()); + } + std::vector expect_cos; + for (const auto &piece : pieces) { + expect_cos.push_back(piece.cos_blocks.front()); + } + + const auto merged = merge_scan_ranges(pieces, window, me, capture); + + BOOST_TEST_CONTEXT("capture=" << capture) { + BOOST_TEST((merged.leader_queries.window() == window)); + BOOST_TEST((merged.follower_src.window() == window)); + BOOST_TEST(merged.leader_val.size() == (capture ? window.count : 0U)); + BOOST_TEST(merged.follower_val.size() == (capture ? window.count : 0U)); + for (const auto wi : window.indices()) { + BOOST_TEST((merged.leader_queries[wi] == expect.leader_queries[wi])); + BOOST_TEST((merged.follower_queries[wi] == expect.follower_queries[wi])); + BOOST_TEST((merged.leader_src[wi] == expect.leader_src[wi])); + BOOST_TEST((merged.follower_src[wi] == expect.follower_src[wi])); + if (capture) { + BOOST_TEST(bits_equal(merged.leader_val[wi], expect.leader_val[wi])); + BOOST_TEST(bits_equal(merged.follower_val[wi], expect.follower_val[wi])); + } + } + // Self wire stays empty; the stages carry the self slot's queries in order. + BOOST_TEST(merged.leader_queries.at_slot(me).empty()); + BOOST_TEST(merged.follower_queries.at_slot(me).empty()); + BOOST_TEST(expect.leader_self.size() > 0U); + BOOST_TEST(expect.follower_self.size() > 0U); + BOOST_TEST(same_stage(merged.leader_self, expect.leader_self)); + BOOST_TEST(same_stage(merged.follower_self, expect.follower_self)); + BOOST_TEST(merged.leader_self.size() == merged.leader_src.at_slot(me).size()); + BOOST_TEST(merged.follower_self.size() == merged.follower_src.at_slot(me).size()); + BOOST_TEST_REQUIRE(merged.cos_blocks.size() == expect_cos.size()); + for (size_t p = 0; p < expect_cos.size(); ++p) { + BOOST_TEST((merged.cos_blocks[p].blocks == expect_cos[p].blocks)); + BOOST_TEST(merged.cos_blocks[p].total_count == expect_cos[p].total_count); + } + // Drained pieces hold no allocation: clear() alone would have kept their capacity. + size_t retained = 0; + for (const auto &piece : pieces) { + for (const auto wi : window.indices()) { + retained += piece.leader_queries[wi].capacity() + piece.follower_queries[wi].capacity(); + retained += piece.leader_src[wi].capacity() + piece.follower_src[wi].capacity(); + if (capture) { + retained += piece.leader_val[wi].capacity() + piece.follower_val[wi].capacity(); + } + } + for (const auto *stage : {&piece.leader_self, &piece.follower_self}) { + retained += stage->pos_flat.capacity() + stage->pos_off.capacity() + stage->k_of.capacity() + + stage->phase_of.capacity(); + } + retained += piece.cos_blocks.capacity(); + } + BOOST_TEST(retained == 0U); + } + } +} + +// The merge moves a buffer instead of copying it when one piece holds a whole stream, or when the first +// nonempty piece already has room for the total; otherwise it reserves the exact total once. +BOOST_AUTO_TEST_CASE(openmp_scan_merge_moves_the_first_useful_buffer) { + const mpi::SlotWindow window{.base = 0, .count = 2}; + const size_t me = 0; + std::vector pieces(3); + for (auto &piece : pieces) { + reset_streams(piece, window, false); + } + // Slot 0 sources: only piece 1 has any, so the merged stream is that very buffer. + auto &only = pieces[1].leader_src.at_slot(0); + only.reserve(100); + only.assign({4, 5, 6}); + const auto *only_data = only.data(); + // Slot 1 sources: piece 0 has room for all of them, so its buffer is kept and piece 2 appended. + auto &roomy = pieces[0].follower_src.at_slot(1); + roomy.reserve(64); + roomy.assign({1, 2}); + const auto *roomy_data = roomy.data(); + pieces[2].follower_src.at_slot(1).assign({9}); + // Slot 1 leader sources: no piece has room, so the result is one exact reservation. + pieces[0].leader_src.at_slot(1).assign({1, 2}); + pieces[0].leader_src.at_slot(1).shrink_to_fit(); + pieces[2].leader_src.at_slot(1).assign({8, 9, 10}); + // Self stage, aligned with slot 0's sources: only piece 1 has queries, so its stage is moved whole. + const std::vector pos{3, 9}; + for (int q = 0; q < 3; ++q) { + pieces[1].leader_self.push(pos, 1); + } + pieces[0].leader_self.reserve(8, 4); // capacity without content is not a query + const auto *stage_data = pieces[1].leader_self.pos_flat.data(); + + auto merged = merge_scan_ranges(pieces, window, me, false); + BOOST_TEST(merged.leader_src.at_slot(0).data() == only_data); + BOOST_TEST(merged.follower_src.at_slot(1).data() == roomy_data); + BOOST_TEST((merged.follower_src.at_slot(1) == std::vector{1, 2, 9})); + BOOST_TEST((merged.leader_src.at_slot(1) == std::vector{1, 2, 8, 9, 10})); + BOOST_TEST(merged.leader_src.at_slot(1).capacity() == 5U); + BOOST_TEST(merged.leader_self.size() == 3U); + BOOST_TEST(merged.leader_self.pos_flat.data() == stage_data); +} + +// --- checked record boundaries and decoded-position validation (resolver input is untrusted) --------------- + +namespace { + +// A single hand-packed header word: phase field, k field (and the wide-k field when escaped), gw field. +template +auto header_word(uint64_t phase_field, uint64_t k_field, std::optional long_k, uint64_t gw) -> size_t { + using QW = QueryWire; + uint64_t w = phase_field | (k_field << QW::kPhaseBits); + size_t at = QW::kPhaseBits + QW::kKBits; + if (long_k) { + w |= *long_k << at; + at += QW::kLongKBits; + } + w |= gw << at; + return static_cast(w); +} + +template +auto expect_malformed(const VecZ &buf, QueryForm form, size_t off, const char *label) -> void { + BOOST_TEST_CONTEXT(label) { + BOOST_CHECK_THROW((void)QueryWire::checked_extent(buf, form, off), MalformedQueryStream); + } +} + +} // namespace + +// Every record push() can produce passes the check, which reports exactly the words the decoder consumes, plus +// the fused value word. Covers the whole (k, gw) surface at 128 modes, the escape boundary and full width. +BOOST_AUTO_TEST_CASE(sparse_record_checked_extent_accepts_every_canonical_record) { + const auto check = [](std::integral_constant, const std::vector &pos, int phase) { + using QW = QueryWire; + for (const auto form : {QueryForm::Plain, QueryForm::Fused}) { + VecZ buf; + const size_t words = QW::push(buf, pos, phase); + if (form == QueryForm::Fused) { + QW::push_value(buf, -0.25); + } + const auto ext = QW::checked_extent(buf, form, 0); + BOOST_TEST(ext.k == pos.size()); + BOOST_TEST(ext.words == buf.size()); + BOOST_TEST(ext.words == words + (form == QueryForm::Fused ? 1U : 0U)); + std::vector out(pos.size()); + const auto d = QW::read_query(buf, form, 0, out); + BOOST_TEST(d.next == ext.words); + BOOST_TEST(out == pos, boost::test_tools::per_element()); + } + }; + size_t cells = 0; + for (size_t k = 0; k <= 40U; ++k) { + for (size_t gw = 0; gw <= QueryWire<128>::kPosBits; ++gw) { + const auto pos = gap_shaped(k, gw, QueryWire<128>::kBits); + if (pos.size() != k || (k < 2U && gw > 0U)) { + continue; + } + check(std::integral_constant{}, pos, static_cast(k % 3U) - 1); + ++cells; + } + } + BOOST_TEST(cells == 353U); + for (const size_t k : {size_t{30}, size_t{31}, size_t{32}, size_t{200}, size_t{2048}}) { + check(std::integral_constant{}, strided(k, 0, k == 2048 ? 1 : 3, 2048), 1); + } + check(std::integral_constant{}, std::vector{0, 257, 499}, -1); +} + +// The plan's fixture: one Plain word at 128 modes with phase +1, an escaped k = 6, gw = 8 and no payload. The +// decoder would consume 20 header bits + 8 + 5*8 = 68 bits (two words), but the canonical k = 6 header is 11 +// bits, so the unchecked size calculation claims 59 bits, one word, and the decode would read past the buffer. +BOOST_AUTO_TEST_CASE(sparse_record_checked_extent_rejects_the_noncanonical_escape) { + using QW = QueryWire<128>; + const VecZ buf{size_t{0x8037e}}; + const auto h = QW::header_at(buf, 0); + BOOST_TEST(h.phase == 1); + BOOST_TEST(h.k == 6U); + BOOST_TEST(h.gw == 8U); + BOOST_TEST(h.bits == 20U); + BOOST_TEST(QW::header_bits_for(h.k) == 11U); + BOOST_TEST(QW::words_at(buf, 0) == 1U); // the undercount the check exists to catch + expect_malformed<128>(buf, QueryForm::Plain, 0, "0x8037e alone"); + // Padding that makes the decoder's read fit does not make the header canonical. + expect_malformed<128>(VecZ{size_t{0x8037e}, 0, 0}, QueryForm::Plain, 0, "0x8037e padded"); + expect_malformed<128>(VecZ{size_t{0x8037e}, 0}, QueryForm::Fused, 0, "0x8037e fused"); +} + +BOOST_AUTO_TEST_CASE(sparse_record_checked_extent_rejects_malformed_headers_and_boundaries) { + using QW8 = QueryWire<8>; // kBits 16, kPosBits 4, kLongKBits 5 + expect_malformed<8>(VecZ{}, QueryForm::Plain, 0, "offset at the end"); + expect_malformed<8>(VecZ{header_word<8>(3, 0, std::nullopt, 0)}, QueryForm::Plain, 0, "phase field 3"); + expect_malformed<8>(VecZ{header_word<8>(1, 31, 20, 0)}, QueryForm::Plain, 0, "escaped k above kMaxPositions"); + expect_malformed<8>(VecZ{header_word<8>(1, 31, 3, 0)}, QueryForm::Plain, 0, "escaped k below kKEscape"); + expect_malformed<8>(VecZ{header_word<8>(1, 2, std::nullopt, QW8::kPosBits + 1)}, QueryForm::Plain, 0, "gw"); + // A fused record without its value word, and a two-word record cut to one. + VecZ fused; + QW8::push(fused, std::vector{1, 4}, 1); + expect_malformed<8>(fused, QueryForm::Fused, 0, "fused value word missing"); + using QW = QueryWire<1024>; + VecZ wide; + const size_t words = QW::push(wide, strided(40, 0, 50, 2048), -1); + BOOST_REQUIRE(words >= 2U); + wide.pop_back(); + expect_malformed<1024>(wide, QueryForm::Plain, 0, "truncated multiword record"); + // A valid record followed by a lone partial word: the second record's header claims more than remains. + VecZ tail; + QW::push(tail, std::vector{3}, 1); + const auto first = QW::checked_extent(tail, QueryForm::Plain, 0); + tail.push_back(wide[0]); + expect_malformed<1024>(tail, QueryForm::Plain, first.words, "trailing partial record"); +} + +// read_positions checks the cumulative size_t position before narrowing it to the output type: each must stay +// below 2*NumModes, which also bounds the next addition and keeps the positions strictly ascending. +BOOST_AUTO_TEST_CASE(sparse_record_decode_rejects_positions_outside_the_width) { + // 250 modes: a 9-bit first position reaches 511 although only 0..499 exist. + { + using QW = QueryWire<250>; + VecZ buf; + QW::push(buf, std::vector{505}, 1); + std::vector out(1); + BOOST_CHECK_THROW((void)QW::read_positions(buf, 0, out), MalformedQueryStream); + VecZ ok; + QW::push(ok, std::vector{499}, 1); + BOOST_CHECK_NO_THROW((void)QW::read_positions(ok, 0, out)); + } + // 32 modes: a gap walks past position 63. + { + using QW = QueryWire<32>; + VecZ buf; + QW::push(buf, std::vector{10, 70}, -1); + std::vector out(QW::kBits); // room for any k the header could claim + BOOST_CHECK_THROW((void)QW::read_positions(buf, 0, out), MalformedQueryStream); + } + // 128 modes into the store's uint8: 257 would narrow to 1, and 300 to 44, both "valid" after narrowing. + { + using QW = QueryWire<128>; + for (const auto &pos : {std::vector{1, 257}, std::vector{100, 200, 300}}) { + VecZ buf; + QW::push(buf, pos, 1); + BOOST_REQUIRE_NO_THROW((void)QW::checked_extent(buf, QueryForm::Plain, 0)); + std::vector out(QW::kBits); + BOOST_CHECK_THROW((void)QW::read_query(buf, QueryForm::Plain, 0, out), MalformedQueryStream); + } + } +} diff --git a/cpp/tests/sparse_resolve_tests.cpp b/cpp/tests/sparse_resolve_tests.cpp index d88008c0c..5208b5cef 100644 --- a/cpp/tests/sparse_resolve_tests.cpp +++ b/cpp/tests/sparse_resolve_tests.cpp @@ -18,15 +18,24 @@ #include #include +#include #include #include +#include #include #include #include #include +#include +#include +#include #include +#include "KernelTestSupport.h" + +#include "monoprop/algebra/MajoranaAlgebra.h" #include "monoprop/core/Monomial.h" +#include "monoprop/detail/evolution/layer_build/Engine.h" #include "monoprop/detail/evolution/layer_build/QueryWire.h" #include "monoprop/detail/evolution/layer_build/Resolve.h" #include "monoprop/detail/mpi/Comm.h" @@ -408,6 +417,1190 @@ BOOST_AUTO_TEST_CASE(sparse_resolve_finds_dense_inserted_keys) { BOOST_TEST(genuinely_absent > 0); } +/* ── Task 6: checked, threaded incoming resolution and bounded self probing ─────────────────────── */ + +namespace { + +using detail::KernelRange; +using kernel_test::AccumulatingObserver; +using kernel_test::PhaseLog; + +template +auto key_of(const Monomial &m) -> std::vector { + std::vector key; + for (size_t w = 0; w < Monomial::num_words(); ++w) { + key.push_back(m.word(w)); + } + return key; +} + +template +auto rows_of(const detail::MPOperator &op) -> std::vector> { + std::vector> rows; + for (size_t i = 0; i < op.store->size(); ++i) { + rows.push_back(op.store->row(i)); + } + return rows; +} + +// Every array exactly, including its length: the probe allocates each once, at the query count. +template +auto same_probe(const detail::IncomingProbe &a, const detail::IncomingProbe &b) -> bool { + const auto eq = [](const auto &x, const auto &y) { return std::ranges::equal(x, y); }; + return a.window == b.window && a.base == b.base && a.nq_total == b.nq_total && eq(a.goff, b.goff) + && eq(a.sender_wi, b.sender_wi) && eq(a.phase_of, b.phase_of) && eq(a.off_of, b.off_of) + && eq(a.idx_of, b.idx_of) && eq(a.miss_g, b.miss_g) && eq(a.pos_flat, b.pos_flat) && eq(a.pos_off, b.pos_off) + && eq(a.k_of, b.k_of) && eq(a.hash_of, b.hash_of); +} + +template +auto exact_sizes(const detail::IncomingProbe &p, size_t positions) -> bool { + const size_t n = p.nq_total; + return p.goff.size() == p.window.count + 1 && p.sender_wi.size() == n && p.phase_of.size() == n + && p.off_of.size() == n && p.idx_of.size() == n && p.pos_off.size() == n && p.k_of.size() == n + && p.hash_of.size() == n && p.pos_flat.size() == positions && p.miss_g.capacity() == p.miss_g.size(); +} + +// The rows a column holds, whichever representation it is in. +template +auto column_rows(const detail::InvertedIndex &index, size_t c) -> std::vector { + std::vector rows; + if (index.column_is_dense(c)) { + const uint64_t *w = index.dense_column_data(c); + for (size_t r = 0; r < index.rows(); ++r) { + if (((w[r >> 6] >> (r & 63U)) & 1U) != 0U) { + rows.push_back(r); + } + } + } + else { + for (const auto r : index.sparse_column_rows(c)) { + rows.push_back(r); + } + } + return rows; +} + +// An index kept up to date by reindex_after_growth against one built from scratch over the same rows. +template +auto same_inverted_index(const detail::MPOperator &grown, const detail::MPOperator &fresh) -> bool { + const auto &a = grown.inverted_index(); + const auto &b = fresh.inverted_index(); + if (a.rows() != b.rows()) { + return false; + } + for (size_t c = 0; c < 2 * N; ++c) { + if (column_rows(a, c) != column_rows(b, c)) { + return false; + } + } + return true; +} + +// Distinct keys for one resolution: `hits` drawn from the seeded store, the rest fresh, in shuffled order. +template +struct QueryPlan { + std::vector> seeds; + std::vector>> per_sender; + size_t hits = 0; + size_t misses = 0; +}; + +enum class HitMix { none, all, mixed }; + +template +auto plan_queries(uint64_t seed, const std::vector &per_sender, HitMix mix, size_t n_seed = 300) + -> QueryPlan { + std::mt19937_64 rng(seed); + QueryPlan plan; + size_t total = 0; + for (const auto n : per_sender) { + total += n; + } + // Enough stored terms for every planned hit: all of them, or one query in three. + plan.seeds = draw_distinct(rng, std::max(n_seed, mix == HitMix::all ? total : (total / 3) + 1)); + std::set> used; + for (const auto &m : plan.seeds) { + used.insert(key_of(m)); + } + size_t next_seed = 0; + size_t i = 0; + plan.per_sender.resize(per_sender.size()); + for (size_t s = 0; s < per_sender.size(); ++s) { + for (size_t q = 0; q < per_sender[s]; ++q, ++i) { + const bool hit = mix == HitMix::all || (mix == HitMix::mixed && i % 3 == 0); + if (hit) { + BOOST_REQUIRE(next_seed < plan.seeds.size()); + plan.per_sender[s].push_back(plan.seeds[next_seed++]); + ++plan.hits; + continue; + } + for (;;) { + const auto m = draw_distinct(rng, 1)[0]; + if (used.insert(key_of(m)).second) { + plan.per_sender[s].push_back(m); + ++plan.misses; + break; + } + } + } + } + return plan; +} + +// Serial reference (budget 1) against budgets 2, 3, 4 and 8 on identical fresh stores: the whole probe, the +// published rows in row-ID order, every row findable at its ID, and -- with a prebuilt inverted index -- the +// index maintained by reindex_after_growth equal to one built from scratch. +template +auto check_incoming_budgets(const char *label, + const QueryPlan &plan, + mpi::SlotWindow window, + bool fused, + bool prebuilt_index) -> void { + BOOST_TEST_CONTEXT(label << " N=" << N << " base=" << window.base << " fused=" << fused + << " prebuilt=" << prebuilt_index) { + const auto incoming = serialize(plan.per_sender, fused, window); + const auto form = fused ? detail::QueryForm::Fused : detail::QueryForm::Plain; + std::vector> expect; + std::vector expect_sender; + std::vector expect_phase; + std::vector expect_value; + for (size_t s = 0; s < plan.per_sender.size(); ++s) { + for (size_t q = 0; q < plan.per_sender[s].size(); ++q) { + expect.push_back(plan.per_sender[s][q]); + expect_sender.push_back(s); + expect_phase.push_back(((q % 2) == 0) ? 1 : -1); + expect_value.push_back(0.5 + static_cast(q)); + } + } + size_t positions = 0; + for (const auto &m : expect) { + positions += m.count(); + } + + auto serial_op = make_op(plan.seeds); + if (prebuilt_index) { + (void)serial_op.inverted_index(); + } + const auto serial = detail::probe_incoming_queries(incoming, serial_op, form, {.threads = 1}); + BOOST_TEST(exact_sizes(serial, positions)); + BOOST_REQUIRE_EQUAL(serial.nq_total, expect.size()); + BOOST_TEST(serial.base == plan.seeds.size()); + size_t hits = 0; + std::vector expect_misses; + for (size_t g = 0; g < serial.nq_total; ++g) { + BOOST_TEST((serial.mono_at(g) == expect[g])); + BOOST_TEST(serial.sender_index(g).value == expect_sender[g]); + BOOST_TEST(serial.window.slot(serial.sender_index(g)) == window.base + expect_sender[g]); + BOOST_TEST(serial.phase_of[g] == expect_phase[g]); + BOOST_TEST(serial.hash_of[g] == detail::OperatorIndex::fold_hash_positions(serial.positions_at(g))); + if (fused) { + const double v = detail::QueryWire::value_at(incoming[serial.sender_index(g)], serial.off_of[g]); + BOOST_TEST(std::bit_cast(v) == std::bit_cast(expect_value[g])); + } + const bool hit = serial.idx_of[g] < serial.base; + if (hit) { + ++hits; + BOOST_TEST((serial_op.store->row(serial.idx_of[g]) == expect[g])); + } + else { + BOOST_TEST(serial.idx_of[g] == serial.base + expect_misses.size()); + expect_misses.push_back(g); + } + } + BOOST_TEST(hits == plan.hits); + BOOST_TEST(std::ranges::equal(serial.miss_g, expect_misses)); + detail::insert_incoming_misses(serial_op, serial, {.threads = 1}); + const auto serial_rows = rows_of(serial_op); + BOOST_TEST(serial_rows.size() == plan.seeds.size() + plan.misses); + for (size_t j = 0; j < expect_misses.size(); ++j) { + BOOST_TEST((serial_rows[serial.base + j] == expect[expect_misses[j]])); + } + for (size_t i = 0; i < serial_rows.size(); ++i) { + const auto found = serial_op.store->find(serial_rows[i]); + BOOST_TEST((found.has_value() && *found == i)); + } + if (prebuilt_index) { + BOOST_TEST(same_inverted_index(serial_op, make_op(serial_rows))); + } + + for (const int threads : {2, 3, 4, 8}) { + auto op = make_op(plan.seeds); + if (prebuilt_index) { + (void)op.inverted_index(); + } + const auto pr = detail::probe_incoming_queries(incoming, op, form, {.threads = threads}); + BOOST_TEST(same_probe(serial, pr), "budget " << threads); + detail::insert_incoming_misses(op, pr, {.threads = threads}); + BOOST_TEST((rows_of(op) == serial_rows), "budget " << threads); + if (prebuilt_index) { + BOOST_TEST(same_inverted_index(op, make_op(serial_rows))); + } + } + } +} + +} // namespace + +// The plan's fixture: a window at base 4, senders 4 and 6 (5 is empty), predicted and published IDs. Repeated +// at base zero. This preserves behavior that existed before Task 6 and may pass before it. +BOOST_AUTO_TEST_CASE(openmp_incoming_miss_order) { + using namespace monoprop; + const auto a = indices_to_bitset<8>({0, 1}); + const auto b = indices_to_bitset<8>({2, 3}); + const auto c = indices_to_bitset<8>({4, 5}); + for (const size_t window_base : {size_t{4}, size_t{0}}) { + for (const auto threads : {1, 3}) { + auto op = make_op<8>(std::vector>{a}); + const mpi::SlotWindow window{.base = window_base, .count = 3}; + mpi::WindowVec incoming(window); + detail::QueryWire<8>::push(incoming.at_slot(window_base), positions_of<8>(b), +1); + detail::QueryWire<8>::push(incoming.at_slot(window_base + 2), positions_of<8>(a), -1); + detail::QueryWire<8>::push(incoming.at_slot(window_base + 2), positions_of<8>(c), +1); + const auto pr = + detail::probe_incoming_queries<8>(incoming, op, detail::QueryForm::Plain, {.threads = threads}); + BOOST_TEST(pr.window.slot(pr.sender_index(0)) == window_base); + BOOST_TEST(pr.window.slot(pr.sender_index(1)) == window_base + 2); + const std::vector offsets{0, 1, 1, 3}, ids{1, 0, 2}, misses{0, 2}; + BOOST_TEST(pr.base == 1U); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.goff.begin(), pr.goff.end(), offsets.begin(), offsets.end()); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.idx_of.begin(), pr.idx_of.end(), ids.begin(), ids.end()); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.miss_g.begin(), pr.miss_g.end(), misses.begin(), misses.end()); + detail::insert_incoming_misses(op, pr); + BOOST_TEST(op.size() == 3U); + BOOST_REQUIRE(op.store->find(a).has_value()); + BOOST_REQUIRE(op.store->find(b).has_value()); + BOOST_REQUIRE(op.store->find(c).has_value()); + BOOST_TEST(op.store->find(a).value() == 0U); + BOOST_TEST(op.store->find(b).value() == 1U); + BOOST_TEST(op.store->find(c).value() == 2U); + } + } +} + +// miss_g holds query ordinals, which are not TermIndex values and may exceed their range. +BOOST_AUTO_TEST_CASE(openmp_incoming_miss_ordinals_are_size_t) { + static_assert(std::is_same_v::miss_g), std::vector>); + BOOST_TEST(true); +} + +// Serial against threaded over the input shapes the resolver meets: empty, all-hit, all-miss and mixed, block +// boundaries 255/256/257 and larger partial blocks, empty intervening senders, window bases zero and non-zero, +// narrow and wide positions with inline and spilled rows and multiword/escaped records, Plain and Fused codecs, +// and a prebuilt inverted index maintained across the publication. +BOOST_AUTO_TEST_CASE(openmp_incoming_probe_threaded_matches_serial) { + const mpi::SlotWindow base0{.base = 0, .count = 1}; + check_incoming_budgets<32>("empty", plan_queries<32>(1, {0}, HitMix::none), base0, false, false); + check_incoming_budgets<32>("empty senders", + plan_queries<32>(2, {0, 0, 0}, HitMix::none), + {.base = 3, .count = 3}, + true, + false); + check_incoming_budgets<32>("255 all-hit", plan_queries<32>(3, {255}, HitMix::all), base0, false, true); + check_incoming_budgets<32>("256 all-miss", plan_queries<32>(4, {256}, HitMix::none), base0, true, true); + check_incoming_budgets<32>("257 mixed", plan_queries<32>(5, {257}, HitMix::mixed), base0, false, false); + check_incoming_budgets<250>("empty intervening senders", + plan_queries<250>(6, {0, 300, 0, 413}, HitMix::mixed), + {.base = 5, .count = 4}, + false, + true); + check_incoming_budgets<250>("fused partial blocks", + plan_queries<250>(7, {1, 600, 130}, HitMix::mixed), + {.base = 2, .count = 3}, + true, + false); + check_incoming_budgets<128>("narrow store, wide wire", + plan_queries<128>(8, {513, 0}, HitMix::mixed), + {.base = 0, .count = 2}, + true, + true); + // Escaped counts: fully paired 20-mode terms carry k >= 31 on the wire. + { + auto plan = plan_queries<1024>(9, {40, 40}, HitMix::mixed, 60); + std::mt19937_64 rng(10); + for (auto &sender : plan.per_sender) { + for (size_t q = 1; q < sender.size(); q += 2) { + auto &m = sender[q]; + if (m.count() < 31U) { + auto wide = random_paired_monomial<1024>(rng, 16 + (rng() % 20)); + if (std::ranges::none_of(plan.seeds, [&](const auto &s) { return s == wide; })) { + m = wide; + } + } + } + } + // Rebuild the hit/miss counts from the store: some hits were replaced by fresh escaped terms. + auto op = make_op<1024>(plan.seeds); + plan.hits = 0; + plan.misses = 0; + size_t escaped = 0; + for (const auto &sender : plan.per_sender) { + for (const auto &m : sender) { + (op.store->find(m).has_value() ? plan.hits : plan.misses) += 1; + escaped += static_cast(m.count() >= detail::QueryWire<1024>::kKEscape); + } + } + BOOST_TEST(escaped > 0U); + check_incoming_budgets<1024>("escaped counts", plan, {.base = 1, .count = 2}, true, false); + } +} + +// Invalid headers and boundaries are rejected on the caller before any decode worker starts: nothing is +// decoded, probed or published, and the store is untouched. +BOOST_AUTO_TEST_CASE(openmp_incoming_probe_rejects_malformed_streams_before_decode) { + using QW = detail::QueryWire<128>; + std::mt19937_64 rng(11); + const auto seeds = draw_distinct<128>(rng, 20); + // Three one-word records: read as Fused, the second becomes the first's value word and the third lacks one. + VecZ good; + for (const auto &pos : {std::vector{1, 2}, std::vector{3, 9}, std::vector{0, 5}}) { + BOOST_REQUIRE(QW::push(good, pos, 1) == 1U); + } + VecZ multiword; + std::vector spread; + for (uint16_t p = 0; p < 250; p += 7) { + spread.push_back(p); + } + const size_t words = QW::push(multiword, spread, -1); + BOOST_REQUIRE(words >= 2U); + VecZ truncated = multiword; + truncated.pop_back(); + VecZ trailing = good; + trailing.push_back(multiword[0]); + VecZ fused_missing_value = good; // Fused form: every record needs a value word, and none has one + VecZ bad_phase{size_t{3}}; + struct Bad { + const char *label; + VecZ stream; + detail::QueryForm form; + }; + const std::vector cases{{"noncanonical escape 0x8037e", VecZ{size_t{0x8037e}}, detail::QueryForm::Plain}, + {"truncated multiword record", truncated, detail::QueryForm::Plain}, + {"trailing partial record", trailing, detail::QueryForm::Plain}, + {"fused stream without value words", fused_missing_value, detail::QueryForm::Fused}, + {"phase field 3", bad_phase, detail::QueryForm::Plain}}; + for (const auto &bad : cases) { + for (const int threads : {1, 3}) { + BOOST_TEST_CONTEXT(bad.label << " budget " << threads) { + auto op = make_op<128>(seeds); + const mpi::SlotWindow window{.base = 3, .count = 3}; + mpi::WindowVec incoming(window); + if (bad.form == detail::QueryForm::Plain) { + incoming.at_slot(3) = good; // a valid sender first, an empty one, then the malformed one + } + incoming.at_slot(5) = bad.stream; + PhaseLog log; + BOOST_CHECK_THROW((void)detail::probe_incoming_queries<128>(incoming, + op, + bad.form, + {.threads = threads}, + AccumulatingObserver{&log}), + detail::MalformedQueryStream); + BOOST_TEST(log.calls.empty()); // no decode, no probe + BOOST_TEST(op.size() == seeds.size()); + } + } + } +} + +// Valid headers with invalid decoded positions fail inside a decode worker; the failure joins and is rethrown +// before the probe starts or anything is published. +BOOST_AUTO_TEST_CASE(openmp_incoming_decode_rejects_invalid_positions_before_probing) { + using QW = detail::QueryWire<250>; + const auto plan = plan_queries<250>(12, {400, 300}, HitMix::mixed); + auto incoming = serialize<250>(plan.per_sender, false, {.base = 0, .count = 2}); + QW::push(incoming[mpi::WindowIndex{1}], std::vector{7, 505}, 1); // query 700, decode block 2 + for (const int threads : {1, 3}) { + BOOST_TEST_CONTEXT("budget " << threads) { + auto op = make_op<250>(plan.seeds); + PhaseLog log; + BOOST_CHECK_THROW((void)detail::probe_incoming_queries<250>(incoming, + op, + detail::QueryForm::Plain, + {.threads = threads}, + AccumulatingObserver{&log}), + detail::MalformedQueryStream); + BOOST_TEST(log.of(KernelRange::decode).size() == 1U); + BOOST_TEST(log.of(KernelRange::decode)[0]->worker.size() == 3U); + BOOST_TEST(log.of(KernelRange::incoming_probe).empty()); + BOOST_TEST(op.size() == plan.seeds.size()); + } + } +} + +// A decode worker's own failure (not a malformed input) is joined and rethrown before probing, too. +BOOST_AUTO_TEST_CASE(openmp_incoming_decode_worker_failure_joins_before_probing) { + const auto plan = plan_queries<250>(13, {600, 424}, HitMix::mixed); + const auto incoming = serialize<250>(plan.per_sender, true, {.base = 1, .count = 2}); + auto op = make_op<250>(plan.seeds); + PhaseLog log; + BOOST_CHECK_THROW((void)detail::probe_incoming_queries<250>(incoming, + op, + detail::QueryForm::Fused, + {.threads = 4}, + AccumulatingObserver{&log, KernelRange::decode, 0, 3}), + std::runtime_error); + BOOST_TEST(log.of(KernelRange::decode).size() == 1U); + BOOST_TEST(log.of(KernelRange::decode)[0]->visits.at(3) == 1); + BOOST_TEST(log.of(KernelRange::incoming_probe).empty()); + BOOST_TEST(op.size() == plan.seeds.size()); +} + +// Decode and probe each run in their own region, one logical range per 256 queries. +BOOST_AUTO_TEST_CASE(openmp_incoming_decode_and_probe_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + const auto plan = plan_queries<250>(14, {500, 0, 524}, HitMix::mixed); + const auto incoming = serialize<250>(plan.per_sender, false, {.base = 0, .count = 3}); + auto op = make_op<250>(plan.seeds); + PhaseLog log; + const auto pr = detail::probe_incoming_queries<250>(incoming, + op, + detail::QueryForm::Plain, + {.threads = 4}, + AccumulatingObserver{&log}); + BOOST_TEST(pr.nq_total == 1024U); + BOOST_REQUIRE(log.of(KernelRange::decode).size() == 1U); + BOOST_REQUIRE(log.of(KernelRange::incoming_probe).size() == 1U); + kernel_test::check_participation(kernel_test::slots_of(*log.of(KernelRange::decode)[0]), 4, "decode"); + kernel_test::check_participation(kernel_test::slots_of(*log.of(KernelRange::incoming_probe)[0]), 4, "probe"); + // Budget 1 and a single block stay on the caller. + for (const auto &[threads, senders] : + std::vector>>{{1, {500, 0, 524}}, {4, {200, 0, 56}}}) { + const auto small = plan_queries<250>(15, senders, HitMix::mixed); + auto op_small = make_op<250>(small.seeds); + PhaseLog serial_log; + (void)detail::probe_incoming_queries<250>(serialize<250>(small.per_sender, false, {.base = 0, .count = 3}), + op_small, + detail::QueryForm::Plain, + {.threads = threads}, + AccumulatingObserver{&serial_log}); + const size_t blocks = detail::logical_ranges(senders[0] + senders[2], detail::probe_block_queries); + kernel_test::check_serial(kernel_test::slots_of(*serial_log.of(KernelRange::decode)[0]), blocks, "decode"); + kernel_test::check_serial(kernel_test::slots_of(*serial_log.of(KernelRange::incoming_probe)[0]), + blocks, + "probe"); + } +} + +// probe_frozen_positions: 256-query blocks over absolute offsets into one shared buffer give exactly the single +// find_batch_positions call, hashes included, at every budget and with more workers than blocks. +BOOST_AUTO_TEST_CASE(openmp_probe_frozen_positions_matches_one_batch) { + const auto run = [](std::integral_constant, uint64_t seed) { + using PosT = typename detail::OperatorIndex::PosT; + std::mt19937_64 rng(seed); + const auto seeds = draw_distinct(rng, 400); + const auto op = make_op(seeds); + const auto fresh = draw_distinct(rng, 700); + for (const size_t n : {size_t{0}, size_t{1}, size_t{255}, size_t{256}, size_t{257}, size_t{1000}}) { + std::vector flat(5, PosT{1}); // unreferenced prefix + std::vector off(n); + std::vector k_of(n); + // Queries laid out in reverse, so offsets are absolute and not monotonic in the query ordinal. + for (size_t q = n; q-- > 0;) { + const auto &m = (q % 2 == 0) ? seeds[q % seeds.size()] : fresh[q % fresh.size()]; + off[q] = flat.size(); + for (const auto p : positions_of(m)) { + flat.push_back(static_cast(p)); + } + k_of[q] = static_cast(m.count()); + } + std::vector want(n); + std::vector want_hash(n); + op.store->find_batch_positions(flat, off, k_of, want, want_hash); + for (const int threads : {1, 2, 3, 4, 8}) { + std::vector out(n, 17); + std::vector hash(n, 17); + detail::probe_frozen_positions(*op.store, flat, off, k_of, out, hash, {.threads = threads}); + BOOST_TEST(out == want, "N=" << N << " n=" << n << " budget " << threads); + BOOST_TEST(hash == want_hash, "N=" << N << " n=" << n << " budget " << threads); + std::vector no_hash(n, 17); + detail::probe_frozen_positions(*op.store, flat, off, k_of, no_hash, {}, {.threads = threads}); + BOOST_TEST(no_hash == want); + } + } + }; + run(std::integral_constant{}, 16); + run(std::integral_constant{}, 17); // wide positions, spilled rows +} + +// The helper's contracts are checked on the caller, before any block runs. +BOOST_AUTO_TEST_CASE(openmp_probe_frozen_positions_rejects_bad_spans) { + using PosT = detail::OperatorIndex<32>::PosT; + const auto op = make_op<32>(std::vector>{indices_to_bitset<32>({1, 2})}); + const std::vector flat{1, 2, 3, 4}; + std::vector out(2); + std::vector hash(2); + const auto expect_throw = [&](std::vector off, std::vector k, size_t n_out, size_t n_hash) { + std::vector o(n_out); + std::vector h(n_hash); + PhaseLog log; + BOOST_CHECK_THROW(detail::probe_frozen_positions<32>(*op.store, + flat, + off, + k, + o, + h, + {.threads = 2}, + AccumulatingObserver{&log}, + KernelRange::incoming_probe), + std::invalid_argument); + BOOST_TEST(log.calls.empty()); + }; + expect_throw({0, 2}, {2}, 2, 2); // counts shorter than offsets + expect_throw({0, 2}, {2, 2}, 1, 2); // output shorter + expect_throw({0, 2}, {2, 2}, 2, 1); // hash output neither empty nor full + expect_throw({0, 3}, {2, 2}, 2, 2); // span past the buffer + expect_throw({0, std::numeric_limits::max()}, {2, 1}, 2, 2); // offset + count wraps +} + +BOOST_AUTO_TEST_CASE(openmp_probe_frozen_positions_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + using PosT = detail::OperatorIndex<32>::PosT; + std::mt19937_64 rng(18); + const auto seeds = draw_distinct<32>(rng, 300); + const auto op = make_op<32>(seeds); + std::vector flat; + std::vector off; + std::vector k_of; + for (size_t q = 0; q < 1024; ++q) { + off.push_back(flat.size()); + for (const auto p : positions_of<32>(seeds[q % seeds.size()])) { + flat.push_back(static_cast(p)); + } + k_of.push_back(static_cast(seeds[q % seeds.size()].count())); + } + std::vector out(off.size()); + std::vector hash(off.size()); + PhaseLog log; + detail::probe_frozen_positions<32>(*op.store, + flat, + off, + k_of, + out, + hash, + {.threads = 4}, + AccumulatingObserver{&log}, + KernelRange::self_probe); + BOOST_REQUIRE(log.of(KernelRange::self_probe).size() == 1U); + kernel_test::check_participation(kernel_test::slots_of(log.calls[0]), 4, "frozen probe"); +} + +/* ── Phase-3 scatter: capability-gated parallel on_resolved ─────────────────────────────────────── */ + +namespace { + +// A generic sink with no parallel capability: on_resolved appends to a shared vector, so a scatter that ignored +// the missing capability would race (TSan) and scramble the order. +template +struct AppendingSink { + using Response = TermIndex; + static auto init_response() -> Response { return std::numeric_limits::max(); } + [[nodiscard]] auto incoming_form() const -> detail::QueryForm { return detail::QueryForm::Plain; } + std::vector order; + auto prepare(const detail::IncomingProbe &, + detail::MPOperator &, + const mpi::WindowVec> &) -> void {} + auto on_resolved(size_t g, + mpi::WindowIndex, + size_t, + size_t ip, + const detail::IncomingProbe &, + detail::QueryWire::WireView) -> Response { + order.push_back(g); + return static_cast(ip); + } +}; + +template +struct ExplicitlySerialSink : AppendingSink { + static constexpr bool parallel_resolve = false; +}; + +struct ScatterRun { + std::vector> responses; // bit patterns compared through bit_cast + std::vector> sink_state; + std::vector marks; + std::vector> rows; + PhaseLog log; +}; + +template +auto run_scatter(const QueryPlan &plan, + mpi::SlotWindow window, + bool leader, + int threads, + Sink &sink, + detail::MPOperator &op, + Capture &&capture) -> ScatterRun { + ScatterRun run; + const auto incoming = serialize(plan.per_sender, sink.incoming_form() == detail::QueryForm::Fused, window); + detail::MatchedEpochSet matched; + const size_t combined = op.size(); + matched.begin_gate(combined); + const auto responses = detail::resolve_incoming(incoming, + op, + leader, + matched, + combined, + sink, + {.threads = threads}, + AccumulatingObserver{&run.log}); + for (const auto wi : window.indices()) { + std::vector r; + for (const auto v : responses[wi]) { + r.push_back(static_cast(v)); + } + run.responses.push_back(r); + } + for (size_t i = 0; i < combined; ++i) { + run.marks.push_back(static_cast(matched.is_marked(i))); + } + for (const auto &m : rows_of(op)) { + run.rows.push_back(key_of(m)); + } + run.sink_state = capture(sink); + return run; +} + +auto same_scatter(const ScatterRun &a, const ScatterRun &b) -> bool { + if (a.responses.size() != b.responses.size()) { + return false; + } + for (size_t s = 0; s < a.responses.size(); ++s) { + if (a.responses[s].size() != b.responses[s].size() + || !std::ranges::equal(a.responses[s], b.responses[s], [](double x, double y) { + return std::bit_cast(x) == std::bit_cast(y); + })) { + return false; + } + } + return a.sink_state == b.sink_state && a.marks == b.marks && a.rows == b.rows; +} + +template +auto graph_state(const detail::GraphSink &sink) -> std::vector> { + std::vector> out; + for (const auto &a : sink.acc) { + std::vector v; + for (const auto &e : a.in_entries) { + v.push_back(e.idx); + v.push_back(static_cast(static_cast(e.phase))); + } + out.push_back(v); + } + return out; +} + +inline auto contract_state(const detail::FusedContract &fc) -> std::vector> { + std::vector v; + for (const auto &h : fc.cross_half) { + v.push_back(h.local_idx); + v.push_back(std::bit_cast(h.v_partner)); + v.push_back(static_cast(static_cast(h.phase_signed))); + v.push_back(static_cast(h.is_insert)); + } + return {v}; +} + +} // namespace + +// GraphSink and the Heisenberg ContractSink certify parallel on_resolved; their scatter runs on workers and +// matches the serial scatter exactly: responses, sink records, leader marks and the published rows. +BOOST_AUTO_TEST_CASE(openmp_resolve_scatter_threaded_matches_serial) { + constexpr size_t N = 250; + const mpi::SlotWindow window{.base = 2, .count = 3}; + const auto plan = plan_queries(19, {1300, 0, 1250}, HitMix::mixed); + VecD coeffs; + for (size_t i = 0; i < plan.seeds.size(); ++i) { + coeffs.push_back(0.125 + (0.001 * static_cast(i))); + } + for (const bool leader : {true, false}) { + std::vector graph; + std::vector contract; + std::vector contract_swept; + for (const int threads : {1, 2, 3, 4}) { + { + auto op = make_op(plan.seeds); + detail::GraphSink sink(window.stop(), 0); + graph.push_back(run_scatter(plan, window, leader, threads, sink, op, [](const auto &s) { + return graph_state(s); + })); + } + for (const bool swept : {false, true}) { + auto op = make_op(plan.seeds); + detail::FusedContract fc; + detail::ContractSink sink{.R = window.stop(), + .my_rank = 0, + .fc = fc, + .op_coeffs = coeffs, + .fused_scale = swept, + .inv_cos = swept ? 1.25 : 1.0, + .schrodinger = false, + .basis = Basis::Majorana}; + (swept ? contract_swept : contract) + .push_back(run_scatter(plan, window, leader, threads, sink, op, [&](const auto &) { + return contract_state(fc); + })); + } + } + for (size_t t = 1; t < graph.size(); ++t) { + BOOST_TEST(same_scatter(graph[0], graph[t]), "graph leader=" << leader << " budget " << t + 1); + BOOST_TEST(same_scatter(contract[0], contract[t]), "contract leader=" << leader << " budget " << t + 1); + BOOST_TEST(same_scatter(contract_swept[0], contract_swept[t]), "swept leader=" << leader); + } + BOOST_TEST(std::ranges::count(graph[0].marks, 1) == (leader ? static_cast(plan.hits) : 0)); + } +} + +BOOST_AUTO_TEST_CASE(openmp_resolve_scatter_workers_participate, + *boost::unit_test::precondition(kernel_test::runtime_offers_two_workers)) { + constexpr size_t N = 250; + const mpi::SlotWindow window{.base = 0, .count = 2}; + const auto plan = plan_queries(20, {2048, 2048}, HitMix::mixed, 1400); + VecD coeffs(plan.seeds.size(), 0.5); + { + auto op = make_op(plan.seeds); + detail::GraphSink sink(2, 0); + const auto run = + run_scatter(plan, window, true, 4, sink, op, [](const auto &s) { return graph_state(s); }); + BOOST_REQUIRE(run.log.of(KernelRange::scatter).size() == 1U); + kernel_test::check_participation(kernel_test::slots_of(*run.log.of(KernelRange::scatter)[0]), 4, "graph"); + } + { + auto op = make_op(plan.seeds); + detail::FusedContract fc; + detail::ContractSink sink{.R = 2, + .my_rank = 0, + .fc = fc, + .op_coeffs = coeffs, + .fused_scale = false, + .inv_cos = 1.0, + .schrodinger = false, + .basis = Basis::Majorana}; + const auto run = + run_scatter(plan, window, true, 4, sink, op, [&](const auto &) { return contract_state(fc); }); + kernel_test::check_participation(kernel_test::slots_of(*run.log.of(KernelRange::scatter)[0]), 4, "contract"); + } +} + +// Sinks that do not certify parallel on_resolved keep the serial scatter at any budget: a generic sink without +// the member, one declaring it false, and the Schrodinger ContractSink (state scoring stays serial). +BOOST_AUTO_TEST_CASE(openmp_resolve_scatter_stays_serial_without_the_capability) { + constexpr size_t N = 128; + const mpi::SlotWindow window{.base = 1, .count = 2}; + const auto plan = plan_queries(21, {1500, 1100}, HitMix::mixed); + const size_t blocks = detail::logical_ranges(2600, detail::resolve_scatter_block); + const auto order_state = [](const auto &s) { + return std::vector>{std::vector(s.order.begin(), s.order.end())}; + }; + for (const int threads : {1, 4}) { + auto op = make_op(plan.seeds); + AppendingSink generic; + const auto a = run_scatter(plan, window, true, threads, generic, op, order_state); + kernel_test::check_serial(kernel_test::slots_of(*a.log.of(KernelRange::scatter)[0]), blocks, "no member"); + std::vector in_order(2600); + std::iota(in_order.begin(), in_order.end(), 0); + BOOST_TEST(a.sink_state[0] == in_order); + + auto op2 = make_op(plan.seeds); + ExplicitlySerialSink declared; + const auto b = run_scatter(plan, window, true, threads, declared, op2, order_state); + kernel_test::check_serial(kernel_test::slots_of(*b.log.of(KernelRange::scatter)[0]), blocks, "false"); + BOOST_TEST(b.sink_state[0] == in_order); + } + // Schrodinger: fresh cross-rank misses are state-scored from their positions (paired-only mono_at). + const auto plan_s = plan_queries(22, {1400, 1200}, HitMix::mixed); + VecD coeffs(plan_s.seeds.size(), 0.25); + std::vector runs; + for (const int threads : {1, 4}) { + auto op = make_op(plan_s.seeds); + op.initial_state = {0, 3, 5, 9}; + detail::FusedContract fc; + detail::ContractSink sink{.R = 3, + .my_rank = 0, + .fc = fc, + .op_coeffs = coeffs, + .fused_scale = false, + .inv_cos = 1.0, + .schrodinger = true, + .basis = Basis::Majorana}; + runs.push_back( + run_scatter(plan_s, window, true, threads, sink, op, [&](const auto &) { return contract_state(fc); })); + kernel_test::check_serial(kernel_test::slots_of(*runs.back().log.of(KernelRange::scatter)[0]), + blocks, + "schrodinger"); + } + BOOST_TEST(same_scatter(runs[0], runs[1])); +} + +// The parallel scatter's matched marks are disjoint uint16 writes only because distinct queries resolve to +// distinct rows: queries are source XOR G over distinct sources. Pinned on generator-derived queries, with a +// duplicated query as the negative control the debug assertion would catch. +BOOST_AUTO_TEST_CASE(openmp_resolve_leader_marks_are_unique) { + constexpr size_t N = 32; + std::mt19937_64 rng(23); + const auto store_terms = draw_distinct(rng, 500); + const auto gen = indices_to_bitset({3, 17}); + // Sources whose partner is stored (hits) and others (misses): partners are pairwise distinct. + std::vector>> per_sender(2); + std::set> seen; + for (size_t i = 0; i < store_terms.size(); ++i) { + const auto query = (i % 4 == 0) ? store_terms[i] : (store_terms[i] ^ gen ^ indices_to_bitset({i % 64})); + if (seen.insert(key_of(query)).second) { + per_sender[i % 2].push_back(query); + } + } + auto op = make_op(store_terms); + const auto incoming = serialize(per_sender, false, {.base = 0, .count = 2}); + const auto pr = detail::probe_incoming_queries(incoming, op, detail::QueryForm::Plain, {.threads = 3}); + BOOST_TEST(detail::leader_marks_unique(pr, op.size())); + auto dup = per_sender; + dup[1].push_back(dup[0][0]); + const auto pr_dup = detail::probe_incoming_queries(serialize(dup, false, {.base = 0, .count = 2}), + op, + detail::QueryForm::Plain, + {.threads = 3}); + BOOST_TEST(!detail::leader_marks_unique(pr_dup, op.size())); +} + +/* ── Self resolution in bounded windows, and the publication order across passes ────────────────── */ + +namespace { + +template +using ObservedEngine = detail::LayerBuildEngine; + +template +auto stage_self(Eng &eng, const std::vector> &keys, const std::vector &srcs) -> void { + using PosT = typename Eng::RowPosT; + for (size_t q = 0; q < keys.size(); ++q) { + std::vector pos; + for (const auto p : positions_of(keys[q])) { + pos.push_back(static_cast(p)); + } + eng.self_stage_.push(pos, (q % 3 == 0) ? -1 : 1); + } + eng.src_idx_r.at_slot(eng.my_rank) = srcs; +} + +struct SelfRun { + std::vector> state; // sink entries, deferred misses, marks, rows + PhaseLog log; +}; + +// Leader then follower self passes over stages spanning several 4096-query windows, then finish(): the sink's +// hits in order, every deferred miss's positions, hash, source, phase and v_src, the leader marks, and the rows +// published after both passes. +template +auto run_self(const std::vector> &seeds, + const std::vector> &leader_keys, + const std::vector &leader_srcs, + const std::vector> &follower_keys, + const std::vector &follower_srcs, + int threads) -> SelfRun { + SelfRun run; + auto op = make_op(seeds); + detail::MatchedEpochSet matched; + const size_t combined = op.size(); + detail::FusedContract fc; + VecD coeffs; + for (size_t i = 0; i < seeds.size(); ++i) { + coeffs.push_back(1.0 / static_cast(i + 3)); + } + const mpi::SlotWindow window{.base = 0, .count = 1}; + auto body = [&](auto &eng) { + const auto values = [](const std::vector &srcs) { + std::vector v; + for (const auto s : srcs) { + v.push_back(0.5 + static_cast(s)); + } + return v; + }; + stage_self(eng, leader_keys, leader_srcs); + if constexpr (Contract) { + eng.src_val_r.reset(window); + eng.src_val_r.at_slot(0) = values(leader_srcs); + } + eng.resolve_self_queries(true); + stage_self(eng, follower_keys, follower_srcs); + if constexpr (Contract) { + eng.src_val_r.at_slot(0) = values(follower_srcs); + } + eng.resolve_self_queries(false); + std::vector deferred; + for (const auto &m : eng.deferred_self_misses) { + deferred.insert(deferred.end(), + {m.k, + m.hash, + m.src, + static_cast(static_cast(m.phase)), + std::bit_cast(m.v_src)}); + for (size_t j = 0; j < m.k; ++j) { + deferred.push_back(eng.deferred_pos_flat_[m.pos_at + j]); + } + } + run.state.push_back(deferred); + std::vector marks; + for (size_t i = 0; i < combined; ++i) { + marks.push_back(static_cast(matched.is_marked(i))); + } + run.state.push_back(marks); + (void)eng.finish(CosMask{}, nullptr); + }; + if constexpr (Contract) { + ObservedEngine> eng(op, + 1, + 0, + matched, + combined, + detail::ContractSink{.R = 1, + .my_rank = 0, + .fc = fc, + .op_coeffs = coeffs, + .fused_scale = false, + .inv_cos = 1.0, + .schrodinger = false, + .basis = Basis::Majorana}, + window, + {.threads = threads}, + AccumulatingObserver{&run.log}); + body(eng); + std::vector recs; + for (const auto *list : {&fc.hits, &fc.inserts}) { + for (const auto &r : *list) { + recs.insert(recs.end(), + {r.src, + r.tgt, + std::bit_cast(r.v_src), + std::bit_cast(r.v_tgt), + static_cast(static_cast(r.phase))}); + } + } + run.state.push_back(recs); + } + else { + ObservedEngine> eng(op, + 1, + 0, + matched, + combined, + detail::GraphSink(1, 0), + window, + {.threads = threads}, + AccumulatingObserver{&run.log}); + body(eng); + std::vector entries; + for (const auto *list : {&eng.sink.acc[0].in_entries, &eng.sink.acc[0].out_entries}) { + for (const auto &e : *list) { + entries.push_back(e.idx); + entries.push_back(static_cast(static_cast(e.phase))); + } + } + run.state.push_back(entries); + } + const auto rows = rows_of(op); + size_t findable = 0; + for (size_t i = 0; i < rows.size(); ++i) { + const auto found = op.store->find(rows[i]); + findable += static_cast(found.has_value() && *found == i); + run.state.push_back(key_of(rows[i])); + } + // A deferred miss published with a wrong cached hash is a correct row that no lookup can find. + BOOST_TEST(findable == rows.size()); + return run; +} + +} // namespace + +// Stages spanning two full 4096-query windows plus a short final one; followers whose source a leader matched +// are filtered before windowing. Serial and threaded self resolution agree exactly, and each window probes in +// at most 16 blocks of 256 queries. +BOOST_AUTO_TEST_CASE(openmp_self_windows_match_serial) { + constexpr size_t N = 128; + std::mt19937_64 rng(24); + const auto seeds = draw_distinct(rng, 6000); + std::set> used; + for (const auto &m : seeds) { + used.insert(key_of(m)); + } + const auto fresh = [&] { + for (;;) { + const auto m = draw_distinct(rng, 1)[0]; + if (used.insert(key_of(m)).second) { + return m; + } + } + }; + const size_t n_leader = (2 * detail::self_probe_window) + 37; + std::vector> leader_keys; + std::vector leader_srcs; + for (size_t q = 0; q < n_leader; ++q) { + leader_keys.push_back(q % 2 == 0 ? seeds[q / 2] : fresh()); + leader_srcs.push_back(q % seeds.size()); + } + // Followers: every fifth source is a row a leader hit (seeds[q/2] for even q), so it is filtered. + std::vector> follower_keys; + std::vector follower_srcs; + size_t filtered = 0; + for (size_t q = 0; q < 5000; ++q) { + const bool matched_source = q % 5 == 0; + follower_keys.push_back(q % 3 == 0 ? seeds[4200 + (q % 1700)] : fresh()); + follower_srcs.push_back(matched_source ? (q % 4000) : 5000 + (q % 900)); + filtered += static_cast(matched_source); + } + // Duplicate hit keys across the two passes would be two rotations with one target; keep them distinct. + { + std::set> keys; + for (const auto &m : leader_keys) { + BOOST_REQUIRE(keys.insert(key_of(m)).second); + } + for (const auto &m : follower_keys) { + BOOST_REQUIRE(keys.insert(key_of(m)).second); + } + } + const size_t kept_followers = follower_keys.size() - filtered; + for (const bool contract : {false, true}) { + const auto run_at = [&](int threads) { + return contract + ? run_self(seeds, leader_keys, leader_srcs, follower_keys, follower_srcs, threads) + : run_self(seeds, leader_keys, leader_srcs, follower_keys, follower_srcs, threads); + }; + const auto serial = run_at(1); + for (const int threads : {2, 3, 4}) { + const auto threaded = run_at(threads); + BOOST_TEST((serial.state == threaded.state), "contract=" << contract << " budget " << threads); + const auto calls = threaded.log.of(KernelRange::self_probe); + const size_t leader_windows = detail::logical_ranges(n_leader, detail::self_probe_window); + const size_t follower_windows = detail::logical_ranges(kept_followers, detail::self_probe_window); + BOOST_REQUIRE(calls.size() == leader_windows + follower_windows); + BOOST_TEST(calls[0]->worker.size() == 16U); + BOOST_TEST(calls[leader_windows - 1]->worker.size() == 1U); // the 37-query tail + BOOST_TEST(calls[leader_windows]->worker.size() == 16U); + BOOST_TEST( + calls.back()->worker.size() + == detail::logical_ranges(kept_followers % detail::self_probe_window, detail::probe_block_queries)); + for (const auto *c : calls) { + BOOST_TEST(c->worker.size() <= detail::self_probe_window / detail::probe_block_queries); + } + } + if (!!kernel_test::runtime_offers_two_workers(0)) { + const auto threaded = run_at(4); + const auto calls = threaded.log.of(KernelRange::self_probe); + kernel_test::check_participation(kernel_test::slots_of(*calls[0]), 16, "self window"); + kernel_test::check_serial(kernel_test::slots_of(*calls[2]), 1, "self tail"); + } + } +} + +// Mixed self and remote hits and misses across the leader and follower passes, as run_exchange orders them: +// remote leader misses publish in the leader round, remote follower misses in the follower round, and the +// deferred self misses (leader, then follower) only after both passes. Identical at every budget. +BOOST_AUTO_TEST_CASE(openmp_mixed_self_and_remote_publication_order) { + constexpr size_t N = 250; + std::mt19937_64 rng(25); + const auto seeds = draw_distinct(rng, 3000); + std::set> used; + for (const auto &m : seeds) { + used.insert(key_of(m)); + } + size_t next_seed = 0; + const auto pick = [&](bool hit) { + if (hit) { + return seeds[next_seed++]; + } + for (;;) { + const auto m = draw_distinct(rng, 1)[0]; + if (used.insert(key_of(m)).second) { + return m; + } + } + }; + const auto keys = [&](size_t n) { + std::vector> out; + for (size_t q = 0; q < n; ++q) { + out.push_back(pick(q % 3 == 0)); + } + return out; + }; + const auto self_leader = keys(900); + const auto remote_leader = keys(700); + const auto self_follower = keys(600); + const auto remote_follower = keys(650); + std::vector self_leader_srcs(self_leader.size()); + std::iota(self_leader_srcs.begin(), self_leader_srcs.end(), 0); + // Follower sources 0..299 are targets a leader hit? Leaders hit seeds[0..], i.e. rows 0,1,2,..; filter a few. + std::vector self_follower_srcs; + for (size_t q = 0; q < self_follower.size(); ++q) { + self_follower_srcs.push_back(q % 7 == 0 ? q / 7 : 2800 + (q % 190)); + } + const mpi::SlotWindow window{.base = 0, .count = 2}; + const auto remote = [&](const std::vector> &k) { + return serialize(std::vector>>{{}, k}, false, window); + }; + std::vector>> results; + for (const int threads : {1, 2, 3, 4}) { + auto op = make_op(seeds); + detail::MatchedEpochSet matched; + const size_t combined = op.size(); + detail::LayerBuildEngine> + eng(op, 2, 0, matched, combined, detail::GraphSink(2, 0), window, {.threads = threads}); + stage_self(eng, self_leader, self_leader_srcs); + eng.resolve_self_queries(true); + const auto resp_leader = detail::resolve_incoming(remote(remote_leader), + op, + true, + matched, + combined, + eng.sink, + {.threads = threads}); + const size_t after_leader = op.size(); + stage_self(eng, self_follower, self_follower_srcs); + eng.resolve_self_queries(false); + const auto resp_follower = detail::resolve_incoming(remote(remote_follower), + op, + false, + matched, + combined, + eng.sink, + {.threads = threads}); + const size_t after_follower = op.size(); + (void)eng.finish(CosMask{}, nullptr); + + // Expected physical order: remote leader misses, remote follower misses, self leader, self follower. + std::vector> expect(seeds); + const auto misses = [&](const std::vector> &k, const std::vector *srcs) { + for (size_t q = 0; q < k.size(); ++q) { + if (srcs != nullptr && matched.is_marked((*srcs)[q])) { + continue; + } + if (std::ranges::find(seeds, k[q]) == seeds.end()) { + expect.push_back(k[q]); + } + } + }; + misses(remote_leader, nullptr); + BOOST_TEST(after_leader == expect.size()); + misses(remote_follower, nullptr); + BOOST_TEST(after_follower == expect.size()); + misses(self_leader, nullptr); + misses(self_follower, &self_follower_srcs); + BOOST_TEST((rows_of(op) == expect), "budget " << threads); + std::vector> r; + r.emplace_back(resp_leader[mpi::WindowIndex{1}].begin(), resp_leader[mpi::WindowIndex{1}].end()); + r.emplace_back(resp_follower[mpi::WindowIndex{1}].begin(), resp_follower[mpi::WindowIndex{1}].end()); + for (const auto &m : rows_of(op)) { + r.push_back(key_of(m)); + } + for (const auto &slot : graph_state(eng.sink)) { + r.push_back(slot); + } + results.push_back(r); + } + for (size_t t = 1; t < results.size(); ++t) { + BOOST_TEST((results[0] == results[t]), "budget " << t + 1); + } +} + namespace { // monomial_hash_positions equals monomial_hash at every tested width. diff --git a/cpp/tests/term_expectation_values_tests.cpp b/cpp/tests/term_expectation_values_tests.cpp index fd31f496e..a5b06933f 100644 --- a/cpp/tests/term_expectation_values_tests.cpp +++ b/cpp/tests/term_expectation_values_tests.cpp @@ -39,10 +39,8 @@ constexpr unsigned int kFullCutoff = 2 * kNumModes; constexpr unsigned int kTruncatedCutoff = 4; // A propagator with the case's circuit built as a graph, ready to query. -auto built_sim(const CaseData &data, - unsigned int cutoff, - size_t partitions = 1, - std::optional schrodinger_cutoff = std::nullopt) -> MonomialPropagator { +auto built_sim(const CaseData &data, unsigned int cutoff, std::optional schrodinger_cutoff = std::nullopt) + -> MonomialPropagator { auto sim = MonomialPropagator(data.hamiltonian, cutoff, data.initial_state, @@ -53,8 +51,7 @@ auto built_sim(const CaseData &data, CutoffType::Length, std::nullopt, kNumModes, - Basis::Majorana, - partitions); + Basis::Majorana); sim.build_graph(data.majoranas, data.param_inds, data.gen_coeffs); return sim; } @@ -83,7 +80,7 @@ auto reassembled(const OperatorDict &op, const std::vector> // The initial coefficients times the values sum to expectation_value(), with and without truncation; exact // either way, since expectation_value() is linear in the coefficients. The operator includes an identity // term, so this also checks that it returns 1. -BOOST_AUTO_TEST_CASE(values_reassemble_the_expectation_value) { +BOOST_AUTO_TEST_CASE(term_expectation_values_values_reassemble_the_expectation_value) { auto data = load_case_data(kCase); data.hamiltonian[VecZ{}] = std::complex{0.75, 0.0}; for (const auto cutoff : {kFullCutoff, kTruncatedCutoff}) { @@ -98,7 +95,7 @@ BOOST_AUTO_TEST_CASE(values_reassemble_the_expectation_value) { } // Each value is what expectation_value() reports once the initial operator is that term alone. -BOOST_AUTO_TEST_CASE(each_value_matches_a_single_term_reweight) { +BOOST_AUTO_TEST_CASE(term_expectation_values_each_value_matches_a_single_term_reweight) { const auto data = load_case_data(kCase); auto sim = built_sim(data, kTruncatedCutoff); const auto keys = keys_of(data.hamiltonian); @@ -121,7 +118,7 @@ BOOST_AUTO_TEST_CASE(each_value_matches_a_single_term_reweight) { // For a row created mid-circuit, the value is the derivative of expectation_value() with respect to the row's // coefficient, as documented: it matches expectation_value() with that row alone. It is not the row's ⟨P⟩. -BOOST_AUTO_TEST_CASE(mid_circuit_row_matches_a_single_term_reweight) { +BOOST_AUTO_TEST_CASE(term_expectation_values_mid_circuit_row_matches_a_single_term_reweight) { const auto data = load_case_data(kCase); auto sim = built_sim(data, kTruncatedCutoff); // Re-weight with the row's own evolved coefficient, which is guaranteed to be Hermitian-encodable. @@ -142,33 +139,13 @@ BOOST_AUTO_TEST_CASE(mid_circuit_row_matches_a_single_term_reweight) { BOOST_TEST(near(expected.imag(), 0.0)); } -// Partitioned values match the single-partition ones. Partitions hash-split the rows, so each key is found -// in exactly one partition and the gather must not double-count it. -BOOST_AUTO_TEST_CASE(partitioned_values_match_single_partition) { - const auto data = load_case_data(kCase); - const auto keys = keys_of(data.hamiltonian); - const auto expected = built_sim(data, kTruncatedCutoff).term_expectation_values(data.parameters, keys); - - for (const size_t partitions : {size_t{2}, size_t{4}}) { - auto sim = built_sim(data, kTruncatedCutoff, partitions); - const auto values = sim.term_expectation_values(data.parameters, keys); - BOOST_REQUIRE_EQUAL(values.size(), expected.size()); - for (size_t q = 0; q < expected.size(); ++q) { - BOOST_TEST_CONTEXT("partitions=" << partitions << " query " << q) { - BOOST_TEST(near(values[q].real(), expected[q].real())); - BOOST_TEST(near(values[q].imag(), expected[q].imag())); - } - } - } -} - // Untruncated, Schrodinger and Heisenberg give the same values. In Schrodinger they are also the evolved // state's coefficients, complex-conjugated: a coefficient multiplies M in the state, while ⟨M⟩ picks up the // coefficient of M's adjoint. -BOOST_AUTO_TEST_CASE(schrodinger_values_are_the_evolved_state_and_match_heisenberg) { +BOOST_AUTO_TEST_CASE(term_expectation_values_schrodinger_values_are_the_evolved_state_and_match_heisenberg) { const auto data = load_case_data(kCase); const auto keys = keys_of(data.hamiltonian); - auto schrodinger = built_sim(data, kFullCutoff, 1, kFullCutoff); + auto schrodinger = built_sim(data, kFullCutoff, kFullCutoff); const auto values = schrodinger.term_expectation_values(data.parameters, keys); const auto state = schrodinger.evolved_operator_coefficients(data.parameters, keys); const auto heisenberg = built_sim(data, kFullCutoff).term_expectation_values(data.parameters, keys); @@ -184,31 +161,25 @@ BOOST_AUTO_TEST_CASE(schrodinger_values_are_the_evolved_state_and_match_heisenbe } // Heisenberg throws for a term absent from the evolved operator. The term is past the cutoff and not fully -// paired (fully-paired terms are exempt from truncation), so it cannot be present. With partitions, every -// partition throws right after a collective; this case exposed the poison race fixed in run_on_all(). -BOOST_AUTO_TEST_CASE(heisenberg_absent_term_throws) { +// paired (fully-paired terms are exempt from truncation), so it cannot be present. The miss is an argument error, +// found before any state changes, so the propagator stays usable. +BOOST_AUTO_TEST_CASE(term_expectation_values_heisenberg_absent_term_throws) { const auto data = load_case_data(kCase); const VecZ absent{0, 2, 4, 6, 8, 10, 12, 14}; - for (const size_t partitions : {size_t{1}, size_t{2}}) { - auto sim = built_sim(data, kTruncatedCutoff, partitions); - BOOST_TEST_CONTEXT("partitions=" << partitions) { - BOOST_CHECK_THROW(sim.term_expectation_values(data.parameters, {absent}), - monoprop::detail::OperatorTermNotFound); - } - } + auto sim = built_sim(data, kTruncatedCutoff); + BOOST_CHECK_THROW(sim.term_expectation_values(data.parameters, {absent}), monoprop::detail::OperatorTermNotFound); + BOOST_CHECK_NO_THROW(sim.term_expectation_values(data.parameters, keys_of(data.hamiltonian))); } // Heisenberg throws once an in-place contraction has absorbed the gates into the operator, since the values -// would miss those gates. With partitions, the facade throws before fanning out. -BOOST_AUTO_TEST_CASE(heisenberg_after_inplace_contraction_throws) { +// would miss those gates. A copy inherits the refusal. +BOOST_AUTO_TEST_CASE(term_expectation_values_heisenberg_after_inplace_contraction_throws) { const auto data = load_case_data(kCase); - for (const size_t partitions : {size_t{1}, size_t{2}}) { - auto sim = built_sim(data, kTruncatedCutoff, partitions); - (void)sim.contract_partially(data.parameters, true); - BOOST_TEST_CONTEXT("partitions=" << partitions) { - BOOST_CHECK_THROW(sim.term_expectation_values({}, keys_of(data.hamiltonian)), GraphStateConflict); - } - } + auto sim = built_sim(data, kTruncatedCutoff); + (void)sim.contract_partially(data.parameters, true); + BOOST_CHECK_THROW(sim.term_expectation_values({}, keys_of(data.hamiltonian)), GraphStateConflict); + auto copy = sim; + BOOST_CHECK_THROW(copy.term_expectation_values({}, keys_of(data.hamiltonian)), GraphStateConflict); } // An out-of-range index throws rather than writing past the monomial, since keys are user input. diff --git a/cpp/tests/unit_tests.cpp b/cpp/tests/unit_tests.cpp index 06ceb530f..49fd6e262 100644 --- a/cpp/tests/unit_tests.cpp +++ b/cpp/tests/unit_tests.cpp @@ -25,8 +25,9 @@ static auto init() -> bool { } auto main(int argc, char* argv[]) -> int { - // overwrite=0, so an explicit environment override still wins; why it is off: tests/cpp/README.md. - setenv("monoprop_PARTITIONS", "off", 0); + // The thread budget is the launch's: CTest registers ordinary cases at monoprop_NUM_THREADS=1 and fixed-T launches + // with their own value, and one launch leaves the variable unset on purpose, so the runner never supplies it + // (cpp/tests/README.md). monoprop::mpi::init(&argc, &argv); int result = boost::unit_test::unit_test_main(&init, argc, argv); monoprop::mpi::finalize(); diff --git a/docs/content/docs/benchmarks.mdx b/docs/content/docs/benchmarks.mdx index 06369bf30..801bb1da0 100644 --- a/docs/content/docs/benchmarks.mdx +++ b/docs/content/docs/benchmarks.mdx @@ -210,7 +210,7 @@ just bench serial # serial run, column "serial" just bench-smoke # quick sanity check (tiny sizes) just bench serial --num-modes 64 --bench-rounds 10 -monoprop_NUM_THREADS=10 just bench serial-t10 # cap the partition count +monoprop_NUM_THREADS=10 OMP_NUM_THREADS=10 OMP_DYNAMIC=FALSE just bench serial-t10 --runtime-shape=openmp # T=10 shards uv run --group bench monoprop-bench-report benches/results # rebuild report, no re-run ``` @@ -235,14 +235,147 @@ Barriers delimit each timed operation, so runtime is the **makespan** across ran of per-rank peak RSS: an upper bound, shared pages counted once per rank and rank-local peaks summed even when they never coexisted. -`monoprop_PARTITIONS` is mandatory above one rank — the engine's default is `ranks == 1 ? cores : 1`, -which would silently measure one partition per rank — and the declared shape is recorded in `meta`. +Declare a multi-rank run's shape with `--runtime-shape=openmp` (below); the declared shape is recorded in `meta`. +Without a declaration the suite keeps the removed partition runtime's guard and asks for `monoprop_PARTITIONS` +above one rank, which the library itself ignores. The guard stays because this suite is a frozen measurement +overlay that also serves the preserved partition baseline. + +`mpi4py` is imported only when the imported extension reports `monoprop.has_mpi`: importing +`mpi4py.MPI` initializes MPI, which an MPI-off run must never do, even with `mpi4py` installed. An +MPI-off build runs one process, launched directly. + +Two options declare what a run measures and fail before anything is measured when the process +contradicts them. `meta` records the declaration, `has_mpi` and the extension's SHA-256: + +- `--runtime-shape=openmp` (every current build) requires `monoprop_PARTITIONS` unset, and a positive + `monoprop_NUM_THREADS`, or with it unset a single positive `OMP_NUM_THREADS`. `--runtime-shape=partitions` + serves only a binary of the removed partition runtime (the preserved baseline) and requires + `monoprop_PARTITIONS` and `monoprop_NUM_THREADS`, equal and positive. MPI-off builds must run one rank. +- `--build-mode=mpi` or `--build-mode=mpi-off` must match the imported extension's `has_mpi`. + +The `openmp` shape's requirement that `monoprop_PARTITIONS` be unset is measurement hygiene in the preflight: +the library ignores the variable. ```bash just bench-build-mpi # build once (MPI on) -monoprop_PARTITIONS=2 monoprop_NUM_THREADS=2 just bench-mpi r5t2 5 --map-by slot:PE=2 --bind-to core +monoprop_NUM_THREADS=2 OMP_NUM_THREADS=2 OMP_DYNAMIC=FALSE just bench-mpi r5t2 5 --map-by slot:PE=2 --bind-to core +``` + +### Memory exactness + +Each memory window records whether its kernel reset worked on every rank, independently: +`opmemexact` for the operation window (`opmempeak`, `opmembase`, `opmemdelta`) and `memhwmexact` for +the outer window (`memhwm`, `memhwm_max`). One never certifies the other. + +In `REPORT.md`, a cell from a `false` window reads *non-exact* (the peak fell back to the RSS at +exit, a lower bound). A cell with no flag, as in artifacts from before the flags existed, reads +*exactness unknown*. `monoprop-bench-bmf` publishes `peak-memory` only when `memhwmexact` is true +and `operation-memory` only when `opmemexact` is true, and warns on stderr otherwise. Latency and +`terms` are unaffected. + +An exact window is not byte-exact on a many-core host. The kernel keeps `VmHWM` from per-CPU RSS +counters that it reads approximately. When threads on many CPUs fault in memory, the recorded peak +can fall short of a transient that really was resident, by tens of MiB. On a 96-core EC2 host the +shortfall ranged from 2 to 51 MiB, independent of the transient's size. A single thread loses +under 1 MiB, and `VmRSS` itself stays exact. Size memory comparisons so that peaks are well above +this bound. + +### Parity evidence for the rank-local OpenMP refactor + +`tools/benchmark-rank-local-openmp.py` wraps the pytest suite above to collect the evidence the +rank-local OpenMP plan (`docs/superpowers/plans/2026-09-18-rank-local-openmp.md`) requires. It adds no benchmarks of its own, apart from the paring-construction measurement below. +The launcher, CPU binding and environment stay external. The driver checks them against a +campaign inventory and an external placement file, and refuses to run when they disagree. + +It has three modes, each run in a fresh process: + +- `observe` measures one campaign cell and writes `DIR/ARM/CELL/SAMPLE/timed.json`. A historical + node runs in-process through pytest, and its raw artifacts are kept under `SAMPLE/timed/`. Add + `--whole-process` for a separate, untimed run that writes `construction.json`. That window opens + before any input is built and closes after the operation's outputs, graph and callable are all + still alive. +- `validate` rebuilds the cell's workload in an untimed process. It records the energy, the full + gradient where the operation has one, and, for unpared operations, per-rank term files. Each + term file is sorted, gzip-compressed JSON Lines (`.jsonl.gz`, level 1, with a reproducible + header), `{"key": [...], "real": x, "imag": y}`, exported with + `evolved_operator(parameters, 0.0)`, so coefficients are rounded to `1e-12`. Pared operations + record scalars only. For `propagate` without coefficient truncation, validation also checks the + result against a graph replay of the same inputs. +- `compare --campaign FILE --manifest FILE --output FILE` joins baseline and candidate artifacts + against the inventory. It exits `0` when every cell passes, `1` for an unmet gate or incomplete + coverage, and `2` for malformed or incompatible evidence. + +`compare` applies exactly five gates per cell, each a median ratio that must be at most 1.00: + +- runtime; +- operation peak, summed over ranks and per-rank maximum; +- whole-construction peak, summed over ranks and per-rank maximum. + +Both exactness flags must be true. Outer-window peaks are reported as labelled diagnostics and never +gate. `compare` wants five paired observations per arm and cell, or ten after the prescribed +repetition, and says when a first failure needs five more. + +It rejects evidence that cannot be joined: + +- run IDs or measurement files reused anywhere; +- mismatched identities or build modes; +- placement files that do not certify the full team; +- artifacts from another campaign, such as calibration runs; +- an arm whose binary was built with the wrong runtime. + +Each arm is bound to the runtime its extension was compiled with, read from the measured file +itself. A build sets `_core.__runtime_identity__` and embeds the same NUL-terminated literal: every +current build `monoprop-runtime=sharded-openmp`; builds from before the partition runtime was removed +`monoprop-runtime=sharded-openmp-prototype` (the development selector) or +`monoprop-runtime=legacy-partitions`. The preflight matches each literal with its terminator, so the +final marker is never mistaken for a prefix of the prototype's, and a binary carrying two markers, or +an attribute its bytes do not carry, is refused. `observe` and `validate` require the attribute to agree +with the bytes of the file whose SHA-256 they record, and record both. Only a sharded build (final or +prototype) can serve `--runtime-shape openmp` (the candidate arm); a legacy build, including the +preserved partition baseline, which predates the markers, serves only `partitions`. `compare` refuses +candidate evidence without a marker-backed sharded identity for its own binary, and baseline evidence +that records a sharded one. The baseline's frozen records carry no identity field and remain +consumable; archived candidate records carry the prototype attribute, so evidence of the final binary +is told apart from them by its attribute and binary hash, never by an arm label. This is read-only +provenance, not a runtime switch: relabelling `--arm`, `--runtime-shape` or the environment cannot +change which runtime a binary contains. The benchmark suite's `--runtime-shape` preflight applies +the same check. + +Numerically, it merges each arm's rank files into a global map and requires equal key sets, with +coefficients agreeing under `abs(a-b) <= 1e-9 + 1e-7*max(|a|,|b|)`. The Heisenberg identity term, +replicated on every rank, is checked for agreement and counted once. + +`driver::pare_functional_construct[heisenberg|schrodinger]` is the one measurement the driver owns. +It times only `expectation_value_and_gradient_functional(1e-10)` on a graph built beforehand, and +keeps the returned callable alive until both memory windows close. `validate` then evaluates that +callable at the recorded parameters. + +```bash +# Baseline (the preserved partition baseline's own environment), MPI-off full machine: direct launch, no mpiexec. +monoprop_PARTITIONS=96 monoprop_NUM_THREADS=96 OMP_NUM_THREADS=96 taskset -c 0-95 \ + .venv/bin/python tools/benchmark-rank-local-openmp.py observe --arm baseline \ + --runtime-shape partitions --campaign "$CAMPAIGN" --cell-id "$CELL" --sample-id s01 \ + --placement "$PLACEMENT" --output "$RESULTS" +# Repeat with --whole-process in a new process; run `validate` without --sample-id. ``` +The candidate arm uses its own environment and the ordinary public constructors, with no partition selector +and no constructor overlay (the constructor has no partition argument, and the preflight refuses a candidate +launch that sets `monoprop_PARTITIONS`): + +```bash +# Candidate, MPI-off full machine: an ordinary MPI-off build, exact team. +monoprop_NUM_THREADS=96 OMP_NUM_THREADS=96 OMP_DYNAMIC=FALSE taskset -c 0-95 \ + "$CANDIDATE_ENV"/bin/python tools/benchmark-rank-local-openmp.py observe --arm candidate \ + --runtime-shape openmp --campaign "$CAMPAIGN" --cell-id "$CELL" --sample-id s01 \ + --placement "$PLACEMENT" --output "$RESULTS" +``` + +`tools/rank-local-openmp-workloads.json` holds each profile's complete configurations and +parameter vectors. `tools/rank-local-openmp-campaign.json` is the owner-approved, frozen inventory of +required cells, each with its identity and digests. `compare` treats that file as authoritative, so +evidence cannot pass by regenerating either file. + ### Benchmarking ladder `benches/LADDER.md` holds the development ladder: five groups of configurations, each row's flags @@ -257,10 +390,10 @@ reports threshold violations. The ladder itself lives in `bench.yml`. To reprodu declare its shape and take its selector and flags from the tables below — the label is your own: ```bash -export monoprop_PARTITIONS=1 monoprop_NUM_THREADS=1 # the shape, declared -just bench l1-hubbard --bench-rounds 3 -k "test_model_propagate and hubbard" +export monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 OMP_DYNAMIC=FALSE # the shape, declared +just bench l1-hubbard --runtime-shape=openmp --bench-rounds 3 -k "test_model_propagate and hubbard" -export monoprop_PARTITIONS=2 monoprop_NUM_THREADS=2 # P sizes the pinning +export monoprop_NUM_THREADS=2 OMP_NUM_THREADS=2 OMP_DYNAMIC=FALSE OMP_PROC_BIND=close OMP_PLACES=cores just bench-mpi l2b 4 --map-by slot:PE=2 --bind-to core monoprop-bench-bmf benches/results l2b # results as Bencher Metric Format JSON ``` @@ -271,8 +404,8 @@ monoprop-bench-bmf benches/results l2b # results as Bencher Metric Format JSO | Measure | Unit | Source | Threshold | |---|---|---|---| | `latency` | nanoseconds | pytest-benchmark mean ± 1σ | Student's t-test, upper 0.99, ≥12 samples | -| `peak-memory` | bytes | the process's peak RSS (`VmHWM`) while the row ran, summed over ranks | +10% | -| `operation-memory` | bytes | what the timed call added above its own floor, summed over ranks | +10% | +| `peak-memory` | bytes | the process's peak RSS (`VmHWM`) while the row ran, summed over ranks; only when `memhwmexact` | +10% | +| `operation-memory` | bytes | what the timed call added above its own floor, summed over ranks; only when `opmemexact` | +10% | | `terms` | count | terms in the evolved operator | any change | The `~GiB` columns below are `peak-memory`. `terms` is deterministic for a fixed seed and size, so @@ -287,16 +420,20 @@ one shape. It is the rung name up to the first `.`: `-c-`, f cores**, 32 GiB nominal (~30 usable), no swap — three testbeds: `intel-xeon-platinum-8488c-8c-L1`, `-L2a` and `-L2b`. -| rungs | testbed | `R` × `P` | launch | `monoprop_PARTITIONS` | rounds | +| rungs | testbed | `R` × `P` | launch | `monoprop_NUM_THREADS` | rounds | |---|---|---|---|---:|---:| | `L1.ops`, `L1.propagate` | L1 | 1 × 1 | `pytest benches` | 1 | 3 | | `L2a.ops`, `L2a.propagate` | L2a | 1 × 8 | `pytest benches` | 8 | 1 | | `L2b.ops`, `L2b.propagate` | L2b | 4 × 2 | `mpiexec -n 4 --map-by slot:PE=2 --bind-to core` | 2 | 1 | +Every rung declares `--runtime-shape=openmp` with an exact team (`OMP_DYNAMIC=FALSE`) and binds one thread per +core through `OMP_PROC_BIND=close OMP_PLACES=cores`; the series recorded before the partition runtime was +removed ran `P` pinned partitions per rank instead, so a step at that commit is the runtime change. + - One node, one MPI-enabled build, so an edge moves the shape and nothing else. - `R × P` = 8 at both L2 shapes. 4 × 2 pays the per-rank cost four times on one node, so it is the memory-worst shape and the one the size ceiling is set against; `P` = 2 keeps more than one - partition inside a rank, which 8 × 1 would not. + shard inside a rank, which 8 × 1 would not. - Rounds is 1 above L1 because `pedantic` builds round *k+1*'s propagator before releasing *k*'s — a memory setting, not a statistics one. @@ -361,10 +498,11 @@ resident; on a floor of its own it is 2.10, so `peak-memory` now means at L1 wha #### Inputs -`rungs` is one rung per line, ` | `: `partitions` an -integer or `per-rank` for one per physical core of the rank's share, and a `.suffix` picking the -process within a shape. A shape that does not fit is refused rather than oversubscribed, and every -rung reads its `ranks`, `partitions_env` and thread placement back out of its result JSON. Once a +`rungs` is one rung per line, ` | `: `threads` (OpenMP threads, +and so shards, per rank) an integer or `per-rank` for one per physical core of the rank's share, and a `.suffix` +picking the process within a shape. A shape that does not fit is refused rather than oversubscribed, and every +rung reads its `ranks` and threads back out of its result JSON, requires `monoprop_PARTITIONS` unset, and logs the +thread placement. Once a testbed has history these should stay fixed — a new shape is a new prefix, so a new testbed. **No benchmark workflow gates a pull request.** It tracks `main`, and a threshold is a report, not a diff --git a/docs/content/docs/building.mdx b/docs/content/docs/building.mdx index c1b310531..b217f43b7 100644 --- a/docs/content/docs/building.mdx +++ b/docs/content/docs/building.mdx @@ -21,11 +21,28 @@ without MPI, so a from-source build is required for multi-rank runs. ## Prerequisites -- a C++23-compliant compiler; on Linux the minimum supported versions are **GCC 14** and **Clang 18** +- a C++23-compliant compiler; on Linux the minimum supported versions are **GCC 14** and **Clang 18**. + GCC builds pass `-ffp-contract=on`, so GCC, like Clang by default, fuses a multiply and an add only within one + expression; the test suite's bitwise comparisons between code paths rely on it. A project that compiles the + installed headers itself uses its own contraction setting. - CMake and Ninja - Python 3.11 or newer and the `uv` package manager (for the bindings) - an MPI implementation such as Open MPI (only for MPI builds) -- `hwloc` (version 2.9+) and `pkg-config` (required so CMake can locate `hwloc`) +- OpenMP for C++, in every build — with or without MPI, and even when you only + run one thread; there is no OpenMP-disabled build. Use the runtime that matches + your compiler: + - **GCC** ships its own `libgomp`; nothing extra is needed; + - **Clang on Linux** needs the `libomp` of the same LLVM version, e.g. + `libomp-18-dev` for `clang++-18` on Debian/Ubuntu; + - **Apple Clang** needs Homebrew's `libomp`. It is keg-only, so point CMake at + it with `export OpenMP_ROOT="$(brew --prefix libomp)"` before building, and + before configuring a project that consumes the installed package. + + The library calls the OpenMP runtime itself, so both `libmonoprop` and the Python + extension link it. The runtime therefore starts when monoprop loads. If you set + `OMP_PLACES` and leave binding enabled, it binds the importing thread to the first + place at that point. That is OpenMP applying your placement request, not monoprop: + monoprop never changes affinity. Set `OMP_PROC_BIND=false` if you want no binding. The repository ships a [DevContainer](https://containers.dev/) with all of the above pre-configured; opening the folder in VS Code and rebuilding the container is @@ -46,6 +63,10 @@ Flake outputs support x86_64 Linux, aarch64 Linux, and aarch64 macOS. The | `nix build .#monoprop-py313` | the package for a specific interpreter: `py312`, `py313` or `py314`, with a `-mpi` suffix for the MPI build | | `nix run` | a Python interpreter with `monoprop` importable | +The packages and the shell take OpenMP from the compiler: GCC's `libgomp` on Linux, +and `llvmPackages.openmp` wherever the standard environment's compiler is Clang +(macOS). + The unsuffixed packages, `nix run` and `nix develop` use Python 3.14, the interpreter nixpkgs builds and caches; older interpreters are compiled locally. @@ -70,7 +91,7 @@ The packaged build deviates from the `uv` build in three places, all in Use the exported overlay when monoprop must share a downstream project's Python and native dependency package set. Following the downstream `nixpkgs` input avoids -introducing a second Python, Boost, hwloc, or MPI closure: +introducing a second Python, Boost, OpenMP, or MPI closure: ```nix { @@ -207,8 +228,8 @@ build) and pass `comm=MPI.COMM_WORLD` to the simulator: mpiexec -n 8 uv run python your_script.py ``` -See [Parallelism and distribution](/features/parallelism) for the communicator options and the -operator-partitioning controls. +See [Parallelism and distribution](/features/parallelism) for the communicator options, the thread budget and +the launch configuration. ## Building the C++ unit tests @@ -237,7 +258,34 @@ link dependencies merely because the consuming project already has an `MPI::MPI_ package through `find_package(monoprop CONFIG)` — and runs it. That is the only leg that exercises `monopropConfig.cmake`, so it is what catches a missing `find_dependency` or a usage requirement the package fails to export. It compiles the same source as the in-tree -probe, so the two cannot drift. +probe, so the two cannot drift. The probe drives both the Majorana and the Pauli basis and +instantiates the internal OpenMP worksharing template in its own translation unit, so the +consumer only builds if the imported `monoprop::monoprop` target carries OpenMP's compile +and link flags — the consumer project adds none of its own. Its constructor calls reach the library's out-of-line +thread-budget and MPI thread checks, so an unexported helper fails the consumer's link. The consumer also checks +that it inherited the build's MPI setting and no runtime selector: pass `-DSMOKE_EXPECT_MPI=ON|OFF` to make a +mismatched package fail to compile. It checks the constructor's shape at compile time, runs every public operation +family, an ordinary derived class through the base interface and the raw-access rule (at one and at two threads), +and, when launched on several ranks with `monoprop_TEST_EXPECT_RANKS` set, compares the root on `MPI_COMM_WORLD` +with one process's. It also runs the +header-only fixed-team phase primitive (`detail/sharded/Team.h`), whose barriers must compile through +the same imported OpenMP flags, and the owner-initialized shard state (`detail/sharded/State.h`) and the +direct-buffer construction engine (`detail/sharded/Construction.h`), each from a separate translation unit that +includes only that header, so the installed headers must be self-contained. A further translation unit starts with +the sharded evaluation header (`detail/sharded/Evaluation.h`) and calls the exported `ev_sharded` and +`ev_and_grad_sharded` at run time, after a coefficient-informed build whose seed replay crosses the shared-library +boundary. + +The exported low-level entry points (`evolve_step`, `evolve_operator` and +`state_operator_derivative_local`) take a trailing thread budget with a serial default. Removing the +partition runtime changed the `MonomialPropagator` constructor and layout, `mpi::Comm`, and the installed +headers (the partition, topology and in-process communicator headers are gone; see +[Parallelism and distribution](/features/parallelism)). Rebuild C++ consumers, `libmonoprop` and the Python +bindings together, from the same monoprop build, and install into a fresh prefix: an old prefix keeps the +deleted headers. + +monoprop links no topology library: the package config's dependencies are Boost, Threads, OpenMP and, in an +MPI build, MPI. An MPI or OpenMP implementation may itself use hwloc; that is its own dependency. ### Debug tree @@ -321,8 +369,31 @@ uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-ca just test-cpp-tsan ``` -TSan is scoped to the concurrency surface — the partition and `ShmComm` tests — -which is where its reports are worth the runtime cost. +`just test-cpp-tsan` is scoped to the concurrency surface, which is where its reports are worth the runtime cost: +the sharded runtime's team, state, construction, evaluation, exchange and root tests (`sharded_`), the fused-record +launches (`fused_env_t*`), every `openmp_*` test (the workshare primitive, the threaded kernels, scan and resolve +phases, and the budget launches), the multi-shard coefficient launches (`evolved_coefficients_env_t*`) and the threaded +lazy-fold checks (`combined_threaded_*`). The OpenMP caveat below applies to all of them, +and CI runs this leg with Clang, libomp and Archer for that reason. + +The OpenMP kernels (`openmp_*` tests) need an OpenMP runtime that TSan understands. GCC's libgomp is +not instrumented, so TSan reports false races even on correct parallel loops; do not read its +output as evidence either way. Clang with LLVM's libomp and its Archer tool works, with the runtime's +own internal locking suppressed: + +```bash +CXX=clang++ CC=clang SKBUILD_CMAKE_BUILD_TYPE=Tsan \ +SKBUILD_CMAKE_DEFINE="monoprop_SANITIZER=tsan;monoprop_ENABLE_MPI=OFF" \ +uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-cache -v + +OMP_TOOL_LIBRARIES="$(clang++ -print-resource-dir)/../../libarcher.so" \ +TSAN_OPTIONS="suppressions=$PWD/.github/tsan-libomp.supp" \ + just test-cpp-tsan +``` + +Check a new toolchain first with a small known-racy and known-safe OpenMP program: the safe one must +stay silent and the racy one must be reported in your own frames. The Archer path varies by +distribution (on Debian and Ubuntu, `/usr/lib/llvm-/lib/libarcher.so`). There is no ThreadSanitizer equivalent of the ASan Python leg. TSan does not support `dlopen` of instrumented libraries, so loading an instrumented `_core` @@ -345,6 +416,27 @@ Python API under TSan would require a TSan-instrumented CPython. those files, so there is one list per platform to keep current. +## The runtime a build contains + +There is one runtime and no build option that selects another: a default build is the sharded OpenMP runtime +([Parallelism and distribution](/features/parallelism)). Its extension carries a read-only identity, the attribute +`_core.__runtime_identity__` and the same NUL-terminated literal in the extension file: + +```bash +uv run --no-sync python -c 'from monoprop import _core; print(_core.__runtime_identity__)' +# monoprop-runtime=sharded-openmp +just check-runtime # attribute, literal, MPI setting, and no stale selector in the C++ tree +``` + +The parity driver and the benchmark preflight read both, so evidence names the binary it came from. Builds from +before the partition runtime was removed carried `monoprop-runtime=sharded-openmp-prototype` (the development +selector `monoprop_SHARDED_OPENMP_PROTOTYPE=ON`) or `monoprop-runtime=legacy-partitions`, and the preserved +partition baseline carries none; the tools still read those for archived evidence. A build tree configured before +the removal keeps a `monoprop_SHARDED_OPENMP_PROTOTYPE` cache entry that no longer does anything: delete the tree. + +Linux x86_64 with GCC/libgomp, Clang/libomp, ASan/UBSan and Clang TSan with Archer, with and without MPI, has been +run. macOS, Linux aarch64, wheels from CI, Nix and the minimum compiler versions are pending their own runs. + ## See also - [Getting Started](/getting-started) — installing a prebuilt release from PyPI. diff --git a/docs/content/docs/features/parallelism.mdx b/docs/content/docs/features/parallelism.mdx index a56727ca9..db882fc21 100644 --- a/docs/content/docs/features/parallelism.mdx +++ b/docs/content/docs/features/parallelism.mdx @@ -1,76 +1,123 @@ --- title: Parallelism and distribution -description: Scaling by partitioning the operator across cores and across MPI ranks. +description: Shared-memory threads and MPI ranks in the sharded OpenMP runtime. --- -monoprop scales the same way on one node and on many: it **partitions the -operator** into disjoint pieces and applies each gate to every partition in -lock-step, exchanging only the terms that cross a partition boundary. Two axes -compose: - -- **Across cores (default):** one single-threaded partition per physical core, - within a single process. This is on automatically, and can be configured with - environment variables (see below). -- **Across nodes (opt-in):** MPI ranks, each of which partitions further across its - own cores. Distributing the operator and its graph across ranks lets you - simulate and variationally optimise systems larger than one node's memory. - -## Operator partitioning (single node) - -By default monoprop splits the operator into one partition per physical core and -gives each partition a pinned worker thread that runs it serially. Each partition -keeps its own small, cache-resident term index; a gate is applied to all -partitions at once, synchronised by a lightweight barrier, and anticommuting terms -whose partner lives in another partition are resolved through a per-gate exchange. - -A partition addresses its own terms with a 32-bit index, so one partition holds at -most 2^32 (about 4.3 billion) terms. That is a few hundred gigabytes of resident -operator in a single partition, so the ceiling is reached by running too few -partitions rather than by problem size, and it is not a build option. An append that -would cross it is refused before it happens — a `RuntimeError` naming the ceiling and -the term counts, with the operator left as it was, rather than a silent wrap. The fix -is more partitions or more MPI ranks — both divide the terms per partition — or a -larger `lower_atol` (see [Truncation and cutoffs](/features/cutoff)). - -## Runtime environment variables +monoprop splits the operator into disjoint pieces and applies each gate to every piece at once, exchanging only +the terms whose partner lives in another piece. Two axes compose: pieces within a process, and MPI ranks across +processes (and nodes). Distributing the operator and its graph across ranks lets you simulate and variationally +optimise systems larger than one node's memory. -| Variable | Default | Meaning | -| --- | --- | --- | -| `monoprop_NUM_THREADS` | one partition per physical core | Caps the number of partitions. Set it to run fewer partitions than cores. | -| `monoprop_PARTITIONS` | `auto` | `auto` = one partition per core (capped by `monoprop_NUM_THREADS`); an integer `N` = exactly `N` partitions; `off` = one partition holding the whole operator. | +Within a process, each MPI rank owns T *shards*, one per OpenMP thread, and one OpenMP team works on all of them for +the whole of an operation. OpenMP is required in every build, with or without MPI; MPI is optional and off by default. +There is no other in-process runtime and no build or runtime option that selects one. -Which MPI rank owns a term is set separately; see [Rank routing](#rank-routing). +## Sharded OpenMP runtime -```bash -# Run 8 partitions instead of one-per-core: -export monoprop_NUM_THREADS=8 -``` +### Launch geometry -## Placement report +A calculation runs as P MPI processes (or one process without MPI) with exactly T OpenMP threads each, fixed for +the whole calculation and the same on every rank. Configure both before launching, and before `import monoprop`: -Building a propagator writes one `COMMPLACE` line per rank to stderr, naming the -CPUs the launcher gave that rank and whether co-located ranks got disjoint masks. -It is report-only — no placement decision reads it — and there is no knob: redirect -stderr to drop it. - -```text -COMMPLACE rank=0 node_rank=0 node_size=2 masks=private cpus=64 node_cpus=128 cpu_list=0-63 +```bash +export monoprop_NUM_THREADS=8 OMP_NUM_THREADS=8 OMP_DYNAMIC=FALSE +mpiexec -n 4 python your_script.py # or, without MPI: python your_script.py ``` -`masks` is `private` when the co-located ranks' affinity masks are pairwise -disjoint, `shared` when two ranks can land on the same CPU, `alone` when this rank -is the only one on its host (which is *not* evidence a multi-rank launcher bound -correctly), and `unknown` when a mask did not fit the exchanged window. - -Pinning each partition to a core is not configurable: leaving placement to the -launcher measured `propagate[hubbard]` 2.90x slower, so the disabled arm is gone. +- **Thread budget.** A propagator reads `monoprop_NUM_THREADS` once, when it is constructed. A set value must be + decimal digits for a number from 1 to 2147483647 (leading zeros are fine); an empty value, a sign, spaces, a + list, zero or an overflowing value raises a `RuntimeError` naming the variable, and never falls back. When the + variable is unset, the OpenMP runtime default (`omp_get_max_threads()`, normally `OMP_NUM_THREADS`) applies. + That budget T is also the rank's shard count; there is no separate shard, thread or partition argument. + Copies and the callables a propagator returns keep its T, and a later change to the environment only affects + new propagators. +- **Exact teams.** Every operation opens one team of T threads and needs all of them. Set `OMP_DYNAMIC=FALSE`, + give each process enough cores, and set no limit below T (such as `OMP_THREAD_LIMIT`). The library does not + check the team it receives, does not repair a smaller one and never changes OpenMP settings: other settings, + or different T on different ranks, are unsupported launches whose failure is not controlled. +- **Placement.** monoprop does not query the topology, count physical cores, clamp T or pin threads, and has no + topology library of its own. Placement belongs to the launcher and to the OpenMP settings (`OMP_PLACES`, + `OMP_PROC_BIND`); one thread per physical core is the recommended layout. Importing `monoprop` loads the OpenMP runtime, so with `OMP_PLACES` set and + binding enabled, the importing thread is bound to the first place. +- **Shard ownership.** Shard t of rank r is routing owner `r * T + t` (see [Rank routing](#rank-routing)). + Changing T between launches changes which shard owns a term. +- **Removed controls.** There is no partition count, child factory, shard or thread argument. The variable + `monoprop_PARTITIONS` of the removed partition runtime is not read at all. + +A store (a shard) addresses its own terms with a 32-bit index, so one shard holds at most 2^32 (about 4.3 billion) +terms. That ceiling is reached by running too few shards rather than by problem size, and it is not a build option. +An append that would cross it is refused before it happens — a `RuntimeError` naming the ceiling and the term +counts, with the operator left as it was. The fix is more threads or more MPI ranks, or a larger `lower_atol` (see +[Truncation and cutoffs](/features/cutoff)). + +### What is per rank and what is global + +Energies and gradients are global: each rank folds its shards' contributions in ascending shard order, the +existing MPI reduction combines the ranks, and the replicated identity term is added once. Everything that +describes stored terms is rank-local and covers all of the rank's shards: `size()`, `graph_size()`, the memory +breakdowns, [evolved_operator][monoprop.monomial_propagator.MonomialPropagator.evolved_operator] (each shard's +terms decoded by its own index) and +[contract_partially][monoprop.monomial_propagator.MonomialPropagator.contract_partially], which concatenates the +shards' coefficient blocks in ascending shard order. That order is stable for a fixed (P, T), but coefficient +positions are not comparable across geometries; compare decoded terms instead. + +### C++ interface + +The C++ `MonomialPropagator` constructor ends with `basis_change`, `logical_num_modes` and `basis`. The partition +count, the child factory (`PartitionChildFactory`) and the partition helpers for derived classes are gone, without +aliases, and the runtime never reads `monoprop_PARTITIONS`. Ordinary extension keeps working: the virtual +destructor, `clone_()`, `update_initial_operator()` and the protected `apply_initial_operator_()`, which returns +this rank's share of the update. + +The raw-layout accessors `mp_op()`, `indexing()`, `graph()` and `graph_data()` return the rank's sole shard when +the launch has T = 1, whatever P is. With T > 1 there is no single store to return, so they throw +`MultiShardUnsupported`, whose message names the remedy: launch with `monoprop_NUM_THREADS=1`. The rejection +changes nothing, and an invalidated propagator reports that first. + +Removing the partition runtime is a C++ source and ABI break: the constructor lost its partition and factory +arguments, `MultiPartitionUnsupported` and `PartitionCountMismatch` are gone (the raw accessors throw +`MultiShardUnsupported`), the installed package no longer ships the partition, topology and in-process +communicator headers (`detail/partition/`, `detail/mpi/ShmComm.h`, `HybridComm.h`, `PartitionBarrier.h`, +`CpuRelax.h`), `mpi::Comm` holds only an ordinary communicator, and `mpi::geometry()` is gone. Rebuild C++ +consumers, `libmonoprop` and the Python bindings together, from the same build. + +### MPI and threads + +The thread that initialized MPI must make every call that can use MPI (construction, operations, retained +callables), from outside any OpenMP region. Inside an operation that thread is the team's primary and is the +only one that calls MPI; the other threads prepare and consume the exchanged data. So `MPI_THREAD_FUNNELED` is +enough: when monoprop initializes MPI (`mpi::init` in C++), it requests FUNNELED and aborts if MPI provides less; +when your program (mpi4py, for example) initialized MPI first, monoprop never reinitializes it, and construction +checks the level actually provided, refusing `MPI_THREAD_SINGLE` before any communication. mpi4py's default, +`MPI_THREAD_MULTIPLE`, is fine, and so is `mpi4py.rc.thread_level = "funneled"` (or `"serialized"`) set before +`from mpi4py import MPI`. A higher level changes nothing: worker threads still never call MPI, and a call from +another thread still prints a diagnostic and aborts that process without communicating. + +## Failed operations + +An operation that fails after it started changing a propagator (building or extending a graph, +propagating, evaluating, contracting, or updating the initial operator) leaves the propagator +unusable. Every later operation on it, copying it, and every callable it returned then raise a +`RuntimeError` that asks you to construct a new propagator. Copies made before the failure keep +working. Argument errors (wrong lengths, out-of-range indices, crossed tolerances and so on) are +raised before anything changes, so the propagator stays usable. + +With MPI and more than one rank, a failure on one rank cannot be handed back to your program +without leaving the other ranks blocked in communication. monoprop therefore prints the rank and the +error to stderr and calls `MPI_Abort`. On a single rank, and without MPI, you get the original +exception. + +A callable returned by `expectation_value_functional` or `expectation_value_and_gradient_functional` borrows its +propagator: keep the propagator alive while you use it. It refuses to run once the propagator is invalid, once +the graph it was made from has changed, or once `update_initial_operator` has changed the operator; make a new +one then. Calls on one propagator are not safe to make concurrently, from threads or from the callables, and the +runtime makes no new reentrancy promise: an operation's team is the whole budget, so do not call into monoprop from +inside an OpenMP region or from a callback of another monoprop call. ## MPI distribution (multi-node) -MPI partitions the operator and graph across ranks, composing with per-rank -partitioning into one flat world of `R × S` partitions (`R` ranks, `S` partitions each). -MPI communication is serialised through each rank's first partition, bracketed by -the intra-rank barriers. +MPI distributes the operator and graph across ranks, composing with the in-process shards into one flat world of +`R × S` routing owners: `R` ranks with `S = T` shards each. ### Rank routing @@ -83,7 +130,7 @@ $$ Every query a rank sends for $G$ therefore goes to the single peer $r \oplus h_d(G)$, whatever $R$ is. XOR is an involution, so both ends derive the pairing without communicating, and the exchange runs -point-to-point. Partitions within a rank keep the `splitmix` hash $q$, because shared-memory fanout +point-to-point. Stores within a rank keep the `splitmix` hash $q$, because shared-memory fanout is free: $$ @@ -104,8 +151,8 @@ over XOR, so a gate's queries still reach several ranks rather than one. | `monoprop_ROUTING` | `linear` | `linear` routes as above; `splitmix` keeps the dense all-to-all. Any other value raises. | | `monoprop_ROUTE_SEED` | `6768574230969066775` | Decimal `uint64` seed for the basis $\{v_i\}$. | -Both variables, and the partition count, must match on every rank. A mismatch would deadlock the -exchange, so it is checked collectively at startup and raised. +Both variables must match on every rank. A mismatch would deadlock the exchange, so it is checked collectively at +startup and raised. There is no such check for T, which the launch must keep uniform. ### Single-node (`MPI.COMM_SELF`) @@ -127,9 +174,9 @@ sim = MajoranaPropagator(..., comm=MPI.COMM_WORLD) mpiexec -n 8 uv run python your_script.py ``` -A pure-MPI rank keeps its whole share in one partition unless -`monoprop_NUM_THREADS` is set, so an MPI user who has not asked for threads gets -one partition per rank. +Each rank runs T shards, so set `monoprop_NUM_THREADS` (and `OMP_NUM_THREADS`) to the threads you give each rank, +for example `T=1` for a pure-MPI run. Without either, each rank takes the OpenMP default, which is usually every +core it can see: give each rank its own cores (`--map-by slot:PE=T --bind-to core` with Open MPI, for example). ### Enabling MPI diff --git a/docs/content/docs/testing.mdx b/docs/content/docs/testing.mdx index 505b1a977..a88414a4d 100644 --- a/docs/content/docs/testing.mdx +++ b/docs/content/docs/testing.mdx @@ -19,6 +19,9 @@ just test # build, then the Python and C++ suites just test-py # the Python suite against whatever is installed ``` +The recipes run the suites at one thread per rank (`monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 +OMP_DYNAMIC=FALSE`, on the test command only); tests that need other thread counts launch their own processes. + This collects two suites: monoprop's own (`tests/`) and the one belonging to the `monoprop-bench-tools` workspace member (`packages/monoprop-bench-tools/tests/`). The member has to be installed for the second to import, which is what the @@ -109,6 +112,123 @@ one entry per rank count rather than per case, because the ranks have to reach the same collectives. Each MPI entry times out after 600 seconds, preventing a collective deadlock from occupying the runner indefinitely. +The launch-time thread policy is part of the registration, not of the test runner: every discovered case, per +case and in the whole-suite variants, runs with `monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 OMP_DYNAMIC=FALSE`, so +ordinary cases see one store per rank. The runner itself never sets `monoprop_NUM_THREADS`, so running the binary +by hand uses your environment: set those variables yourself to reproduce a CTest case. + +A few cases also run again as separate `openmp_env_*` entries, each a fresh process with its own +launch environment. They cover `monoprop_NUM_THREADS=1` and `=3`, the OpenMP default with the +variable removed from the environment (the case checks that nothing supplied it), and +`OMP_PROC_BIND=false` for the affinity check, and `openmp_env_root_t4` runs the runtime contract at an exact team of +four. They are needed because a propagator reads its thread budget once, when it is constructed. +Kernel cases rerun under `OMP_THREAD_LIMIT=2` and `OMP_DYNAMIC=TRUE` (`openmp_env_kernels_*`) to test the +worksharing helper's reduced-team contract; no sharded operation runs under those settings, which its launch +contract excludes. + +The fixed-team phase primitive (`detail/sharded/Team.h`), with its section limit, has its own fresh-process entries, +`sharded_team_env_t1`, `_t2`, `_t4` and `_t4_passive` (the last with `OMP_WAIT_POLICY=PASSIVE`). Each +sets `monoprop_NUM_THREADS` and `OMP_NUM_THREADS` to T with `OMP_DYNAMIC=FALSE`, because the primitive +assumes the launch supplies exactly T workers. Run them with +`ctest --test-dir --no-tests=error -R '^sharded_team_env_'`; the T=4 entries need four cores. + +The owner-initialized shard state (`detail/sharded/State.h`), the direct-buffer construction and propagation +engine (`detail/sharded/Construction.h`) and the snapshot-safe evaluation (`detail/sharded/Evaluation.h`) follow +the same pattern with `sharded_state_env_t1`, `_t2` and `_t4`, `sharded_construction_env_t1`, `_t2` and `_t4` +and `sharded_evaluation_env_t1`, `_t2` and `_t4`. Select all of them with `-R '^sharded_'`. +`evolved_coefficients_env_t2` and `_t4` rerun the `evolved_operator_coefficients()` cases with several shards per +rank, against `evolved_operator_terms()` at the same T; `term_expectation_values_env_t2` and `_t4` likewise rerun +the `term_expectation_values()` cases against `expectation_value()`. The unit-test +executable replaces the global `operator new`/`delete` family (`cpp/tests/AllocationProbe.cpp`) so these +cases can inject real allocation failures; the library and installed consumers are unaffected. Their oracles are +independent references and the root at the same launch; bitwise comparisons with a separately built runtime run +from separate processes (below). + +#### The sharded runtime's own launches + +`sharded_api_tests.cpp` checks the public C++ surface at compile time: the removed partition interface, selector +and headers are absent, each negative probe paired with a positive control that shows the probe can see what it +looks for. `sharded_root_tests.cpp` drives the root's public operations with private shard +inspection, actual worker observation and failure injection, rerun as `sharded_root_env_t1`, `_t2` and `_t4` +and `sharded_root_env_openmp_default_t3` (no `monoprop_NUM_THREADS`, `OMP_NUM_THREADS=3`). The fused-sweep cases +rerun as `fused_env_t2` and `_t4`. Each fixed-T entry is a fresh process; no case changes T inside a process. + +```bash +ctest --test-dir build/editable/Release --output-on-failure --no-tests=error -LE mpi +ctest --test-dir build/editable/Release --output-on-failure --no-tests=error -R '^(sharded_root|fused)_env' +``` + +Multi-process cases of the sharded runtime have their own launches, one per P ranks x T threads, labelled `mpi`. +Each sets `monoprop_NUM_THREADS`, `OMP_NUM_THREADS` and `monoprop_TEST_EXPECT_TEAM` to T with `OMP_DYNAMIC=FALSE`, +leaves binding to OpenMP (`OMPI_MCA_hwloc_base_binding_policy=none`), and names the world size in +`monoprop_TEST_SHARDED_RANKS`; a multi-rank case skips without it, so the whole-suite variants never run one at +their default budget. + +| Entries | Build | What they run | +| --- | --- | --- | +| `sharded_exchange_mpi2_t*`, `_mpi3_t*` | every MPI build | the physical round (`detail/sharded/Exchange.h`): closed-form blocks over both transports, with exchanged and known counts (count blocks of every wire width), the XOR peer, empty rounds, staging reuse, request moves | +| `sharded_root_mpi2_t*`, `_mpi4_t*`, `_mpi3_t2_splitmix`, `_mpi3_t2_linear_rejected` | every MPI build | the root over P ranks against one process, raw access and derived hooks per rank, MPI on the primary only, the owner's physical rounds reused across operations and never shared by a copy, construction rounds unmapping their staging when a construction ends while the replay round keeps its staging, and the unroutable-geometry rejection | +| `sharded_world_equivalence_mpi2_t*`, `_mpi4_t*` | every MPI build | the serial-against-world cases of `mpi_distributed_layer_equivalence.cpp` and `mpi_fresh_insert_equivalence.cpp` | +| `monoprop_link_export_probe_mpi2_t1`, `_t2` | every MPI build | the shared-library consumer on two ranks, comparing the root on the world communicator with one process | +| `monoprop_unit_tests.x_mpi_2`, `_mpi_4` | every MPI build | the whole suite at T = 1 per rank | + +```bash +ctest --test-dir build/editable/Release --output-on-failure --no-tests=error -L mpi +``` + +The Python integration tests launch a fresh interpreter per T from a parent that is not itself an MPI job (they +skip inside one), so run them from an ordinary `pytest` process: + +```bash +monoprop_TEST_LEGACY_PYTHON="$LEGACY_ENV/bin/python" uv run --no-sync pytest tests/test_sharded_openmp.py -v +``` + +`monoprop_TEST_LEGACY_PYTHON` names the interpreter of an environment built separately from a checkout from +before the partition runtime was removed (with its default, legacy build). Its processes run at +`monoprop_PARTITIONS=T` for a bitwise comparison at the same (1, T) geometry — an independently implemented +runtime, never loaded into the same process; without it those cases skip, so a verification run sets it. The +same file checks the extension's runtime identity (the attribute and the NUL-terminated literal, and neither +pre-removal marker), that the removed constructor arguments raise `TypeError` at every level (the `_core` +classes, [MajoranaPropagator][], [PauliPropagator][] and the generated adapters) and that a `monoprop_PARTITIONS` +set before launch changes nothing, bit for bit. In an MPI build with an MPI launcher on `PATH`, the +`test_multirank_*` cases also run the probe on several processes, each rank writing its own document: at two +ranks against a legacy MPI environment rank by rank and bit for bit, across the geometries (1, 4), (2, 2) and +(4, 1), and on three processes under splitmix, with the gathered maps, energies and gradients within the map +tolerance. + +`tests/test_mpi_thread_support.py` (MPI builds) launches `tests/mpi_thread_support_probe.py`, which sets +`mpi4py.rc.thread_level` and imports `mpi4py.MPI` before `monoprop`, as a host would: FUNNELED, SERIALIZED and +MULTIPLE complete real construction, graph, energy, gradient and retained-functional work on one rank and on two +ranks of two threads, against a `COMM_SELF` reference; SINGLE is refused on one rank and aborts two ranks promptly; +and a call from another Python thread fails fast at FUNNELED and MULTIPLE. Each case reads back the level MPI +actually provided and skips, saying so, when it does not exercise the case's route. + +The MPI build also registers `mpi_failure_*` entries, labelled `mpi` and `mpi_failure`, from +`cpp/tests/mpi_failure_driver.cpp`. That driver has its own `main()`, so the unit-runner glob +excludes it. Each entry launches two ranks through `cpp/tests/run_mpi_failure_scenario.cmake`, with +a 30-second timeout. Rank 0 fails in one of these ways while its peer follows the normal phase: + +- `mpi_failure_sharded_*` drive the sharded root at two threads per rank: rank 0's worker 1 throws in a replay + finish; the primary throws in a traversal before any physical round, in a cosine pass while a replay round is + posted, or while packing queries with the count round posted; rank 1 sends malformed query blocks; +- the low-level engine and the exported replay functions, on the sole shard of a root launched with one thread + per rank, with explicit two-worker kernel budgets: a failure (on the caller or on worker 1 of the cosine kernel) + while a replay `Ticket` is posted, or while a `PendingAlltoallv` is posted; a worker of the threaded scan or self + probe throws before the first query exchange; a worker of the incoming decode, frozen probe or scatter throws + while the peer enters the response round; +- host-initialized MPI provides only `SINGLE` (`mpi_failure_insufficient_thread_level_single`); +- a guarded call arrives from a `std::thread` under `FUNNELED`, `SERIALIZED` or `MULTIPLE`. + +A failure entry passes only on a prompt nonzero exit with the expected diagnostic. The thread-support acceptance +entries (`mpi_failure_library_initialized_level`, `mpi_failure_host_initialized_funneled`, `_serialized`, +`_multiple`) instead require a clean exit after real operations at two threads per rank, on the world and on a +reordered split communicator, checked against a single-process root; every entry prints the requested and the +provided level. A timeout means a rank was left blocked, and it fails the entry. The wrong-thread cases rely on the launcher ending the +peers of a process that aborts, which is Open MPI's default. `ctest -L mpi_failure` runs just +these entries. If the MPI library grants a different thread level than a scenario needs (more than SINGLE for the +refusal, or not exactly the requested level for an acceptance run), that scenario reports itself as skipped +instead of passing. + CI runs the non-MPI and MPI labels separately with `--no-tests=error`, so an MPI lane fails if its MPI variant was not registered. Standalone MPI-enabled C++ test executables initialize MPI before constructing propagators and destroy the diff --git a/docs/superpowers/plans/2026-09-18-rank-local-openmp.md b/docs/superpowers/plans/2026-09-18-rank-local-openmp.md new file mode 100644 index 000000000..a35e5f5e4 --- /dev/null +++ b/docs/superpowers/plans/2026-09-18-rank-local-openmp.md @@ -0,0 +1,2888 @@ +# Rank-local OpenMP Implementation Plan + +> **Superseded implementation path — 2026-09-29.** Use the +> [sharded OpenMP plan](2026-09-29-sharded-openmp.md) and its +> [design](../specs/2026-09-29-sharded-openmp-design.md) for new work. Tasks 0–6, their outcomes and the failed +> architecture checkpoint are preserved below. Task 1's frozen inventory, formal baseline and evidence protocol +> remain the reference imported by the replacement plan. Tasks 7–13 and conflicting historical execution directions +> are superseded, not an authorized queue. New implementation and measurements require separate approval. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or +> superpowers:executing-plans to implement this plan task-by-task. Read both documents completely; full-plan execution +> requires explicit approval. Task 0 now precedes all downstream work and ends with an owner architecture decision. +> After that gate, separately authorized baseline-only calibration may precede campaign approval, as Task 1 describes. +> Do not infer missing contracts. Checkboxes track future work. + +**Goal:** First compare MPI-off index ownership/scaling with a mini-app. Reopen the one-store-per-rank choice using its +evidence; implement the conditional OpenMP replacement only after architecture approval and full-library validation. + +**Conditional architecture (Tasks 1–13):** Keep packed position-list rows, keyless indexing, inverted indices, graphs +and MPI routing algorithms/settings. Setting the router's partition dimension to one can change historical physical-rank +ownership under splitmix routing; Task 10 makes that compatibility consequence explicit. Workers traverse frozen +structures or update preallocated disjoint ranges; the caller initially owns structural publication. A bounded Boost +concurrent-index trial may change hash publication, not row/graph ownership or canonical IDs. Final MPI calls belong to +MPI's initializing thread under FUNNELED. Prototype replay first, then construction, and delete the old runtime last. + +**Tech Stack:** C++23, OpenMP CXX, existing Boost and optional MPI, nanobind, Python 3.11+, uv/scikit-build-core, +Boost.Test, pytest. Monoprop's direct hwloc dependency is removed with the legacy partition runtime. + +**Spec:** [Rank-local OpenMP design](../specs/2026-09-18-rank-local-openmp-design.md). Read both documents before +execution. + +**Decision after Task 6 (owner, 2026-09-29):** the one-store design is not pursued further. The post-Task-6 +architecture go/no-go showed it clearly trailing the partition baseline at full machine, with no credible bounded route +to parity (see "Architecture go/no-go outcome" after Task 6). The architecture will be re-designed and the re-design +keeps the sharding: thread-owned operator stores stay. Tasks 7–13 below describe the abandoned one-store path and are +superseded; do not execute them. Tasks 0–6 remain as the record of the experiment, and Task 1's frozen campaign and +formal baseline stay the reference for the re-design. A new spec and plan are required before further implementation. + +**Status:** Planning only. Refreshed source baseline: current `origin/main` at +`90d57177c2b0cd93503f88dd931d2f8dd409aa23`, inspected in rebased `refactor-parallelization` at +`35f359f095fa3f1d628ca2fdc66c93fef6ea7564`. Only CONTEXT.md and these planning documents differ between those commits. +The original planning publication was `38eb13d80a95dd1ac60e762f14f68c9a5925f755`. This owner-requested current-main +refresh preserves the subsequent approved runtime, compatibility, numerical and failure policies and uncommitted edits. +The owner has now explicitly reopened the one-store architecture decision through Task 0. Tasks 1–13 retain the +previously approved one-store proposal as a CONDITIONAL path, not an automatic follow-up. Stop after the mini-app for +an owner decision; reconcile both documents before proceeding. The follow-up interview fixed full-machine MPI-off +library acceptance for small/medium and selected large profiles before seeing results. Task 0 is a staged index-phase +pipeline study, not an isolated hash-container contest or a replacement for those library gates. +On the retained one-store path, OpenMP is mandatory; partition configuration is removed immediately; thread budgets are +launch-environment-only and captured at construction. The fixed32 capacity reduction remains an explicit consequence, +not a restriction that predetermines Task 0's architectural conclusion. +The owner selected one AWS EC2 `c8a.metal-24xl` for implementation/testing and initial acceptance. Production targets +are dual-socket Sapphire Rapids and AMD Zen3/Zen4/Zen5 HPC nodes, commonly four NUMA domains/socket. Hardware SMT may +be on or off, but SMT workers are not recommended: always configure one worker per physical core. Passing the frozen +single-instance EC2 campaign is sufficient for initial acceptance; HPC performance remains separately unqualified. +Multi-node qualification is pending a proper HPC cluster/interconnect, not a cloud-network substitute. Record actual EC2 +topology/software and agree representative workload sizes before freezing the campaign or collecting acceptance data. +The owner approved baseline-only, unscored sizing trials before the freeze, once separately authorized. Small/medium +profiles cover the full geometry sweep plus MPI-off full-machine operation. Selected large profiles cover the two +full-node MPI decompositions and MPI-off full-machine operation, not a new all-sizes Cartesian product. + +No implementation, engine builds/tests or benchmarks are authorized by this revision. All future execution commands +below are planned, not evidence of passing gates. EXISTING labels describe the inspected baseline; PROPOSED labels +describe future work. Full implementation and any publication of this revision require separate authorization. + +**Source-drift audit completed:** this refresh replaces the superseded +`290112c8289ab8015eb9a2c7651ac409839c5a88` pin with the main revision above, as requested by the owner. All 43 changed +paths were audited; affected contracts, source anchors, examples and preservation tests are updated below. The previous +d93d63f source-refresh blocker is resolved, not carried forward merely because HEAD contains planning commits. No +runtime gate is thereby passed or authorized. Both arms must use the refreshed pre-OpenMP engine as their common +starting point; samples from superseded engines cannot establish its baseline. Recheck subsequent source drift before +execution, and retain separate build/calibration authorization, campaign freeze and implementation gates. + +## Global Constraints + +The storage constraints describe the conditional one-store path in Tasks 1–13. Task 0 alone permits the explicit +per-thread-store experiment, not production shards. Full-machine MPI-off measurements are required both in Task 0 +and in full-library acceptance, including cross-NUMA placement. Reconcile the storage design with the owner's +post-experiment decision without silently weakening that coverage or any numerical/performance gate. + +- C++23; retain the GCC 14 / Clang 18 minimum compiler versions and Python 3.11 floor. +- MPI remains optional and OFF by default; retain working non-MPI wheels. +- OpenMP becomes a required implementation dependency for the replacement runtime; support Linux x86_64, Linux aarch64, + and macOS builds covered by the current project. +- Each independent propagator instance owns one packed operator store per MPI rank; threads are never communicator + ranks. Preserve independent copies, immutable graph-core sharing, retained coefficient/state snapshots and pared + graphs. +- Remove monoprop's direct hwloc dependency and topology/affinity machinery; do not replace it with another + library-owned hardware-discovery or pinning mechanism. +- Users choose P processes and T threads per process and configure resource allocation, binding and pinning; monoprop + does not clamp T to physical cores or validate NUMA placement. +- SMT use is not recommended: allocate at most one monoprop worker per physical core, including on SMT-enabled hosts. + Enforce this through user launch/binding settings and acceptance evidence, not a library hardware policy. +- Preserve packed rows, keyless indexing through row references, fixed uint32_t TermIndex, sparse state storage, and + inverted-index cosine recomputation. The Task 11 index A/B may replace the index, not rows. +- Do not introduce hidden operator shards, a dense duplicate store, a second persistent full-key store, per-thread + operators, or full-vector coefficient double buffering. Select any concurrent index only after correctness/memory/time + evidence; its container name is neither a rejection criterion nor proof of suitability. +- Required MPI-enabled cells keep each rank within one NUMA domain; MPI-enabled cross-NUMA L2a is diagnostic-only. + Actual MPI-disabled full-machine operation is required for small/medium and selected large profiles, even across + NUMA domains. Diagnostic results cannot replace, excuse or average away any required cell; correctness still applies. +- Preserve Majorana and Pauli bases, Heisenberg and Schrödinger pictures, cutoffs, graph build/replay/paring, gradients, + partial contraction, copying, and initial-operator updates. +- Final MPI calls execute on the MPI-initializing thread, outside library-created OpenMP regions. Request/validate at + least MPI_THREAD_FUNNELED, accepting higher provided levels without relaxing the caller rule. Retain + MPI_THREAD_SERIALIZED during legacy Hybrid coexistence; lower the minimum only with its removal in Task 12. +- No exception escapes an OpenMP structured region; distributed operations fail coherently rather than leaving peers in + collectives. Enforce invalidation after failed mutation; dependent functionals cannot reuse an invalid owner. +- No global omp_set_num_threads/omp_set_dynamic/omp_set_nested mutation; each worksharing region receives an explicit + per-object thread budget. +- Use bitmap traversal or anticommutation pass for the existing operation; reserve prefix offsets for cumulative counts + used to allocate output ranges. +- Do not change numerical tolerances, truncation rules, generator order, source ordering, or MPI routing to make a + regression disappear. +- Do not declare success without target-hardware runtime and peak-memory measurements at the same rank count, CPU + allocation, workload, and fixed 32-bit TermIndex as the baseline. +- No automatic regression allowance: the acceptance target is candidate/baseline <= 1.00 for the median runtime and + median peak memory of every required workload cell. Noise triggers more measurement, not a silent waiver. +- Update AGENTS.md, README.md, and affected user/contributor docs in the same implementation changes that alter their + documented behavior. +- No permanent legacy/new-backend selector. Temporary coexistence is confined to the development branch; remove it + before accepting the replacement. + +Additional execution rules: + +- **No runtime work or implementation is authorized by this planning amendment.** Task 0's 2-hour and + min(8 GiB, 25% RAM) study limits are approved, not permission to launch. Authorize implementation, host/allocation and + setup/build separately. Task 1 retains its independent 2-hour/15-minute/75% baseline-only pilot. After Task 0's + architecture gate, obtain separate authorization for that baseline calibration/build work before campaign approval. + Candidate engine changes require + implementation authorization and an owner-approved frozen campaign. Formal baseline collection also waits for that + freeze. Simplification without regressions is the goal; reject the architecture if it cannot earn acceptance. The + campaign bounds performance claims, not mathematical correctness across the supported API. +- P (called R in existing code/benchmark geometry) is the user-selected process count; T is the requested threads per + process, normally set with `monoprop_NUM_THREADS`. Only an unset override selects the OpenMP runtime default. Users + must allocate enough resources and configure rank/thread binding + with OpenMP, `srun --cpu-bind`, and/or `mpiexec` options. Required MPI-enabled cells keep each rank within one NUMA + domain; MPI-enabled cross-NUMA L2a stays diagnostic. Required MPI-off full-machine cells instead span the allocation + in one process. Neither permits a parity waiver or calling a smaller NUMA-local run full-machine. Recommend one + worker per physical core, including when hardware SMT is enabled. Do not create a topology service/NUMA scheduler, + silently change P/T, or enforce that + recommendation through library clamping or affinity changes. +- Do not touch pre-existing `.codegraph/` or `.direnv/`. Use an isolated implementation checkout and a separately built + baseline. Follow the worktree/GitButler skills for version-control writes; no raw git write commands. +- New behavior follows RED → minimal implementation → GREEN → diff review. Run tests before production edits and record + actual results. Preservation fixtures can already pass after an earlier task; do not manufacture a failure or remove + completed plumbing. Each threading task also needs a kernel-specific worker-participation check, not only equal + serial/threaded numerical output. For configuration/documentation use a failing integration check. +- Commit only when authorized, after the task gate. Use `(): ` and the required + `Assisted-by: Pi:` trailer. Suggested boundaries are the numbered tasks, not every checkbox. +- Do not use a broad formatter over unrelated files. Public declarations receive a one-line summary; header + declarations/members follow the repository's documentation convention. + +## Execution map and gates + +| Stage | Tasks | Gate before proceeding | +| --- | --- | --- | +| Architecture experiment | 0 | Done: evidence inconclusive for ownership; one store per rank is the working hypothesis (see Task 0 outcome) | +| Baseline (conditional) | 1 | Task 0 decision, authorized baseline-only calibration, frozen inventory, fresh formal baseline evidence | +| Small prototype | 2–4 | OpenMP packaging works; one-store serial and threaded replay agree | +| Shared-store construction | 5–7 | Tasks 5–6 done; post-Task-6 go/no-go ran and failed (see its outcome); Task 7 superseded | +| Remaining evaluation kernels | 8–9 | Superseded by the post-Task-6 decision to re-design while keeping the sharding | +| Integration cutover | 10–11 | Superseded: the one-store path does not proceed to the API cutover | +| Removal/release | 12–13 | Superseded: the partition runtime is kept | + +One writer per checkout. Kernel and storage tasks are ordered: do not have agents concurrently edit `Engine.h`, +`Evolution.cpp`, or `MonomialPropagator.inl`. Independent read-only reviews are safe. Do not batch past a failed gate. + +Expected RED/GREEN checkpoints (record real results; these are planned conditions, not claims of execution): + +| Task | RED before the task's change | GREEN required | +| --- | --- | --- | +| 0 | standalone target/self-tests and three variants absent | exact index results, honest scaling/memory evidence, owner decision; no library-parity claim | +| 1 | exactness/shape schema and new driver tests fail | complete baseline/validator evidence; no parity claim | +| 2 | missing Workshare/OpenMP exported dependency | helper and installed consumer pass | +| 3 | options overload/config/failure tests fail | serial behavior preserved; failures join/abort promptly | +| 4 | cosine/fused work-range participation remains serial; preservation tests may pass | exact coefficients and measured replay scaling | +| 5 | ordered range/window/self-stage tests fail | byte/order/ID equivalence and bounded scratch | +| 6 | new position helper/checked-boundary tests fail; Task 3 options already exist | position/hash probe and ordered serial publication | +| 7 | arithmetic-only capacity/new guard tests fail | uint32 count/ID guards; optional fill passes only if measured | +| 8 | endpoint work-range participation remains serial; preservation tests may pass | current stable derivatives and sparse transport preserved | +| 9 | named threaded reductions absent | fixed-block association approved and deterministic | +| 10 | one-store/default/compatibility tests fail | approved S=1 API/resource semantics | +| 11 | absent/incomplete evidence is not a pass | every runtime/peak cell <=1.00 after prescribed repetitions | +| 12 | static legacy/direct-dependency checks fail | runtime/topology removed LAST; parity rerun | +| 13 | clean wheel/Nix/consumer/docs checks expose missing integration | complete evidence and independent reviews | + +## Baseline-drift checkpoint and handoff + +Before authorized runtime work or implementation, record HEAD, status and these EXISTING anchors at source pin +90d57177c2b0cd93503f88dd931d2f8dd409aa23. Line numbers identify that snapshot; symbol names are durable anchors. The +current audit is complete. If later source differs semantically, report the changed contract and refresh/approve the +plan before coding. Missing PROPOSED symbols are expected RED, not drift. + +The audit compared the superseded pin with current main across routing/MPI, construction, Python API/tests, build/CI, +version and docs tooling. Packed storage/QueryWire, fixed32, streamed paired initialization, stable-gradient formulas, +benchmark families/windows, dependency manifests and justfile recipes retain their earlier contracts. Main's routing +basis, communicator agreement/cache, request ownership and Python incremental-axis changes are now baseline behavior, +not optimizations to implement or credit to OpenMP. VERSION is 0.9.2a1; record source/binary hashes, not just that +label. + +| EXISTING anchor | Must match before proceeding | +| --- | --- | +| `cpp/monoprop/detail/mpi/Routing.h:100–164,265–296` | full-rank basis draw, width-bound boolean router, default linear, Config and pure routes_pairwise predicate | +| `cpp/monoprop/detail/mpi/Pairwise.h:70–145`; `MPICompat.cpp:91–130` in the same directory | exact mode/partitions/seed agreement; communicator-attribute transport cache; request pre-sizing | +| `cpp/monoprop/detail/mpi/Comm.h:70–175`; `MPICompat.h:135–263` in the same directory | typed window iteration; sparse-plan validation; move-only PendingAlltoallv; WindowVec-only blocks; known-count width check | +| `cpp/monoprop/Evolution.cpp:102–117`; `cpp/monoprop/detail/mpi/Exchange.h:88–148` | rank-uniform routes_pairwise boolean, dense peer walk/nonempty legs, owned request vector | +| `cpp/monoprop/detail/mpi/HybridComm.h::derived_wire_plan_ / exchange_payload_` | fold every partition's send/receive peers; multi-peer fallback retains pairwise transport | +| `cpp/monoprop/detail/evolution/layer_build/Scan.h:208–249` | FusedScanResult has per-stream windows, no result.window; precomputed gen_shift and two self stages | +| `cpp/monoprop/detail/evolution/layer_build/Resolve.h:37–182` | sender_index plus window.slot, position/hash probing and ordered publication | +| `cpp/monoprop/detail/evolution/layer_build/Engine.h:348–372,470–574,600–612` | explicit plan/window constructor, position/hash deferred publication and bounded self probe | +| `cpp/monoprop/detail/operator/OperatorIndex.h:282–288` | five-span find_batch_positions, optional hash output | +| `cpp/monoprop/TypeAliases.h:39`; `cpp/include/monoprop/Evolution.h:54–63` | uint32_t; derivative has LayerAngle and optional CosRecordView | +| `cpp/monoprop/Evolution.cpp:437–464`; `cpp/monoprop/MPFunctions.cpp:44–172` | stable derivative snapshot/predivide/restore and record decisions | +| `cpp/include/monoprop/MonomialPropagator.h:288–306` | virtual update/clone, nonvirtual apply helper | +| `src/monoprop/monomial_propagator.py:231–270`; `tests/test_circuit.py:768–1008` | picture-correct incremental axis and post-call seed, both bases and shared-index gates | +| `justfile:75–122,169–171,371–397`; `pyproject.toml:162–168`; `cpp/tests/boost-test.cmake:1–9` | centralized recipes/MPI build override; CTest default 2;4, distinct from just's default 2 | +| `benches/bench_random.py:22–139`; `benches/conftest.py:248–274,411–493` | four operation families, shape guard, existing op/outer windows | +| `packages/monoprop-bench-tools/src/monoprop_bench_tools/memory/cpu.py:75–107,213–233` | nested peak preservation already implemented | + +The updated CI action versions and docs package/lockfile are part of this source snapshot, not runtime-design changes. +Qiskit conversion also gained sparse-observable/single-Pauli and Pauli-product-rotation support: retain its existing +conversion tests and version-gated optional coverage; this OpenMP work neither rewrites conversion nor raises its floor. + +Handoff checklist: exact revision/binary and overlay hashes; owner approvals; last passed task; actual RED and GREEN +commands/results; remaining gate/capacity/placement blockers; fixed routing and workload manifest; raw artifact paths. +Never hand off only “tests pass”. No builds/tests/imports/installs/VCS writes are authorized by this document refresh. + +### Remote entry point: Task 0, then conditional Task 1 + +The initial handoff is Task 0's standalone mini-app, not a full monoprop build or the baseline pilot. Transfer both +planning documents and CONTEXT.md; verify the source pin, authorize the small implementation and execution within +Task 0's 2-hour/min(8 GiB, 25% RAM) limits, accounting for setup/build separately. Use c8a.metal-24xl unless +the owner approves another host. Return results and stop for the architecture decision. No Task 1 command below runs +until that decision and the corresponding document reconciliation. +No delegation, provisioning, commits or publication are implied by this handoff. + +After that gate, the baseline-only pilot belongs to Task 1. It need not run on the planning machine before handing this +plan to an implementer. Run it on the authorized execution host, currently c8a.metal-24xl. If the host differs, +clarify the hardware decision before treating its sizing results as applicable to the target campaign. +No remote access, provisioning or runtime execution is authorized merely by reading these documents. + +Transfer the latest spec, plan and CONTEXT.md with the source checkout. The historical planning commit alone does not +include subsequent interview decisions. Verify document contents/digests on the execution host, including the 2-hour, +15-minute and 75% calibration limits. Documentation-only commits above the source anchor are not engine drift; inspect +semantic source/build/benchmark changes against the baseline-drift checkpoint rather than require HEAD to equal a +planning commit. The refreshed source pin is 90d57177c2b0cd93503f88dd931d2f8dd409aa23. Build the baseline from that +engine, with separately recorded measurement overlays; do not auto-retarget to a later HEAD or reuse superseded-engine +measurements. The owner-requested current-main source refresh is complete; execution authorization is still required. + +A bounded starter authorization may cover only host/source preflight, documented project-local setup, an isolated +baseline build, existing baseline API/benchmark sizing trials and their report. It does not authorize candidate engine +changes, formal acceptance samples, commits, pushes or a PR. Read both documents completely before starting; distinguish +EXISTING baseline behavior from PROPOSED replacement behavior rather than making the baseline conform to the new design. +Do not infer subagent/delegation permission from the plan's choice of execution skills. + +Before expensive commands, state the observed host/allocation, source/doc identities, permitted scope, resource limits +and a durable artifact directory outside the source checkout. Preserve the source checkout and record any approved +measurement overlay separately. On interruption, retain commands/results, trial outcomes and remaining budget in a +checkpoint; resuming does not reset the pilot limits. + +The pilot handoff contains absolute artifact/report paths and digests; host/toolchain/binary provenance; verified +placement; all tried configurations and outcomes; time/memory observations including export/validation; proposed exact +small/medium/large workload and geometry/routing cells; and the estimated time/cost of the formal campaign, including +prescribed extra repetitions. State blockers and unmeasured items explicitly. Stop for owner approval of that campaign +and budget, and authorization for the remaining work. Task 1 is not complete until its tools/tests and fresh post-freeze +formal baseline evidence also pass. No pilot result is a parity claim or an acceptance sample. + +## File responsibilities + +Paths in task lists are relative to the repository root. + +| Area | Existing anchors | New files | +| --- | --- | --- | +| Preliminary index study | `detail/operator/OperatorIndex.h`, `core/Monomial.h`; read-only reuse | `tools/index-miniapp/{CMakeLists.txt,main.cpp,README.md}`; no installed interface or engine changes | +| Runtime configuration | `detail/EnvConfig.h`, constructor/copy in `MonomialPropagator.inl`; delete `detail/partition/CpuTopology.*`, do not move it | `cpp/monoprop/detail/parallel/{Options.h,Workshare.h,ThreadBudget.h,ThreadBudget.cpp,CMakeLists.txt}` (budget validation only; no hardware discovery) | +| Replay | `Evolution.cpp`, `detail/evolution/CosineRecompute.h`, `MPFunctions.cpp` | no executor or new graph layer | +| Construction | `detail/evolution/layer_build/{Scan,Resolve,Engine,FusedApply}.h` | range helpers stay beside current kernels | +| Storage | `detail/operator/{OperatorIndex,MPOperator,InvertedIndex}.h` | bounded Boost index variant only; no duplicate full-key store | +| MPI failure | `detail/mpi/{Comm,MPICompat,MPIUtils,Exchange}.h`, `MPICompat.cpp` | `cpp/monoprop/detail/mpi/OperationFailure.h` | +| Tests | existing suites listed per task | `cpp/tests/{openmp_workshare_tests,openmp_runtime_tests,openmp_kernel_tests,openmp_equivalence_tests}.cpp`, `tests/test_openmp_config.py`, `cpp/tests/mpi_failure_driver.cpp` | +| Benchmark evidence | `benches/conftest.py`, both benchmark-tool renderers | `tools/benchmark-rank-local-openmp.py`, `tools/rank-local-openmp-workloads.json`, `tests/test_rank_local_openmp_benchmark.py` | +| Documentation | `AGENTS.md`, `README.md`, docs listed in Task 13 | `docs/content/docs/openmp-migration.mdx` | + +`detail/...` in the table means `cpp/monoprop/detail/...`. Task file lists below use full paths. New installed headers +must enter CMake FILE_SETs, not merely compiler include paths. + +## Shared invariants: reviewer checklist + +1. `OperatorIndex::find_batch_positions` (and dense `find_batch`) is read-only **only while rows, overflow storage, and + hash slots are frozen**. `set()` is not a disjoint-row parallel setter: it touches a shared overflow map, even + through `erase` for inline rows. +2. `MPOperator::inverted_index() const` and `InvertedIndex::row_parity_words() const` can mutate caches. Prepare them on + the controlling thread before a team. Never retain a parity pointer across store growth. +3. Distinct inverted-index rows or graph records can share a packed word. Keep bitmap append and graph phase packing + serial initially. +4. Leaders finish local resolution, remote resolution/insertion, responses, and matched marks **before** follower work. + Deferred self misses are published after both passes. Physical insertion order is remote leader misses → remote + follower misses → self leader misses → self follower misses. +5. XOR by one fixed generator is injective, and source terms are unique across owners. This is the reason query + keys/targets are unique; the hash table does not deduplicate bulk input. Debug tests must verify this invariant. + Generic duplicate input to a helper requiring distinct keys must not be silently accepted. +6. Query words, source IDs and captured values form parallel streams. Merge each owner/leader-or-follower stream in + source-range order, not worker completion order. +7. Fused cosine scaling captures old source values before multiplying. Each rotation record owns its entire coefficient + pair. No coefficient atomics. +8. A graph layer's `scaled_count` currently follows post-insertion finalization. Preserve it; the pre-gate source count + is a different boundary. +9. Self derivative entries pair `k` with `k + count/2`, not `2*k` with `2*k+1`. Snapshot both required pre-update values + before writing either endpoint. +10. Gate loops, reverse-gradient loops, repeated-parameter accumulation, graph append and publication phase boundaries + remain caller-ordered. Only the bounded index trial may parallelize hash publication within a phase; MPI stays on + its initializing thread in the final runtime. +11. Empty local stores still follow the same distributed collective sequence. Launch-time thread budgets may differ by + rank without changing ownership; each object's captured budget is immutable. +12. Internal numerical kernels fall back to serial inside an existing OpenMP region. The supported host enters through + one controlling caller per rank: MPI's initializing thread in MPI builds, outside active host OpenMP teams, with no + overlapping dependent-state or host MPI calls. Higher provided support does not relax this rule. The fallback does + not extend that calling contract. +13. A post-mutation failure invalidates the affected owner and its dependent functionals. Check validity before later + state access or copying; independent earlier copies remain valid. Pre-mutation validation errors do not poison. +14. Keep the index swappable for Task 11's A/B trial: + - Nothing outside `OperatorIndex` touches the hash table's internals (slot layout, `insert_slot_`, probe chains). + - No caller depends on the table's iteration order. Clone and memory accounting go through the public API. + - Publication goes through `bulk_insert_hashed`, whose parallel-options argument the packed index may ignore. + - The duplicate-key behaviour of `insert_absent_terms` / `bulk_insert` is pinned by a test. Today the packed index + indexes both rows; any change is an explicit, reviewed contract change. + +## Build/test command library + +These commands are to run **during implementation**, not evidence of passing tests now. Run from the relevant checkout. +Unset stale sanitizer/build-type and MPI override settings before Release commands; R explicitly uses +`monoprop_ENABLE_MPI=OFF` as well as its CMake OFF setting. A configuration change requires forced reinstall and +regenerated tests; `uv sync` alone does not reliably relink the C++ executable. For ordinary suite runs explicitly set +`monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1`; this is test-runner policy, not a library multi-rank default. Dedicated +kernel tests pass internal Options; propagator tests launch fresh processes with a selected environment budget on an +adequate allocation. Unset removed partition settings in candidate launches. OpenMP-fallback tests specifically unset +monoprop_NUM_THREADS. + +### R: non-MPI Release + +```bash +export monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 +monoprop_ENABLE_MPI=OFF uv sync --all-extras --group workspace-test --group bench \ + --reinstall-package monoprop --no-cache \ + --config-settings-package='monoprop:cmake.define.monoprop_ENABLE_MPI=OFF' -v +cmake --build --preset skbuild-release +OMP_NUM_THREADS=1 just test-cpp +OMP_NUM_THREADS=1 just test-py +``` + +For a targeted C++ gate after rebuilding: + +```bash +ctest --test-dir build/editable/Release --label-exclude mpi --no-tests=error \ + --output-on-failure -R 'openmp_|operator_index|fused_|combined_recompute' +``` + +Use `ctest --test-dir build/editable/Release -N -V` to verify discovery. A regex matching zero tests is not a pass. New +Boost cases must be flat `BOOST_AUTO_TEST_CASE` names; CMake discovers `cpp/tests/*.cpp` automatically after +regeneration. + +### M: MPI Release + +```bash +export monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 +# Environment switch also adds mpi4py>=4.1.0 to the isolated build requirements. +monoprop_ENABLE_MPI=ON uv sync --all-extras --group workspace-test --group bench \ + --reinstall-package monoprop --no-cache \ + --config-settings-package='monoprop:cmake.define.monoprop_MPI_TEST_PROCS=1;2;3;4' -v +uv run --no-sync python -c 'import monoprop; assert monoprop.has_mpi, monoprop.__file__' +cmake --build --preset skbuild-release +OMP_NUM_THREADS=1 just test-cpp +# Every success matrix including three ranks uses explicit splitmix. +monoprop_ROUTING=splitmix OMP_NUM_THREADS=1 just test-cpp-mpi +monoprop_ROUTING=splitmix OMP_NUM_THREADS=1 just test-py-mpi '1;2;3;4' +# Rebuild/register a separate default-linear success matrix, excluding unsupported R=3. +monoprop_ENABLE_MPI=ON uv sync --all-extras --group workspace-test --group bench \ + --reinstall-package monoprop --no-cache \ + --config-settings-package='monoprop:cmake.define.monoprop_MPI_TEST_PROCS=1;2;4' -v +cmake --build --preset skbuild-release +monoprop_ROUTING=linear OMP_NUM_THREADS=1 just test-cpp-mpi +monoprop_ROUTING=linear OMP_NUM_THREADS=1 just test-py-mpi '1;2;4' +``` + +Run only on an allocation supporting those ranks; smaller local runs are smoke tests, not substitutes. Do not apply the +serial CTest fabric exclusions to real MPI/performance runs. `just bench-build-mpi` is the existing MPI benchmark build +recipe; `just test-mpi` is the centralized full test recipe. Verify executable mtimes after sync; use the explicit +`cmake --build` above before trusting C++ results (or update the recipe to guarantee relinking in its authorized task). +The CTest default is `2;4`, but `just test-mpi`/`just test-py-mpi` default to `2` unless given ranks or the environment +setting. Supply the intended matrix explicitly; a default two-rank run is not coverage for four-rank multi-peer replay. +If direct CMake settings are used instead of the MPI environment switch, also supply +`--config-settings-package='monoprop:build.requires=mpi4py>=4.1.0'`. `--all-extras` supplies runtime extras, not this +isolated-build requirement. Use `--label-exclude mpi` for the non-MPI leg: the export probe is labeled `unit`, not +`serial`. Test linear R=3 rejection directly with `Router::for_modes`, not by treating a full-suite failed launch as +success. The `just bench-mpi` companion forwards extra arguments to **mpiexec**, not pytest. + +There is no alternate index-width build. `cpp/monoprop/TypeAliases.h:39` fixes TermIndex to uint32_t; +`operator_index_tests.cpp:31–34` asserts it. Inspect cache and imported binary identity for every configuration. + +### S: sanitizer profiles + +```bash +export monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 +SKBUILD_CMAKE_BUILD_TYPE=AsanUbsan \ +monoprop_ENABLE_MPI=OFF SKBUILD_CMAKE_DEFINE='monoprop_SANITIZER=asan-ubsan;monoprop_ENABLE_MPI=OFF' \ +uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-cache -v +cmake --build build/editable/AsanUbsan --target monoprop_unit_tests.x +ASAN_OPTIONS=detect_leaks=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 \ + ctest --test-dir build/editable/AsanUbsan --output-on-failure +LD_PRELOAD="$(g++ -print-file-name=libasan.so):$(g++ -print-file-name=libstdc++.so.6)" \ +ASAN_OPTIONS=detect_leaks=0 uv run --no-sync python -m pytest -s + +SKBUILD_CMAKE_BUILD_TYPE=Tsan monoprop_ENABLE_MPI=OFF \ +SKBUILD_CMAKE_DEFINE='monoprop_SANITIZER=tsan;monoprop_ENABLE_MPI=OFF' \ +uv sync --group workspace-test --all-extras --reinstall-package monoprop --no-cache -v +cmake --build build/editable/Tsan --target monoprop_unit_tests.x +TSAN_OPTIONS=halt_on_error=1:history_size=4 \ + ctest --test-dir build/editable/Tsan --output-on-failure --no-tests=error -R 'openmp_' +``` + +Migrate `justfile::test-cpp-tsan` from `(partition_|shm_comm_)` to retained concurrency tests plus `openmp_`, with +`--no-tests=error`, before deletion; `.github/workflows/qa-analysis.yml` invokes that recipe. Direct selection above is +a focused gate, not a replacement for CI migration. Use existing recipes in workflows; +`tools/check-workflow-commands.py` (enforced by `prek.toml`) forbids duplicating ordinary workflow build/test commands. + +ASan and TSan are separate Linux profiles. Qualify the compiler/OpenMP-runtime TSan combination using a known-racy and +known-safe small test first; unsupported runtime reports are a tooling limitation, not race-free evidence. No +stock-Python TSan gate, no blanket suppression of engine frames, and no sanitizer performance measurements. + +--- + +### Task 0: MPI-off index mini-app and architecture decision + +**Question:** For one process using T physical cores, how do per-thread stores, a shared single-writer index and a +shared concurrent index scale under monoprop-like batches? The owner explicitly permits the answer to overturn the +one-store-per-rank choice. Compare the index-phase pipeline: ownership transfers and packed-row initialization, +not isolated containers. Owners initialize private rows in parallel; both shared variants initialize rows on the caller. +The concurrent index only parallelizes publication, so it need not fix a serial row-initialization bottleneck. +This is NOT a propagation benchmark or production cutover. Exact key/ID checks establish index-result agreement, not +retained-operator equivalence, which also requires coefficients. + +**Size boundary and files:** Create only `tools/index-miniapp/{CMakeLists.txt,main.cpp,README.md}`. One translation unit +contains the generator, three small adapters, driver and `--self-test`; split no framework out of it. Reuse +`cpp/monoprop/detail/operator/OperatorIndex.h`, `cpp/monoprop/core/Monomial.h` and their header-only dependencies +without editing them. Do not build/link/import monoprop, add Python bindings, use MPI/ShmComm, copy the engine, +simulate gates, coefficients/inverted indices/graphs, capture library traces, or implement a scheduler/container +framework. Aim for a few hundred lines of C++, not a second library. If it needs a production storage extraction or +substantial additional machinery, stop and simplify or request a narrower experiment. It must not depend on Tasks 1–3 +being implemented first. + +**PROPOSED interface:** `index-miniapp --self-test`, or one variant/case per fresh process: + +```bash +OMP_NUM_THREADS="$T" OMP_DYNAMIC=FALSE OMP_PLACES=cores OMP_PROC_BIND=close \ + "$BUILD/index-miniapp" --variant shared-serial --mode grow --rows 65536 \ + --batch 4096 --batches 16 --miss-percent 10 --key-length 6 --modes 128 --seed 0 +``` + +This is a proposed smoke-size invocation, not a frozen measurement size or an executed command. Supported variants are +`per-thread`, `shared-serial`, `shared-concurrent`; modes are `lookup` and `grow`. The first performance pass uses only +modes=128, key-length=6 and inline width=11. Width 1024 and spilled length-16 fixtures remain small correctness checks; +their performance studies are deferred, not automatic extra cells. Keep any supporting dispatch minimal, not a generic +container/type framework. Capture omp_get_max_threads once and record +the actual team. Optional `--profile` enables phase timers in a separate diagnostic run; normal runs time complete +batches only, leaving phase fields empty rather than reporting invented zeros. No library environment parser, global +OpenMP mutation or new public monoprop option. Reject malformed numbers, out-of-range percentages, impossible key/row +counts and checked-arithmetic overflow before allocation. Keep study sizes below the common one-store fixed32 ceiling; +report the architectural capacity difference without billion-row stress tests. A worker failure is caught inside its +region, all workers join, then the process reports failure; do not emit a success row. + +- [ ] Obtain separate implementation, host/allocation and execution authorization. The approved Task 0 study limit is + 2 hours total for calibration and measurements, including process initialization, input generation, validation and + phase profiles. Account for setup/build time and costs separately; they need separate authorization. Limit each fresh + process's peak RSS, with setup/growth/checks, to min(8 GiB, 25% of observed usable allocation RAM). Record the + RAM basis and fixed byte ceiling before trials; do not recompute it from falling free memory. Use external monitoring + and + stop on a limit; add no library resource clamp. Keep all attempts and consumed/remaining time across interruptions. + If limits prevent completion, report incomplete evidence; do not extend them or omit losing cells. + Record source/header and executable hashes, compiler/options, Boost/OpenMP versions, commands and placement. Task 0 + neither consumes nor extends Task 1's pilot; its 8-GiB cap does not restrict Task 1's calibrated workloads. + Keep CSV/logs outside the checkout, separate from full-library calibration/acceptance. These limits do not authorize + runtime measurements, setup or launch. +- [ ] Add the standalone self-test entry before implementing variants. RED is the absent target/adapter or incorrect + expected result, not a broken production test. Use ordinary runtime checks that remain active in Release, not assert + alone. The same driver executes tiny cases through all three variants at requested teams 1/2/3/4 on suitable hardware: + +| Tiny fixture | Required result | +| --- | --- | +| Initial A={0,1}; query B={2,3}, A, C={4,5}; publish misses | Hit mask false/true/false; shared IDs 1/0/2; final three exact keys | +| Next batch C,B,A | All hits; shared IDs 2/1/0, including with the concurrent index | +| Repeated existing A,A | Both queries resolve correctly and preserve their output ordinals | +| Repeated absent B,B in a distinct-miss publication fixture | Reject the invalid fixture; do not silently create duplicate rows | +| Empty input/store, all hits, all misses; 15/16/17 and 63/64/65 query tails | Correct sentinels, no dropped work and correct growth | +| Two distinct keys with equal cached 32-bit hash | Exact comparison distinguishes them before and after table growth | +| Spilled rows and a reallocation of their backing store | Earlier lookups still work; no dangling key/query views | + + Across shards compare decoded keys and owner/local-ID pairs, not global numeric IDs. Also compare each variant with + its own serial execution at fixed shard count, so scheduling cannot change IDs. Check every tiny result against a + simple serial key oracle; keep that oracle out of measured processes. Test runtime-limited teams, joined worker + exceptions and actual participation separately; fewer observed workers cannot be reported as T-core scaling. +- [ ] Add only this independent build configuration, enforcing the repository's compiler floors as well. It uses no + root add_subdirectory, binding generation or monoprop runtime dependency; Task 2 still owns production OpenMP setup: + +```cmake +cmake_minimum_required(VERSION 3.28...4.0) +project(index_miniapp LANGUAGES CXX) +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) + message(FATAL_ERROR "GCC 14 or newer is required") +elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18) + message(FATAL_ERROR "Clang 18 or newer is required") +endif() +find_package(Boost 1.85 CONFIG REQUIRED) +find_package(OpenMP REQUIRED COMPONENTS CXX) +find_package(Threads REQUIRED) +add_executable(index-miniapp main.cpp) +target_compile_features(index-miniapp PRIVATE cxx_std_23) +target_include_directories(index-miniapp PRIVATE "${CMAKE_CURRENT_LIST_DIR}/../../cpp") +target_link_libraries(index-miniapp PRIVATE Boost::boost OpenMP::OpenMP_CXX Threads::Threads) +enable_testing() +add_test(NAME index-miniapp-self-test COMMAND index-miniapp --self-test) +``` + + Keep monoprop_ENABLE_MPI undefined; reject a build where it is defined. Confirm no MPI library is linked. The owner + waived compile/test verification against the Boost 1.85 floor after the first pass; building with the host's Boost + (>= 1.85, as `find_package` requires) suffices. Do not silently raise the floor or substitute a different container. +- [ ] Generate identical deterministic global keys/query batches for every variant and T. Use an injective synthetic + encoding of logical key ordinals into sorted positions: divide the 2*NumModes positions into key_length disjoint + bands, encode ordinal digits in that radix, and apply a seed-derived cyclic offset within each band. Reject an + ordinal that does not fit; each band contributes one position. This avoids an O(N) key-deduplication table or trace + corpus. Stream initial rows and retain only one bounded query batch. For hits choose distinct earlier ordinals; for + misses use fresh ordinals, unique within each publication batch. Use floor(batch*miss_percent/100) misses, report the + actual count, and reject a requested distinct-hit count above the existing population. Mix order deterministically. + Repeated hits are a separate self-test, not an accidentally shrinking working set. In lookup mode misses stay absent; + in grow mode they become available to later batches. This structured synthetic family is NOT a claim about real + workload ratios or ownership skew; report those limits rather than adding synthetic physics or a trace service. +- [ ] Implement exactly these ownership variants using the same OpenMP harness. T means one team including its primary + thread; no multiple teams, per-NUMA shared stores, extra processes or fourth hybrid architecture in this task: + +| Variant | Required implementation | +| --- | --- | +| per-thread | T disjoint current OperatorIndex stores; owner is monomial_hash(key)%T, as in R=1 routing; each owner is a single writer | +| shared-serial | One current OperatorIndex; parallel disjoint find_batch_positions output ranges; caller-ordered missing-ID assignment, grow_rows_geometric/set_positions and bulk_insert_hashed | +| shared-concurrent | One packed row backing and one boost::concurrent_flat_map index using compact row handles/cached hashes; ordered IDs and caller row initialization, then parallel index publication | + + Reuse public row access on OperatorIndex for the concurrent adapter; call grow_rows_geometric/set_positions but never + populate its reference table or call reserve(), which would grow that unused table. Its constant empty 16-slot table + is permitted ONLY as measured mini-app overhead, not a second populated index or a production design. OperatorIndex + is noncopyable/nonmovable: keep stores in unique_ptr objects with stable addresses. Store row IDs, not raw row + pointers or query spans; transparent hash/equality consult that same frozen backing store and compare monomial + identity exactly. Retain cached hashes for publication. Never re-enter Boost from a visitor or + assume bulk visitors arrive in query order. A bounded per-query visitor is sufficient; no bulk-API tuning project. + Instantiate only the selected variant. Boost also changes table layout/probing, so its difference from the reference + is not a pure measurement of locking overhead; report its T=1 overhead as well as parallel scaling. +- [ ] Replay only the index phase: prepare one query batch outside timing, then route/bucket → frozen probe → ordered + missing-ID assignment → grow/fill packed rows → publish → return results in original query order → join. Hashing and + ownership calculation belong inside timing, not input preparation; retain/reuse hashes across probe/publication. + In lookup mode omit growth/publication, retaining hit/miss lookup work. Reference variants keep find_batch_positions + and its internal 16-query prefetch pipeline. For shared lookup use balanced contiguous query spans over the actual + team, including short spans/tails; use the same spans for Boost. Do not artificially cap participation at batch/256 + and mistake that chunking limit for map scalability. Small batches may have fewer active workers; record that. + Preserve stable per-owner query subsequences for the sharded arm. That arm may grow/fill/publish different stores in + parallel; shared rows/overflow remain caller-written even in the concurrent arm. Do not overlap reads/inserts or + allocate IDs by scheduling. Use native table growth; time/report any reserve. No free final-capacity + reservation for only one arm. Row initialization completes before concurrent map operations, and joins precede any + row growth/relocation or dependent reads. +- [ ] Model owner bucketing with bounded flat arrays of ordinals/offsets and result scatter, not a copy of ShmComm. + Parallel source chunks may use O(T*T) counts/offsets plus O(batch) records; use ordered prefixes, not one serial + dispatcher doing all work or a T-fold copy of the keys. No producer-consumer queues or custom barriers. Include this + overhead in complete-batch timing, and report index-only probe/publication timings separately. Label bucketing as a + simplified shared-memory model; neither its cost nor its omission from kernel timing predicts legacy transport time. +- [ ] Check results outside timing without a second resident operator: every declared hit/miss, returned row's exact + key, expected owner, final count and insertion order. A final streaming lookup of every expected key plus the exact + row count checks the retained set without an O(N) shadow map. Checksums may prevent dead-code elimination and help + compare runs, but are not the correctness oracle. A timed run with incorrect results is invalid, never fast evidence. + Run separate ASan/UBSan checks and a qualified OpenMP race check on tiny cases; unsupported tooling stays a reported + limitation. Do not use sanitizer timings for scaling or expand into the whole library's test matrix. +- [ ] Run the approved first pass: fixed GLOBAL initial rows and total queries across T, never N rows per thread. + At modes=128/key-length=6/batch=4096/seed=0, use exactly three cases: frozen all-hit lookup, growth with 10% misses, + and growth with 100% misses. Select one small and one larger memory-resident size before comparative timing, within + the common study cap including growth/checking. Verify the accessed footprint exceeds the relevant last-level cache + if claiming DRAM scaling; if the cap prevents that, state the limitation rather than enlarging the budget. Choose + batches to exercise the footprint and record initial/final sizes. Sweep 1/2/4/... physical cores plus exact domain + and allocation endpoints, deduplicated. Repeat within one domain and across the allocated machine; a one-domain host + needs no duplicate series. Use the same core sets and external memory policy for all variants at a given T. The + full-machine MPI-off series is required, even across NUMA domains; an MPI run cannot replace it. +- [ ] Report first-pass results before expanding scope. Defer wider-key, spilled-row performance, batch=64 and other + hit/miss-distribution sensitivities until those results justify an owner-approved follow-up. Keep tiny correctness + fixtures for collisions, both position widths, spilled rows and backing growth; deferring performance cases does not + permit incorrect adapters. Qualify conclusions to measured inputs. Do not add a Cartesian product, weak-scaling + campaign, containers or a trace subsystem; follow-up cannot silently extend the remaining time/RAM limits. +- [ ] Verify active worker masks against physical-core/NUMA topology using external tools and OpenMP diagnostics. + Record natural first-touch: per-owner and caller-built rows can occupy memory differently under the same external + policy. Count that difference as a real ownership/construction consequence; do not normalize away an architectural + disadvantage. Separate locality effects from hash-table synchronization in the report. An explicitly matched + interleaved-memory rerun may diagnose a result within the budget, but cannot replace the original comparison or apply + only to one arm. No internal affinity setter, hwloc dependency or automatic geometry repair. Unknown placement makes + scaling evidence incomplete. +- [ ] Emit a plain CSV row per process with full knobs, requested/actual team sizes, initial/final rows, hits/misses, + complete-batch and optional phase seconds, queries/second, min/max shard rows, row/spill/index byte estimates where + available, and validation outcome. Report setup separately. Measure complete batches without fine-grained timers, + then collect --profile diagnostics in separate matching processes to avoid timer overhead bias. Label per-owner + phase sums/maxima as nonadditive diagnostics when owners overlap; do not present their sum as elapsed batch time. + Profiles also record active-worker counts from the actual probe/publication bodies, not merely team creation. + Use three fresh-process observations per variant/cell with rotated order; retain every sample, median and range. + Report speedup versus each arm's T=1 and time/memory ratios versus per-thread at the SAME T; do not select from + throughput alone or hide losing cases. + Capture whole-process peak RSS with an external tool in fresh processes, including setup/growth. Input generation + and checking use bounded scratch, not a large reference map that masks index peaks. RSS includes allocator/runtime + overhead; current memory_bytes/index_estimated_memory_bytes are estimates, not exact allocations or Task 1 windows. + Archive compiler/runtime/placement metadata beside CSV rather than building a report schema/framework into the app. + +**Planned standalone smoke commands, only after authorization:** BUILD is an absolute, isolated build directory; the +selected allocation must support the requested workers. No uv sync, monoprop imports or engine build is needed: + +```bash +cmake -S tools/index-miniapp -B "$BUILD" -DCMAKE_BUILD_TYPE=Release +cmake --build "$BUILD" --parallel +OMP_NUM_THREADS=1 ctest --test-dir "$BUILD" --output-on-failure --no-tests=error +for T in 1 2 3 4; do + OMP_NUM_THREADS="$T" OMP_DYNAMIC=FALSE "$BUILD/index-miniapp" --self-test +done +``` + +**Gate and handoff:** Deliver the three-file mini-app, self-test evidence, raw CSV/peak-RSS/placement logs and a short +comparison with phase bottlenecks, uncertainty and omitted costs. Distinguish an ownership-model result from a measured +current-library baseline. Do not claim propagation, replay, gradients, MPI or full-library parity from this study. +If an arm is unavailable, placement is unverifiable, correctness fails or the budget ends, report incomplete evidence; +never declare the remaining arm the winner by default. Stop for the owner's decision to retain a shared-store path, +retain/reconsider sharding, or gather specified additional evidence. Record that decision and revise the spec and +Tasks 1–13 accordingly BEFORE proceeding. Preserve the already-approved required MPI-off full-machine library coverage +across all three size tiers; do not choose protected configurations after seeing which architecture wins. No fixed +mini-app speedup threshold replaces the owner decision or weakens the five full-library gates. Task 0 outputs are never +formal Task 1 samples, and its whole-process RSS cannot certify the library's operation/construction windows. + +**Task 0 outcome (owner decision):** the evidence is inconclusive for the ownership axis. One packed store per rank +remains the working hypothesis for Tasks 1–13, and the per-thread (sharded) design stays available through the +existing partition runtime until Task 12. The ownership decision moves to Task 11's full-library comparison against the +old partitions, with an earlier diagnostic go/no-go after Task 6. The index axis — current packed index with serial +publication versus `boost::concurrent_flat_map` with parallel publication — stays open until Task 11's A/B trial. +Evidence (measured on the target host, not acceptance samples): +- The mini-app favoured neither design uniformly. Per-thread won growth with 100% new terms; the shared stores won + lookups; the concurrent index lost 1.2–2.1× on one core. +- Instrumented `propagate` on the benchmark models: lookups hit 80–89% on Hubbard and Pauli, and 0% on the random + circuits. The scan takes 66–95% of one-core time; lookups take 19–21% on the physics models and 2% on random. +- In-library Boost trial: +15–18% end-to-end on one core for Hubbard/Pauli at L1 size. Parallel publication gave no gain + at per-gate batch sizes; the scan was still serial, so there is no full-machine result. + +The results, overlays and budget ledger are archived outside the repository with the Task 0 report, in +`/home/ubuntu/task0-artifacts/` on the c8a.metal-24xl host (`report.md`, `ledger.md`, `budget.jsonl`). They are +intentionally uncommitted and will be removed before merging. The owner waived the Boost 1.85 compile check, as stated +above. + +### Task 1: Freeze baselines and repair measurement trustworthiness + +**Prerequisite:** Task 0 results have been reviewed by the owner and recorded (see the Task 0 outcome), and both +documents have been reconciled. The following remains the conditional one-store campaign, not an automatic continuation +of the mini-app. The formal baseline observations of the partition runtime, including MPI-off full-machine cells, +are also the sharded reference for the post-Task-6 go/no-go and Task 11's ownership decision. No extra measurement +framework is needed. + +**Benchmark roadmap: use [benches/LADDER.md](../../../benches/LADDER.md), not a new suite.** N is nodes, C is usable +physical cores/node, R is ranks/node, and the ladder's P is old partitions/rank (not the generic process-count P used +elsewhere in this plan). Map old P to candidate T workers/rank. On the single target instance N=1: + +| Rung/control | Candidate geometry | Classification | +| --- | --- | --- | +| L1 | R=1, T=1 | Required small/medium single-thread control | +| NUMA-local shared-memory control | R=1, T=C_dom | Required small/medium; not canonical full-node L2a | +| MPI-only control | R=D, T=1 | Required small/medium, with fewer active cores than full-node L2b | +| L2b | R=D, T=C_dom | Required one-rank/domain production shape | +| L2b variant | R=2D, T=floor(C_dom/2) | Required two-ranks/domain shape when C_dom>=2 | +| L2a (MPI-enabled) | R=1, T=C | Full-node diagnostic; cross-NUMA allocation expressly allowed | +| MPI-off full machine | R=1, T=C, has_mpi=false | Required small/medium and selected large profiles, including cross-NUMA | +| L3/L4 | N>1 | Pending proper-interconnect HPC qualification | + +D is allocated NUMA domains, C_dom the selected cores/domain, C=D*C_dom. Record actual balanced subsets and unused cores +rather than infer topology. If D=1 makes MPI-enabled L2a identical to required L2b, deduplicate within that build mode +without downgrading required status. Never deduplicate MPI-off against MPI-enabled evidence, even at the same R/T. +Selected large profiles use both L2b decompositions AND actual MPI-off full-machine operation. Calibrate common profile +sizes using the MPI-off baseline too; a post-freeze capacity failure blocks acceptance, not permission to shrink a +workload or drop its cell. MPI-enabled diagnostic L2a retains the same flags even if it cannot fit. The NUMA-local +control cannot substitute for full-machine operation in either build mode. + +The practical workflow is baseline-only ladder sizing → owner-approved frozen cells/flags → the same pytest nodes on +both arms at matched shapes → existing reports plus the five parity metrics and numerical evidence below. Prefer the +ladder's Hubbard/Pauli lower_atol, random-Heisenberg obs_terms and random-Schrödinger num_generators sizing controls; +record seed/structure and the resulting gradient length. These pre-freeze workload choices never relax oracle +comparison tolerances or permit post-regression retuning. Size graph rows using the largest build/energy/gradient peak, +including export/validation resources, within the already agreed pilot limits. + +The ladder recommends grouped graph runs to save setup. Use that for pilot/diagnostic work where fixtures actually +share a graph; audit the fixtures rather than assume all models share build output. Formal per-operation observations +still use fresh processes and --bench-rounds=1 to avoid prior live graphs and fixture order affecting their peaks. Their +separate construction worker measures setup explicitly. Reuse the existing functions, builders and report files; +observe/validate/compare only supply the missing evidence joins/checks, not another benchmark framework. + +Keep MPI-enabled cross-NUMA L2a in a separately labelled diagnostic inventory/output directory, outside the frozen +required-cell manifest supplied to compare. MPI-off full-machine cells belong IN that manifest, with unchanged +numerical checks and all five parity gates. Never reclassify a failed required cell as diagnostic or use L2a ratios +to waive it. +Numerical mismatches in diagnostic runs still require investigation. Historical ladder results are sizing examples, +not target-machine acceptance evidence. Update its shape/declaration guidance with the candidate cutover. + +**Files:** +- Modify: `benches/conftest.py` (exactness and shared shape preflight); preserve existing windows and timed operations + in `benches/{bench_random,bench_models}.py`. No historical node renaming or restoration of removed pytest nodes. +- Modify: `packages/monoprop-bench-tools/src/monoprop_bench_tools/{report,bmf}.py`. +- Test: `packages/monoprop-bench-tools/tests/{test_memory,test_report,test_bmf}.py`. +- Create: `tools/benchmark-rank-local-openmp.py`, `tools/rank-local-openmp-workloads.json`, + `tests/test_rank_local_openmp_benchmark.py`. +- Reference/migrate guidance: `benches/LADDER.md`; supplement with `tools/capture-baseline.py`. +- Modify: `AGENTS.md`, `README.md`, `docs/content/docs/benchmarks.mdx` where measurement commands/interpretations are + documented. + +**EXISTING:** The four pytest families in each benchmark file are `build_graph`, `propagate`, `energy`, `gradient`. All +have operation windows. `propagate` is graph-free, not a separate inplace node. Functional creation in energy/gradient +is outside their timed window. Nested peaks are preserved by `memory/cpu.py::_fold_open_windows`; keep its tests. + +**PROPOSED interfaces:** One bounded driver and shared overlay used unchanged on both arms, preserving historical node +IDs. Add independent `opmemexact[nodeid]` and `memhwmexact[nodeid]` booleans (all ranks exact in that window); absent +means unknown. The separate construction worker records its own `construction_exact`. Each fresh process writes one +kind-specific artifact; `compare` joins a timed/construction pair into one observation as specified below. The new +paring-construction measurement below is deliberate additional measurement work requiring full-plan approval, not +instrumentation-only. + +**Selected platform, not observed topology:** AWS's +[exact-size table](https://docs.aws.amazon.com/ec2/latest/instancetypes/co.html#co_hardware) specifies c8a.metal-24xl as +AMD EPYC 9R45, 96 physical cores/96 vCPUs, one thread/core and 192 GiB. Its +[C8a description](https://aws.amazon.com/ec2/instance-types/c8a/) explicitly says no SMT. These are published facts, not +measurements. Socket/NUMA counts remain unverified. Do not label oversubscription as SMT coverage or assume this machine +has production's usual eight domains. Once execution is authorized, archive the following read-only observations from +inside the actual allocation; these commands have not been run here: + +```bash +lscpu +lscpu -e=CPU,NODE,SOCKET,CORE,ONLINE +numactl --hardware +grep -E 'Cpus_allowed_list|Mems_allowed_list' /proc/self/status +``` + +Use these observations to select NUMA-contained MPI acceptance and full-allocation MPI-off geometries. A 1x96 shape is +full-machine only with all 96 allocated physical cores. If an MPI-enabled run spans domains, it is diagnostic L2a; +in an actual MPI-disabled build, full-machine operation is required whether or not it crosses domains. Missing +diagnostic tools require another external observation method, not a library topology dependency. Record hardware SMT +state, but configure one worker per physical core. Do not add SMT-sibling profiles; HPC/multi-node qualification remains +separate and pending. + +- [ ] Obtain separate authorization for baseline build/calibration work. On c8a.metal-24xl, record sockets/NUMA domains, + physical cores, hardware SMT state, memory allocation and software stack before choosing geometries. Agree + representative workloads/sizes and single-instance routing/rank/thread shapes. Initial acceptance is the EC2 campaign, + including a two-ranks-per-domain case and actual MPI-off full-machine operation in all three selected size tiers; + all workers use distinct physical cores. Production-HPC performance and + multi-node qualification on a proper interconnect remain pending, not prerequisites for initial EC2 acceptance. + Production's common eight-domain layout is not a hard-coded EC2 topology. Historical profiles below are candidates, + not an approved campaign. + Confirm the bounded new paring-construction measurement in that campaign. Record baseline + revision, allocation, compiler/flags, has_mpi, MPI version where applicable, index width, OpenMP runtime, allocator + environment, imported extension path/hash, seed and complete workload configuration before collecting evidence. +- [ ] Before changing engine code, separately build the baseline using R/M. Preserve its environment and binary. Apply + only the identical measurement-harness changes to both arms; record this overlay's diff/hash. Use the refreshed + main pin for the baseline and the candidate's starting engine, not the former routing-branch snapshot. Main's basis + redraw, transport and parameter-axis changes must not be attributed to OpenMP. Baseline numerical failures stop this + task. After the freeze, any required baseline failure blocks collection, not permission to resize or remove the cell. +- [ ] Once authorized, run unscored baseline-only feasibility trials before freezing workload sizes. Use existing + baseline APIs/benchmark entrypoints; add no fourth driver mode or general calibration framework. Check elapsed + time and memory for construction, evaluation and numerical export/validation, including their temporary storage. + The approved pilot budget is 2 hours total, excluding builds/setup, and 15 minutes per trial including all those + phases. If insufficient, stop and report; do not silently extend either limit. Target at most 75% of observed usable + allocation RAM across all ranks, including construction and validation/export temporary storage. Record the RAM basis + and byte ceiling before trials; do not keep recomputing it from falling free memory. Exceeding this target requires + owner review before larger trials. This is operator-side sizing, not a library memory clamp or an OOM guarantee. + Record every attempted configuration, geometry/placement, baseline binary/overlay identity, elapsed time, memory + observations, outcome and reason for retaining/rejecting a size. Keep raw trials in a separate calibration archive, + without acceptance sample IDs. Capacity/time failures may inform pre-freeze sizing; numerical inconsistencies stop. + Do not run/consult candidate performance to select the inventory. Estimate formal campaign time/cost from the pilot, + then seek owner approval of sizes, coverage and campaign expenditure before the freeze and formal collection. Do not + reduce required samples/coverage to fit an unapproved budget. Collect fresh formal baseline samples after the freeze. + Never promote calibration runs, even with unchanged parameters. Include actual MPI-disabled baseline feasibility + for small/medium and selected large profiles before freezing shared sizes. No candidate-influenced resizing or + removal after freeze. Task 1 limits are independent of Task 0's 8-GiB study; approval is not execution permission. +- [ ] Add renderer tests independently varying operation and outer exactness: true/false/absent and mixed flags. + `bmf.py::build_bmf` publishes `peak-memory` from `memhwm`, so gate it on `memhwmexact`; `operation-memory` comes from + `opmemdelta`, so gate it on `opmemexact`. `report.py::build_report` must label each corresponding metric correctly. + False means non-exact warning and no exact metric for that window; absent means legacy/unknown, never assumed true. + RED: current renderers claim exactness without flags. GREEN: no cross-window certification; time/term fields survive. +- [ ] In `OpMemory.close`, collectively reduce `HighWaterMark.exact` across every rank into `opmemexact`; independently + do the same in `record_memory` for `memhwmexact`. Retain peak/base/delta sum/max, outer memhwm/memhwm_max, opbytes, + opmembreak, opsize and existing provenance. Tiny integration checks require complete fields for all four families. No + rank-zero-only collective. Extend existing nested transient/fallback tests, not the reset implementation. +- [ ] Migrate `_require_shape` in the identical shared overlay. Add explicit manifest/CLI runtime declaration + `--runtime-shape=partitions|openmp` to pytest and driver; this is PROPOSED measurement configuration, not a backend + selector. Baseline declares partitions with `monoprop_PARTITIONS=T` and `monoprop_NUM_THREADS=T`; candidate declares + openmp with `monoprop_NUM_THREADS=T` and removed partition settings unset. OMP_NUM_THREADS may also be T for the host + runtime. Accept the documented candidate fallback with monoprop_NUM_THREADS unset and OpenMP resolving T; verify it + separately in configuration tests. Require positive declared R/T and equal allocation. + Record declared and observed shape separately; a renamed `--arm` label must not bypass validation. Before acceptance, + diagnostic runs must verify actual baseline partition participation and candidate worker participation on + representative large kernels, accounting for documented small/nested/runtime-limit paths. Reject missing or + contradictory evidence. Tests: missing baseline declaration, invalid candidate thread override, leftover candidate + partition setting in the measurement environment, wrong observed R/T, mislabeled arm, and valid explicit/fallback + candidate declarations. This strict measurement preflight does not restore a removed production getenv reader. + RED: current multi-rank guard rejects candidate configuration. +- [ ] Include the build mode in shared preflight and evidence. Read monoprop.has_mpi from the imported extension, + record it with the extension hash, and require it to equal identity.has_mpi for both arms. Actual MPI-off cells have + has_mpi=false and R=1; a single rank of an MPI-enabled binary cannot substitute. The current conftest imports mpi4py + whenever installed: gate that import on monoprop.has_mpi BEFORE touching mpi4py.MPI. For MPI-off, use comm=None and + launch directly, with no MPI launcher/initialization or invented MPI thread-support/version values. Apply the same + guard in the new driver. Use the identical overlay on both arms, without changing the installed Python API. + Tests must cover mpi4py present/absent with has_mpi=false and a trap on attempted MPI import, unchanged MPI-enabled + fixture behavior, requested build-mode mismatch, missing build-mode evidence and MPI-off declarations with R>1. + Neither an arm label nor absence of mpiexec proves the build mode. +- [ ] Separate whole-construction measurement is intentional policy, not a repair for nesting. A fresh untimed worker, + without pytest memory fixtures/session graphs, opens one `HighWaterMark`, constructs inputs/propagator/graph/callable + as required, runs the operation and retains outputs until close. Record peak sum/max, floor, delta and exactness. Use + existing model builders; no duplicate physics or general benchmark framework. +- [ ] After unscored calibration, freeze an owner-approved campaign inventory BEFORE formal baseline collection. Its + JSON schema is + `{"schema_version":1,"workloads_path":str,"workloads_sha256":str,"cells":[cell]}`. Each cell has unique slug `id`, + `profile`, `node_id`, `operation`, `measurement_kind` (`pytest` or `driver`), `identity`, `config_digest` and + `parameters_digest`. `identity` contains boolean `has_mpi`, `ranks`, requested `threads`, `index_bits=32`, + `cpu_allocation`, `compiler_flags`, resolved `config`, and `routing` (`mode`, derived `bits`, numeric `seed`). Expand + the size-band-specific inventory below: small/medium profiles use the full geometry sweep and existing routing + requirements plus MPI-off full-machine operation. Selected large profiles use the two full-node MPI decompositions + and MPI-off full-machine operation. Do not generate an all-sizes Cartesian product. Archive the inventory/digest + with owner approval. This file, not later observations, is the authoritative expected-cell list. A full cell missing + from BOTH arms must fail completeness. Reject duplicate/extra cell IDs and mismatched identities. Do not regenerate or + shrink the inventory to make observations pass. +- [ ] `tools/rank-local-openmp-workloads.json` maps `profiles[profile][node_id]` to `operation`, `measurement_kind`, + `basis`, `picture`, complete `config`, deterministic `parameters`, and `pare_threshold`. `tiny` is the small band and + calibrated `reference` is medium; each band's `-pared` variant shares its geometry policy. Preserve the existing + unpared four-family and threshold-1e-10 energy/gradient plus driver-construction coverage. Add `large`/`large-pared` + entries for the owner-selected large workloads at both full-node MPI decompositions and MPI-off full-machine shape. + Profile names do not freeze historical reference sizes: record every calibrated override, with no ambiguity within a + profile/node pair. Define + config/parameter digests as SHA-256 of UTF-8 + `json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)`. File digests are SHA-256 of exact file + bytes. Resolve relative JSON paths against the containing file's directory. Do not put binary or placement-evidence + hashes inside cross-arm `identity`: those legitimately differ, while allocation/routing/workload must match. +- [ ] Give the driver exactly three argparse modes: `observe` requires `--arm baseline|candidate`, + `--runtime-shape partitions|openmp`, `--campaign FILE`, `--cell-id CELL`, `--sample-id SAMPLE`, `--placement FILE`, + `--output DIR`, plus optional `--whole-process`; `validate` takes the same options except + `--sample-id`/`--whole-process`; `compare --campaign FILE --manifest FILE --output FILE`. Resolve profile, historical + node ID and complete workload from the campaign cell and digest-checked workload file; reject an observed + geometry/routing/configuration that differs. The pytest overlay also takes `--runtime-shape`; never rewrite historical + node IDs. Environment/launcher/allocation stay external; the driver installs nothing. +- [ ] Define the external placement JSON consumed by `--placement`: `schema_version=1`, `artifact_kind="placement"`, + `arm`, `cell_id`, `binary_hash`, `identity`, `runtime_shape`, `declared`, and `observed`. `declared` contains `ranks`, + `threads`, `cpu_allocation` and `settings` (the exact OpenMP/library environment). `observed` contains `has_mpi`, + `ranks`, `threads_per_rank` and `cpu_allocation`, from Task 11's binary-linked large-kernel diagnostic. Preserve + raw launcher/affinity evidence alongside this summary. Baseline shape is `partitions`; candidate shape is `openmp`; + both must match the campaign R/T/allocation. A limited-team diagnostic cannot establish the requested full-team + benchmark cell. Small-kernel serial fallback remains allowed and reported. Each artifact records `placement_path` and + its exact `placement_sha256`; compare reads and validates that file, not just a caller boolean. Cross-arm + placement-file hashes differ legitimately. Diagnostic provenance can be reused for matching samples of the same + cell/arm/binary, never substituted for measured operation or construction artifacts. +- [ ] Normal `observe` writes `DIR/ARM/CELL/SAMPLE/timed.json`; `observe --whole-process` writes + `DIR/ARM/CELL/SAMPLE/construction.json` in a SEPARATE fresh process with the same sample ID. Cell/sample IDs are + path-safe slugs. Create files exclusively; reject overwrite. Each invocation generates a fresh unique `run_id` and + records schema version, artifact kind, arm, cell/sample IDs, campaign digest, binary hash, profile/node/operation, + identity, config/parameter digests and placement reference/hash. Keep pytest raw files in that sample's timed + subdirectory so labels cannot overwrite another sample. Timed artifacts contain runtime, operation and outer-window + fields only; construction artifacts contain construction-window fields only. Every timed, construction and validation + artifact records the imported binary's observed has_mpi as well as the expected identity; compare both to the cell and + placement evidence. Do not invent timing for the untimed worker. `validate` writes a separate uniquely named + validation artifact under `DIR/ARM/CELL`, without sample ID. +- [ ] Define comparison input as + `{"schema_version":1,"campaign_sha256":str,"cells":[{"id":str,"baseline":[sample],"candidate":[sample]}]}`. Each + `sample` contains `sample_id`, `timed_path`, `construction_path`, `validation_path`. `compare` first verifies the + campaign/workload digests and exact expected cell inventory. It then reads the three references and joins only if arm, + cell, binary hash, profile/node/operation, identity, configuration/parameter digests, campaign digest and placement + evidence agree; timed/construction sample IDs must also agree. Binary hashes need not match across arms. Require five + distinct timed runs AND five distinct construction runs per arm/cell initially, ten after prescribed repetition. + Reject missing/mismatched pairs, duplicate sample IDs within an arm/cell, reused measurement paths or run IDs, + including reuse across cells/arms. A validation artifact may serve multiple samples only for exactly the same + cell/arm/binary/configuration/parameters. Never count copied paths or five references to one run as five samples. +- [ ] The joined observation exposes `runtime_seconds`, operation `peak_sum_bytes`/`peak_max_bytes`, + `construction_peak_sum_bytes`/`construction_peak_max_bytes`, and `op_exact`/`construction_exact`. Both exactness flags + must be true for parity. Preserve `outer_peak_sum_bytes`/`outer_peak_max_bytes` and independent `outer_exact` as + DIAGNOSTICS, not parity gates. Outer false/unknown must be labeled non-exact/unknown, not upgraded by op_exact; it + does not invalidate otherwise exact operation/construction evidence. Missing outer flags in legacy raw files map to + unknown; missing required operation/construction exactness fails acceptance. Include operation/construction + floor/delta sum/max and persistent bytes as diagnostics, never substitutes for absolute peaks. Keep original raw + artifact metric names: operation peaks come from opmempeak; outer peaks from memhwm/memhwm_max. Compare exactly FIVE + per-cell medians: runtime, operation peak sum/max, construction peak sum/max. All five ratios must be <=1.00. At R=1 + with MPI disabled, sum and max coincide; retain all five fields/gates and the same numerical oracles. Report + outer ratios separately; an outer-only regression cannot fail this five-metric gate. Reject missing/non-finite + required measurements rather than use zeros. Derive numerical success by reading validation products below; a + caller-supplied boolean is insufficient. Exit 0 means all cells pass, 1 means unmet gates (including incomplete + campaign/sample coverage), and 2 means malformed/incompatible evidence. Output `failed_cells`, missing coverage, five + gate ratios and separately labeled diagnostic ratios. +- [ ] Implement `validate` in a **separate untimed/unmeasured process**, reconstructing the same + model/configuration/operation with existing builders. Include the shared provenance above with + `artifact_kind="validation"` and its own run ID; record `schema_version`, `node_id`, `operation` (one of the five + operation rows below), `config_digest`, `parameters_digest`, `binary_hash`, `basis`, `picture`, finite `energy`, full + `gradient` when relevant, `global_term_count`, `graph_layers`, and `term_files` paths to decoded term/coefficient + files. Each sorted term file is gzip-compressed JSON Lines (`.jsonl.gz`) with + `{"key":[indices],"real":number,"imag":number}`; gradient arrays are in optimizer parameter order. Uncompressed, + truncated or otherwise unreadable term files are malformed evidence. `pare_functional_construct` and pared evaluation + artifacts omit term files because the pared callable owns a subgraph rather than an independently exported operator; + validate its scalar/full gradient against the baseline pared callable. Use deterministic input parameters; a graph + benchmark replays at those parameters for validation. The required products are: + +| Operation | Validation products | +| --- | --- | +| build_graph | energy, gradient, decoded evolved term/value map and graph layer count after replay | +| propagate | immediate energy and decoded evolved term/value map; independent graph-replay comparison on the same inputs where both paths use the same cutoff settings | +| energy | scalar result plus graph decoded map if unpared; pared scalar against same-threshold baseline | +| gradient | scalar/full gradient plus graph decoded map if unpared; pared scalar/full gradient otherwise | +| pare_functional_construct | evaluate the returned pared energy/gradient callable at the recorded parameters; compare to the baseline pared callable at the same threshold | + +For random graph-free propagation, EXISTING `build_random_propagator(..., lower_atol=None)` is the current default; +record it explicitly. Fixed Hubbard/Pauli atols are below. Freeze separate energy/gradient profiles with +`pare_threshold=None` and `pare_threshold=1e-10`; the latter times evaluation, not construction. Pared profiles validate +the same returned scalar/full-gradient callable on both arms, not an unpared operator map. Do not compare different +truncation settings. Existing tiny/exact and finite-difference tests remain independent physics oracles; large-cell +evidence compares baseline/candidate of the identical operation, not an infeasible dense exact simulator. +- [ ] Implement exactly one NEW bounded measurement identity per picture/profile: + `driver::pare_functional_construct[heisenberg]` and `driver::pare_functional_construct[schrodinger]`. In the workload + manifest set `operation="pare_functional_construct"`, `measurement_kind="driver"`, + `functional="expectation_value_and_gradient_functional"`, `pare_threshold=1e-10`, picture, full random config and + parameter digest. Use the proposed `observe`/`validate` interface: `--cell-id` resolves this node/profile from the + frozen campaign, not a new framework or pytest family. Workload key is `(profile,node_id)`; a campaign cell also fixes + geometry/routing and gets a unique slug. Freeze tiny-pared, reference-pared and selected large-pared entries before + formal baseline collection, respecting each band's geometry coverage. + Example planned invocation after selecting the matching cell and binary-linked placement file (launcher/allocation + supplied externally): + +```bash +monoprop_NUM_THREADS="$T" OMP_NUM_THREADS="$T" \ +uv run --no-sync python tools/benchmark-rank-local-openmp.py observe \ + --arm candidate --runtime-shape openmp --campaign "$CAMPAIGN" --cell-id "$CELL" \ + --sample-id s01 --placement "$PLACEMENT" --output "$RESULTS" +# Repeat in a separate fresh process with the same arguments plus --whole-process. +# Run validate separately; omit --sample-id and --whole-process. +``` + + Unset removed partition settings before this candidate command; if launched at three ranks, explicitly export + `monoprop_ROUTING=splitmix` to every rank. Use `--runtime-shape partitions` and both legacy T declarations for + baseline. In normal driver observe, build random inputs first, then open an outer `HighWaterMark` before + propagator/graph construction (the driver analogue of pytest `record_memory`). Construct graph fully before barrier, + timer and operation `HighWaterMark(settle=False)`. Close operation then outer window, retaining graph/callable through + both; publish outer and operation exactness independently. Time ONLY + `built_graph.expectation_value_and_gradient_functional(1e-10)`, using existing barrier semantics (entry untimed, + completion included); one call per fresh process, retain returned callable until the window closes. Record time as + rank maximum and peaks/floors/deltas as rank sum/max with independent exactness. Do not evaluate inside timing. A + separate `validate` process invokes the retained callable at deterministic parameters and records scalar/full gradient + plus graph/config/binary provenance; compare the same pared callable on both arms. No fabricated term export from a + private pared graph. `observe --whole-process` opens before random inputs/propagator/graph construction, builds the + same callable and keeps graph/callable live until close; it is untimed and records construction peaks separately. + Tests instrument event order and callable lifetime, forbid evaluation during timing, check exact CLI/manifest + identity, threshold/config parity, missing outputs and numerical mismatch. RED: driver identities absent. GREEN: + bounded driver and validation artifacts complete on both arms. This retains original paring-construction runtime/peak + coverage. +- [ ] Export maps with the existing low-level `propagator._simulator.evolved_operator(parameters, 0.0)`; use an empty + parameter vector after graph-free propagation and the repeated circuit parameters for graph replay. Sort raw encoded + index tuples and store real/imaginary coefficient components in per-rank gzip files: level 1, with no timestamp or + file name in the gzip header, so equal maps give equal bytes. The line format, sort order and rounding are unchanged + by compression. The owner approved compression instead of adding storage. On the pilot's term files it measured + 4.4-16x smaller, and the draft campaign needs about 24 GB per arm instead of about 122 GB on the host's 128 GB volume. + These exported coefficients are currently rounded to 1e-12; state that limitation and retain unrounded scalar/gradient + checks. Stream-merge sorted rank files when comparing global maps instead of requiring historical rank ownership. For + Heisenberg, the binding adds the replicated identity/core term on each rank: retain one copy after checking agreement, + **do not sum it R times**. Other duplicated nonidentity terms fail unique ownership; Schrödinger identity follows + ordinary owner semantics. Require equal global key sets, including zero-coefficient stored terms; compare values with + the existing `test_utils::near` rule from `cpp/tests/TestUtilities.h.in`: `abs(a-b) <= 1e-9 + + 1e-7*max(abs(a),abs(b))`, componentwise for complex values and gradients. Reject missing/non-finite outputs. Preserve + all stricter existing unit-test tolerances; this benchmark comparison does not weaken them. During pre-freeze + calibration, export failures may require an approved smaller equal-arm profile. After freeze they block acceptance, + not permission to resize or supply an unchecked `numerical_ok=true`. +- [ ] Link every performance observation to a successful validation artifact for the same binary/configuration/operation + and compare baseline/candidate validation artifacts. Add tests with wrong energy, a changed gradient entry, one + missing/extra term, duplicate ownership, replicated identity, non-finite data, stale binary hash and absent + validation. A completed pytest benchmark with only type/length assertions is not numerical evidence. +- [ ] Test the comparator without running benchmarks: + +```python +import hashlib +import json +from pathlib import Path +import subprocess +import sys + + +def test_comparison_does_not_hide_one_regression(tmp_path): + script = ( + Path(__file__).resolve().parents[1] / "tools/benchmark-rank-local-openmp.py" + ) + + def encoded(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + + def digest(value): + return hashlib.sha256(encoded(value)).hexdigest() + + def save(name, value): + path = tmp_path / name + path.write_bytes(encoded(value)) + return str(path) + + config = { + "gen_length": 4, + "obs_terms": 16, + "num_generators": 8, + "num_modes": 8, + "cutoff": 6, + "seed": 0, + "lower_atol": None, + } + parameters = [0.0] * 8 + identity = { + "has_mpi": False, + "ranks": 1, + "threads": 2, + "index_bits": 32, + "cpu_allocation": "test-cpus", + "compiler_flags": "test", + "config": config, + "routing": {"mode": "linear", "bits": 0, "seed": 0}, + } + expected, entries = [], {} + for name in ("build", "replay"): + node = f"energy-fixture-{name}" + entries[node] = { + "operation": "energy", + "measurement_kind": "pytest", + "basis": "majorana", + "picture": "heisenberg", + "config": config, + "parameters": parameters, + "pare_threshold": None, + } + expected.append( + { + "id": name, + "profile": "tiny", + "node_id": node, + "operation": "energy", + "measurement_kind": "pytest", + "identity": identity, + "config_digest": digest(identity["config"]), + "parameters_digest": digest(parameters), + } + ) + workloads = {"schema_version": 1, "profiles": {"tiny": entries}} + campaign = { + "schema_version": 1, + "workloads_path": save("workloads.json", workloads), + "workloads_sha256": digest(workloads), + "cells": expected, + } + campaign_path = save("campaign.json", campaign) + cells = [] + for target, candidate_time in zip(expected, (10.1, 5.0), strict=True): + name = target["id"] + cell = {"id": name} + for arm, elapsed in (("baseline", 10.0), ("candidate", candidate_time)): + shape = "partitions" if arm == "baseline" else "openmp" + binary_hash = ("b" if arm == "baseline" else "c") * 64 + settings = { + "monoprop_NUM_THREADS": "2", + "OMP_NUM_THREADS": "2", + "OMP_DYNAMIC": "FALSE", + } + if arm == "baseline": + settings.update(monoprop_PARTITIONS="2") + placement = { + "schema_version": 1, + "artifact_kind": "placement", + "arm": arm, + "cell_id": name, + "binary_hash": binary_hash, + "identity": identity, + "runtime_shape": shape, + "declared": { + "ranks": 1, + "threads": 2, + "cpu_allocation": "test-cpus", + "settings": settings, + }, + "observed": { + "has_mpi": False, + "ranks": 1, + "threads_per_rank": [2], + "cpu_allocation": "test-cpus", + }, + } + placement_path = save(f"{name}-{arm}-placement.json", placement) + common = { + "schema_version": 1, + "campaign_sha256": digest(campaign), + "arm": arm, + "has_mpi": False, + "cell_id": name, + "profile": target["profile"], + "node_id": target["node_id"], + "operation": "energy", + "binary_hash": binary_hash, + "identity": identity, + "config_digest": target["config_digest"], + "parameters_digest": target["parameters_digest"], + "placement_path": placement_path, + "placement_sha256": digest(placement), + } + validation = { + **common, + "artifact_kind": "validation", + "run_id": f"{name}-{arm}-validation", + "basis": "majorana", + "picture": "heisenberg", + "energy": 0.0, + "global_term_count": 0, + "graph_layers": 0, + "term_files": [], + } + validation_path = save(f"{name}-{arm}-validation.json", validation) + cell[arm] = [] + for i in range(5): + sample_id = f"{name}-{arm}-{i}" + timed = { + **common, + "artifact_kind": "timed", + "sample_id": sample_id, + "run_id": f"{sample_id}-timed", + "runtime_seconds": elapsed, + "peak_sum_bytes": 1000, + "peak_max_bytes": 1000, + "op_exact": True, + "outer_peak_sum_bytes": 1200, + "outer_peak_max_bytes": 1200, + "outer_exact": True, + "operation_floor_sum_bytes": 800, + "operation_floor_max_bytes": 800, + "operation_delta_sum_bytes": 200, + "operation_delta_max_bytes": 200, + } + construction = { + **common, + "artifact_kind": "construction", + "sample_id": sample_id, + "run_id": f"{sample_id}-construction", + "construction_exact": True, + "construction_peak_sum_bytes": 1200, + "construction_peak_max_bytes": 1200, + "construction_floor_sum_bytes": 800, + "construction_floor_max_bytes": 800, + "construction_delta_sum_bytes": 400, + "construction_delta_max_bytes": 400, + } + cell[arm].append( + { + "sample_id": sample_id, + "timed_path": save(f"{sample_id}-timed.json", timed), + "construction_path": save( + f"{sample_id}-construction.json", construction + ), + "validation_path": validation_path, + } + ) + cells.append(cell) + manifest = save( + "manifest.json", + {"schema_version": 1, "campaign_sha256": digest(campaign), "cells": cells}, + ) + output = tmp_path / "comparison.json" + result = subprocess.run( + [ + sys.executable, + str(script), + "compare", + "--campaign", + campaign_path, + "--manifest", + manifest, + "--output", + str(output), + ], + check=False, + ) + assert result.returncode == 1 + assert json.loads(output.read_text())["failed_cells"] == ["build"] +``` + +These two miniature inventory entries use empty-map energy fixtures, not real benchmark measurements. They follow the +same schema, reference loading and provenance checks as real artifacts; no synthetic-mode bypass is allowed. The +comparison test does not run physics, acquire placement, or establish actual hardware participation. Add tests for a +whole required cell deleted from both arms (including a large MPI-off full-machine cell), missing samples, mismatched +timed/construction pairs, reused files/run IDs and calibration output offered as acceptance evidence. Also reject +missing/mismatched has_mpi between inventory, artifacts, placement and binaries; MPI-off declared at R>1; and a D=1 +attempt to deduplicate MPI-off cells against MPI-enabled cells. Check missing placement provenance (exit 2), +each false/absent exactness flag, unequal +shape/configuration/index width, and sum-versus-max peaks. For only an outer-peak regression or unknown outer exactness, +report the diagnostic accurately while allowing the five-metric gate to pass. Operation/construction inexactness still +fails. The comparator reports that five more observations are needed on first ratio failure; it launches no jobs. +- [ ] Use the baseline-only trials and observed allocation to freeze owner-approved sizes for these coverage families in + `tools/rank-local-openmp-workloads.json`: fixed Hubbard and Pauli models and random + Heisenberg/Schrödinger × build_graph/propagate/energy/gradient. Energy/gradient have separate unpared and + threshold-1e-10 profiles; do not change node IDs to distinguish the profile. Also require random Heisenberg and + Schrödinger `pare_functional_construct` driver cells; no removed pytest node is revived. The following are EXISTING + source-reference configurations, not the selected acceptance workloads: + +```json +{ + "hubbard": {"num_sites": 60, "hopping": 1.0, "interaction": -2.0, + "chemical_potential": 0.0, "trotter_dt": 0.2, "trotter_steps": 29, + "observable_site": 46, "observable_spin": "up", "neel_start_spin": "down", + "cutoff": 6, "lower_atol": 0.0001}, + "pauli": {"num_qubits": 127, "num_layers": 20, "observable_qubit": 62, + "theta": 0.7853981633974483, "coupling": 0.7853981633974483, + "cutoff": 8, "lower_atol": 0.0001}, + "random": {"gen_length": 4, "obs_terms": 10000, "num_generators": 100, + "num_modes": 128, "cutoff": 6, "seed": 0, "lower_atol": null} +} +``` + +Add a tiny profile with random options `--num-generators 8 --num-modes 8 --cutoff 6 --obs-terms 16`; other random values +stay fixed. Both fixed models use their existing picture; random cases cover both pictures. Keep these complete +configurations in artifacts, not just a profile label. +- [ ] Collect node IDs with `uv run --no-sync python -m pytest benches --collect-only -q`; record exact IDs and run one + node per process. The existing fixed-model guard skips retained graphs with more than two repeated steps + (`MAX_GRAPH_STEPS=2`), so the 29-step Hubbard reference is intentionally a capacity decision: use + `monoprop_BENCH_ALLOW_BIG_GRAPH=1` only on adequately provisioned hardware, or obtain approval for a fixed lower + `--hubbard-trotter-steps` profile on both arms before formal baseline collection. A skipped cell is not a pass. Do not + silently omit graph-heavy cells or bypass the guard on an unverified allocation. Owner decision after the pilot: + Hubbard graph cells (`build_graph`, `energy`, `gradient`) use 2 Trotter steps on smaller lattices, within the guard. + Hubbard `propagate` keeps its 29 steps. The 29-step graph did not fit this host: the pilot killed it above the + 75% memory target, with at least 147 GiB resident. It may be checked separately on a larger machine; that check is + not part of this campaign. +- [ ] Select required geometries from observed physical cores, not vCPU/SMT-sibling counts. For D allocated domains with + C_dom cores each, keep `(R,T)=(1,1),(1,C_dom),(D,1),(D,C_dom)` for small/medium profiles. These R=1 controls use one + domain. Add `(2*D,floor(C_dom/2))` with disjoint cores when C_dom>=2; record unused cores. Preserve MPI routing + coverage. Add actual MPI-off `(1,C)` for every small/medium profile and the owner-selected large/large-pared profiles. + Selected large acceptance uses `(D,C_dom)` and `(2*D,floor(C_dom/2))` with MPI, plus `(1,C)` without MPI. Freeze + has_mpi and geometry per cell; deduplicate only within the same build mode and complete identity. In particular, + D=1 never lets an MPI-enabled sample replace required MPI-off evidence. Freeze allowed cells per profile; + do not reclassify profiles or drop cells after a regression. Uneven domains require approved balanced subsets, not + assumed C_dom=96/D. Each pair has identical build mode, R/T/cores and inputs. Baseline uses T partitions per process; + never compare only against forced-serial old code. Launch MPI-off directly after verifying not monoprop.has_mpi; + its one process spans the allocation without an MPI launcher or fictitious NUMA containment. +- [ ] Run MPI-enabled canonical full-node L2a separately as a labelled diagnostic with the same model flags at R=1,T=C. + It may cross NUMA domains; verify complete allocation and one worker/core rather than claiming containment. Keep its + separately budgeted outputs outside required-cell artifacts, not in an extra driver mode. Do not substitute a smaller + NUMA-local run or relabel the required MPI-off `(1,C)` cell as diagnostic. Record failed/unavailable diagnostics; + no diagnostic ratio is a parity gate or waiver. Required MPI L2b stays NUMA-contained; L3/L4 remain pending. Preserve + existing node IDs/term-count fields and add runtime-shape/build-mode/actual-team metadata. Counts alone cannot replace + global term-map/value checks. +- [ ] Use `just capture-baseline` / `just diff-baseline` and `tools/capture-baseline.py --compare` only as supplementary + small-fixture regression evidence. Its keys include rank, tolerance differs and it lacks operation + gradients/provenance; it cannot replace the ownership-aware global-map comparator or independent + exact/finite-difference tests. + +**Frozen campaign (owner-approved 2026-09-25):** `tools/rank-local-openmp-campaign.json` (sha256 +`ffde870c77a66c8ce37fb2cc07b39abef976e9d7938b7cea59f43cb407de068a`) lists 250 cells over +`tools/rank-local-openmp-workloads.json` (sha256 `508b65f1f0eb2171fc4fbd79feba2381054c6dcdd38a614600f5b957ab814416`). +Neither file may be regenerated or shrunk to make observations pass. The unscored pilot and its size decisions are +archived outside the repository with the Task 1 handoff report; no pilot output is a formal sample. +- Artifact locations: all Task 1 artifacts are in `/home/ubuntu/task1-artifacts/` on the c8a.metal-24xl host. + They are intentionally uncommitted and will be removed before merging. `ledger.md` indexes them and records every + hash. + - `report.md`: the handoff report. `preflight/`, `provenance/` (overlay patches and hashes) and `findings/` + (VmHWM shortfall, gzip measurement) hold its supporting evidence. + - `formal/results/baseline//`: the formal baseline evidence (s01..s05 timed and construction artifacts plus + one validation per cell, about 20 GB). `formal/placements/` and `formal/diagnostics/` hold the per-cell placement + files and their raw evidence. + - `formal/baseline-index.json` (sha256 `82479d6ff83deb3113306ac1caa1720d25f15a400cbc89b5693f85ffad47fff5`): the + baseline half of the Task 11 comparison manifest. + - `formal/formal.py`, `formal/verify.py`, `formal/consistency.py` and `pilot/pilot.py`: the operator-side code + that collected and checked the evidence, kept as run rather than as reusable tooling. + - `pilot/` and `proposal/`: the unscored pilot and the inventory proposal. +- The observed host has one NUMA domain (D=1, C=C_dom=96). Within each build mode, L1 and the MPI-only control + coincide at 1x1, and the NUMA-local control, L2b and L2a coincide at 1x96. There is therefore no separate L2a + diagnostic inventory. MPI-off 1x96 cells stay separate and required. The single-thread control runs in the MPI + build. +- `tiny` and `reference` (plus their `-pared` variants) run at MPI 1x1, 1x96 and 2x48 and at MPI-off 1x96. + - Each band has 16 unpared nodes. The `-pared` variants cover energy/gradient at threshold 1e-10 for all four + families, plus `driver::pare_functional_construct` in both pictures. + - Random reference `propagate` and `build_graph`, in both pictures, add 3x32 splitmix, 4x24 splitmix and + 4x24 linear cells. +- `large` and `large-pared` run at MPI 1x96 and 2x48 and at MPI-off 1x96. Their sizes (13-68M terms) are set by the + cost of validation export, not by RAM. +- Owner decision on memory: the measured `VmHWM` shortfall is accepted as documented measurement noise for the + five gates. With threads on many CPUs, a window can miss up to about 50 MiB of a real transient (see + `docs/content/docs/benchmarks.mdx`, "Memory exactness"). The strict <=1.00 gates are not relaxed. +- The pilot killed one trial above the 75% memory target: the rejected 29-step Hubbard graph. The owner reviewed it; + no cell in the campaign needs that graph. + +**Gate:** Calibration archive and owner-approved size-band inventory/digest are retained; benchmark tools' unit tests +and baseline numerical suites pass. Fresh post-freeze baseline observations and placement evidence cover every required +cell; no calibration sample is counted. Historical benchmark keys remain unchanged. No parity claim yet. + +### Task 2: Add the smallest OpenMP primitive and its build dependency + +**Files:** +- Create: `cpp/monoprop/detail/parallel/{Options.h,Workshare.h,CMakeLists.txt}`. +- Modify: `CMakeLists.txt`, `cpp/monoprop/CMakeLists.txt`, `cpp/monoprop/detail/CMakeLists.txt`, + `cmake/monopropConfig.cmake.in`. +- Create/test: `cpp/tests/openmp_workshare_tests.cpp`. +- Modify: `pyproject.toml`, `tools/install-deps.sh`, `tools/packages/{apt,brew}.txt`, + `.github/actions/setup/action.yml`, `.devcontainer/Dockerfile`, `nix/{monoprop,devshell}.nix`, `flake.nix`, + `.github/workflows/{test,nix,deploy}.yml` for required OpenMP across supported dependency routes. +- Extend: `cpp/tests/find_package_smoke/CMakeLists.txt`, `cpp/tests/link_export_probe/link_export_probe.cpp`; reuse + `just test-find-package`, not a second consumer harness. +- Modify: `AGENTS.md`, `README.md`, `docs/content/docs/building.mdx` for required OpenMP. + +**PROPOSED interfaces:** In namespace `monoprop::detail::parallel`: + +```cpp +struct Options { + int threads = 1; //!< Validated positive per-object budget; not a global runtime setting. +}; + +template +auto for_blocks(size_t count, Options options, Fn &&body) -> void; +// body(block_id), one invocation per logical block; returns only after all work joins. +// No MPI, no shared append, no escaping worker exception. Serial if nested or budget==1. +``` + +`Options.h` contains no OpenMP include; `Workshare.h` includes ``. No pool, scheduler, task queue, type-erased +jobs, or persistent worker-owned operator state. + +- [ ] Add tests that require the new helper: every block visited exactly once; count 0/1/19; budget 1/2/3; more + requested threads than blocks; nested invocation; original runtime max-thread setting unchanged; a worker throws + `std::runtime_error` and caller catches it after all work has joined. +- [ ] Use this test pattern; each element has one writer: + +```cpp +BOOST_AUTO_TEST_CASE(openmp_blocks_visit_once) { + std::vector visits(19); + monoprop::detail::parallel::for_blocks(19, {.threads = 3}, [&](size_t b) { + ++visits[b]; + }); + for (const auto n : visits) { BOOST_TEST(n == 1); } +} +``` + +For the exception test also record visited/completed blocks without asserting which ones finish after a failure. Never +use `BOOST_TEST` inside workers; store results and assert on the caller. +- [ ] Run the new test build: expect missing header/helper. Add required dependency integration: + +```cmake +# Root discovery +find_package(OpenMP REQUIRED COMPONENTS CXX) +# Compilation and link/export requirements are separate: the shared library is +# assembled from TARGET_OBJECTS and does not inherit the object target's linkage. +target_link_libraries(monoprop-objs PUBLIC OpenMP::OpenMP_CXX) +target_link_libraries(monoprop PUBLIC OpenMP::OpenMP_CXX) +# Installed cmake/monopropConfig.cmake.in: +find_dependency(OpenMP REQUIRED COMPONENTS CXX) +``` + +Register `Options.h`/`Workshare.h` in the existing `headers` FILE_SET via the new subdirectory. Use imported target +flags, not hard-coded `-fopenmp` or `-lgomp`. There is no OpenMP-disabled build in this design; a one-thread build/run +is still OpenMP-enabled. +- [ ] Implement serial and parallel paths. Serial path calls `body(b)` in ascending order without allocating error + slots. Parallel path requests `min(options.threads,count)` workers, uses + `#pragma omp parallel for schedule(static) num_threads(team)`, catches into preallocated `std::exception_ptr` slots + **indexed by actual worker ID**, joins, then rethrows the first populated slot. Allocate at most `team` errors, not + one per row/block. Validate positive budget and loop-bound representability before the region. Correctness must not + depend on actual team size equaling the request. +- [ ] Do not invoke throwing setup outside the worker's catch. Exceptions from allocation of error slots occur before a + region and propagate normally. The helper cannot provide transactional rollback. +- [ ] Add `libomp` to the Homebrew manifest and the matching Clang OpenMP development package to the apt/setup route; + GCC uses its selected libgomp. `.github/actions/setup/action.yml` and `.devcontainer/Dockerfile` consume the shared + package manifests: update the source of truth rather than duplicate installs. Wire compiler-compatible OpenMP inputs + into both Nix package and devshell, preserving Nix's explicit sandbox build-time mpi4py provisioning. Extend + `flake.nix` package/MPI package/check/devshell checks and `.github/workflows/nix.yml` smoke validation. Verify OpenMP + configure/link/run in both Nix packages and shell. Hwloc stays until Task 12; do not delete it early. +- [ ] Install `libomp` for macOS and Clang Linux jobs; verify keg-only Homebrew discovery with the actual toolchain. + Keep GCC's selected libgomp when using GCC. Linux wheel dependency repair and macOS delocation must resolve the chosen + runtime in a clean environment. Keep hwloc only while the legacy partition runtime still requires it; Task 12 deletes + that dependency and its topology code rather than porting them. Audit Threads separately for remaining non-partition + uses; OpenMP discovery/linking must not add a direct hwloc requirement. +- [ ] Run R and the new helper tests with `OMP_DYNAMIC=TRUE` and `FALSE`, plus `OMP_THREAD_LIMIT=1`. A limit of one must + still visit every block. Add a test-only worker-ID observation proving more than one worker participates when + hardware/runtime permits; report a skip when fewer than two are available, not a false success. + +**Gate:** Helper tests and `just test-find-package` pass; extend its shared link-export probe source to instantiate an +OpenMP template and both basis APIs using installed targets only. RED: missing imported OpenMP dependency/header; GREEN: +clean consumer finds/links/runs with no manual flags or source includes. No engine behavior changes yet. + +**Task 2 outcome (2026-09-27):** gate passed on GCC 15.2 and Clang 18.1.8 Linux; other platforms pending. +- Helper contract as implemented, which later tasks rely on: + - `for_blocks` throws `std::invalid_argument` for a budget below one and `std::length_error` for + `count > PTRDIFF_MAX`, before any block runs, on every path. + - It runs serially and in ascending order when `min(threads, count) <= 1` or `omp_get_level() > 0`, so any + enclosing region counts, active or not. + - In the parallel path each worker stops taking blocks after its own first exception. The lowest worker ID's + exception is rethrown after the join. +- Tests: `cpp/tests/openmp_workshare_tests.cpp` has 13 flat cases. The participation case is skipped by a + Boost precondition when the runtime cannot provide a two-worker region. + - RED: missing header, then `undefined reference to omp_get_thread_num` without the CMake dependency, then + the installed consumer missing `Workshare.h`. + - GREEN: 4 of 4 requested workers observed under `OMP_DYNAMIC=TRUE/FALSE` with both libgomp and libomp; + `OMP_THREAD_LIMIT=1` stays correct with the participation case skipped; hundreds of fresh processes + without failure. Injected helper mutations were all caught. +- R and M suites pass: splitmix R=1–4 and linear R=1,2,4, in C++ and Python. `just test-find-package` passes + for the R, M, Clang and repaired-wheel installs. A copy of the installed config without + `find_dependency(OpenMP)` fails to configure, so that line is load-bearing. +- The engine has no OpenMP code yet, and GNU ld's default `--as-needed` drops libgomp. The GCC `_core.abi3.so` + and `libmonoprop.so` are byte-identical to the Task 1 baselines, and a locally repaired wheel bundles no + OpenMP runtime. Verify runtime bundling and delocation once library code uses OpenMP (Task 4 onward, and + Task 13). +- Dependency routes: + - Homebrew `libomp`, found through `OpenMP_ROOT=$(brew --prefix libomp)` in the setup action, the macOS + cibuildwheel environment and the docs. Downstream macOS consumers need the same variable. + - The setup action installs `libomp-dev` whenever `CXX=clang++` on Linux. The devcontainer adds + `libomp-dev` for its default clang. + - Nix adds `llvmPackages.openmp` when `stdenv.cc.isClang`, and the flake checks include `monoprop-mpi`. + `nix.yml` gains a dev-shell `just build` + `just test-find-package` step. + - The clang-tidy job in `qa-analysis.yml` sets `CXX: clang++` for setup, so it receives libomp. + - Hwloc is unchanged. +- Verified only on Linux x86_64 (GCC, Clang, local wheel with auditwheel). Nix, macOS/Homebrew, the devcontainer + and cibuildwheel/delocate have not run; they are pending CI or a suitable host. +- Build-route note: `cmake --build --preset skbuild-release` cannot rebuild a tree that `uv sync` created, + because its cached `CMAKE_MAKE_PROGRAM` points at the deleted build-isolation ninja. Rebuild through the R/M + `uv sync` commands instead, and check binary mtimes and hashes. + - Keep M and Clang builds apart from R with `UV_PROJECT_ENVIRONMENT=` and + `--config-settings-package='monoprop:build-dir=build//{build_type}'`. + - Use `--all-extras` for comparable pytest counts: without mpi4py the communicator-parametrized cases are + not collected. +- Artifacts: `/home/ubuntu/task2-artifacts/` on the c8a.metal-24xl host. `report.md` indexes the numbered + logs, build scripts, mutant runs and the wheel. They are intentionally uncommitted and will be removed before + merging. The frozen campaign/workload files and the Task 1 artifacts and baseline binaries were verified + unchanged. + +### Task 3: Plumb per-object budgets and distributed failure boundaries + +**Files:** +- Create: `cpp/monoprop/detail/parallel/{ThreadBudget.h,ThreadBudget.cpp}`. +- Create: `cpp/monoprop/detail/mpi/OperationFailure.h`. +- Modify: `cpp/monoprop/detail/parallel/CMakeLists.txt`, `cpp/monoprop/detail/mpi/CMakeLists.txt`. +- Modify: `cpp/include/monoprop/{MonomialPropagator,Evolution,MPFunctions}.h`. +- Modify: `cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl`, `cpp/monoprop/{Evolution,MPFunctions}.cpp`. +- Modify: `cpp/monoprop/detail/evolution/{CosineRecompute,CosineRecomputeCallbacks}.h`, + `cpp/monoprop/detail/evolution/layer_build/{Scan,Resolve,Engine,FusedApply}.h`. +- Modify: `cpp/monoprop/detail/mpi/MPICompat.{h,cpp}`; audit request/collective boundaries in `Exchange.h` and + `Pairwise.h`. + Final environment cleanup is Task 10, MPI-level cutover Task 12. +- Create/test: `cpp/tests/openmp_runtime_tests.cpp`, `cpp/tests/mpi_failure_driver.cpp`, `tests/test_openmp_config.py`; + modify `cpp/tests/CMakeLists.txt` to exclude the failure-driver main from the unit-runner glob and build a separate + executable. + +**PROPOSED interfaces:** + +```cpp +// ThreadBudget.h; include , and Options.h. +namespace monoprop::detail::parallel { +// nullopt selects runtime_default; a present value must be decimal digits and in [1, INT_MAX]. +// Throw std::invalid_argument on invalid input; no hardware query or global runtime mutation. +auto resolve_thread_budget(std::optional configured, int runtime_default) -> Options; +// Read monoprop_NUM_THREADS once and obtain omp_get_max_threads(); called only during construction. +auto capture_thread_budget() -> Options; +} +// No new public constructor argument or binding keyword. +// Private object members: parallel::Options parallel_; bool invalid_ = false; +// Private helpers: auto require_valid_() const -> void; auto invalidate_() noexcept -> void; +// MonomialPropagator.h: declare monoprop::InvalidPropagatorError deriving std::runtime_error. +// EvalRequest: append detail::parallel::Options parallel = {}; +``` + +All the following existing functions gain a trailing `parallel::Options options = {}` (fully qualify namespace in public +headers): `build_layer`, `fused_find_and_collect`, `probe_incoming_queries`, `resolve_incoming`, `apply_fused_contract`, +`scale_cos_lazy`, `accumulate_cos_lazy`, `scale_cos_mask`, `accumulate_cos_mask`, `evolve_step`, `evolve_operator`, +`state_operator_derivative_local`, and `build_cos_callbacks`. Private helpers receive/forward the same value. Keep +callback types `LayerCosScale`/`LayerCosAccumulate`/`LayerCosIndices` unchanged: closures capture options. EXISTING +derivative signature in `cpp/include/monoprop/Evolution.h:54–63` is the following signature (retain its existing export +annotation): + +```cpp +auto state_operator_derivative_local(VecD&, VecD&, const MPGraphView&, size_t, + LayerAngle, mpi::Comm, const detail::LayerCosAccumulate&, + const detail::CosRecordView& = {}) -> double; +``` +PROPOSED options follow `record`, not `angle` or `cos_acc`; preserve `LayerAngle{.gen_coeff=g,.param=theta}` and record +forwarding. `MPFunctions.cpp:44–172` records pre-layer values based on accumulated amplification AND vanishing-cosine +criteria; `Evolution.cpp:437–464` snapshots self endpoints, predivides, accumulates, restores and combines +pre-layer-unit sums. Keep record append/predivide/restore serial because duplicate indices are allowed. Do not replace +this stable protocol with a division-only reverse pass. + +- [ ] Add compile/integration tests for old low-level numerical call sites (their default Options stay serial), + prototype budgets 1/3 in separately launched processes, copied budget and retained-functional capture. Prove legacy + children stay serial even when their process environment requests multiple threads. +- [ ] During coexistence, initialize `parallel_` to `{.threads=1}`. Only when the EXISTING constructor has explicit + `partitions==1` AND `comm.kind==mpi::Comm::Kind::Mpi`, capture the environment budget before work. This also covers + non-MPI builds, whose ordinary Comm has Kind::Mpi. Shm/Hybrid children never activate OpenMP; multi-partition facades + keep their legacy path. Prototype tests use existing `partitions=1` and launch-time monoprop_NUM_THREADS; no new + constructor keyword, runtime switch or test-only production setter is introduced. Task 10 removes the old arguments + and makes capture unconditional for independent instances. +- [ ] Rebuild bindings through R/M without adding a thread keyword. Configuration already invokes binder generation + and installation invokes dispatch generation. For manual regeneration of the default Release configuration only: + +```bash +uv run --no-sync python tools/generate-binders.py --max-num-modes 1024 \ + --batch-size 8 --output-dir build/editable/Release/src/monoprop/bindings/generated \ + --binding-kind core +``` + +For a nondefault maximum use the actual `monoprop_MAX_NUM_MODES` cache value (current default 1024). Generated files +stay in the build tree and are not committed. Never hand-edit generated bindings, invent a mode-width policy, or modify +`tools/_binding_layout.py` for this task. +- [ ] Implement `resolve_thread_budget(configured,runtime_default)` as a pure parser: nullopt selects a positive runtime + default; present input must be nonempty ASCII decimal digits, with the whole value in [1, INT_MAX]. Use checked + parsing before narrowing; reject signs, whitespace, lists, junk, zero and overflow. Leading zeros are harmless. Do not + retain the old arbitrary million-thread parser cap. Translate std::invalid_argument to PropagatorConfigError at the + constructor boundary, then apply the coherent distributed failure policy if peers can be stranded. Tests cover nullopt + with 1/3, nullopt with 0/-1 rejection, "8" with default 2 returning 8, and "1" with default 0 returning 1. + Reject empty, "0", "-1", "abc", "2,3", " 3" and "+3". Test INT_MAX text acceptance and overflow rejection without + creating teams. +- [ ] `capture_thread_budget()` reads getenv("monoprop_NUM_THREADS") once, distinguishes absent from present-empty, + and calls the pure resolver with omp_get_max_threads(). No function-static budget cache or operation-time reload. + During coexistence keep the old config parser solely for legacy partition selection; remove that cached budget path + in Task 10. Copy resolved options unchanged. The supported application fixes ranks/threads at launch; no live + configuration mechanism is provided or required. +- [ ] OpenMP supplies the fallback at any rank count, not an actual team size or hardware count. Do not independently + parse OMP_NUM_THREADS or derive T from omp_get_num_procs, hardware_concurrency, affinity masks, places, hwloc, sysfs + or NUMA. A request may exceed the allocation; resource suitability is the user's responsibility. Small-work/nested + internal-helper fallback and OpenMP limits may reduce actual teams without changing the captured budget. +- [ ] In this task test parser/copy/callable capture and runtime-limited teams through the one-store prototype; + final constructor surface and full default-selection subprocess tests belong to Task 10. Verify library calls leave + OpenMP settings unchanged and issue no affinity-setting calls. A no-affinity-change test must disable OpenMP binding + (`OMP_PROC_BIND=FALSE`), so it does not mistake the runtime's user-requested placement for library pinning. No + hardware-topology query is needed by these tests; Task 11 placement checks are external benchmark-operator work. +- [ ] Forward options through ALL paths: graph-only build; graph-with-coefficients stored-mask closure; immediate fused + apply; replay; retained functional capture and `EvalRequest`; `ev`/`ev_and_grad` → `prepare_evolved_operator` → + evolution; derivative callback; partial contraction. `build_cos_callbacks` captures `[cache, sc, options]`. Keep + parity pointers ephemeral. Do not alter the GIL policy or external callable lifetime contract. +- [ ] Implement a caller-side `mpi::operation_failed(comm, std::exception_ptr)` as a `[[noreturn]]` helper. For an + ordinary real MPI communicator with size>1, report rank/error and call `MPI_Abort`; if MPI_Abort returns, terminate. + For size one/non-MPI, rethrow the original exception. Keep transitional legacy communicator failures on their existing + poison/abort paths until Task 12. +- [ ] Guard every newly throwable distributed phase, not merely the outermost function. Especially catch a worker + failure in overlapped cosine work **inside the lifetime of an active exchange ticket** and abort before stack + unwinding can wait on peers that never posted. This covers BOTH replay `Exchange.h::Ticket` and construction + `MPICompat.h::PendingAlltoallv`: both are already move-only/self-draining; move assignment drains the old destination. + Preserve unified request-vector ownership, moved-from posted=0 and pre-sized request storage. Normal successful paths + still explicitly wait. Extend the active-ticket driver scenario to both handle types; source lifetime tests alone do + not establish coherent distributed failure. Do not rely on a ticket + destructor to coordinate distributed failure; do not add a per-kernel allreduce to agree on allocation failure. Wrap + constructor/build/replay/evaluation phases that can strand another rank, including retained-callable entry points. + Single-rank failures preserve the original exception type. Add private `invalid_`, `invalidate_()` and + `require_valid_()` to enforce non-reuse. The latter throws InvalidPropagatorError with a stable recreate-the-object + diagnostic. On the caller, separate validation from the mutation phase using a local `mutation_started` flag; set it + before the first mutable cache/state/workspace change. A catch after that boundary calls invalidate_() before the + existing abort/rethrow path, including inside active-ticket lifetime. Successful calls never invalidate; pre-mutation + validation errors preserve validity. No atomic flag, rollback framework or new persistent workspace is needed. +- [ ] Call require_valid_() before state-consuming/mutating public operations and dependent functional invocations. + Retained closures capture the existing non-owning owner reference and check it before touching caches/snapshots; + they invalidate that owner on their own post-mutation failure. Do not change C++ callable lifetime or Python owner + retention. Validate a copy/clone source before any store/graph copy; cloning must not revive an invalid object. + Already independent copies remain valid. Destruction is always allowed; immutable configuration diagnostics need + not inspect invalid computational state. No production reset/setter clears invalid_. +- [ ] Add single-rank test-only failure injection at a real mutation boundary and during functional evaluation. Catch + the original exception; then assert build/replay/update/functional creation, a previously retained functional and + copy/clone reject the invalid owner. An independent earlier copy still evaluates. A validation-only failure leaves + the original usable. RED: reuse currently reaches state; GREEN: entry guards reject before further work. Injection + remains test-only and no production arbitrary-callback API is added. +- [ ] Validate host-owned and library-owned MPI with MPI_Query_thread. During coexistence, keep the required/requested + MPI_THREAD_SERIALIZED level: Hybrid still makes MPI calls from non-initializing workers. Task 12 lowers both checks + to MPI_THREAD_FUNNELED only after removing that path. Do not reinitialize host-owned MPI or assume the provided level + equals the requested one. Higher levels are valid; no speedup is presumed from requesting a lower minimum. +- [ ] Add an internal `mpi::require_initializing_thread()` guard in MPICompat, using MPI_Is_thread_main after MPI is + initialized. Apply it before MPI-using constructor/operation/retained-functional/cleanup work on the new ordinary-MPI + one-store path, including implicit communicator work before constructor bodies. Do not impose it on still-supported + legacy Shm/Hybrid worker dispatch. MPI-off is a no-op; public operations still enter outside active host OpenMP teams. + Keep `mpi::routes_pairwise` behind the caller guard on the ordinary-MPI path: first use may allreduce, warm use + still calls MPI_Comm_get_attr, and a new/duplicated communicator must agree independently. Preserve the + communicator-attribute + cache and Hybrid's constructor agreement; do not replace them with per-gate agreement or a raw-handle global map. + At Task 10 every new propagator uses this contract, even if provided support is SERIALIZED or MULTIPLE. Do not capture + whichever thread first touched an object as a substitute for MPI's actual initializing thread. +- [ ] On wrong-thread entry, print an actionable initializing-thread diagnostic and fail fast locally without calling + MPI_Abort, communicator queries, collectives or the normal operation_failed helper from that thread. This unsupported + host-contract violation is distinct from valid-caller distributed failures. Test it under a launcher configured to + terminate peers on a process's abnormal exit; do not promise peer cleanup under an arbitrary launcher policy. Add no + process-wide lock, dispatch service or nested-execution machinery. Preserve no-overlap rules for dependent state and + host MPI calls; an internal helper's nested fallback does not widen the public contract. +- [ ] Add a separate failure driver with `worker-throw`, `before-exchange`, `active-ticket`, `insufficient-thread-level` + and `wrong-thread-entry`. Inject on rank zero while peers follow the normal phase. For wrong-thread-entry, invoke a + real guarded entry from a host std::thread (outside OpenMP) after main-thread MPI initialization, at supplied + SERIALIZED/MULTIPLE now; add FUNNELED in Task 12 after lowering the minimum. Never relax the caller rule. + Use a 30s subprocess timeout under two ranks, with launcher abort-on-bad-exit enabled for local-fatal cases. Require + prompt nonzero failure with the expected diagnostic, not timeout. Keep abort cases outside the ordinary unit runner. + Test all required/provided-level decisions in a pure validator; real initialization tests record the actual provided + value rather than assuming a request guarantees an insufficient level. Test single-rank worker throws separately. + +**Gate:** R/M compile and pass with kernels still serial. Focused gates include `combined_recompute_equivalence.cpp` +record restoration and `tests/test_deep_circuit_gradient.py` deep non-singular, vanishing-cosine and no-record cases, +before and after threading. Preserve `tests/test_circuit.py` incremental-axis/seed behavior too; its existing passing +cases are baseline preservation evidence, not RED for a feature already shipped on main. Options reach every intended +entry point; runtime global settings unchanged; failure tests do not hang. Public/exported signature ABI change is +recorded for a coordinated library/bindings rebuild. + +**Task 3 outcome (2026-09-27):** gate passed on GCC 15.2 (R and M) and Clang 18.1.8/libomp (R) Linux, after one +fix to the Task 1 harness that the change required (see below). Kernels are still serial. Other platforms pending. +- Contracts as implemented, which later tasks rely on: + - `resolve_thread_budget`/`capture_thread_budget` (exported, installed) follow the parser rules above. Only + explicit `partitions == 1` on a `Kind::Mpi` communicator captures a budget, once per object. Everything else, + including Shm/Hybrid children and facades, keeps `{.threads = 1}`. Copies and retained functionals keep the + captured value. A parse error is `PropagatorConfigError` on one rank and `MPI_Abort` on several. + - `run_operation_` separates validation from mutation with a `mutation_started` flag. A later failure calls + `invalidate_()` and then `mpi::operation_failed()`. That applies to every non-facade object on an ordinary + communicator, not only the prototype. It aborts when the communicator has more than one rank, and rethrows + the original exception otherwise. Facades rethrow through their legacy path. Validation that used to be + interleaved with mutation now runs first: generator, initial-operator and basis-change indices. + - Guards sit inside the `Ticket` lifetime in `evolve_step` and `state_operator_derivative_local`, around local + work before each allreduce in `ev`/`ev_and_grad`, and around each `wait_into` inside its `PendingAlltoallv` + lifetime in `run_exchange`. `build_layer` has no whole-body guard: the propagator's operation guard covers it. + - The requested and required MPI level stay `MPI_THREAD_SERIALIZED`. `mpi::init` checks the level actually + provided whether MPI was host- or library-initialized, and the prototype constructor calls + `require_thread_support()`. + - `require_initializing_thread()` uses `MPI_Is_thread_main`. On a wrong thread it prints a diagnostic and calls + `std::abort()` with no MPI calls. It guards only the MPI-using prototype entries, retained functionals included. + - White-box access for tests goes through a befriended `detail::PropagatorTestAccess`, which the library never + defines. +- Tests: + - `openmp_runtime_tests.cpp` has 23 cases. + - Five fresh-process `openmp_env_*` CTest entries cover budgets 1 and 3, the OpenMP default, + `OMP_THREAD_LIMIT=2`, and `OMP_PROC_BIND=false` for the affinity check. + - `tests/test_openmp_config.py` has 13 cases, and `mpi_utils_tests.cpp` gains a `routes_pairwise` cache case. + - `mpi_failure_driver.cpp` is excluded from the unit-runner glob and registered as eight `mpi_failure_*` entries. + - RED: invalid budgets were silently accepted (9 Python failures), and the C++ build failed on missing headers. + - Six R mutants and three failure-driver mutants were all killed. Without the abort, or without the guard + inside the Ticket lifetime, the scenarios hang until the 30 s timeout. +- Final R and M runs on fresh rebuilds: + - R: ctest 333/333; pytest 840 passed, 13 skipped (after the harness fix). + - M serial: ctest 363/363; pytest 827 passed, 26 skipped. + - Splitmix R=1-4 and linear R=1,2,4: C++ suites and failure scenarios pass; `--with-mpi` pytest passes. + - `just test-find-package` passes on R and M, including a new prototype chain. + - Clang: 333/333 and 840 passed. + - Generated bindings are byte-identical to Task 2's. +- Failure scenarios, two ranks with a 30 s timeout: all eight end with a nonzero exit in 2 s or less and print + the expected diagnostic. + - Provided levels equal the requested ones (SINGLE, FUNNELED, SERIALIZED, MULTIPLE), so the + insufficient-level case really runs; mpi4py 4.1.2 provides MULTIPLE. + - A wrong-thread entry exits with SIGABRT (134) and prints no `MPI_Abort` text. +- The exported `evolve_step`, `evolve_operator` and `state_operator_derivative_local` changed signature (trailing + `Options`, serial default). Rebuild the library, the bindings and C++ consumers together (`building.mdx`). +- Linkage: `_core.abi3.so` and `libmonoprop.so` now NEED the OpenMP runtime (`libgomp.so.1` or `libomp.so.5`), + so Task 2's "no libgomp" finding no longer holds. Check wheel bundling and delocation again in Task 13. +- Harness finding, fixed with owner approval: + - With `OMP_PLACES` set and binding on, the runtime binds the initial thread to the first place. libgomp does + it when the library loads, libomp at the first OpenMP call. This is OpenMP applying the user's placement; + monoprop calls no affinity API. + - `tools/benchmark-rank-local-openmp.py` read `sched_getaffinity(0)` after `import monoprop`, so it rejected the + documented candidate launch on GCC builds, and `test_observe_rejects_contradictory_evidence[change2]` failed. + - `open_context` now reads the allocation before importing monoprop. Afterwards R pytest gives 840 passed; the + harness tests pass on R, Clang, M and MPI with 2 ranks. + - For the baseline binary, which loads no OpenMP runtime, both orders read the same mask, so collected evidence is + unaffected. The same edit is applied to the tool copy in both baseline worktrees, which now match the committed + tool (sha256 `2207529f…`). Their harness tests pass, including MPI with 2 ranks, and their binaries are unchanged. + - Still open: `memory/cpu.py` reports the diagnostic `affinity_cpus` after the import, so under libgomp binding it + reports 1. It is metadata, not a gate. +- Not run: `just build-docs`, the Nix, macOS/Homebrew and devcontainer routes, and the wheel repair routes. +- Artifacts: `/home/ubuntu/task3-artifacts/` on the c8a.metal-24xl host (`report.md`, `ledger.md`, `logs/`). + They are intentionally uncommitted. The frozen campaign and workload files, Task 1 artifacts and baseline + binaries were verified unchanged. + +### Task 4: Prototype cosine and fused updates without structural concurrency + +**Files:** +- Modify: `cpp/monoprop/detail/evolution/CosineRecompute.h`, `cpp/monoprop/detail/evolution/layer_build/FusedApply.h`. +- Create/test: `cpp/tests/openmp_kernel_tests.cpp`. +- Extend: `cpp/tests/{combined_recompute_equivalence,fused_cos_sweep_tests}.cpp`. + +**PROPOSED interfaces:** Consume Task 3 options. No further public interface. `for_blocks` handles region/exception +joining; caller chooses semantic blocks and the serial threshold. + +- [ ] First add exact vector-equivalence tests using a stored mask spanning 4097 words; final mask word has only bit + zero. Test budgets 1/2/3/4, zero words, 63/64/65-row tails and more workers than work. + +```cpp +BOOST_AUTO_TEST_CASE(openmp_cos_mask_exact) { + monoprop::CosMask mask; + for (size_t w = 0; w < 4097; ++w) { + const auto bits = w == 4096 ? uint64_t{1} : uint64_t{0x5555555555555555}; + mask.blocks.emplace_back(w * 64, bits); + mask.total_count += static_cast(std::popcount(bits)); + } + monoprop::VecD s(4097 * 64, 1.25), p = s; + monoprop::detail::scale_cos_mask(s.data(), mask, 0.625, {.threads = 1}); + monoprop::detail::scale_cos_mask(p.data(), mask, 0.625, {.threads = 3}); + BOOST_CHECK_EQUAL_COLLECTIONS(s.begin(), s.end(), p.begin(), p.end()); +} +``` + +- [ ] Add lazy-mask tests in `combined_recompute_equivalence.cpp` with an `InvertedIndex<8>` exceeding three + `kColumnBlockWords` blocks, mixed dense/sparse columns, odd/even Majorana, Pauli transformed fold, empty columns, zero + generator, and partial `scaled_count`. Clear `row_parity_` before the **parallel-first** call to expose cold-cache + races. Compare coefficients exactly against the existing serial oracle. +- [ ] Parallelize `scale_cos_lazy` over existing 1024-word fold blocks after caller-side parity preparation. Obtain + scratch inside each worker invocation; use a fixed `std::array` for no-allocation fold + scratch or the existing TLS under the helper's catch. Never capture a reference to the caller's TLS buffer in the team + lambda. +- [ ] Parallelize `scale_cos_mask` over disjoint mask-block ranges. Keep accumulate/reduction routines serial in this + task. Keep `fold_to_cos_mask` and `fold_to_indices` serial to preserve diagnostics/ordering cheaply. +- [ ] In fused apply, size snapshot arrays on the caller, parallel gather insert-record snapshots, join, complete + optional cosine phase, join, then workshare hit/insert/cross-half records. Each iteration performs the entire existing + rotation formula; retain phase order. In debug tests record all updated row IDs and assert exactly one add-owner per + destination before using this invariant. +- [ ] Keep first prototype thresholds conservative: at least two logical work blocks; a cosine block is 1024 words and a + record block is 1024 records. Tiny cases run serial. Thresholds may be tuned only by Task 11 evidence, not semantic + changes. Preservation fixtures above may already pass with Task 3's serial options plumbing. For RED/GREEN of actual + threading, use test-build-only observations at each affected kernel work-range body: pre-size one worker-ID slot per + logical range, have its sole owner record `omp_get_thread_num()`, and check distinct IDs after join on adequate + hardware. Do not count a team created elsewhere as this kernel's participation. A runtime-limited skip is not scaling + evidence. These observations must not become a production callback API or persistent per-term storage. +- [ ] Run R, M targeted fused/cosine tests, S, and repeated stress with runtime-limited teams. Measure serial versus + threaded one-store replay before investing in construction. Archive timings/peak/worker scratch; this prototype cannot + establish construction scalability. + +**Gate:** Exact coefficient agreement and measured replay scaling on large cases; no coefficient-sized additional +persistent buffer. If replay is already bandwidth-bound with regressions, stop/profile rather than proceed on assumed +speedup. + +**Task 4 outcome (2026-09-28):** the owner accepted the gate, with the high-budget losses and the lack of a +`propagate` gain on record. Correctness gates pass on GCC 15.2 (R, M) and Clang 18.1.8/libomp (R) Linux. Other +platforms are pending. +- Contracts as implemented, which later tasks rely on: + - `scale_cos_lazy` runs one logical range per `kColumnBlockWords` (1024) fold words. + - Row parity is fetched on the caller for each call. + - Scratch is a `std::array` on the worker's stack, never the caller's `column_block_scratch()`. + - Tail masking stays global through `apply_fold_mask`. + - `scale_cos_mask` runs one range per 1024 stored mask blocks. It relies on ascending, distinct, word-aligned + bases. + - `apply_fused_contract` keeps its phases, each joined before the next: + 1. The insert snapshots are gathered into the existing `RotationRec::v_tgt`. + 2. The optional cosine phase runs. + 3. The records are applied in ranges of 1024. + - Every record range calls one out-of-line `[[gnu::noinline]] apply_fused_record_range`, so the compiler's + FMA contraction of `cos·c + sin·φ·v` is identical at every budget. Without it, fused-scale `propagate` differed + from Task 3 by 1 ulp. Task 9 evaluates explicit `std::fma`. + - A kernel stays serial below two ranges; the thresholds are fixed until Task 11. + - Debug builds assert `fused_add_owners_unique(fc)` on every apply. + - The kernels take a defaulted, test-only range observer (`NoRangeObserver` in production); there is no + production callback. Accumulators, index materialization and every other routine stay serial. +- Tests: + - `openmp_kernel_tests.cpp` has 11 cases, with `KernelTestSupport.h`. + - `combined_recompute_equivalence.cpp` adds two synthetic multi-block lazy-fold cases: tails, cold parity, + Majorana, Pauli, empty columns and index growth. + - `fused_cos_sweep_tests.cpp` adds four cases: real-path records with one add-owner per slot in both pictures + (cross-rank halves under MPI), exact serial/threaded prototypes, and fused and two-pass agreement with replay at + the existing tolerances. + - Two `openmp_env_kernels_*` reruns cover `OMP_THREAD_LIMIT=2` and `OMP_DYNAMIC=TRUE`. + - The failure scenario `active-ticket cosine-worker` throws from worker 1 while a Ticket is posted. + - RED: participation failed with the kernels serial, while the exactness cases already passed as preservation. + GREEN: 3–4 distinct workers observed in every kernel's own ranges, on libgomp and libomp. + - Six mutants were all killed, three of them by TSan. Stress: 800 fresh processes, no failures. + - Coefficients are bitwise equal across budgets. Six library paths are bitwise equal to Task 3. +- Final runs on fresh rebuilds: + - R: ctest 352/352; pytest 840 passed, 13 skipped. + - Clang: 352/352; 840 passed. + - ASan/UBSan: 352/352; 819 passed. + - M: ctest serial 382/382; linear R=1,2,4 and splitmix R=1-4 MPI ctest and `--with-mpi` pytest pass. + - `just test-find-package` passes on R and M. +- TSan qualifies only with Clang + libomp + Archer and one `called_from_lib:libomp.so.5` suppression. GCC's libgomp + reports false races on known-safe code, so it is unsupported (`building.mdx`). +- Replay diagnostics (owner-approved budget; 102 of 180 min used; diagnostic, not acceptance evidence): + - Setup: Task 4 R build, prototype selected, allocation read before import, one worker per core. + - One-store speedup at the best budget: + + | Case | Speedup | Budget | + | --- | --- | --- | + | random energy, Heisenberg | 5.3× | T=48 | + | random energy, Schrödinger | 24× | T=96 | + | Pauli-127 energy | 2.3× | T=16 | + | gradient | 1.44× | — | + | pared energy | 1.2× | — | + | `propagate` | none | — | + + - Losses: 7% (E1, T=96 against T=48), 9% (E4, T≥48 against T=16), 0.8% (`propagate`, T≥48 against T=1). + `OMP_WAIT_POLICY=PASSIVE` made them worse, so worker spinning is not the cause. + - Synthetic kernels scale 20–54× at T=48, so replay is limited by serial work (Tasks 5, 8, 9) and small layers, + not by the kernels. Peak memory is flat across budgets. + - Against the baseline partition runtime (90d5717) at the same T: + - One thread: within 0.95–1.07. + - T≥16: the partitions are 1.2–35× faster, the largest gaps in gradient and `propagate`. + - The baseline's peak grows with T by up to 0.38 GiB. +- Not run: `just build-docs`, and the Nix, macOS/Homebrew, devcontainer and wheel-repair routes. + - Wheel bundling of the now-used OpenMP runtime is still to be checked (Task 13). +- Artifacts: `/home/ubuntu/task4-artifacts/` on the c8a.metal-24xl host (`report.md`, `ledger.md`, + `replay-results.md`, `diag/`). They are intentionally uncommitted. The frozen campaign and workload files, the + Task 1 artifacts, and the baseline binaries and worktrees were verified unchanged. + +### Task 5: Deterministic parallel bitmap traversal and query generation + +**Files:** +- Modify: `cpp/monoprop/detail/evolution/layer_build/{Scan,Engine}.h`, `cpp/monoprop/algebra/AlgebraCommon.h`. +- Extend: `cpp/tests/{fused_cos_sweep_tests,sparse_query_tests,pauli_build_layer_tests}.cpp`, + `cpp/tests/{exact_upper_atol_rescue,evolution_detail_tests}.cpp`. + +**PROPOSED interfaces:** Add `CutoffEvaluator::parallel_safe() const noexcept -> bool`, initially true only for existing +typed `LengthCutoff`/`SupportCutoff` targets. Ordinary Python/C++ constructors take cutoff settings and basis-change +data, not arbitrary functions. Opaque predicates remain possible through protected C++ `cutoff_fn_` or low-level +helpers; keep their serial path. Current basis-change closures capture data by value, with no mutable capture found; +their initial serial path is conservative, not evidence of unsafety. Do not add a new callback API or claim a const +std::function alone proves safety. + +EXISTING `Scan.h::fused_find_and_collect` returns `FusedScanResult` and takes `SlotWindow window`, `my_rank`, +router, precomputed `gen_shift`, then capture/scaling arguments. `Engine.h::build_layer` derives the plan/window once +from that shift and passes BOTH explicitly to LayerBuildEngine after the sink argument. Keep that constructor contract +in direct-engine fixtures; it no longer derives geometry itself or supplies a default plan. FusedScanResult has no +separate window member: compare each populated stream's window(), with value streams empty when capture is disabled. +PROPOSED: extract the following range helper in Scan.h; preserve these inputs/result type, prepare lazy data once, and +merge private results. Do not reintroduce rank-count-shaped query vectors. The range helper must never call a lazy +accessor itself: + +```cpp +template +auto fused_find_and_collect_range(const MPOperator &op, + const Monomial &gen, const CutoffEvaluator &cutoff_eval, + const CutoffContext &cut_st, const VecD &coeffs, + std::optional only_rotate_len_k, mpi::SlotWindow window, size_t my_rank, + const routing::Router &router, size_t gen_shift, const InvertedIndex &index, + const uint64_t *row_parity, size_t wlo, size_t whi, + bool capture_values, double *fused_scale_coeffs, + double fused_scale_cos) -> FusedScanResult; +``` + +Recomputing the small immutable generator context per range is acceptable initially; do not introduce a persistent +prepared-state cache. Tail masking uses the full index's final word, not each range's final word. + +- [ ] Before changing traversal, add a serial/threaded result comparison helper checking each owner's leader/follower + query words, source IDs, captured values, cosine indices, coefficients and both self stages (logical position spans, + offsets, counts and phases). Include nonzero window bases, linear zero/nonzero shifts and splitmix dense windows. + Cases must exercise dense and sparse pivots, odd Majorana correction even when selected fold columns are empty, Pauli + J(G), no length cap, cap 0/positive, lower-atol equality, upper-atol rescue, cos=0 fallback, capture-values on/off, + and no emitted queries. +- [ ] Extend the EXISTING stateless opaque C++ lambda fixture in `majorana_cutoff_tests.cpp:171–196`: evaluator + classification must be non-parallel-safe and the kernel's test-only work-range observations must show its serial + path. Do not invent a stateful Python callback or a callback-heavy performance cell for a nonexistent constructor + surface. Preserve actual basis-change results; this first version leaves that traversal serial. Audit real cosine + callbacks/parity-cache preparation separately. Serial fallback is not an exemption from approved campaign gates. +- [ ] On caller: validate arguments, prepare algebra/generator columns, obtain router once, initialize lazy + index/parity, snapshot row/word count, check skip/tail conditions. Partition words into at most + `min(options.threads,ceil(words/1024))` contiguous nonempty ranges. Range IDs increase with source word index and do + not depend on actual worker count. +- [ ] Move `EvenParityNzWord` scratch, `CosineWordBuilder`, counters and the `emit` lambda **inside** each range + invocation. Reuse `even_parity_scan_pass1(wlo,whi)` then immediately emit its results. Workers read/update only their + assigned source rows. They may not resolve target coefficients or grow stores. +- [ ] If cutoff is not parallel-safe or work is small, call the original serial body without chunk metadata. Otherwise + `for_blocks(range_count,options,...)` fills one `FusedScanResult` per range. No shared push_back/critical section + and no per-bitmap-block × rank vector table. +- [ ] Join, calculate checked total lengths, reserve once and merge reachable slots in ascending range ID using + `window.indices()` and `.at_slot(flat_slot)`; never use a flat slot as a window index. All pieces share the exact + `window` (base and count). Move the first useful buffer where safe and release drained buffers immediately. + `leader_queries/src/val` and follower equivalents preserve each stream's source-order subsequence. Wire records have + variable boundaries, not a fixed stride; use `QueryWire` and preserve Plain/Fused semantics. Self wire stays empty. +- [ ] Merge `leader_self` and `follower_self` separately, aligned with the self slot's src/val streams. EXISTING + `SelfQueryStage` in `PartnerMerge.h:81–140` uses logical `size()`/`positions()`, not backing vector sizes. Use its + existing `push(span,phase)` in source order initially: for each q below `piece.size()`, pass the span at + `piece.pos_off[q]` of length `piece.k_of[q]` and `piece.phase_of[q]`. This appends to the logical position end and + rebases offsets correctly without a new helper or accessing private counters. Check total queries/positions and + reserve bounds first. Do not copy capacity tails. Assert self stage length equals self src length; values match only + when capture is enabled. Keep cosine word seam handling and ascending disjoint cos_blocks in Engine. + +```cpp +// PROPOSED merge loop shape; append concrete streams as specified above. +for (const auto wi : window.indices()) { + const auto flat_slot = window.slot(wi); + for (size_t range = 0; range < pieces.size(); ++range) { + // pieces[range].leader_queries[wi] belongs to flat_slot, not flat slot wi.value. + // Append leader src/val in the same range order; handle followers separately. + // If flat_slot==my_rank, merge self positions, never synthesize self wire. + } +} +``` + + Metadata bound is O(T*window.count), payload O(total emitted records), never per-block×R or per-thread full-store + dense keys. At S=1, linear zero shift is all self; nonzero shift has exactly one reachable remote slot. +- [ ] Keep leader/follower resolution unchanged in this task. Compare graph data and row IDs at fixed rank count after + full construction, not just energy. +- [ ] Measure simultaneous private+merged query capacities and worker TLS. If merge duplication fails peak parity, use a + two-pass exact-fill variant **only for pure cutoffs**: count with no scaling/capture mutation, caller prefix/reserve, + then fill and scale once. Count and fill must use identical pre-scale coefficients and predicates; do not scale in + both passes. If this CPU/memory tradeoff still fails, stop. Do not replace compact storage to fix output buffering. + +**Gate:** RED: new range helper absent; Task 3 options already exist. Exercise the new multi-range merge path in ordered +window/self regressions, not only the old serial path. GREEN: exact serial/threaded query words, self positions, IDs and +graph fields at fixed geometry. Ordered query/row/graph equivalence, no callback-thread contract change, no lazy-cache +races. Construction bitmap-traversal time and peak scratch are recorded separately from resolution. + +**Task 5 outcome (2026-09-28):** the owner accepted the gate, with the high-budget losses and the unexplained P1 +peak growth on record. Correctness gates pass on GCC 15.2 (R, M) and Clang 18.1.8/libomp (R) Linux. Other platforms +are pending. +- Contracts as implemented, which later tasks rely on: + - `fused_find_and_collect` prepares the inverted index and row parity on the caller, then splits the words into + `scan_ranges(words, options)` = `min(threads, ceil(words/1024))` contiguous runs of whole fold blocks + (`scan_range_words`). Range IDs depend on the word count and budget only, never on the actual team. + - Each range runs `fused_find_and_collect_range` into a private `FusedScanResult`. It never calls a lazy accessor, + acquires its thread_local scratch on the executing thread, masks only the global tail word and, in fused mode, + scales only its own sources, once, after capturing the pre-scale value. + - `merge_scan_ranges` joins the pieces in range order, stream by stream: checked totals, one reservation, the + first buffer moved when it holds the stream or has room, drained buffers released (swapped, not cleared). + Self stages merge through `SelfQueryStage::push`; cosine sets stay one per range, ascending and disjoint, and + `build_layer` concatenates them. + - `CutoffEvaluator::parallel_safe()` is true only for typed `LengthCutoff`/`SupportCutoff`. Opaque predicates, + basis-change closures and scans below two ranges run the unchanged serial scan on the caller. + - The test-only observer seam (`KernelRange`, `NoRangeObserver`, `logical_ranges`) moved to + `layer_build/Common.h` and gained `KernelRange::scan`. `build_layer` forwards a defaulted observer. + - Resolution, missing-ID assignment, insertion, publication, graph packing and `scaled_count` are unchanged. +- Tests: + - `cpp/tests/ScanTestSupport.h` builds a four-fold-block synthetic operator per rank and compares every stream, + both self stages, the cosine set and the coefficients byte for byte at budgets 2-4 against budget 1. + - The `openmp_scan_*` cases span one rank, linear zero/non-zero shifts with non-zero window bases, dense splitmix + at three and four ranks, dense/sparse/empty pivots, Majorana and Pauli J(G), caps, cutoff 0, atol boundaries, + capture, the fused sweep, cold caches, the merge itself, and opaque/basis-change serial fallback. + - Full construction at budgets 1-4 gives identical rows, row IDs, graph layers and coefficients in both pictures. + - Failures: a scan worker throw joins before rethrow and invalidates the propagator; the new two-rank scenario + `before-exchange scan-worker` aborts promptly under both routings. + - RED: the tests did not compile without the helper, then failed every multi-range expectation with traversal + forced serial while exactness passed. GREEN: 2-4 distinct workers in the scan's own ranges. + - Seven mutants were all killed, one by TSan. Stress: 800 fresh processes, no failures. Serial scan digests and + Task 4's cross-revision probe are bitwise equal to Task 4 at every budget checked. +- Final runs on fresh rebuilds: R, Clang and ASan/UBSan ctest 366/366 (pytest 840, 840, 819 passed); TSan (Clang + + libomp + Archer, same single suppression) 85/85 with no reports; M serial 396/396, linear R=1,2,4 and splitmix + R=1-4 MPI ctest and `--with-mpi` pytest pass; `just test-find-package` passes on R and M. +- Construction diagnostics (owner-approved budget; 18 of 60 min used; diagnostic, not acceptance evidence): + - End-to-end speedup at the best budget: random Heisenberg `build_graph` 4.8× (T=48), random Schrödinger + `build_graph` 11.8× (T=96), Pauli-127 `build_graph` 1.55× (T=4), random Heisenberg `propagate` 5.0× (T=48; + Task 4 had no gain). + - Traversal alone scales 18-21× on the random cases and 3× on Pauli-127. Resolution, deferred insertion, + finalisation and the replay step stay serial and now bound B1 and B4. + - Losses: `propagate` 18% slower at T=96 than T=48; Pauli-127 4.5% slower at T≥48 than T=16; random Heisenberg + `build_graph` 2% slower at T=96 than T=48. + - The serial ordered merge grows with the range count, to 28-37% of traversal time at high T. It is a Task 11 + bounded-optimisation candidate. + - Memory: the threaded scan's simultaneous private+merged heap peak never exceeded the serial scan's, so the + two-pass exact fill was not needed. Process peak grows 0.5-7% at T>1; `MALLOC_ARENA_MAX=1` removes it for the + Schrödinger and Pauli cases (glibc per-thread arenas), but `propagate`'s +2.9% remains unexplained. +- Not run: `just build-docs`, and the Nix, macOS/Homebrew, devcontainer and wheel-repair routes. +- Artifacts: `/home/ubuntu/task5-artifacts/` on the c8a.metal-24xl host (`report.md`, `ledger.md`, + `diag-results.md`, `diag/`). They are intentionally uncommitted. The frozen campaign and workload files, the + Task 1 artifacts, and the baseline binaries and worktrees were verified unchanged. + +### Task 6: Parallel frozen lookup while retaining serial insertion + +**Files:** +- Modify: `cpp/monoprop/detail/evolution/layer_build/{Resolve,Engine,QueryWire}.h`. +- Extend: `cpp/tests/{operator_index_tests,sparse_resolve_tests,sparse_query_tests,mpi_fresh_insert_equivalence}.cpp`. + +**PROPOSED interfaces:** + +```cpp +// PROPOSED Resolve.h helper; per-query spans have equal lengths; store frozen until return. +template +auto probe_frozen_positions(const OperatorIndex &store, + std::span::PosT> pos_flat, + std::span pos_off, std::span k_of, + std::span out, std::span hash_out, + parallel::Options options = {}) -> void; +``` + +EXISTING `OperatorIndex::find_batch_positions(pos_flat,pos_off,k_of,out,hash_out={}) const -> void` +(`OperatorIndex.h:282–288`) consumes position spans; use 256-query blocks, with absolute pos_off values into the shared +pos_flat and sliced query/output/hash spans. Retain batched lookup and cached hashes, not scalar find or dense keys. +Change incoming `miss_g` from `vector` to `vector` because it contains query ordinals, not term IDs. +Count that memory change explicitly. + +- [ ] Add this preservation test to `sparse_resolve_tests.cpp`, using its EXISTING `make_op` and `positions_of` + helpers/includes. Task 3 already added the trailing options argument, so this test may pass before Task 6. Its purpose + is exact predicted/published IDs and nonzero-window addressing, not a fabricated RED. Use the missing + `probe_frozen_positions` helper and malformed-boundary tests as this task's RED targets. Public Majorana indices are + reversed when encoded; `QueryWire::push` needs raw ascending bit positions obtained with `positions_of`: + +```cpp +BOOST_AUTO_TEST_CASE(openmp_incoming_miss_order) { + using namespace monoprop; + const auto a = indices_to_bitset<8>({0, 1}); + const auto b = indices_to_bitset<8>({2, 3}); + const auto c = indices_to_bitset<8>({4, 5}); + for (const auto threads : {1, 3}) { + auto op = make_op<8>(std::vector>{a}); + const mpi::SlotWindow window{.base = 4, .count = 3}; + mpi::WindowVec incoming(window); + detail::QueryWire<8>::push(incoming.at_slot(4), positions_of<8>(b), +1); + detail::QueryWire<8>::push(incoming.at_slot(6), positions_of<8>(a), -1); + detail::QueryWire<8>::push(incoming.at_slot(6), positions_of<8>(c), +1); + const auto pr = detail::probe_incoming_queries<8>( + incoming, op, detail::QueryForm::Plain, {.threads = threads}); + BOOST_TEST(pr.window.slot(pr.sender_index(0)) == 4U); + BOOST_TEST(pr.window.slot(pr.sender_index(1)) == 6U); + const std::vector offsets{0, 1, 1, 3}, ids{1, 0, 2}, misses{0, 2}; + BOOST_TEST(pr.base == 1U); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.goff.begin(), pr.goff.end(), offsets.begin(), offsets.end()); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.idx_of.begin(), pr.idx_of.end(), ids.begin(), ids.end()); + BOOST_CHECK_EQUAL_COLLECTIONS(pr.miss_g.begin(), pr.miss_g.end(), misses.begin(), misses.end()); + detail::insert_incoming_misses(op, pr); + BOOST_TEST(op.size() == 3U); + BOOST_REQUIRE(op.store->find(a).has_value()); + BOOST_REQUIRE(op.store->find(b).has_value()); + BOOST_REQUIRE(op.store->find(c).has_value()); + BOOST_TEST(op.store->find(a).value() == 0U); + BOOST_TEST(op.store->find(b).value() == 1U); + BOOST_TEST(op.store->find(c).value() == 2U); + } +} +``` + +Repeat at window base zero, empty/all-hit/all-miss inputs, >256 queries and short tails, fused-value codec, overflow +rows, and a prebuilt inverted index. +- [ ] Prepare safe decode on caller BEFORE workers. EXISTING `QueryWire::count_queries`/`read_query` assume valid input; + do not use an unchecked walk as a safety check. In `Resolve.h::probe_incoming_queries`, walk each sender by window + index, requiring off::header_bits_for(h.k)` before calculating payload size; reject noncanonical escaped + counts below `kKEscape`. Otherwise `words_of_header` can undercount the actual header consumed by `read_positions`. + Compute payload words using the validated header, add the Fused value word with checked arithmetic, and require the + entire record to fit `buffer.size()-off`. Reject malformed/truncated streams; require final off==buffer.size. Add a + pre-worker rejection fixture for `QueryWire<128>`: `VecZ{size_t{0x8037e}}`, one Plain word with phase +1, escaped k=6, + gw=8 and zero payload bits. Its actual decode needs 68 bits but a canonical k=6 header implies 59; it must be rejected + before any payload read. Keep valid escaped-count and Fused coverage. Checked-prefix query and decoded-position + totals, plus goff, off_of, pos_off, k_of and sender_wi on caller. Check uint32 sender/count narrowing and vector + max_size. Allocate all output arrays once; no worker vector growth. +- [ ] Decode disjoint records into their prepared `pos_flat` spans via `read_query(buf,form,off,span)`, preserving + phase_of and Fused value offsets. Check returned next matches the prepared boundary. Extend EXISTING + `QueryWire::read_positions` to check cumulative decoded size_t positions before narrowing to PosT: each must be + strictly ascending and below 2*N. Checking only already-narrowed bytes could miss overflow. Headers already bound + payload reads; invalid positions throw inside the worker catch before hashing/publication. No dense key + reconstruction. Join and surface exceptions through Task 3's failure boundary, then `probe_frozen_positions`, join, + normalize not-found and assign misses serially in ascending sender-window/query order. Retain hash_of for every query. + Check ID/count limits before exposing predicted IDs. RED malformed-boundary/overflow tests must reject before worker + decode; GREEN includes empty, multiword, escaped-popcount, Fused, overflow-row and partial-block cases without dense + reconstruction. +- [ ] Keep `insert_incoming_misses` publication serial: grow → `set_positions(base+j,pr.positions_at(g))` → + `bulk_insert_hashed(...pr.hash_of[g])` → `reindex_after_growth`. Preserve phase-3 scatter before insertion because it + reads pre-insert coefficients. All ordinary set/overflow/hash/inverted/state-extension mutation stays single-writer. + Existing paired-only `IncomingProbe::mono_at` for Schrödinger state scoring is a bounded per-record exception already + in ContractSink; preserve it, never turn it into a dense batch/store. Keep that scoring path serial initially. +- [ ] Parallelize incoming scatter only after `responses` sizing and sink `prepare`. PROPOSED capability: + `static constexpr bool parallel_resolve = true` in `GraphSink`/`ContractSink`; it certifies only independent + `on_resolved` writes after prepare, not other sink methods. Missing/false capability means serial for generic sinks; + use nested `if constexpr (requires { Sink::parallel_resolve; })` then test its value, avoiding absent-member lookup. + For `ContractSink`, also require `!sink.schrodinger` initially; scoring remains serial. Keep shared + `ContractSink::on_response_block` push_back serial. Matched leader marks are unique uint16 elements and bounded by + pre-gate `combined_size`; join before followers read them. +- [ ] For self lookup, EXISTING `Engine.h::resolve_range_` uses 64-query batches of gathered offsets/counts into + `SelfQueryStage` and returns cached hashes. PROPOSED: use bounded windows up to 4096 queries for parallel work; serial + stable follower filtering → gather absolute position offsets/counts → parallel position probe → serial hit/deferred + emission in original order. Leaders skip filtering. No self wire encode/decode, no full-store keys, and no store + insertion between windows. Preserve deferred positions AND hashes, reuse bounded scratch and retain recording-sink + compatibility. +- [ ] Add assertions/tests for mixed self hit/miss, remote hit/miss, and leader/follower ordering. Do not coalesce all + insertion into a final phase. Keep remote-leader then remote-follower insertion and both-pass deferred self + publication. + +**Gate:** R/M lookup and fresh-insert tests pass with exact fixed-geometry IDs. Report +probe/prefix/publication/inverted-index times separately. If serial hash publication dominates, report its Amdahl limit +and carry the evidence into the Task 11 index A/B. Do not improvise a custom concurrent table here. Give +`bulk_insert_hashed` a trailing `parallel::Options` argument now (the packed index publishes serially and ignores it), +so the Task 11 variant needs no call-site changes (shared invariant 14). + +**Architecture go/no-go (diagnostic, after Task 6 and before Task 10):** with the Task 1 harness, compare the +one-store prototype against the frozen partition baseline at matched geometries. The prototype has threaded replay +(Task 4), traversal (Task 5) and frozen lookup (Task 6). Cover L1 (R=1,T=1), the NUMA-local control and MPI-off full +machine for representative small/medium profiles. These are fresh diagnostic observations, never formal samples, and +they cannot waive a Task 11 cell. Report per-phase time and peak memory. If the one-store path trails clearly and no +bounded Task 11 optimization explains a route to parity, stop for the owner's decision before Task 10 removes the +partition API. This is the last cheap point to revert: only the additive one-store path would be dropped. + +**Task 6 outcome (2026-09-29):** correctness gates pass on GCC 15.2 (R, M) and Clang 18.1.8/libomp (R) Linux; +other platforms are pending. The performance findings below are recorded, not accepted: the work ended at the +architecture decision that follows. +- Contracts as implemented: + - `probe_frozen_positions` splits position-list queries into 256-query blocks, each one `find_batch_positions` + call with absolute offsets into the shared buffer and sliced counts, outputs and hashes. It checks equal + lengths and in-bounds spans before any block runs. The store stays frozen until every worker joins. + - `probe_incoming_queries` validates every sender stream on the caller before any decode, with + `QueryWire::checked_extent`: ternary phase, k and gap-width bounds, canonical escaped counts, whole records + including the fused value word, and a final offset equal to the buffer size. It checks the prefixes and + allocates each array once at its exact size. Workers decode disjoint blocks; `read_positions` rejects any + size_t position at or above 2N before narrowing. After the frozen probe, the caller checks the row-ID ceiling + and assigns misses in sender-window/query order. `miss_g` is `vector`. Malformed input throws + `MalformedQueryStream`; the plan's `0x8037e` fixture is rejected before any payload read. + - `resolve_incoming` scatters on workers only for a sink with `static constexpr bool parallel_resolve = true` + and a true `parallel_resolve_enabled()` (`GraphSink`, Heisenberg `ContractSink`). Schrödinger scoring and + generic sinks stay serial; debug builds assert unique leader marks. It runs inside `guard_distributed`. + - The self lookup runs in windows of at most 4096 queries (at most 16 probe blocks), with serial filtering, + gathering and emission, and no insertion between windows. Publication order and the single-writer + grow → `set_positions` → `bulk_insert_hashed` → reindex sequence are unchanged. `bulk_insert_hashed` gained + a trailing `parallel::Options` that the packed index ignores. The duplicate-key behaviour is pinned by + `bulk_insert_duplicate_keys_are_both_indexed`. + - The observer seam gained `KernelRange::{decode,incoming_probe,self_probe,scatter}`, forwarded through + `build_layer` and `LayerBuildEngine`. +- Tests: + - New and extended cases in `sparse_resolve_tests`, `sparse_query_tests`, `operator_index_tests`, + `mpi_fresh_insert_equivalence` and `fused_cos_sweep_tests`. + - Four `mpi_failure_resolve_worker_*` scenarios: the self probe fails before the exchange; the decode, incoming + probe and scatter fail during the response round. All abort in about 0.4 s. + - RED: compile failure without the helper and API, then a malformed-stream test failing with the unchecked + walk, and participation failing with the regions forced serial while exactness passes. Nine mutants were + all killed. + - Serial and threaded results are bitwise equal at budgets 1–4 and 8, and at world sizes 2–4. The Task 5 + cross-revision probe is bitwise equal at budgets 1, 4 and 16. + - Final runs: R, Clang and ASan ctest 392/392; TSan (Clang + libomp + Archer, same suppression) 117/117 with + no reports; M serial 422/422, MPI splitmix 18/18, linear 17/17; `--with-mpi` pytest passes. Stress: 800 + fresh processes, no failures. +- Phase diagnostic (owner-approved; 23 of 60 min): end-to-end build at T=96 is 4.1× (B1), 8.4× (B2) and 1.5× (B4) + over T=1. Serial fractions at T=1 bound speedup at 5.9×, 15.5× and 4.4×. Hash publication and reindex take 6% of + B1's time at T=1 and 28% at T=96. Under libgomp at T≥48, Task 6 is slower than Task 5 (B1 +19%, B2 +42% at T=96) + because many 16-thread self-probe regions interleave with 96-thread ones. libomp shows no regression, and the + serial self probe restores Task 5 times. This was left unfixed. +- Artifacts: `/home/ubuntu/task6-artifacts/` (`report.md`, `ledger.md`, `design.md`, `diag-results.md`, + `arch-results.md`, `diag/`, `arch/`, `logs/`). They are intentionally uncommitted. The frozen campaign and + workload files, the Task 1 artifacts and the baseline binaries and worktrees were verified unchanged. + +**Architecture go/no-go outcome (owner decision, 2026-09-29): re-design, keeping the sharding.** +- The diagnostic (owner-approved; 150 of 180 min) covered 96 frozen cells (`tiny` and `reference`, all 16 nodes, + at mpi-1x1, mpi-1x96 and off-1x96). Each cell had 3 alternating fresh pairs through the Task 1 driver. The + prototype was selected by a recorded `partitions=1` adapter, and results were written outside the formal + evidence. + - Numerics: every cell matched the formal baseline (global keys and coefficients, energies, gradients), with + exact windows. + - L1: runtime ratios 0.94–1.04 (reference); memory at parity. + - Full machine (mpi-1x96, and MPI-off 1x96): 0 of 16 reference cells at runtime ≤ 1.00, with ratios of + 2.0–62× (gradient Schrödinger 62×, gradient Heisenberg 31×, build_graph Heisenberg 7.2×). Peaks were mostly + lower (0.5–0.96), except Hubbard (up to 1.15). +- No credible bounded route: at T=96 the prototype's parallel scan alone takes longer than the baseline's whole + operation (build_graph Heisenberg 0.465 s vs 0.399 s; Schrödinger 1.34 s vs 0.50 s). Removing every remaining + serial phase in Tasks 7–11 would therefore still leave these cells above 1.00. Evaluation would need to be + 11–62× faster. +- Decision: the one-store-per-rank design is abandoned before Task 10's API cutover, so the partition API and + runtime are unchanged. The architecture is re-designed and keeps the sharding. Tasks 7–13 are superseded. The + Task 2–6 changes stay on this branch as the record of the experiment, and the re-design's spec and plan decide + what to keep or revert. Pending that plan, carry forward: + - OpenMP is a required dependency. + - Task 3's per-object budgets and its invalidation and distributed-failure contracts apply to every + ordinary-communicator object. + - The exported functions' trailing `Options` parameter is a C++ ABI change. + - The prototype kernels are reachable only through explicit `partitions=1`. + +### Task 7: Harden rank-wide capacity and optionally parallelize append-only row fill + +**Files:** +- Modify: `cpp/monoprop/detail/operator/{OperatorIndex,MPOperator}.h`. +- Modify only if row-fill optimization is justified: `cpp/monoprop/detail/evolution/layer_build/{Resolve,Engine}.h`. +- Extend: + `cpp/tests/{operator_index_tests,inverted_index_tests,evolution_detail_tests,exchange_layout_precondition_tests}.cpp`, + `cpp/tests/{graph_encoding_tests,ctor_validation_tests,majorana_cutoff_tests}.cpp`. + +EXISTING `grow_rows_geometric` already checks append counts (`OperatorIndex.h:111–122,544–563`); +`operator_index_tests.cpp:68–99` checks ceiling/wrap refusal without huge allocation. Extend, do not replace, these +tests. + +**PROPOSED required interface:** static `OperatorIndex::checked_append_end(size_t base,size_t count) -> size_t` +validates row-count/index limits without allocating. Keep capacity arithmetic distinct from valid row IDs. + +**PROPOSED optional measured interface:** + +```cpp +// PROPOSED OperatorIndex members, only if measured row fill justifies them. +[[nodiscard]] auto inline_width() const noexcept -> size_t; +auto set_new_inline_positions(size_t i, std::span positions) noexcept -> void; +auto reserve_index_for_size(size_t final_size) -> void; +``` + +- [ ] Add boundary tests before mutation: `(ceiling-1,1)` succeeds; `(ceiling-1,2)`, `(ceiling,1)` and arithmetic + overflow fail; `(ceiling,0)` succeeds as a count boundary. No billion-row allocations. Extend the existing + fixed-uint32 guard tests. Operator row IDs exclude the sentinel ceiling while some graph count fields can equal their + representable max: keep those domains separate. +- [ ] Implement checked append addition as `base>ceiling || count>ceiling-base` before evaluating `base+count`. Check + row-storage multiplication, vector max_size, table load-factor sizing and next-power-of-two rounding before + allocations. Geometric spare capacity cannot wrap. Update diagnostics from per-partition to per-rank store at cutover. +- [ ] Use these checks before incoming predicted IDs and deferred insertion. Audit existing MPI checked counts without + changing transport or count width. The owner accepts that consolidating old partitions reduces aggregate per-rank + addressability. A required campaign cell above the new ceiling is still a **capacity acceptance blocker**, not + permission to widen only the candidate, change ranks or introduce hidden shards. Do not infer a universal RAM limit + from the illustrative 512-GiB/eight-domain machine. C8a has 192 GiB advertised RAM; per-rank budgets still + require the observed allocation and actual memory demands, not a hard-coded equal split. +- [ ] Profile row fill. If it is not significant, leave row initialization and publication serial and mark the optional + steps below skipped with evidence; the required capacity checks still ship. +- [ ] If justified, add tests proving simultaneous new inline writes do not modify old overflow entries. + `set_new_inline_positions` requires already-grown new rows, each with one owner and position count<=inline width; it + must contain no overflow lookup/erase, allocation, resize, table operation, or exception. Keep ordinary `set` serial. +- [ ] Do not introduce a dense KeyAt/Monomial append layer. Operate on EXISTING `IncomingProbe::positions_at(g)` and + `deferred_pos_flat_` spans, preserving cached `hash_of[g]` / `deferred_self_misses[j].hash`. Check ranges before + growth; input positions must outlive growth and never alias invalidated store views. +- [ ] Apply this order directly at `Resolve.h::insert_incoming_misses` and `Engine.h::insert_deferred_self_misses`: + serial checked reserve/grow → parallel inline-position fill → join → serial `set_positions` for overflow rows → serial + deferred `sink.emit_deferred` in k order (no incoming metadata emission) → serial `bulk_insert_hashed` using retained + hashes → serial `reindex_after_growth`. No readers before publication; no shared overflow lookup/erase from workers. + There is no production assign_row callback here to remove. Preserve the independent dense reference in + `sparse_resolve_tests.cpp`; do not change unrelated `insert_absent_terms` callback contracts. +- [ ] Preserve streaming initialization during capacity changes: `for_each_paired_monomial` enumeration order, + `paired_op_size` saturation guard, local reserve estimate and pre-worker rejection. At S=1 reserve/check against real + rank share, not T. Keep `majorana_cutoff_tests.cpp` enumeration/count tests, `ctor_validation_tests.cpp` oversized + constructor rejection and initial-basis partition-invariance coverage when migrating it to thread budgets. +- [ ] Compare all rows, keys, hash lookup, inverted bits/parity, sparse/dense state extension and `scaled_count` against + serial append after every growth. Include boundaries 0/1/15/16/17/63/64/65/127/128/129 and appends starting at + unaligned rows. + +**Gate:** Capacity guards pass R/M. Optional fill must improve measured bottleneck without peak regression. Hash +insertion, overflow mutation, packed graph phases and inverted append remain serial at this stage. + +### Task 8: Parallel replay endpoints and derivative pairs + +**Files:** +- Modify: `cpp/monoprop/Evolution.cpp`, `cpp/include/monoprop/Evolution.h` only for signature consistency. +- Extend: `cpp/tests/{combined_recompute_equivalence,evolution_detail_tests}.cpp`, + `cpp/tests/{mpi_distributed_layer_equivalence,flat_exchange_tests}.cpp`, `tests/test_deep_circuit_gradient.py`. + +**PROPOSED interfaces:** Use Task 3 options in snapshot, pack, remote apply and self-pair helpers. If range flattening +needs a descriptor, define local `EndpointRange { size_t slot; size_t begin; size_t end; }` in `Evolution.cpp`; this is +proposed transient metadata, not a new graph field. Flatten **ranges within active slots**; merely parallelizing ranks +underutilizes sparse linear-routing peers. + +- [ ] Write serial/threaded forward and derivative comparisons with multiple self pairs, nonadjacent row IDs, + remote+self slots, multiple remote peers, empty peers, zero-angle, deep non-singular circuits, vanishing-cosine and + no-record paths. Retain + exact recorded restoration, including duplicate indices, indices outside the cosine set and cos rounding to one. + Compare energy/gradient to current finite-difference/exact oracles. Assert even self endpoint count and unique output + ownership. +- [ ] Record baseline PASS for existing mathematical/record fixtures where appropriate. Add test-build-only worker-ID + observations inside the endpoint work-range bodies, one preallocated slot per range with one writer. Their multiworker + requirement is RED while those bodies are serial; check after joining on an adequate allocation. Keep observations out + of production APIs and distinguish runtime-limited skips from demonstrated participation. +- [ ] Pre-size snapshots/payloads on caller. Parallel gather and pack disjoint ranges, join before posting MPI. Preserve + forward ordering: snapshot → pack → post → cosine → wait → remote apply → self apply. Do not reorder for overlap in + the same patch. +- [ ] For self derivative workshare over `k in [0,pairs)`, keep endpoint `k+pairs` in the same iteration. Read both + values required by the original formula before either overwrite. Never sort entries or independently workshare the two + halves. +- [ ] Retain serial scalar accumulations in this task; if a loop both mutates arrays and produces a scalar, isolate its + independent update portion only where the original formula permits it. Task 9 defines deterministic partials for the + full loop. +- [ ] Keep layer loops and gradient reverse order unchanged. No worker may call `mpi::rank`, `router_for`, exchange + begin/wait, or allreduce. Query rank/layout on caller. +- [ ] Cover worker exceptions after exchange posting using Task 3 active-ticket failure case. Buffers cannot resize or + die while requests are outstanding. Preserve EXISTING `mpi::routes_pairwise(comm)` selection and the boolean + `post_flat_alltoallv(..., pairwise)` argument. Plain MPI walks a dense PeerPlan and posts only nonempty legs; + `sparse_pairwise` sizes requests in its own pre-pass, with no active-leg estimate argument. Hybrid folds send AND + receive summaries across every partition. Several peers or incompatible summaries retain every peer on the pairwise + arm; that local fallback must NEVER switch transport to MPI_Alltoallv. Empty layouts post nothing on that same arm. + Keep communicator-wide agreement/cache on the caller, not workers or a per-layer layout predicate. Do not introduce a + new generator-derived replay plan. Run `flat_exchange_tests.cpp` dense, pairwise, dense-layout-on-pairwise and + empty-layout cases after every transport edit, plus + `hybrid_comm_derived_wire_plan_reads_every_partitions_row` and + `hybrid_comm_derived_wire_plan_falls_back_to_every_peer` during coexistence. The latter needs at least four ranks. + +- [ ] Keep `LayerAngle`, `CosRecordView` and `LayerCosIndices` intact. Serial forward record append, predivide/restore + and snapshot-before-restore ordering are mandatory even if record ordinals differ: index duplication is legal. + Preserve `A = cos_acc(...)*sec` and `-g*(sin*(A-ep.cos_terms)-ep.sin_terms)` in pre-layer units from Evolution.cpp; + threading must not reintroduce division-only inversion or move cancellation after multiplication by sec. + +**Gate:** RED endpoint participation checks; preservation tests need not fail first. GREEN includes the entire +deep-circuit gradient suite at budgets 1/2/3/4. Full replay and immediate-contraction equivalence across both +pictures/bases, gradients and MPI; useful active-slot worksharing even with few peers. No full-vector snapshot beyond +current algorithmic requirements. + +### Task 9: Deterministic reductions without silently breaking serial dot contracts + +**Files:** +- Modify: `cpp/include/monoprop/MPFunctions.h`, `cpp/monoprop/MPFunctions.cpp`, `cpp/monoprop/Evolution.cpp`, + `cpp/monoprop/detail/evolution/CosineRecompute.h`. +- Extend: `cpp/tests/{mpfunctions,combined_recompute_equivalence,openmp_kernel_tests}.cpp`. + +**PROPOSED interfaces:** Keep `inner_product(v,w)` and `EvalState::dot(op)` unchanged. Add explicitly named +`inner_product_threaded(v,w,parallel::Options)` and `EvalState::dot_threaded(op,parallel::Options) const`. They have +documented fixed-block association, not historical left-to-right association. `ev`/`ev_and_grad` use the new helpers +through `EvalRequest.parallel` under the approved numerical policy; implementation remains separately gated. + +- [ ] Add cancellation-heavy finite vectors, zero/signed-zero, sparse gapped rows, very short arrays, >4096 rows, + non-finite inputs, and repeated parameter indices. Check old serial dot tests unchanged; check new helper results + bit-identical across thread budgets at fixed geometry and within existing oracle tolerances. +- [ ] Define fixed logical dot blocks of **4096 row positions**, independent of worker count or dense/sparse + representation. Dense blocks sum in increasing row order; sparse blocks use lower_bound to locate entries in that same + row range and sum in ascending row order. Fold partial doubles on caller in increasing block ID. Use the same + partition even at budget one. Document non-finite/signed-zero limitations; do not strengthen sparse/dense guarantees + beyond what tests and current finite-input contract support. +- [ ] `EvalState::sparse` currently does not enforce all documented ascending/unique conditions. Do not introduce unsafe + parallel scatter. Keep `scatter_into` serial; for the new dot helper detect non-strictly-increasing rows and fall back + to existing `dot` behavior. This preserves historical duplicate/unsorted-input behavior without adding a new public + rejection in this refactor. +- [ ] For cosine accumulators use fixed 1024-word logical bitmap blocks; each block computes its scalar partial and + performs its disjoint existing coefficient updates. Caller folds partials in block order. Stored masks use logical + word ranges, not mask-vector ordinal ranges, when comparing to lazy recomputation. Leave old standalone serial + accumulator oracles in tests; compare according to existing tolerances except binary-exact fixtures. +- [ ] For endpoint derivative contributions keep two partial doubles per fixed 1024-record block. A block owns full self + pairs. Sum both fields in block order. Do not use OpenMP `reduction`, atomics, critical floating accumulation, or O(n) + product materialization to recreate serial sum order. +- [ ] Keep `indices_above`, parameter mapping and export loops serial initially. If a later profile shows + `indices_above` dominant, use count → caller offsets → disjoint exact fill, with logical row order, strict `>`, + negative threshold admitting zeros, and NaN behavior unchanged. No new global threshold policy. +- [ ] Run `tests/test_deep_circuit_gradient.py` and `combined_recompute_equivalence.cpp` across budgets 1/2/3/4, + including deep non-singular amplification, vanishing cosine, no-record and duplicate-record restoration cases. +- [ ] Run retained/direct evaluation, paring, partial contraction, repeated parameter mapping, both pictures/bases, + fixed-rank deterministic repeats, and near-cutoff retained-key comparisons. Include the incremental-axis and + post-call-seed tests from `tests/test_circuit.py`: gradients must use the equivalent composed circuit's parameter + order, including multi-monomial gates sharing one index. Cross-rank-count sums need not be bitwise + equal; don't widen tolerances to pass them. +- [ ] Evaluate explicit `std::fma` for multiply-add updates whose result currently depends on compiler contraction. + Task 4 found that the fused-scale insert arms of `apply_fused_contract` (`cos·c + sin·φ·v`, two products) round + differently depending on where the compiler fuses: same source, 1-ulp differences between inlining contexts. Task + 4 pins them with one out-of-line `apply_fused_record_range`. That makes serial and threaded runs bitwise equal within + one binary, not across binaries or compilers. + - Inventory the affected expressions: fused apply, the scan's fused cosine sweep, replay endpoint and derivative + updates, and the new block partials. Single-product `c += a·b` updates already have only one possible fusion. + - Compare three options on the same inputs: the current contraction-dependent code, explicit `std::fma` with one + documented association, and contraction disabled for the affected translation units. + - Measure each option's retained-key and coefficient differences against the frozen baseline, with existing + tolerances. Also measure runtime on `-march=native` builds and on builds without FMA hardware (sanitizer, Debug, + and the wheel flags actually used), where `std::fma` becomes a software call. + - Adopt `std::fma` only with an owner-approved numerical-policy decision covering those differences and costs. Never + guard it by `__FMA__` silently, and never use it to hide a tolerance failure. Record the decision either way. + +**Gate:** Existing serial API guarantees remain intact; new numerical association is documented and deterministic by +team size. The `std::fma` evaluation is recorded with its measurements and decision. Global retained term keys must agree, including near cutoffs and stored zeros; matching energies/gradients +alone is insufficient. A key-set change blocks the gate. Stop and report, never widen tolerances or change truncation. + +### Task 10: Cut over configuration and API semantics to one store + +**Prerequisite:** the post-Task-6 go/no-go passed, or the owner explicitly chose to continue. Reverting this task +later means reverting the API/configuration cutover; the partition runtime itself remains until Task 12. + +**Files:** +- Modify: `cpp/include/monoprop/MonomialPropagator.h`, `cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl`, + `cpp/monoprop/detail/EnvConfig.h`, `cpp/monoprop/detail/parallel/{ThreadBudget.h,ThreadBudget.cpp}`. +- Modify: `src/monoprop/bindings/binder.h`, `src/monoprop/monomial_propagator.py` docstrings. +- Create/extend: `cpp/tests/openmp_equivalence_tests.cpp`, `tests/test_openmp_config.py`, + `cpp/tests/{env_config_tests,routing_tests,mpi_utils_tests,mpi_distributed_layer_equivalence}.cpp`, + `cpp/tests/{mpi_fresh_insert_equivalence,unit_tests,ctor_validation_tests,hybrid_comm_tests}.cpp`, + `cpp/tests/link_export_probe/link_export_probe.cpp`, + `tests/{test_monoprop_smoke,test_parameter_validation,test_circuit}.py`. +- Migrate now: `cpp/tests/{partition_equivalence_tests,partition_group_clone_tests}.cpp`; preserve useful coverage in + the OpenMP equivalence suite and `cpp/tests/{simulator_copy_tests,update_initial_operator}.cpp` before retiring + obsolete facade/factory assertions. Update `cpp/tests/README.md`; do not defer broken constructor users to Task 12. +- Modify: `AGENTS.md`, `README.md`, `docs/content/docs/features/parallelism.mdx`, + `docs/content/docs/openmp-migration.mdx` (new). + +**PROPOSED interfaces/policy:** Implement immediate removal, as approved in the design interview. No aliases, +deprecation window, public thread-count constructor argument or permanent backend selector. + +- [ ] In fresh subprocesses test the final environment-only policy: + +| Input | Result | +| --- | --- | +| `monoprop_NUM_THREADS=N`, valid decimal 1..INT_MAX | capture N exactly; no hardware clamp | +| valid override and a different OpenMP default | monoprop_NUM_THREADS wins | +| present empty/zero/negative/malformed/overflowing override | actionable PropagatorConfigError; no fallback | +| override unset, any MPI rank count | capture omp_get_max_threads() at construction | +| both thread environment variables unset | capture runtime default; no hardware-suitability promise | +| stale `monoprop_PARTITIONS` in application environment | no production reader, alias or effect | +| C++ partition/factory constructor arguments | removed; old source must migrate | +| low-level Python `partitions` or `num_threads` keyword | rejected as an unknown keyword | + +Only absent monoprop_NUM_THREADS selects the OpenMP default; zero is not a public sentinel. Use Task 3's strict parser +rather than the old cached permissive one. Remove Settings::num_threads and its legacy accessor/parser once their +partition consumers are gone; remove resolve_partition_count_ declaration/definition in this task too, so its old parser +references do not survive as ill-formed dead template code. Retain independent routing settings/cache. Scan readers for +remaining partition-only knobs and remove them. At the inspected source the only partition-specific runtime variable is +monoprop_PARTITIONS; monoprop_ROUTING and monoprop_ROUTE_SEED remain MPI configuration. Do not add a removed-variable +reader for warning or rejection. Candidate benchmark preflight may reject a stale launch environment independently. + +The supported application fixes ranks/threads at launch. Capture once per constructor, never reload during operations or +add a setter. Copies/functionals preserve their budget. OpenMP limits or small-work paths may use fewer actual workers; +this is not a budget change. There is no topology-based fallback or special multi-rank default of one thread. +- [ ] Test defaults in fresh processes with monoprop_NUM_THREADS unset: OMP_NUM_THREADS=1/3, list-valued 2,3 (use the + runtime's outer-level default), and unset OMP_NUM_THREADS (compare against omp_get_max_threads, not assumed cores). + Repeat at one and multiple ranks. Test override precedence, all invalid-input classes from Task 3 and limited teams. + Invalid settings on one rank must fail coherently rather than strand peers. Add a bounded white-box test that changes + the source setting after construction solely to prove existing objects/copies/closures do not reread it; this is not + a supported hot-configuration API. Full-budget matrix tests use separate launches, not mid-calculation changes. + Remove prototype partitions=1 arguments; public tests use environment configuration, internal kernels use Options. +- [ ] Apply the approved splitmix physical ownership change: `floor((hash % (R*S))/S)` becomes `hash % R`. + Linear rank mapping remains unchanged relative to the REFRESHED main baseline; S=1 removes only the partition hash. + Preserve boolean mode/seed, `linear_requested()`, `make_router(R,1)`, router.linear_bits(), generator + rank_shift and dest_from_shift peer selection. Keep the deterministic full-rank basis draw (up to 1024 attempts) and + width bound log2(R)<=min(2*NumModes,64), where NumModes is the template width, not necessarily logical_num_modes_. + The old transposed-plane/geometry-bit helper is gone; do not restore it. The pure routing::routes_pairwise predicate + chooses transport for an agreed Config; its false result does not waive Router's invalid-geometry rejection. + No virtual shards. Exports/load/reduction order can change under splitmix; validate global term/value maps + and resulting per-rank peaks at equal R/T/routing settings. Candidate ownership never depends on T. +- [ ] Extend `routing_tests.cpp`, `mpi_distributed_layer_equivalence.cpp` and `mpi_fresh_insert_equivalence.cpp`: + default linear success at R=1/2/4; explicit splitmix success at R=1/2/3/4. Record `monoprop_ROUTING`, + `monoprop_ROUTE_SEED` and resolved `linear_bits()` as derived metadata, not a requested bit count. Separately assert + `Router::for_modes<8>(3,1,true)` throws `UnroutableGeometry` and splitmix construction succeeds. Keep fresh-process + default tests distinct from splitmix matrix runs; never expect an automatic fallback. Assert unique global ownership, + generator-peer agreement, zero-payload participation, fixed-candidate ordering across budgets and global numerical + equivalence. Do not require old/new rank-local maps equal for splitmix. Preserve current + `routing_more_rank_bits_than_modes_is_refused`, `routing_basis_columns_are_independent_at_any_seed`, + `routing_fibres_are_equal_sized`, `routing_dest_is_bit_identical_to_the_reference_bit_walk` and + `routing_sparse_peer_plans_are_validated_before_use`. Run seed variations in separate processes because EnvConfig + and the basis are cached; do not add environment reload to make tests convenient. +- [ ] Keep exact routing agreement on the caller: `routing::Config` carries raw mode, partitions and seed; + `Pairwise.h::agree_routes_pairwise` reduces each value and its complement with MPI_MAX in one collective. + `MPIUtils.h::check_routing_agreement` delegates to `mpi::routes_pairwise`, whose ordinary-MPI result is cached on + that communicator with no attribute-copy callback. At cutover partitions=1 is fixed and budgets are not agreement + inputs. Add ordinary-MPI tests in mpi_utils_tests.cpp: explicit matching Configs agree, differing valid mode/seed/part + Configs throw routing::RoutingDisagreement on every rank, and cold/warm calls plus duplicated/freed/new communicators + preserve independent agreement. Use explicit test Configs or fresh process environments, not production cache reset. + Keep all ranks in the same cold-use sequence, including empty local stores. Do not reintroduce probabilistic digest + agreement, per-gate allreduces or a cache keyed only by recycled MPI_Comm values. +- [ ] Preserve EnvConfig's separate route-seed parser: uint64 decimal via from_chars; unset/empty uses the default; + signs, whitespace, trailing junk and overflow reject. Do not conflate it with the new present-empty thread error. + Preserve COMMROUTE's ranks_per_coset/rank_cosets diagnostic and invalid-gate retry behavior. These describe reachable + cosets, not observed idle ranks. Equal fibres over all possible monomials do not prove actual retained-term balance; + per-rank load/peak measurements remain required. +- [ ] Migrate every constructor/helper/binding test using removed partition arguments in this same task. Use a source + search over cpp/tests, tests and src; include the installed export probe, constructor-validation cases and Python + parameter-validation/smoke tests. Old calls failing to compile or obsolete keywords still being accepted are + meaningful RED conditions. Move useful numerical/copy/update/empty-rank coverage before retiring facade/factory-only + cases; preserve ordinary-MPI cases currently housed in hybrid suites. Record each retired case's replacement in the + Task 12 ledger. Remove unit_tests.cpp's monoprop_PARTITIONS override and use the serial launch budget instead. + R/M must compile and pass here, not become green only after Task 12 deletes test files. +- [ ] Each independent propagator constructor creates one rank-local MPOperator and primary graph; comm_ represents + only real MPI ranks. Remove facade branches, partitions/child_factory arguments, PartitionChildFactory and + partition-specific error types such as MultiPartitionUnsupported. No diagnostic compatibility shims. Leave now-unused + transport implementation files until Task 12. Independently copied propagators keep their own stores; retained + coefficient/state snapshots, shared immutable graph cores and pared graphs remain legitimate existing objects. +- [ ] Preserve rank-local size/bytes accounting (do not allreduce twice), core term handling, copy independence, + immutable graph-core sharing, virtual destructor, virtual `clone_`, overridden virtual `update_initial_operator`, and + protected nonvirtual `apply_initial_operator_`. Do not add override to or make that helper virtual. Copy budget and + ordinary MPI communicator. Raw accessors expose that instance's rank-local store/primary graph at every budget. + Preserve Task 3 invalidation checks: invalid sources cannot be cloned; an already independent copy stays usable. +- [ ] Preserve `MonomialPropagator.inl:126–151,197–227` streamed paired-basis construction and local reservation: + cardinality rejection remains before worker startup and uses real ranks at S=1, never T. Do not materialize + `generate_paired_op` globally per rank. Migrate `partition_equivalence_tests.cpp` initial-basis invariance alongside + enumeration/cardinality tests from `majorana_cutoff_tests.cpp` and `ctor_validation_tests.cpp`. +- [ ] Migrate numerical tests from partition counts to launch-time budgets 1/2/3/4 in fresh processes; lower-level + kernel tests may pass explicit internal Options. Distinguish stored requests from actual team sizes under OpenMP + limits. Library budget selection never clamps to affinity. Run parallel gates on a user/CI + allocation supporting the requested teams and report limited-team skips honestly; pure budget tests need no extra + CPUs. The default white-box harness selects a serial library budget (not CPU pinning); dedicated launches override + it before construction. Test OpenMP-default selection in fresh processes with that library override absent. +- [ ] Test public Majorana/Pauli mathematical constructors unchanged. Remove binding-only partitions and do not add + num_threads or any other public thread-count argument. Test copy, operator update, retained callbacks after growth, + cutoff/basis-change update, paring, Schrödinger state extension and independent repeated evaluation. +- [ ] Preserve Python incremental graph construction exactly: build_graph(a); build_graph(b) denotes b+a in Heisenberg + and a+b in Schrödinger. New Heisenberg blocks take low parameter indices and lift existing ones, without renumbering + gate arrival IDs or splitting a multi-monomial gate's shared index. seed_parameters is on the POST-call axis; Python + rotates it for the existing C++ build, then relabels the successful graph. C++ build_graph still takes its mapping + as given. Keep this Python adaptation rather than shifting gates/parameters inside worker loops. Run these existing + tests at budgets 1/2/3/4 in fresh launches, with no tolerance relaxation: + `test_incremental_build_wires_the_picture_s_equivalent_circuit`, + `test_incremental_heisenberg_moves_each_block_to_the_front_of_the_axis`, + `test_incremental_reindex_keeps_a_multi_monomial_gate_on_one_index`, + `test_incremental_build_wires_the_equivalent_circuit_for_pauli`, and + `test_extend_seed_parameters_are_read_on_the_post_call_axis`. Compare decoded retained maps for seeded truncation + as well as sizes/scalars; preserve failed-build retry and retained-functional invalidation coverage. +- [ ] Regenerate bindings via the configured generator. Preserve 32-mode template dispatch. Document + low-level/source/ABI breaks and unspecified coefficient-order changes; compare decoded term/value mappings rather than + sorted coefficients alone. + +**Gate:** R/M API/configuration tests pass; one independent propagator across R ranks owns R packed stores, regardless +of T. Copies and retained snapshots/graphs obey the qualified ownership rule. Removed configuration/constructor surfaces +stay absent; invalid objects cannot be reused. Record the approved immediate compatibility breaks in the release ledger. + +### Task 11: Measure parity and apply only bounded optimizations + +**Files:** Task 1 benchmark files and, for bounded optimizations, the kernels below. The concurrent-index trial may +modify `cpp/monoprop/detail/operator/OperatorIndex.h`, its immediate MPOperator/Resolve/Engine callers and +`cpp/tests/{operator_index_tests,sparse_resolve_tests,simulator_copy_tests}.cpp`. Keep any adapter private and narrow; +no container-plugin framework. Do not rename `bench_*` functions or move benchmark files. + +- [ ] Before measuring, check that both arms run identical measurement code. The baseline worktrees hold uncommitted + copies of `tools/benchmark-rank-local-openmp.py`. Their sha256 must equal the candidate's committed copy (after + Task 3: `2207529f…`, which reads the allocation before importing monoprop). Re-sync them after any later tool change. +- [ ] Add metadata for `has_mpi`, MPI requested/provided thread support, requested/effective team size, OpenMP runtime, + OMP_NUM_THREADS/PLACES/PROC_BIND/DYNAMIC/THREAD_LIMIT, any library override, routing mode/effective linear bits/seed, + launcher command, rank/worker CPU masks, NUMA IDs, revision/dirty overlay and extension path/hash. Collect placement + in separate diagnostic runs of the same binary/configuration with OpenMP runtime diagnostics + (`OMP_DISPLAY_AFFINITY=TRUE`, `OMP_DISPLAY_ENV=VERBOSE`), launcher binding reports, and external OS/scheduler tools + (on Linux, `lscpu -e=CPU,CORE,SOCKET,NODE` and `/proc//task/*/status`). Observe actual active-worker masks/team + size, not merely omp_get_max_threads; if a runtime cannot supply that evidence, use external sampling or test-only + observation rather than adding a production topology API. Preserve old single-CPU-thread fields as historical data: + OpenMP core places may contain both SMT siblings, but at most one worker may occupy each physical core in the + recommended/acceptance configuration. A wider affinity mask is not evidence of extra worker capacity. Record distinct + physical-core ownership from verified sibling topology across active ranks/workers, not only disjoint logical CPU IDs. + No required hwloc dependency for the library or measurement helper. +- [ ] For required MPI-enabled cells, verify that rank masks and worker places stay within one NUMA domain and the + reserved allocation, with no overlap. Required MPI-off full-machine cells and diagnostic MPI-enabled cross-NUMA L2a + instead record the complete allocation and actual domain/core masks; do not falsely certify containment. In every + case verify distinct physical-core use. These checks are measurement requirements, not library policy. For MPI + deployments advise at least one process per allocated NUMA domain, with disjoint cores when several share a domain. + Use actual topology, not a constant four domains/socket; partial-node runs cover only their allocated domains. + Do not require one logical CPU per worker mask. Record idle/small-kernel serial fallback versus representative + large-kernel actual team size separately. Unverifiable placement blocks benchmark evidence, not library construction. +- [ ] Compare old partitions vs final new threads at identical R,T and routing settings, using Task 1's frozen + size-band-specific inventory and equal has_mpi. Small/medium cover the full sweep plus MPI-off full-machine operation; + selected large profiles cover both full-node MPI decompositions and MPI-off full-machine operation. Do not add/remove + required cells, change build modes or reinterpret profile sizes after seeing candidate performance. + Freeze the production routing mode in that matrix; additionally require the random reference profile under explicit + splitmix at R=3 and R=4, plus default linear at R=4 so ownership-dependent imbalance is measured, not only + unit-tested. Run five alternating **fresh-process** observations per arm/cell; external repetition only, + `--bench-rounds=1`. Include first-call/cold cache and separately measured repeated warm calls without overlapping + object construction. +- [ ] Treat this comparison as the ownership decision recorded in Task 0: one store per rank against the sharded + partition runtime, on the frozen campaign with all five gates. If a required cell still fails after the prescribed + repetitions, the bounded optimizations and the index A/B below, keep the partition runtime and stop with a + regression report. Task 12 does not start, and the owner decides whether to revise the design. + +For required MPI-off full-machine observations, select the R (MPI-disabled) build on both arms and verify with +`uv run --no-sync python -c 'import monoprop, sys; sys.exit(int(monoprop.has_mpi))'` (exit 0 required). +Use Task 1's direct driver invocation without mpiexec. Set T=C with matching physical-core allocation and OpenMP places; +the baseline still declares both +legacy T variables. The shared overlay must avoid importing mpi4py.MPI even if it is installed. Record has_mpi=false +from the imported binary in timed/construction/validation/placement evidence, not just the requested build setting. +The usual five fresh observation pairs, numerical oracles and all five ratios apply, including selected large profiles. + +Example candidate Open MPI invocation after selecting R (= P), K (ranks/domain), T, LABEL and absolute RESULTS. +Use K=1 or K=2 for the agreed decomposition profiles. Choose a compatible placement for routing-specific R=3/R=4 cells; +do not force ppr:1:numa onto an allocation with too few NUMA domains. All placements require observed disjoint cores. +This demonstrates one raw pytest run, not a complete parity observation. The Task 1 driver uses the same launcher +contract, unique per-sample raw directory, frozen campaign and paired construction/validation artifacts for acceptance. +Unset removed partition configuration and use the supported library override. The OpenMP fallback is tested separately: + +```bash +unset monoprop_PARTITIONS +# This example chooses splitmix explicitly, including if R=3; use linear only in a separate R=1/2/4 cell. +export monoprop_ROUTING=splitmix +mkdir -p "$RESULTS" +monoprop_BENCH_LABEL="$LABEL" monoprop_BENCH_RESULTS="$RESULTS" \ +monoprop_NUM_THREADS="$T" OMP_NUM_THREADS="$T" OMP_DYNAMIC=FALSE \ +OMP_PLACES=cores OMP_PROC_BIND=close \ +uv run --no-sync mpiexec -n "$R" --map-by "ppr:${K}:numa:PE=${T}" \ + --bind-to core --nooversubscribe --report-bindings \ + -x monoprop_BENCH_LABEL -x monoprop_BENCH_RESULTS -x monoprop_ROUTING \ + -x monoprop_NUM_THREADS -x OMP_NUM_THREADS -x OMP_DYNAMIC -x OMP_PLACES -x OMP_PROC_BIND \ + python -m pytest 'benches/bench_models.py::test_model_propagate[pauli]' \ + -o filterwarnings=default --bench-rounds=1 --runtime-shape=openmp \ + --benchmark-json="$RESULTS/time-$LABEL.json" +uv run --no-sync monoprop-bench-report "$RESULTS" +uv run --no-sync monoprop-bench-bmf "$RESULTS" "$LABEL" +``` + +This mapping syntax is Open MPI-specific. Verify scheduler/launcher support; do not escape the allocation. For the +**baseline only**, set `monoprop_PARTITIONS="$T"` and `monoprop_NUM_THREADS="$T"`, export both to ranks with +`-x monoprop_PARTITIONS -x monoprop_NUM_THREADS`, and replace `--runtime-shape=openmp` with +`--runtime-shape=partitions`. Keep OMP settings, routing, allocation and shared overlay identical. The candidate uses +monoprop_NUM_THREADS but never monoprop_PARTITIONS. The shared preflight is a required Task 1 change: the existing +guard would reject this candidate above one rank. Verify declared versus observed shape with binary-linked diagnostics; +labels alone cannot certify shape. After M build assert monoprop.has_mpi before invoking any launcher. Follow the +[physical-core launch recipes](../specs/2026-09-18-rank-local-openmp-design.md#physical-core-launch-recipes) for both +Open MPI and Slurm. Document srun's --threads-per-core=1, compatible core binding and OpenMP budgets/places, and the +required NUMA verification. Do not combine conflicting --hint options or claim that socket/block distribution implies +NUMA containment. Explicit-mask fallbacks require eligible full-CPU step allocation, not merely an exclusive job. +OMP_PLACES=cores is not a worker-count cap, and merely setting OMP_NUM_THREADS does not enlarge a one-core rank mask. +- [ ] Report runtime, exact operation peak/floor/delta, outer peak sum/max, separate whole-construction peak, persistent + operator/graph bytes, query/snapshot/TLS capacities, term/graph sizes, and numerical checks. Rank peak sums are upper + bounds on aggregate footprint, not node provisioning estimates; retain maximum rank peak as another statistic. Reject + non-exact operation/construction windows as parity evidence; outer exactness controls diagnostic labeling only. +- [ ] Compare medians per cell using exactly the five Task 1 gates: runtime, operation peak sum/max and construction + peak sum/max. Outer peak ratios are diagnostics only. All five required ratios must be <=1.00. If any fail, collect + five additional observations per arm for those cells and compare all ten. Do not replace the strict gate with a + geometric mean, a made-up 5% tolerance, or a changed problem size. A noisy failure means parity was not demonstrated, + not proof of architectural slowdown; it still blocks acceptance. Do not drop the cell or quietly waive it. +- [ ] Profile phases before optimization: bitmap traversal, query merge, decode/probe, missing-ID assignment, row fill, + hash publication/rehash, inverted-index maintenance, graph packing, MPI pack/wait, cosine, pairs, dot. Record miss + fraction, overflow fraction and bytes, not just aggregate speed. +- [ ] Allowed bounded optimizations, in order of measured relevance: + 1. Serial thresholds and block sizes; reduce unnecessary tiny teams, keeping logical **reduction** partitions fixed + for numerical determinism. + 2. Ordered query exact-fill from Task 5; bounded self-probe windows from Task 6; optional inline fill from Task 7. + 3. Incoming missing-ID prefix: parallel fixed-block miss counts, serial exclusive offsets, exact resize, parallel + ascending fill. IDs remain sender/query order; scratch O(block count), not threads×queries. + 4. Serial index pre-reserve to reduce rehash (measure simultaneous old/new table capacity); the separate bounded + concurrent-index trial below is also allowed, not assumed to win. + 5. Packed graph phases by **whole-word ownership** only: serial validate/count/allocate; each worker constructs and + assigns one uint64 word, including slot seams and zero padding. No endpoint-wise `|=`. Test slots 63/2/65 and + nonbinary fallback; preserve derived D permutation instead of adding persistent D indices. + 6. Inverted-index append only if measured dominant: bounded windows of 64-row-aligned blocks; per-block/per-column + integer counts, serial column offsets/allocations, disjoint sparse element writes and whole dense-word ownership, + serial handling of an unaligned first word, column-owned promotion after fill. Scratch O(window_blocks×2N), never + full-height bitmap per worker. Tests must compare dense/sparse promotion, parity and sorted posting order against + serial after every append. +- [ ] Each optional optimization gets its own failing regression tests, focused review and full affected parity rerun. + Beyond the bounded index trial below, whole-batch streaming/windowing, a custom concurrent table, new transport, + executor, rank redistribution, full per-thread stores or a persistent duplicate full-key map require a new design. + +#### Index A/B: shared-serial versus shared-concurrent + +Compare two build variants of the same one-store engine, with everything else identical: +- **shared-serial:** the current packed `OperatorIndex`, parallel frozen probes and serial publication. +- **shared-concurrent:** the same packed rows indexed by `boost::concurrent_flat_map` over `{row ID, cached hash}` + handles, with bulk lookup and parallel publication of preassigned IDs. + +The variants differ only inside `OperatorIndex` and at its publication call sites; shared invariant 14 keeps them +local. This is a development-only A/B between an existing container and the reference, not a container search or a +permanent backend selector. It needs authorized implementation/measurement budget. Task 0's in-library trial is the +starting point (archived overlay patch); its timings are not acceptance evidence. + +- [ ] Prerequisite: Tasks 5–6 are complete, so traversal and lookup are parallel. With a serial scan (66–95% of one-core + time in Task 0) the variants cannot be told apart at full machine. +- [ ] Build the variants from one development-branch compile-time option; never keep two resident indices. Run the full + C++/Python suites on both during development. Task 0 found bitwise-identical term maps and energies, and one + duplicate-key fixture that depends on the contract pinned by invariant 14. +- [ ] Boost: the owner waived compile/test verification against the 1.85 floor. Bulk visitation exists at 1.85. + Container statistics (`BOOST_UNORDERED_ENABLE_STATS`) need Boost 1.86 or newer, are for diagnostic builds only, and + serialize on one lock: never time them. +- [ ] Use `boost::concurrent_flat_map` as an index into the packed rows, with compact stored row references and + heterogeneous position-query lookup. Full monomials, query-buffer spans and a second persistent reference index are + forbidden as stored duplicate keys. Hash/equality must represent monomial identity, not compare row IDs as if they + were term keys. Retain exact collision checks and cached query hashes. +- [ ] Preserve the batched lookup/result contract. + - Use bulk visitation in chunks of at most 16 queries. The visitor receives only matched elements, in no documented + order, so it only records `(cached hash, row ID)` pairs. + - After the call and outside Boost's locks, match them to queries: a query alone with that hash in its chunk is the + match; several such queries (duplicates or a collision) are separated by exact row comparison. Misses produce no + call. + - Never use side-effecting hash/equality, re-enter the container from a visitor, or assume callback order. A + per-query visitor is an acceptable fallback. + - Canonical row order comes from row IDs, not bucket iteration. +- [ ] Complete frozen probing and deterministic missing-ID assignment before any publication. Caller checks fixed32 and + allocation limits, reserves rows/table, initializes packed rows and overflow entries, then workers publish only the + index entries for preassigned IDs. No row growth, coefficient resize, overflow mutation, inverted-index mutation, + graph packing or external-key relocation while map workers run. Indexed row contents remain immutable until the + index is cleared/rebuilt. Join before dependent reads, structural maintenance or freeing scratch. Preserve remote + leader → remote follower → deferred self leader → deferred self follower publication phase order and matched timing. + Callbacks supplied by callers of `bulk_insert` (construction) are not assumed thread-safe: publish those serially. +- [ ] Keep one index per independent packed store. Copy/move/clone must bind row-aware hash/equality state to the + destination's rows, never a destroyed source. Container synchronization does not make external-row lifetimes safe. + Keep exceptions inside Workshare, join, then use existing caller invalidation and distributed-failure boundaries, + including during active-ticket lifetime. No worker MPI or scheduling-based ID counter. +- [ ] Extend tests before adoption: deliberate hash collisions with distinct keys; repeated lookup queries; mixed + hit/miss batches and exact row IDs at budgets 1/2/3/4; inline and spilled rows; reserve/growth boundaries; the pinned + duplicate-key contract; clone lookup after source mutation/destruction; allocation/publication failure invalidation; + and end-to-end term/graph equivalence at the same geometry. Add a worker-participation test on the variant path. Run + qualified race checks without blanket suppressions. +- [ ] Measure both variants with the Task 1 harness on identical frozen inputs, allocation and placement: fresh-process + pairs, alternating order, all five metrics, plus per-phase lookup/publication/reindex times and peaks. Cover at least: + - L1 (R=1,T=1) cells: Task 0 measured shared-concurrent 15–18% slower on Hubbard/Pauli L1; + - MPI-off full machine; + - the all-new-terms random profiles, where serial publication is the concern; + - the physics models (80–89% hits). + + No new sizes, higher memory allowance or acceptance waiver. Microbenchmark gains do not override end-to-end + regressions. Obtain approval for any spend beyond the approved budget. +- [ ] Decide from the required cells, not from index-phase speedups. Record the choice, remove the losing variant and + the build option before final acceptance, and rerun the affected gates on the final binary. If neither variant can + pass, stop: the A/B is permission to compare an existing container, not to rescue the architecture at any cost. + +**Gate:** All required performance/memory cells pass with attached raw observations and numerical/placement evidence. +The ownership decision and the index A/B outcome are recorded, and only the chosen variant remains. +Missing required EC2 evidence, capacity failure, persistent serial-publication bottleneck or repeated ratio>1.00 blocks +removal/acceptance. Report it honestly; the design is not a promise of achievable parity. + +### Task 12: Remove legacy partition runtime, topology machinery and direct hwloc dependency + +**Files:** +- Delete: `cpp/monoprop/detail/partition/PartitionGroup.h`, + `cpp/monoprop/detail/mpi/{ShmComm,HybridComm,PartitionBarrier}.h`. +- Delete `cpp/monoprop/detail/mpi/CpuRelax.h` only if no retained caller remains. +- Delete: `cpp/monoprop/detail/partition/CpuTopology.{h,cpp}`, `cpp/monoprop/detail/partition/CMakeLists.txt`, + `cpp/tests/cpu_topology_tests.cpp`. Do not create replacement CpuTopology files under parallel/ or another path. +- Modify for dependency removal: `cpp/monoprop/CMakeLists.txt`, `cpp/tests/CMakeLists.txt`, `pyproject.toml`, + `tools/install-deps.sh`, `tools/packages/{apt,brew}.txt`, `.github/actions/setup/action.yml`, + `.devcontainer/Dockerfile`, `nix/{monoprop,devshell}.nix`, `flake.nix`, `justfile`, + `.github/workflows/{test,qa-analysis,docpages,copilot-setup-steps,bench,nix,deploy}.yml`. +- Modify migration/build guidance in the same change: `AGENTS.md`, `README.md`, + `docs/content/docs/{building,benchmarks,openmp-migration}.mdx`, `docs/content/docs/features/parallelism.mdx`. +- Modify: `cpp/monoprop/detail/CMakeLists.txt`, `cpp/monoprop/detail/{mpi,parallel}/CMakeLists.txt`, + `cpp/include/monoprop/MonomialPropagator.h`, `cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl`, + `cpp/monoprop/detail/mpi/{Comm,MPICompat,MPIUtils,Exchange,Pairwise,CheckedCount,Routing}.h`, + `cpp/monoprop/detail/mpi/MPICompat.cpp`, `cpp/monoprop/MPFunctions.cpp`. +- Migrate/retire test files according to ledger below; update `cpp/tests/README.md` and architecture docs in the same + change. + +**PROPOSED interface:** Simplify `mpi::Comm` to the existing implicit MPI_Comm wrapper with `.mpi`; remove +Kind/Shm/Hybrid and associated pointers/constructors/dispatch. Keep non-MPI behavior, ordinary MPI, PeerPlan routing and +existing typed exchanges. Do not collapse it to a raw alias if that changes overload behavior unnecessarily. + +- [ ] Start only after Task 11 parity, and only if Task 11's ownership decision confirmed the single store, under + separately authorized implementation. This is the irreversible step: afterwards, returning to sharding means + restoring the partition runtime, transport, topology code and hwloc dependency. The owner approved the immediate + partition-related C++ extension/ABI break; confirm no runtime path constructs old groups before deleting definitions. +- [ ] Remove any remaining `partition_group_`, friendship and unused fan-out implementation. Confirm Task 10 already + removed resolve_partition_count_, public partition/factory arguments and facade-only test users; remove remaining + is_partition_facade/protected partition-only helpers. Keep `clone_` and virtual lifecycle/operator-update hooks. No + compatibility facade that invokes old callbacks once with silently different semantics. +- [ ] Remove non-MPI-thread transport branches from utilities/tickets. Production router geometry is `(real_ranks,1)`. + Preserve EXISTING boolean linear/splitmix default/rejection contract and generator shift/peer rules; no + routing-default repair is needed. Keep ordinary MPI sparse query routing and sparse on-wire replay, sender order, + count checks, communicator lifetime and initializing-thread guards (including external initialization). + `MPIUtils.h::check_routing_agreement` still delegates to mpi::routes_pairwise; retain exact Config agreement, + communicator attributes/no-copy lifetime semantics and fixed S=1, never per-rank OpenMP budgets. Preserve + require_routable validation, the flat known_recv_counts width check, and WindowVec-only begin_alltoallv/wait_into. + Keep cpp/tests/SlotBlocks.h as the test-only whole-world adapter where migrated tests use it; do not restore + production vector-of-vectors overloads. Ticket/PendingAlltoallv move/drain semantics survive deletion. Keep both MPI + and non-MPI + implementations. +- [ ] With the last Hybrid caller removed, change MPICompat's requested and required minimum to MPI_THREAD_FUNNELED. + Preserve MPI_Query_thread validation for externally initialized MPI and Task 3's caller guard even at higher levels. + Test library-owned initialization; external FUNNELED, SERIALIZED and MULTIPLE; rejection of actual SINGLE; + wrong-thread entry at higher levels; and valid caller execution with workers. Record requested/provided support + and distinguish unavailable runtime scenarios from exercised cases. In Python subprocesses, configure mpi4py before + importing MPI/monoprop; its default MULTIPLE is accepted, not downgraded. Audit finalization/cleanup and failure paths + as well as hot kernels: no worker MPI calls, including MPI_Abort. Update diagnostics/docs/tests in the same task, then + rerun MPI correctness/failure tests and affected parity cells on the final binary. Changing only Init_thread's + constant does not complete this gate. +- [ ] Delete the entire topology/affinity implementation and its legacy call sites, including automatic physical-core + counting, L3 dealing, pinning and mandatory placement diagnostics. Do not move them into ThreadBudget or reimplement + them with OpenMP-place unions, OS affinity, sysfs or another topology library. User-controlled OpenMP and launcher + behavior replaces these policies, not another internal runtime. +- [ ] Remove `find_package(PkgConfig REQUIRED QUIET)`, `pkg_check_modules(HWLOC ...)`, the hwloc status message and both + `PkgConfig::HWLOC` links in `cpp/monoprop/CMakeLists.txt`; remove the unit-runner's hwloc link in + `cpp/tests/CMakeLists.txt`. PkgConfig is currently required there only for hwloc; do not require it on monoprop's + behalf once that use is gone. Audit installed target/config output for residual direct hwloc requirements. Keep OpenMP + target discovery/link/export intact; a transitive MPI/OpenMP-runtime dependency on hwloc is allowed and is not a + monoprop dependency to remove. +- [ ] Remove hwloc installer implementation/configuration/help/output from `tools/install-deps.sh`, including + `--skip-hwloc`; remove explicit hwloc/libhwloc-dev from `tools/packages/{apt,brew}.txt`, wheel hooks and any direct CI + installs. Shared setup and devcontainer consume those manifests, so do not add duplicate replacement install blocks. + Remove direct hwloc arguments/buildInputs in both Nix files and replace the Nix workflow's + `pkg-config --modversion hwloc` smoke requirement with OpenMP configure/link/run validation via existing recipes. Keep + required OpenMP across flake package/MPI package/devshell checks and the deploy wheel matrix. Remove documentation + claiming hwloc/pkg-config is a direct monoprop prerequisite or that the library pins threads. Do not uninstall or + strip a library needed by the selected MPI/OpenMP runtime, or remove another dependency's legitimate pkg-config use. +- [ ] Complete this migration ledger before deleting obsolete files. Constructor/facade-dependent cases must already + have moved in Task 10; retain transport-only coverage until this task removes its implementation. + +| Old coverage | Replacement/retirement | +| --- | --- | +| `partition_equivalence_tests.cpp`: energies, gradients, propagation, determinism, copies, Pauli, setters, partial contraction | `openmp_equivalence_tests.cpp`, budgets 1/2/3/4, both pictures, decoded term/value equivalence | +| facade raw-accessor errors | single-store raw-accessor consistency at every budget | +| factory throw/child construction | factory API removed; Task 3 validates joined worker exceptions without terminate | +| `partition_group_clone_tests.cpp`: derived children | retire child-existence assertions; retain derived clone/update-initial-operator independence in `simulator_copy_tests.cpp` and `update_initial_operator.cpp` | +| `shm_comm_tests.cpp` / `hybrid_comm_tests.cpp`: private barrier algorithms | retire only after ownership/source-order/empty-payload/count/error assertions map to MPI/OpenMP tests | +| `hybrid_comm_sparse_plan_on_the_plain_mpi_path` | move unchanged semantics to ordinary MPI tests; unknown and known receive counts | +| plain-MPI half of `hybrid_comm_known_recv_counts_are_masked_through_the_plan` | preserve masked non-peer counts and sparse peer payload assertions; add short known-count rejection before posting | +| `hybrid_comm_unroutable_peer_plan_is_refused` | move ordinary-MPI invalid-shift/count-entry checks; retain routing_sparse_peer_plans_are_validated_before_use | +| `hybrid_comm_pending_alltoallv_owns_its_requests` | move ordinary-MPI move/noncopyable/destructor-drain coverage unchanged; extend active-destination move assignment with live buffers | +| `hybrid_comm_derived_wire_plan_reads_every_partitions_row` / `hybrid_comm_derived_wire_plan_falls_back_to_every_peer` | retire only partition-row folding; preserve empty, one-peer and multi-peer replay under ordinary MPI, including four-rank fixtures | +| `flat_exchange_tests.cpp` replay dense/pairwise/dense-layout/empty tests | retain and run under ordinary MPI after deletion; dense gate test is flat_exchange_dense_gate_always_takes_the_collective | +| hybrid MPI energy and size case | retain as MPI×OpenMP matrix case in `mpi_distributed_layer_equivalence.cpp` | +| poisoned waiter release | Task 3 MPI failure-driver, joined exceptions and enforced invalid-owner/functionals tests | +| `cpu_topology_tests.cpp`: topology enumeration, mask classification, pinning/L3 dealing | retire these library-owned policy tests; Task 3/10 count/precedence/no-affinity-mutation tests cover the retained library contract, and Task 11 verifies placement externally | + +Record individual retired case names and replacement names in the implementation acceptance artifact; do not merely +delete files until green. +- [ ] Audit references: + +```bash +rg -n 'PartitionGroup|ShmComm|HybridComm|PartitionBarrier|CpuRelax|is_partition_facade|run_on_all' cpp src +rg -n 'partition_|partitions|monoprop_PARTITIONS' cpp src README.md AGENTS.md docs/content/docs +rg -n -i 'hwloc|CpuTopology|PkgConfig::HWLOC|libhwloc-dev' \ + cpp cmake CMakeLists.txt pyproject.toml tools .github .devcontainer nix flake.nix justfile \ + README.md AGENTS.md docs/content/docs +``` + +First command should have no production legacy-runtime matches; second must resolve only to mathematical/MPI +partition terminology or historical migration notes, never a retained constructor alias, getenv reader or fan-out. +The hwloc audit must find +no direct discovery/link/install requirement or CpuTopology implementation; migration notes about removal and +third-party/transitive dependencies are allowed. Inspect each match rather than deleting mathematical/MPI partition +terminology indiscriminately. +- [ ] Update `justfile::test-cpp-tsan` and `.github/workflows/qa-analysis.yml` selection before retiring files; require + nonempty `openmp_` coverage. Keep `just test-cpp`'s unit-only export probe and `just test-find-package` consumer. +- [ ] Run R/M/S, C++ installed-consumer checks and full Python suite. Re-run Task 11 parity after deletion: removal can + change layout, initialization and measurements. Preserve archived baseline; do not make candidate its own baseline. + +**Gate:** No in-process communication runtime, library-owned worker threads, custom barriers, topology/affinity +subsystem, direct hwloc dependency or permanent legacy backend remain. Build/install/import a non-MPI candidate in an +environment without hwloc development headers/pkg-config metadata; inspect direct linkage instead of requiring the +absence of a runtime's own transitive hwloc dependency. One packed store per independent propagator per rank is verified +by construction, copy/functional tests and memory accounting, not merely a flag. + +### Task 13: Final packaging, documentation and independent acceptance + +**Files:** +- Modify as required: `.github/workflows/{test,bench,bench_bare_metal,deploy,nix,qa-analysis}.yml`, `pyproject.toml`, + `tools/install-deps.sh`, `tools/packages/{apt,brew}.txt`, `.github/actions/setup/action.yml`, `justfile`, + `nix/{monoprop,devshell}.nix`, `flake.nix`, `.devcontainer/Dockerfile`. +- Extend existing consumer/probe: `cpp/tests/find_package_smoke/CMakeLists.txt`, + `cpp/tests/link_export_probe/link_export_probe.cpp`. +- Modify: `AGENTS.md`, `README.md`, `docs/content/docs/{building,benchmarks,openmp-migration}.mdx`, + `docs/content/docs/features/{parallelism,evaluation}.mdx`, `docs/content/docs/concepts/algorithm.mdx`, + `cpp/tests/README.md`, + public Python docstrings affected by ordering/runtime guidance. +- Test: existing wheel jobs; benchmark renderer tests; whole C++/Python suite. + +- [ ] Validate the Nix package, explicit MPI package and devshell paths after dependency removal. Current + `.github/workflows/nix.yml` owns these entrypoints; keep its OpenMP smoke gate consistent with the shared recipes. + Planned commands (separate candidate environment; none were executed in this refresh): + +```bash +nix flake check --print-build-logs +nix build --print-build-logs .#monoprop .#monoprop-mpi +nix run . -- -c 'import monoprop; print(monoprop.__version__)' +nix develop --command bash -euo pipefail -c ' + monoprop_ENABLE_MPI=OFF just build --reinstall-package monoprop --no-cache + cmake --build --preset skbuild-release + just test-find-package +' +``` + + Require installed runtime resolution and the extended OpenMP/both-basis consumer, not a pkg-config hwloc probe. + Archive MPI package import/smoke evidence too; the current default flake check alone covers only non-MPI packaging. +- [ ] Build/install/repair wheels through `.github/workflows/deploy.yml::build-wheels` and the `pyproject.toml` + cibuildwheel hooks for the currently advertised Linux x86_64/aarch64 and macOS matrix. Validate import and a threaded + propagation in a clean environment outside source checkout, MPI disabled. Inspect ELF/dylib dependencies and bundled + runtime licensing; check coexistence with NumPy/other imported OpenMP users. Do not claim Windows or unsupported + sanitizer/runtime combinations tested. +- [ ] Run `just test-find-package` against the installed package, extending its existing shared probe source rather than + creating a duplicate harness; compile a template kernel using OpenMP and both Majorana/Pauli APIs. The consumer must + not inherit source-tree include paths or manual linker flags. +- [ ] Document one packed store per independent propagator per rank, preserving copies, existing functional snapshots + and pared graphs. Users choose P/T at launch and bind with OMP_PLACES+OMP_PROC_BIND, srun --cpu-bind and/or mpiexec. + Normally use monoprop_NUM_THREADS=T; only unset selects the OpenMP runtime default. Present-invalid is an error. + Capture once during construction, with no reload/setter. Describe no hardware clamp/topology discovery/NUMA + validation and possible smaller OpenMP teams. Distinguish NUMA-contained required MPI cells, MPI-enabled cross-NUMA + L2a diagnostics, and required MPI-off full-machine operation spanning domains. Show matching allocation, rank-binding + and thread-place examples, including the one-core-rank-mask pitfall and direct MPI-off execution. Document one + controlling host caller on MPI's initializing thread outside host OpenMP teams, + no overlapping use of dependent state or host MPI calls, FUNNELED-or-higher support, caller checks, callable lifetime, + enforced failed-object invalidation, accepted capacity reduction and numerical contracts. Placement verification for + benchmarks does not add a topology dependency. For MPI deployments advise at least one process per allocated NUMA + domain; do not tell MPI-off users to add unavailable MPI ranks to avoid regressions. Identify topology externally; + production's common four-domains/socket layout is not universal. Distinguish the selected + c8a.metal-24xl initial acceptance from pending Intel/AMD HPC qualification. Explicitly recommend against SMT workers, + even when hardware SMT is enabled, and demonstrate launcher plus OpenMP settings for one worker per physical core. + Multi-node performance qualification remains pending a proper HPC cluster/interconnect; do not claim it from EC2 + same-instance MPI tests. +- [ ] State exact immediate breaks: partitions/child_factory arguments, PartitionChildFactory, partition-only helpers, + errors and environment controls removed without aliases; no replacement public num_threads argument. Old automatic + physical-core selection, multi-rank serial default, library pinning and direct hwloc requirement are gone. OpenMP is + mandatory even without MPI. Raw order/splitmix physical ownership may differ; ABI requires rebuild. Requests are not + hardware-clamped. Record the stronger initializing-thread MPI caller rule as a compatibility break, even at supplied + SERIALIZED/MULTIPLE. Preserve Python mathematical signatures and current incremental-axis/post-call seed semantics + in the evaluation guide/docstrings. Keep Qiskit conversion and its optional-version tests unchanged; do not advertise + those already-shipped additions as OpenMP work. Explain routing geometry and COMMROUTE cosets without claiming a + balanced basis guarantees a balanced retained operator. Use `[Symbol][]` references in prose/docstrings, + not hard-coded API URLs. +- [ ] Run `prek run --all-files` before any separately authorized push, plus relevant tests, `just gen-api` and + `just build-docs`. Targeted hooks during development do not replace the all-files gate. Keep generated bindings/API + artifacts consistent; no hand-edited dispatch. +- [ ] Obtain fresh-context read-only spec/correctness and performance-memory reviews. Reviewers receive both documents, + implementation diff, retired-test ledger, raw baseline/candidate results, placement/capacity report, build logs and + known tool limitations. Any concurrency, ordering, memory or unsupported-claim finding blocks completion until + fixed/retested. +- [ ] Produce a final acceptance artifact containing: revision identities and overlay hashes; configuration matrix; + actual executed commands/results; numerical cases; maximum observed store sizes; MPI failure tests; runtime/library + identities; old/new line/file/runtime-component summary; per-cell time/peak medians and ratios; raw artifact paths; + reviewer findings/fixes; residual risks. Include actual output/artifact paths when delegated, not filenames mentioned + only in prompts. + +**Final acceptance checklist:** + +- [ ] Task 0 evidence and the owner's architecture decision are archived; this conditional implementation and its + MPI-off acceptance coverage match the reconciled spec/plan, not a superseded assumption of one store. +- [ ] Mathematical features, equal global retained term keys, copies/updates/closures and graph paring pass existing + oracles; matching energies/gradients alone is insufficient. +- [ ] Environment-only immutable budgets, strict invalid-input errors, removed partition interfaces and enforced + invalid-object/dependent-functional rejection pass; host calling assumptions and immediate breaks are documented. +- [ ] Thread-count-stable ordering and approved deterministic reductions pass; no shared append, lazy-cache, packed-word + or MPI-worker race remains. +- [ ] Fixed uint32_t TermIndex and optional-MPI/non-MPI packaging pass. +- [ ] Rank-wide limits and verified one-worker-per-physical-core placement satisfy the single-instance campaign, + including its two-ranks-per-domain case. HPC-family performance and proper-interconnect multi-node qualification are + explicitly pending, not falsely passed or silently included in the initial acceptance claim. +- [ ] Every required runtime and exact peak-memory cell passes Task 11, including actual MPI-off full-machine + small/medium and selected large profiles. All five metrics and numerical checks apply; a NUMA-local win cannot + excuse a full-machine regression. Unsupported or missing measurements are not marked passed. +- [ ] No legacy runtime, direct hwloc dependency, replacement topology/pinning service or scheduler remains. A selected + Boost concurrent index passed its bounded tests and affected parity gates, with no duplicate operator/full-key store. + MPI requests/requires FUNNELED and checks the initializing caller even at higher provided levels. +- [ ] Documentation, benchmark schema consumers and API generation agree. Operation/construction exactness gates the + five parity metrics; outer-window exactness controls diagnostic labels only. Frozen campaign coverage, unique paired + measurement artifacts and matching validation/placement provenance pass, including all four historical pytest families + and new bounded paring-construction cells. +- [ ] Shared preflight proves declared/observed equal-allocation shape. Every successful R=3 launch uses explicit + splitmix; linear rejection is tested separately. Current stable-gradient and streamed-initialization regressions pass. +- [ ] Required OpenMP works through shared package/setup, wheel/deploy, Nix package/devshell/check and + installed-consumer routes; centralized TSan selection remains nonempty. `prek run --all-files` is complete before an + authorized push. +- [ ] Owner receives evidence and integration choices; no unrequested commit/push/PR or merge is performed. + +## Stop-and-report template + +Use this instead of making an unapproved design change: + +```text +Blocked task/gate: +Checkout/revision/configuration: +Command or workload cell: +Expected contract: +Observed failure or ratio (raw artifact path): +Smallest reproducer / phase profile: +Changes already made and worktree status: +Approved bounded options attempted: +Decision required from owner: +``` + +Examples requiring a stop: a full rank cannot fit fixed uint32_t TermIndex, serial publication prevents parity, query +buffering exceeds peak baseline, the benchmark operator cannot verify the target allocation/placement externally, +unsupported OpenMP packaging, failure injection hangs, numerical changes alter retained terms, or the owner declines a +compatibility break. Task 0 permits the three standalone index variants and reopens the architecture with the owner; +Task 11 permits only the retained candidate's bounded integration trial. Neither silently authorizes production shards, +an unbounded/custom concurrent store, altered acceptance geometry, a weakened oracle or a waiver. diff --git a/docs/superpowers/plans/2026-09-29-sharded-openmp.md b/docs/superpowers/plans/2026-09-29-sharded-openmp.md new file mode 100644 index 000000000..4b79f687b --- /dev/null +++ b/docs/superpowers/plans/2026-09-29-sharded-openmp.md @@ -0,0 +1,1500 @@ +# Sharded OpenMP Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans for authorized inline execution, or +> superpowers:subagent-driven-development only when the operator explicitly authorizes delegation. Read this plan and +> its spec completely. Steps use checkboxes. A task approval does not authorize the next task, measurements or +> publication. + +**Goal:** Replace the custom partition runtime with coarse-grained OpenMP execution while retaining sharded storage, +owner-local mutation and the frozen numerical/runtime/memory acceptance contracts. + +**Architecture:** One rank-level propagator owns T internal shard states, one per OpenMP worker. A single team spans +each operation; explicit local phases and published buffers replace child-propagator dispatch and in-process +communicators. MPI's initializing thread is the OpenMP primary and owns physical-rank MPI calls inside that team. + +**Tech Stack:** C++23, required OpenMP CXX, existing Boost and optional MPI, nanobind, Python 3.11+, +uv/scikit-build-core, Boost.Test and pytest. No replacement affinity/topology dependency. + +**Spec:** [Sharded OpenMP execution design](../specs/2026-09-29-sharded-openmp-design.md). + +## Status and authority + +Planning revision at `a6a995a4352bc5a9494acc4c13df33aa61ef84ef`, dated 2026-09-29. The owner approved writing the +replacement documents, not executing this plan. No implementation, build, experiment, delegation, commit or publication +is authorized by these checkboxes. Review subsequent source differences before starting; never reset to a historical +pin. + +Progress: S0 was separately authorized, executed and accepted by the owner on 2026-09-29 (outcome under Task S0). +S1 was separately authorized and implemented on 2026-09-29 (outcome under Task S1); owner acceptance is not recorded +here. S2 was separately authorized and implemented on 2026-09-30 (outcome under Task S2); owner acceptance is not +recorded here. S3 was separately authorized and implemented on 2026-09-30 (outcome under Task S3); owner acceptance +is not recorded here. S4 was separately authorized, implemented on 2026-10-01 and accepted by the owner on 2026-10-01 +(outcome under Task S4). Its architecture checkpoint ran on 2026-10-01; the owner decided **rework**, and a first +profiling-driven rework was implemented the same day. The checkpoint re-run on the reworked code (2026-10-01) still +failed; after profiling the owner decided a second **rework** (C, D, E), implemented on 2026-10-02. Its checkpoint +re-run (2026-10-02) left small deviations, which the owner accepted on 2026-10-02 as reasonable, to be re-evaluated at +a later stage (see S4's outcome). That is the owner's **proceed** decision for the S4 checkpoint. S5 was separately +authorized and implemented on 2026-10-02 (outcome under Task S5); its MPI+OpenMP checkpoint (tier A) ran the same day, +and the owner accepted S5 as done on 2026-10-02, with the remaining deviations and the optimization opportunities +recorded under S5 for later. S6 was separately authorized, implemented and accepted by the owner on 2026-10-03 +(outcome under Task S6). Bounded diagnostics and one optimization followed before S7 (recorded after S6's checklist): +a re-check of the deviating checkpoint cells, B-replay (owner-parallel replay layout, landed) and B-memory (staging +change rejected). The owner decided to run S7 with the remaining MPI-off memory cells recorded as known deviations. +S7 was separately authorized and ran on 2026-10-05/06 (outcome under Task S7): evidence complete, strict parity not +demonstrated (102 of 250 cells pass all five gates). Four owner-authorized remedy rounds followed on 2026-10-06/07 +(recorded after S7's checklist); their kept source changes landed without a new campaign. The owner decided on +2026-10-07 to keep glibc malloc: no allocator change. A further owner-authorized round (A, 2026-10-07, recorded after +the remedy rounds) fixed the two largest remaining failure classes and a regression of the remedy rounds. S7 has not +been re-run on the remedied source. On 2026-10-07 the owner waived the S7-pass prerequisite for S8: S8 may proceed +without a passing pre-removal campaign, and one parity campaign on S8's final binary replaces an S7 re-run. The final +acceptance gates are unchanged. S8 was separately authorized on 2026-10-07; its cutover is implemented and locally +verified (outcome under Task S8). Its optimization points were taken up in owner-approved rounds on 2026-10-07/08 and +the kept changes verified; the owner had the work committed and pushed on 2026-10-08. S8's final parity campaign +ran on 2026-10-08/09 under separate authorization: 145 of 250 cells pass all five gates (S7: 102), so strict parity +is not demonstrated (outcome under Task S8). The owner accepted the MPI 1x1 runtime tail and left the T = 96 slow +starts to the medians before the campaign. An owner-approved optimization round on the memory misses (round 5, +2026-10-10) followed, verified and pushed at the owner's request (outcome under Task S8). On 2026-10-11 the owner +had `origin/main` merged into the branch (main's two engine changes ported onto the shards), then approved a +measurement round against round 5 (step 6) and the two fixes it led to; both pushed at the owner's request. No +binary after the campaign has campaign evidence yet. + +This replaces the abandoned one-store Tasks 7–13 in the +[historical plan](2026-09-18-rank-local-openmp.md). Its Tasks 0–6 remain historical evidence, not an unexecuted queue. +Its Task 1 frozen inventory, artifacts and measurement protocol remain authoritative where imported below. New task IDs +are **S0–S8** to prevent confusion with that experiment. Task 6 is not the performance baseline. + +## Global constraints + +- C++23; GCC 14 / Clang 18 minimum; Python 3.11 minimum. +- OpenMP CXX is required, including MPI-off builds and installed consumers. MPI remains optional and OFF by default. +- Retain Linux x86_64, Linux aarch64 and macOS packaging coverage; report untested platforms as pending. +- Keep packed position-list rows, keyless indexing, sparse state, inverted indices and packed graph representation. +- Keep `TermIndex = std::uint32_t`, with its reserved sentinel, per shard. Retain checked count/allocation arithmetic. +- Preserve both bases and pictures, cutoffs, basis changes, propagation, graphs, gradients, paring, copies and updates. +- Remove the custom worker pool, in-process communicator abstractions, custom barriers and direct hwloc dependency. +- Allocation, placement and pinning belong to the launcher/OpenMP configuration. Recommend one worker per physical core. +- Numerical preservation and the frozen runtime/memory gates govern this refactor; neither is waived by a faster kernel. +- Fixed uniform P processes × T threads; T is captured at construction and is exactly the internal shard count. +- No T-consistency collectives/checks, reduced-team handling, resharding, hardware clamp or independent shard-count + knob. +- No nested kernel parallelism initially. Preserve ascending-shard combination and existing MPI reduction association. +- No arithmetic reordering, new floating-point reductions or FMA-policy change during this refactor. These options + remain open after landing, individually or together, under the explicitly separate follow-up at the end of this plan. +- Preserve immutable graph sharing and independent store copies. Raw root accessors work only at T=1. +- C++ naming/style/Doxygen, Python docstrings, generated-file restrictions and API links follow `AGENTS.md`. +- Use traversal for query-producing bitmap work, and prefix-sum for cumulative offsets. Do not call both a scan. +- Build/artifact isolation is mandatory; do not alter baseline environments, archived evidence or unrelated checkout + data. +- Before an authorized push, run `prek run --all-files` and relevant tests, and fix every finding the change + introduces. Standing owner waiver (recorded at S0 acceptance, 2026-09-29), valid for the whole execution of this + plan: ignore the pre-existing `ruff check` findings in `docs/notebooks/chemistry/chemistry.ipynb`, + `docs/notebooks/fermi_hubbard/fermi_hubbard.ipynb` and `docs/notebooks/kicked_ising/kicked_ising.ipynb` (49 at + S0). Do not edit those notebooks to silence them. The waiver covers no other file, hook or finding; report any new + finding in those notebooks, or any change in their count, rather than treating it as waived. + +## Evidence pins and scope + +The preserved production baseline is `90d57177c2b0cd93503f88dd931d2f8dd409aa23`. Baseline tools were subsequently +synchronized by overlay without rebuilding its binaries. At this planning checkpoint the local frozen inputs match: + +| File | SHA-256 | +| --- | --- | +| `tools/rank-local-openmp-campaign.json` | `ffde870c77a66c8ce37fb2cc07b39abef976e9d7938b7cea59f43cb407de068a` | +| `tools/rank-local-openmp-workloads.json` | `508b65f1f0eb2171fc4fbd79feba2381054c6dcdd38a614600f5b957ab814416` | +| `tools/benchmark-rank-local-openmp.py` | `2207529fb45454990dfba3ad0e1cf944ba1ae1d15385489aef4b16a365426697` | +| `CONTEXT.md` | `1eaf2d92f23f3d0f349e281363e846f7baa5235415f20117abcad40eb5249d66` | + +The recorded remote baseline index is `/home/ubuntu/task1-artifacts/formal/baseline-index.json`, hash +`82479d6ff83deb3113306ac1caa1720d25f15a400cbc89b5693f85ffad47fff5`. Its identity and raw artifacts have not been +independently reverified during this redesign. Reverify them during separately authorized preflight; do not reconstruct +or overwrite the formal baseline. Preserve the whole `/home/ubuntu/task1-artifacts/` tree and later task reports. + +The 250-cell inventory and five independent <=1.00 median gates remain: runtime, operation peak sum/max, construction +peak sum/max. Initial observations use five distinct fresh-process pairs; failing cells get the specified additional +five. Numerical validation is separate and includes global retained operators. No workload resizing, cell dropping, +geometric-mean masking, memory allowance or substitution of MPI-enabled P=1 for actual MPI-off operation. + +Task 6's full-machine failure motivates two early architecture checkpoints, S4 and S5. Neither is a formal campaign or +permission to spend the old 180-minute budget. Each experiment requires its own written scope, elapsed/per-trial/RAM +limits, allocation/provenance plan and owner approval. Budgets never roll over. Keep diagnostics and formal results +separate. A checkpoint failure stops the plan for an owner decision; it does not silently move optimization into S6. + +## Module and source map + +New private names below are proposed interfaces, not claims about existing source. Install private headers reachable +from public templates using the existing CMake header/export mechanism. Avoid a broad source reorganization. + +| Path | Responsibility | +| --- | --- | +| New `cpp/monoprop/detail/sharded/Team.h` | Operation-scoped team and checkpoint-stable exception publication; no queue, threads or transport ownership | +| New `cpp/monoprop/detail/sharded/State.h` | Stable shard addresses and owner-initialized/copyable mathematical state | +| New `cpp/monoprop/detail/sharded/Construction.h` | Rank-level gate/pass orchestration over shard-local construction phases | +| New `cpp/monoprop/detail/sharded/Evaluation.h` and `Evaluation.cpp` | Replay, reverse pass and ordered rank-local result combination | +| New `cpp/monoprop/detail/sharded/Exchange.h` and `Exchange.cpp` | Physical-MPI staging/request ownership only; no in-process collective facade | +| New `cpp/monoprop/detail/sharded/CMakeLists.txt` | Source/header registration, including exports needed by installed template consumers | +| `cpp/monoprop/detail/evolution/layer_build/{Engine,Resolve,QueryWire,Scan,FusedApply}.h` | Preserve owner-local algorithms; separate pure phases from communication; read-only input views | +| `cpp/monoprop/{Evolution,MPFunctions}.cpp` | Extract reusable local replay/derivative operations and scratch without changing formulas | +| `cpp/include/monoprop/MonomialPropagator.h`, `cpp/monoprop/detail/monomial_propagator/MonomialPropagator.inl` | Rank-level state, public operations, lifecycle, aggregates and exports | +| `cpp/monoprop/detail/mpi/{Comm,Exchange,MPICompat,MPIUtils,OperationFailure,Pairwise,Routing}.h` | Real-MPI ownership, routing cache and failure contracts; remove fake communicator dispatch at cutover | +| `cpp/monoprop/detail/mpi/MPICompat.cpp`, `cpp/monoprop/detail/EnvConfig.h` | Initialization/thread support and removal of partition configuration | +| `cpp/tests/PropagatorTestAccess.h` | Private shard inspection, actual-phase observers and failure injection | +| New `cpp/tests/sharded_{team,state,construction,evaluation,exchange}_tests.cpp` | Focused flat Boost tests for new seams | +| New `tests/test_sharded_openmp.py` | Process-isolated launch geometry, full API and exporter integration | +| `cpp/tests/CMakeLists.txt`, `justfile`, `.github/workflows/test.yml` | Separate T=1 internals and fixed-T sharded test launches | + +Existing high-risk seams to inspect before editing: + +- `Engine.h::run_exchange`, `resolve_self_queries`, `insert_deferred_self_misses`, `finish`, `build_layer`. +- `GraphSink` / `ContractSink`, `IncomingProbe`, `SelfQueryStage`, `SlotWindow` and the Task 6 checked decoder. +- `Evolution.cpp::CrossRankExchangeHandle`, `begin_flat_exchange`, `wait_flat_exchange`, the two exchange finish paths. +- `MPFunctions.cpp::prepare_evolved_operator`, `ev`, `ev_and_grad` and `eval_scratch`. +- `MonomialPropagator.inl::run_operation_`, constructors/copy, `evolved_operator_terms`, functionals and updates. +- `PartitionGroup.h`, `ShmComm.h`, `HybridComm.h`, `PartitionBarrier.h`: algorithm/order reference, not a runtime to + wrap. + +## Shared correctness contracts + +1. A store and its overflow map have one mutation owner at a time. Reserving capacity does not make concurrent + `set_positions` safe. Different stores may mutate concurrently; lazy caches belong to their owner too. +2. At fixed P/T preserve owner mapping, source/query order, per-shard row IDs and graph records. Keep different local + shards distinct from self; combine local and remote sender blocks in ascending flat-owner order. +3. Publication order is cross-owner leaders, cross-owner followers, deferred self leaders, deferred self followers. + Match followers against the completed leader pass. Keep the source boundary distinct from final `scaled_count`. +4. XOR with a fixed generator gives distinct partners for distinct sources. Preserve the existing duplicate-key behavior + rather than silently deduplicating; see invariant 14 in the historical plan and `operator_index_tests.cpp`. +5. Queries remain compact. Keep absolute position offsets, cached hashes, checked wire boundaries/position narrowing and + shard-local ID/sentinel limits. Query ordinals remain `size_t`. Do not restore a rank-wide single-store capacity + limit. +6. Send and endpoint-snapshot buffers are immutable through consumption. MPI request storage and buffer addresses do not + move after posting. Requests outlive region exit; failure handling precedes any destructor that could wait on a peer. +7. All supported workers reach the same checkpoints. Failure decisions are stable for a checkpoint generation; a fast + worker's next-phase failure cannot make a slower worker branch out of the previous phase. Empty shards participate. +8. Preserve self-pair layout (`k`, `k+count/2`), signs, cosine masks/tails, `LayerAngle`, record/restore order and + derivative `A=cos_acc(...)*sec`, `-g*(sin*(A-ep.cos_terms)-ep.sin_terms)`. Keep sparse/duplicate dot and scatter + semantics. +9. No new arithmetic association in S0–S8. Keep `kVanishingCos=0x1p-20`, `kRecordSpreadBits=53.0`, the contraction + workaround and existing tolerances. Preserve Python incremental axes and post-call seeds in both pictures/bases. +10. Validate full retained maps, including zeros/near-cutoff rows. Replicated Heisenberg identity agrees and is included + once. Duplicate nonidentity ownership or nonfinite values fail. General map comparison remains + `abs(a-b) <= 1e-9 + 1e-7*max(abs(a),abs(b))`; stricter existing tests and unrounded scalar/gradient checks remain. + +## Verification commands and task handoffs + +During authorized execution, use fresh R (MPI OFF) and M (MPI ON) environments/builds, outside baseline directories. +The following is a build/test recipe, not an instruction to execute while reviewing the plan. Set absolute `ARTIFACTS` +to the approved task directory. A stable installed Ninja avoids the expired build-isolation Ninja path in old caches. + +```bash +: "${ARTIFACTS:?Set the approved absolute task artifact directory}" +export R_ENV="$ARTIFACTS/venv-r" R_BUILD="$ARTIFACTS/build-r" +export monoprop_NUM_THREADS=1 OMP_NUM_THREADS=1 OMP_DYNAMIC=FALSE +monoprop_ENABLE_MPI=OFF UV_PROJECT_ENVIRONMENT="$R_ENV" \ + uv sync --all-extras --group workspace-test --reinstall-package monoprop --no-cache -v \ + --config-settings-package="monoprop:build-dir=$R_BUILD" \ + --config-settings-package="monoprop:cmake.define.CMAKE_MAKE_PROGRAM=$(command -v ninja)" +cmake --build "$R_BUILD" --target monoprop_unit_tests.x +ctest --test-dir "$R_BUILD" --show-only +ctest --test-dir "$R_BUILD" --output-on-failure --no-tests=error +UV_PROJECT_ENVIRONMENT="$R_ENV" uv run --no-sync python -c \ + 'import monoprop, sys; sys.exit(int(monoprop.has_mpi))' +UV_PROJECT_ENVIRONMENT="$R_ENV" uv run --no-sync pytest +``` + +For M, use distinct `M_ENV`/`M_BUILD` and `monoprop_ENABLE_MPI=ON` in the build environment; verify `has_mpi` is true. +Use the documented `just test-mpi` routes, adapted to the isolated tree when necessary. MPI-off tooling must not import +`mpi4py.MPI`. Keep `--no-sync` when running an existing build. Verify executable/extension/library paths, hashes, +mtimes, actual linkage and regenerated nonempty Boost discovery; `uv sync` alone does not prove the C++ binary was +relinked. +New sources/tests must be registered/reconfigured before claiming a missing-test RED or a passing run. + +Once sharded test registration exists, launch each supported T in a separate process; do not change T inside a test: + +```bash +monoprop_NUM_THREADS=4 OMP_NUM_THREADS=4 OMP_DYNAMIC=FALSE \ + ctest --test-dir "$R_BUILD" --output-on-failure --no-tests=error -R '^sharded_' +monoprop_NUM_THREADS=4 OMP_NUM_THREADS=4 OMP_DYNAMIC=FALSE \ + UV_PROJECT_ENVIRONMENT="$R_ENV" uv run --no-sync pytest tests/test_sharded_openmp.py -v +``` + +These commands require an allocation suitable for four workers. Ordinary low-level tests stay at T=1. A helper-level +limited-team test may remain for `for_blocks`, but it does not define a supported sharded-operation launch. +Run ASan/UBSan separately from TSan. Qualify Clang/libomp/Archer with known-safe/racy probes; unsupported libgomp TSan +is not race-free evidence. Observe actual workers inside real phases; assert only after joining, not from worker bodies. + +Every task ends with a written handoff outside the checkout: source/build identity, changed interfaces, actual RED/GREEN +commands and outcomes, preservation tests, request/memory/ordering review, pending platforms and the next authorization +boundary. A passing old fixture is preservation evidence, not an invented RED. Commits require separate authority and +repository message/trailer/signing policy; no task here contains an automatic git-write step. + +## Task S0: A fixed-team, phase-safe execution primitive + +**Deliverable:** independently tested operation-scoped team/checkpoint code, with no propagator or transport migration. + +**Files:** create `detail/sharded/Team.h`, its `CMakeLists.txt` and `cpp/tests/sharded_team_tests.cpp`; register through +`cpp/monoprop/CMakeLists.txt` and `cpp/tests/CMakeLists.txt`; clarify `detail/parallel/Options.h` documentation. +Paths under `detail/` here and below are under `cpp/monoprop/`. + +**Interfaces:** in `monoprop::detail::sharded`, introduce `TeamFailure` with a constructor taking `size_t threads`, +`record(size_t shard, std::exception_ptr error) noexcept -> void`, `checkpoint() noexcept -> bool`, and +`first_error() const noexcept -> std::exception_ptr`. One error slot per worker; `first_error` selects the lowest +failing shard after joining. Add these templates: + +```cpp +template +auto run_team(parallel::Options options, Fn &&body) -> std::exception_ptr; + +template +auto phase(TeamFailure &failure, size_t shard, Fn &&body) noexcept -> bool; +``` + +`run_team` constructs error storage before entering `omp parallel num_threads(options.threads)`, calls the noexcept +body with `(size_t shard, TeamFailure&)`, then returns the error without rethrowing. `phase` catches its local body, +records its own slot and returns the collective checkpoint decision. Every phase result is checked; false ends the +common sequence without invoking later phase bodies. No MPI, configuration check or topology query. + +**Outcome:** implemented and **accepted by the owner on 2026-09-29**. Start: `f31174c`. Handoff with the +contract, the barrier/generation/join argument, commands, toolchains and full evidence: +`/home/ubuntu/s0-artifacts/HANDOFF.md` (outside the checkout). + +- Interface as specified, plus a `TeamBody` concept that requires a noexcept orchestration body; `run_team` rejects a + nonpositive budget before the region. The checkpoint uses `omp masked` (the non-deprecated `master`). +- RED: the registered test failed to compile only on the missing `monoprop/detail/sharded/Team.h`. GREEN: 17 flat + `sharded_team_*` cases in fresh `sharded_team_env_t{1,2,4}` and `_t4_passive` launches observed 1/2/4 distinct + workers; GCC/libgomp and Clang 18/libomp R (413/413), GCC ASan/UBSan (413/413), GCC M with Open MPI 5.0.10 + (459/459, compatibility only), R pytest (767 passed, 25 skipped). `for_blocks` preservation passed under dynamic and + thread-limited teams. +- 13 `Team.h` mutants: 12 fail specific assertions (an asynchronously changing stop flag fails the generation case at + T>=2 only); a failing worker that skips the checkpoint shows only as a bounded 120 s timeout. +- Clang/libomp/Archer TSan qualified with known-safe/racy probes; the real tests are silent, and the missing-leading- + barrier mutant is reported in `TeamFailure::record`. The installed `find_package_smoke` consumer instantiates + `Team.h` through `monoprop::monoprop` for the R, M and Clang packages. +- Pending: macOS, Linux aarch64, wheels, Nix and minimum-version compiler routes. + +- [x] Add a real-team test and run it before implementation; the absent helper is the initial compile RED: + +```cpp +namespace parallel = monoprop::detail::parallel; +namespace sharded = monoprop::detail::sharded; + +BOOST_AUTO_TEST_CASE(sharded_team_visits_each_owner) { + const auto options = parallel::capture_thread_budget(); + auto visits = std::vector(static_cast(options.threads), 0); + const auto error = sharded::run_team(options, [&](size_t shard, sharded::TeamFailure &failure) noexcept { + sharded::phase(failure, shard, [&] { ++visits[shard]; }); + }); + BOOST_CHECK(!error); + for (const auto count : visits) { + BOOST_CHECK_EQUAL(count, 1); + } +} +``` + +- [x] Implement a checkpoint-stable decision, not a single asynchronously changing stop flag. A valid initial shape + has owner-written `errors_` and primary-written `proceed_`: + +```cpp +auto TeamFailure::checkpoint() noexcept -> bool { +#pragma omp barrier +#pragma omp master + { + proceed_ = std::none_of(errors_.begin(), errors_.end(), [](const auto &error) { + return static_cast(error); + }); + } +#pragma omp barrier + return proceed_; +} +``` + + The next decision cannot be written until every worker reaches the next leading barrier. (Superseded by the second + S4 rework, fix C: one barrier per checkpoint with a generation-stamped first failure; see `Team.h`.) Integrate these boundaries + with handoff needs instead of layering an unrelated failure protocol on top. Add Qt-style declarations/member docs. + Clarify `Options` as a requested budget: the reduced-team guarantee belongs to `for_blocks`, not every consumer. + The sharded team requires exactly T; this documentation change must not add actual-team checks or weaken the generic + helper's existing behavior. +- [x] Test T=1/2/4, empty work, nonprimary failure, simultaneous failures, primary-only work, skipped later mutation and + delayed-worker checkpoint stability. Add a mutation check that removes synchronization/actual work and makes the + corresponding tests fail; use bounded test timeouts rather than treating a hang as success. +- [x] Run the focused R tests with `-R '^sharded_team_'`, qualified sanitizer probes and installed-header compilation. + Keep whole-body callbacks noexcept; every potentially throwing operation belongs inside a protected phase. +- [x] Hand off the helper and barrier/lifetime argument. Do not add MPI or start an experiment in this task. + +## Task S1: Owner-initialized shard state and copy semantics + +**Deliverable:** internal state can be seeded, copied, counted and destroyed without a pool or child propagators. + +**Files:** create `detail/sharded/State.h`, `cpp/tests/sharded_state_tests.cpp`; adapt initialization/copy extraction in +`MonomialPropagator.inl`, `detail/operator/MPOperator.h` and private `PropagatorTestAccess.h` where needed. Do not +switch the public runtime yet. + +**Interfaces:** `ShardState` contains the existing `MPOperator`, `MPGraph` and `MatchedEpochSet` +state, not another `MonomialPropagator`. Use stable `std::unique_ptr` addresses. Define +`Shards = std::vector>>` and a construction helper: + +```cpp +template +auto make_shards(parallel::Options options, Initialize &&initialize) -> Shards; +``` + +The initializer consumes `size_t shard` and returns `std::unique_ptr>`; it contains only +owner-local initialization. Caller-side configuration/router/MPI setup precedes it. New state and source copies +preserve captured T. + +**Outcome:** implemented on 2026-09-29; **owner acceptance pending**. Start: `4cdffac`. Handoff with the +interfaces, seeding/ownership/copy/cleanup arguments, commands, toolchains, memory model and full evidence: +`/home/ubuntu/s1-artifacts/HANDOFF.md` (outside the checkout). + +- `detail/sharded/State.h`: `ShardState` (the existing `MPOperator`, `MPGraph` and `MatchedEpochSet`), `Shards`, + `make_shards`, `seed_shards`, `copy_shards`, and per-shard `total_size`/`operator_memory_usage`/ + `graph_memory_usage`. Caller-side preparation (`validate_initial_operator`, which returns the identity core term, + `paired_basis_bounds`, `packed_inline_width`, `OperatorSeed`) is separate from the owner-local `seed_operator`. The + legacy single-store constructor now seeds through the same `seed_operator`; `MPOperator::initialize_caches` holds + the extracted Heisenberg-sparse/Schrödinger-dense warm-up. A defaulted test-only `NoShardObserver` is called + inside each owner's seed and copy body. Arguments are checked before the team, including `threads < 1` before any + container size; there is no actual-team or cross-rank check. +- `MPOperator()` was `noexcept` although its store initializer allocates. RED: a real injected allocation failure + hit `std::terminate`; removing only `noexcept` made the sweeps pass. +- RED: the registered test failed to compile only on the missing `State.h`. GREEN: 19 flat `sharded_state_*` cases + in fresh `sharded_state_env_t{1,2,4}` launches (16/19/19 cases). They check a field-by-field partition oracle at + (1,T), plus an independent test-side reference, because the oracle now shares `seed_operator`. They also check flat + ownership at P=1/2/4 linear and P=3 splitmix, identity roles, empty owners, cache selection and pending entries, + exact and independent copies with shared layer cores, the captured budget and the real owner guard (preservation). + Failure cases cover primary, nonprimary and concurrent owners. Allocation-failure sweeps fail every site of the + seed, copy and legacy-constructor paths, with no leak and the source left intact. The test-only + `cpp/tests/AllocationProbe.cpp` replaces global new/delete in the unit-test executable and is compiled out under + Clang sanitizers and GCC TSan. +- 11 `State.h`/`MPOperator.h` mutants (lost ownership, caller-side copy, lost copy mapping, wrong flat owner, + identity per shard, global reserve, swallowed errors, leaked siblings, store theft, Heisenberg densification, + restored `noexcept`) are all detected, with no hangs. A materialize-the-basis-per-owner mutant would be + state-identical and is guarded by review only. +- Results: + - GCC/libgomp R: 435/435 and pytest 767 passed, 25 skipped. + - Clang 18/libomp R: 435/435. + - GCC ASan/UBSan: 435/435. + - GCC M with Open MPI 5.0.10: 465/465 + 16/16 and pytest 821 passed, 32 skipped (compatibility only). + - Clang/libomp/Archer TSan, qualified with known-safe/racy probes: silent on the real cases, and a publication-race + mutant is reported. + - The installed `find_package_smoke` consumer compiles `State.h` alone (chain g) through `monoprop::monoprop` for + the R, M and Clang packages. +- Pending: macOS, Linux aarch64, wheels, Nix, minimum-version compiler routes and actual P>1 sharded runs (S5). + Public runtime integration is S4 work. + +- [x] Add fixtures for each basis/picture, empty owners, replicated identity, source mutation after copy, invalid-source + rejection and constructor failure. Verify the union of shard keys matches the existing partition oracle at fixed P/T. + For each owner, the test checks `router.dest(key)` against its flat owner ID. +- [x] Implement first-touch with only the pointer vector allocated on the caller; the protected owner body allocates the + substantial state. The helper's central pattern is: + +```cpp +auto shards = Shards(static_cast(options.threads)); +const auto error = run_team(options, [&](size_t shard, TeamFailure &failure) noexcept { + phase(failure, shard, [&] { shards[shard] = initialize(shard); }); +}); +if (error) { + std::rethrow_exception(error); +} +return shards; +``` + + Caller distributed guards must still abort peers on failed distributed initialization. No requests may be owned by + this helper when it rethrows. Audit allocating `noexcept` constructors, including the current default `MPOperator` + constructor, before claiming catchable allocation-failure coverage. +- [x] Extract seeding rather than copying full child constructors. Preserve hash ownership, streamed Schrödinger paired + basis generation, per-shard reserve and lazy-cache ownership. Reuse existing deep-store-copy and + immutable-core-sharing semantics. Preserve rank-level virtual clone/update hooks; do not recreate + `PartitionChildFactory` privately. +- [x] Run `-R '^sharded_state_'` at supported separate launches; prove stores are distinct, owner allocation occurs in + the owner body, copied budget is unchanged and earlier independent copies survive invalidation of another instance. +- [x] Hand off memory accounting for state, graph sharing and retained scratch. No global operator mirror or benchmark. + +## Task S2: Direct-buffer construction and graph-free propagation + +**Deliverable:** a complete P=1, T-shard construction/propagation engine with testable local phase boundaries. + +**Files:** create `detail/sharded/Construction.h`, `cpp/tests/sharded_construction_tests.cpp`; extract phases from +`detail/evolution/layer_build/Engine.h`; adapt `Resolve.h`, `QueryWire.h`, `Scan.h` and `FusedApply.h` only as needed. +Retain `sparse_query_tests.cpp`, `sparse_resolve_tests.cpp`, `operator_index_tests.cpp`, `majorana_cutoff_tests.cpp`, +`pauli_build_layer_tests.cpp` and fused tests as preservation coverage. + +**Interfaces:** retain `LayerBuildEngine`, `GraphSink` and `ContractSink`, but move blocking transport out of the owner +engine. Existing legacy calls may use an adapter while the old runtime is retained. Introduce these owner-local methods +on `LayerBuildEngine`; `Response` is `typename Sink::Response`: + +```cpp +auto prepare_exchange(bool leaders, + mpi::WindowVec &&queries, + mpi::WindowVec> &&sources, + mpi::WindowVec> &&values, + SelfQueryStage &&self_stage) -> const mpi::WindowVec &; +auto resolve_published(const mpi::WindowVec> &incoming, bool leaders) + -> mpi::WindowVec>; +auto consume_published(const mpi::WindowVec> &responses) -> void; +``` + +`prepare_exchange` performs the existing stable follower filtering/self resolution and returns the sink's outgoing +buffer. `resolve_published` performs checked decode/probe/scatter/publication within one shard. `consume_published` +preserves response order and leader marks. Existing `finish(CosMask&&, CosMask*)` remains the finalization seam. +These methods perform no MPI, barrier or nested worksharing; `SlotWindow` stays a data-layout type, not a communicator. + +**Outcome:** implemented on 2026-09-30; **owner acceptance pending**. Start: `a56daac`. Handoff with the +interfaces, phase sequence, buffer-lifetime table, ordering/numerical/failure arguments, memory accounting, commands, +toolchains and full evidence: `/home/ubuntu/s2-artifacts/HANDOFF.md` (outside the checkout). + +- `detail/sharded/Construction.h`: `build_graph` and `propagate` over P=1, T shards in one `run_team` for the whole + gate loop, with `ConstructionContext`, `GraphCircuit`/`PropagationCircuit`, the test-only + `NoConstructionObserver` and `gather_published`. Argument errors throw before the team. Phase failures are returned + after the join as `ConstructionOutcome{error, mutation_started}`, never rethrown. `mutation_started` is false only + if the pre-mutation frame phase failed; there is no rollback. +- Per gate: traversal and leader prepare, then leader resolution, then leader consume and follower prepare, then + follower resolution, then follower consume and finish/apply. That is five checkpoints for an exchanging gate; an + identity gate skips the exchange, and T=1 keeps same-shard resolution only. Destinations read other shards' + published blocks as spans in ascending flat-source order. Opaque cutoffs and basis-change closures traverse every + shard on the primary, in an exclusive phase that every worker enters. Within-shard kernels get serial `Options{}`. +- Engine: `prepare_exchange`, `resolve_published`, `consume_published`, the extracted `scan_gate` and + `stamp_layer_metadata`. The communicator left the engine: the legacy transport is the `run_exchange(comm, plan, …)` + adapter over the same phases, with unchanged request lifetimes and guards. The resolver, the response path and the + sink callbacks take read-only views (`mpi::views_of`). `MPOperator::current_picture` and + `extend_from_current_picture` were extracted, and the propagator delegates to them. +- `QueryWire::Writer::put`/`flush` were `noexcept` but call `push_back`. RED: an injected allocation failure while + encoding a query hit `std::terminate`; removing only `noexcept` made the sweeps pass. This also fixes the legacy + path. +- RED: the registered test failed to compile only on the missing `Construction.h`. GREEN: 17 flat + `sharded_construction_*` cases in fresh `sharded_construction_env_t{1,2,4}` launches (15/17/17 cases). At (1,T), + every shard's rows, IDs, coefficients, caches and graph layers are bit-identical to the legacy partition child, and + so are the query/source/value/answer streams of every pass. Because both paths share the phases, the cases also + check an independent insertion-order reference, an independent coefficient-map propagator, the frozen + `random_exact` energy and the cross-T global retained maps. They also cover participation, empty and identity + inputs, the span seam (nonzero windows, empty senders, 255/256/257 queries, narrow and wide positions, the + malformed `QueryWire<128>` fixture), self streams beyond 4096, throws in every phase, kernel and publication + failures, and allocation sweeps. +- 11 mutants were all detected, with no hangs. The shared-engine order and follower-filter mutants pass the legacy + comparison and are caught only by the independent references. A race mutant is reported by TSan/Archer. +- Results: + - GCC/libgomp R: 455/455, the fixed-T launches 10/10, pytest 767 passed, 25 skipped. + - Clang 18/libomp R: 455/455. + - GCC ASan/UBSan: 455/455. + - GCC M with Open MPI 5.0.10: 485/485 + 16/16 (every `mpi_failure_*` scenario), pytest 821 passed, 32 skipped + (legacy compatibility only). + - Clang/libomp/Archer TSan, qualified with known-safe/racy probes: silent on `sharded_*` at T=1/2/4 and on + `openmp_*`. + - The installed `find_package_smoke` consumer compiles `Construction.h` alone (chain h) through + `monoprop::monoprop` for the R, M and Clang packages. +- Not in S2: coefficient-informed `build_graph`, replay, energy, gradients and paring (S3), rank-level integration + (S4) and P>1 (S5). Allocation injection is unavailable under Clang sanitizers and GCC TSan. Pending: macOS, Linux + aarch64, wheels, Nix and minimum-version compiler routes. + +- [x] Add differential fixtures with same-shard, other-local-shard and empty destinations; mixed hits/misses; all-hit, + all-miss, fresh growth/reindex; Plain/Fused forms; both sinks; and fixed-geometry row/graph/query/response equality. + A genuine RED must show missing sharded behavior/owner participation, not merely rename passing serial assertions. +- [x] Split `run_exchange` at the above boundaries and adapt incoming containers to immutable spans without duplicating + whole local query buffers. `QueryWire::WireView` already exists; preserve its checks and generalize the remaining + owning-container callers. Use `window.indices()`/`window.slot(wi)`/`at_slot(flat)` correctly. +- [x] Implement leader then follower orchestration with published views. The pass order is executable structure: + +```text +owner traversal -> publish outgoing -> resolve destination-owned queries -> publish responses +-> consume responses -> complete leader marks -> repeat for followers +-> insert deferred self misses -> finalize graph/fused updates +``` + + Each arrow is a lifetime/synchronization boundary, not a new local collective API. Fixed source order must match + `ShmComm`'s historical order. Pass serial `parallel::Options{}` to reused within-shard kernels. +- [x] Keep same-shard position stages unencoded, bounded self lookup and deferred hashes. Preserve scatter before + insertion, graph versus fused response types, Schrödinger scoring, cosine tails/pivots and paired cutoff exceptions. + Retain the malformed `QueryWire<128>` Plain fixture `VecZ{size_t{0x8037e}}` and checked overflow tests. +- [x] Test throw sites at traversal, decode, publication and finalization. Observers record actual shard owners in those + bodies; no empty-region proxy. Test opaque callback/basis-change supported paths without assuming thread safety. +- [x] Run `-R '^(sharded_construction_|sparse_|operator_index_|fused_|pauli_)'` plus relevant cutoff tests, verifying + the selection is nonempty and broad enough. Hand off exact fixed-geometry/global-map evidence and buffer live ranges. + +## Task S3: Snapshot-safe replay, energy, gradients and paring + +**Deliverable:** complete P=1 sharded evaluation, including reverse derivatives and retained/pared functionals, without +cross-shard live-coefficient reads or per-kernel OpenMP regions. + +**Files:** create `detail/sharded/Evaluation.h`, `Evaluation.cpp`, `cpp/tests/sharded_evaluation_tests.cpp`; extract +local functions from `Evolution.cpp`, `MPFunctions.cpp`, `detail/evolution/CosineRecompute.h` and callback plumbing. +Integrate graph views from `detail/pare/PareGraph.cpp`; preserve `combined_recompute_equivalence.cpp`, +`pare_graph_tests.cpp`, `evolution_detail_tests.cpp` and `tests/test_infinite_cutoff.py`. + +**Interfaces:** use one `EvalRequest` and `detail::CosCallbacks` per shard with this rank-level detail interface: + +```cpp +auto ev_sharded(std::span requests, + std::span callbacks, + parallel::Options options, mpi::Comm comm) -> double; +auto ev_and_grad_sharded(std::span requests, + std::span callbacks, + parallel::Options options, mpi::Comm comm) -> std::pair; +``` + +These own one team for the entire call. Initially the physical communicator has size one; S5 adds remote legs without +changing callers. Requests/callbacks are prebuilt views with T entries and no shard-count setter. Keep existing exported +single-store functions as low-level compatibility/reference paths where appropriate; new detail functions need exports +if public template instantiation calls them across the shared-library boundary. + +**Outcome:** implemented on 2026-09-30; **owner acceptance pending**. Start: `0c2503b`. Handoff with the +interfaces, phase and buffer-lifetime tables, ordering/numerical/callback/failure arguments, memory accounting, +commands, toolchains and full evidence: `/home/ubuntu/s3-artifacts/HANDOFF.md` (outside the checkout). + +- `detail/sharded/Evaluation.h`/`.cpp`: `ev_sharded` and `ev_and_grad_sharded` (one team for forward and reverse; + the communicator is touched only on the caller, and a phase failure reaches `mpi::operation_failed` before the + reduction), the S4 seam `evaluate_shards` (per-shard terms and gradients plus the error, returned after the join), + `combine_contributions`/`combine_gradients` (ascending-shard fold), `replay_shards` (partial contraction), + exported owner-local forward phases with `replay_forward_in_team`, and `prepare_retained`/`RetainedEvaluation` + for retained and pared functionals. `Construction.h` gains coefficient-informed `build_graph_informed` (seed + replay and per-gate new-layer replay in the same team). `detail::pare_graph_owner` takes the flat owner; + `detail::make_cos_callbacks` and `full_cos_mask` moved to `CosineRecompute.h`; `CosCallbacks::owner_parallel` + marks library-built callbacks (anything else runs on the primary in an exclusive phase). +- Each replay step publishes pre-cosine endpoint snapshots into a double buffer: one checkpoint per step, partners + never read live coefficients. The kernels were extracted into the library-internal + `detail/evolution/LayerReplay.h`, shared with the rewired legacy `Evolution.cpp`/`MPFunctions.cpp`. +- Contraction fidelity: the inlined legacy derivative fuses `cos_acc(...)*sec` into `A - ep.cos_terms`. A first + out-of-line derivative changed legacy gradient bits versus HEAD in record-bearing cases; the final form keeps the + legacy code inline and gives the sharded path the raw accumulation. A cross-build public-API probe shows legacy + results bit-identical to HEAD (GCC, Clang, MPI) and the sharded results equal legacy bitwise. +- RED: the registered test failed to compile only on the missing `Evaluation.h`. GREEN: 20 flat + `sharded_evaluation_*` cases in fresh `sharded_evaluation_env_t{1,2,4}` launches: bitwise equality with the legacy + partitions (per-shard evolved operators and terms, energies, every gradient component, pared functionals, partial + contraction in both pictures, informed builds); independent map-propagator, frozen-energy, closed-form and + finite-difference references including the `test_deep_circuit_gradient.py` cases; layouts, identity, records, + lifetimes, participation, opaque callbacks and failures including allocation sweeps. +- 14 mutants (missing snapshots, wrong publication buffer, self pairing, records, restore, contraction, fold order, + identity per shard, empty pruned mask, opaque on owners, informed replay, swallowed error) are all detected, with + no hangs; a single-buffer race mutant is reported by TSan/Archer. +- Results: + - GCC/libgomp R: 478/478 and pytest 767 passed, 25 skipped. + - Clang 18/libomp R: 478/478. + - GCC ASan/UBSan: 478/478, no reports. + - GCC M with Open MPI 5.0.10: 508/508 + 16/16 (every `mpi_failure_*` scenario) and pytest 821 passed, 32 skipped + (legacy compatibility only). + - Clang/libomp/Archer TSan, qualified with known-safe/racy probes: silent on `sharded_*` at T=1/2/4 and on + `openmp_*`. + - The installed `find_package_smoke` consumer (chain i) calls both exported evaluators through + `monoprop::monoprop` for the R, M and Clang packages. +- Also reformatted three S2 files that were not clang-format clean at `0c2503b` (whitespace only). +- Pending: macOS, Linux aarch64, wheels, Nix, minimum-version compiler routes, the Python ASan leg and P>1 sharded + runs (S5). Rank-level wiring of these seams into `MonomialPropagator` is S4 work. + +- [x] Add exact small shard-pair fixtures where one owner overwrites a coefficient before another consumes its old + endpoint; only a published snapshot can pass. Cover empty owners, identity once, sparse/duplicate dot/scatter and + repeated parameter indices, then finite-difference/reference gradients and pared/unpared functionals. +- [x] Extract pure owner-local operations from the existing replay/derivative begin/apply/finish functions. Publish only + needed endpoint values, not a full new operator mirror. Preserve this ordering: + +```text +snapshot/pack all required endpoints -> publish -> independent self/local work +-> consume peer snapshots -> cosine/record/restore phases in their original order +-> local dot/gradient contributions -> ascending-shard fold -> scalar/vector result +``` + + Keep each self pair together; no adjacent-pair reinterpretation. Forward and derivative scratch must stay valid + across phases. Reuse legitimate mathematical snapshots; account for retained TLS versus per-instance scratch. +- [x] Implement ordered local combination. For every result position, the reference fold is: + +```cpp +auto total = 0.0; +for (size_t shard = 0; shard < contributions.size(); ++shard) { + total += contributions[shard]; +} +``` + + Add core/identity only once using existing semantics. Do not parallelize this fold with a new floating-point + association, change `std::fma` policy or remove `[[gnu::noinline]] apply_fused_record_range` without neutrality proof. +- [x] Verify record thresholds, vanishing cosine, restoration after derivative calls, signs in both pictures/bases, + basis changes, incremental graph axes, repeated evaluation, copies, updates and functional owner lifetimes. +- [x] Run `-R '^(sharded_evaluation_|pare_|combined_|evolution_)'`, the named Python numerical suites and qualified + sanitizers. Compare retained operators as well as energy/gradient; performance work is not part of this task. + +## Task S4: Integrated MPI-off prototype and first architecture checkpoint + +**Deliverable:** public mathematical operations use the new P=1 sharded engine in an isolated development build, with +credible end-to-end numerical/memory/runtime evidence or an explicit stop decision. + +**Files:** `MonomialPropagator.h`, `MonomialPropagator.inl`, `detail/sharded/{State,Construction,Evaluation}.h`, +`src/monoprop/bindings/binder.h`, `cpp/monoprop/CMakeLists.txt`, `cpp/tests/PropagatorTestAccess.h`; +create `tests/test_sharded_openmp.py`; adapt `tools/benchmark-rank-local-openmp.py`, +`tests/test_rank_local_openmp_benchmark.py` and +`packages/monoprop-bench-tools/src/monoprop_bench_tools/{models,preflight,bmf}.py` for candidate construction. + +**Interfaces:** connect the S1 state, S2 engine and S3 evaluation to existing root build/propagate/functional/update +entry points. A temporary compile-time `monoprop_SHARDED_OPENMP_PROTOTYPE` option selects the integrated candidate; +normal builds retain the legacy path until cutover. This is development instrumentation, not a public runtime setting +or permanent backend selector. Old constructor controls may remain in the legacy build during coexistence, but cannot +select a different shard count in the candidate. Explicit nondefault legacy controls must not silently be honored there. + +**Outcome:** implemented and **accepted by the owner on 2026-10-01**. Start: `f347e6f`. Handoff with the +integration map, ownership/routing/failure arguments, commands, toolchains, memory accounting, migration ledger and +full evidence: `/home/ubuntu/s4-artifacts/HANDOFF.md`; experiment proposal (not run): +`/home/ubuntu/s4-artifacts/EXPERIMENT-PROPOSAL.md` (both outside the checkout). + +- `monoprop_SHARDED_OPENMP_PROTOTYPE` (default OFF) is a PUBLIC definition of `monoprop-objs` and an INTERFACE + definition of the exported target, so bindings, tests and installed consumers inherit it; `monopropConfig.cmake` + reports it. ON builds define the root in `detail/monomial_propagator/ShardedPropagator.inl`; the class then has no + `mp_op_`, `graph_`, matched marks or partition group, so leftover root-storage use cannot compile. The root owns + configuration, the captured budget (T threads = T shards, copies keep it), a (1, T) router prepared once on the caller + (`make_router`, never derived from the communicator), the replicated identity, the epoch, validity and T shards. +- Every public operation goes through S1-S3: seeding/copying on owners, `build_graph`/`build_graph_informed` (seed + angles as `contract_partially` computes them)/`propagate`, `prepare_retained` + `evaluate_shards` for direct and + retained evaluation (two operation regions for direct evaluation, constant in depth), `replay_shards` for partial + contraction (blocks in shard order), per-shard export in `evolved_operator_terms()`, owner phases for + initial-operator updates (routed by the same router), remapping and empty-parameter pictures. Settings stay + root-only. Outcome mutation flags are transferred before rethrow under `run_operation_`; failures after a team + invalidate the root and go to `mpi::operation_failed`. +- Rejected, before any mutation or collective: a communicator of more than one rank (no legacy fallback), nondefault + `partitions` or `child_factory`, any `monoprop_PARTITIONS`. Raw `mp_op()`/`indexing()`/`graph()`/`graph_data()` + return the sole shard at T = 1 and raise `MultiPartitionUnsupported` with a launch-time T = 1 remedy otherwise; this + stays the design (S6 renames the exception). `_core.__runtime_identity__` plus an embedded literal identify the + runtime a binary contains. +- RED: with only the option wired, the ON build still ran legacy (explicit partitions and `monoprop_PARTITIONS` + accepted, legacy accessor remedy): 16 Python failures and failing `sharded_root_env_*`. GREEN: 20 `sharded_root_*` + cases in `sharded_root_env_t{1,2,4}` (private shard inspection, actual-worker observation and fault injection via + `PropagatorTestAccess` and the test-only `detail/sharded/RootObserver.h`), and `tests/test_sharded_openmp.py` in + fresh fixed-T processes: bitwise equality with a separate legacy build at `monoprop_PARTITIONS=T` for 8 fixtures at + T = 1/2/4 (energies, gradients, functionals, full decoded maps, contraction blocks, every aggregate and memory field) + and cross-T full maps, energies and gradients within tolerance. +- 10 S4 mutants (wrong-shard export, omitted owner update, missing invalidation, repeated identity, lost epoch guard, + wrong-shard picture, functional without invalidation, remap on shard 0, caller-side seeding, legacy selection) are + all detected; a shared-shard race mutant is reported by TSan/Archer. +- Results (final diff `86ff189b…`): + - GCC/libgomp: legacy 478/478, pytest 806 passed; prototype 372/372, pytest 856 passed at T = 1 and the full + suite at T = 2 and 4. + - GCC + Open MPI 5.0.10: prototype 400/400 plus the two-rank rejection; legacy 508/508 + 16/16 (all + `mpi_failure_*`), pytest at one and two ranks. + - Clang 18/libomp: 372/372 and 478/478. GCC ASan/UBSan (prototype): 372/372, no reports. Clang/libomp/Archer TSan, + qualified with known-safe/racy probes: silent on `sharded_root_*` at T = 1/2/4 and on `openmp_*`. + - Installed `find_package_smoke` for five packages at T = 1/2/4; a consumer expecting the other runtime fails to + compile. +- Migration ledger: `cpp/tests/README.md`. Suites needing the legacy root (partition facades, the one-store + prototype, S1-S3 live-oracle seam suites, multi-rank legacy and failure scenarios) stay in OFF builds; other + propagator suites run as T = 1 raw-layout tests in ON builds. +- Measurement tooling: the driver, preflight and `benches/conftest.py` bind each arm to the runtime read from the + measured `_core` file (candidate: positive sharded identity; baseline: legacy, or none for the frozen records). + Driver `2207529f…` -> `0813b7db…`; campaign and workloads unchanged; the modified compare path re-joins all 250 + frozen baseline cells. The shared builders already used the public constructors; the `partitions=1` was Task 6's + external adapter, now obsolete. A both-arm overlay is prepared, not applied to any preserved environment. +- Architecture checkpoint (owner-approved proposal, run 2026-10-01; report `/home/ubuntu/s4-artifacts/experiment/`): + - 32 frozen reference cells, 5 alternating fresh-process pairs each, through the tool overlay on both arms; + 134.7 of 170 measurement minutes; 704 processes, no timeouts or memory kills. Preflight confirmed 96 physical + cores, one NUMA domain and no SMT; frozen inputs, baseline binaries and the Task 1 tree were unchanged + (snapshots); both negative controls exited 2. + - Numerics: all 32 candidate validations equal the formal baseline validations; every memory window exact. + - off-1x96 (MPI-off, 96 threads): runtime ratios 0.90–1.44 (6/16 <= 1.00), operation peaks 0.95–1.05, + construction peaks 0.96–1.005; 3/16 cells meet all five ratios. mpi-1x1: runtime 0.93–1.03, memory <= 1.004; + 11/16 meet all five. Task 6's one-store prototype had measured 2.1–61.6 on the same full-machine cells. +- Owner decision: **rework**. Profiling (`perf` and AMD IBS on 5 cells; `/home/ubuntu/s4-artifacts/profile/`) + showed that barrier waiting was not the gap: the extra work came from a per-partner binary search for slot sizes + in replay, and from construction's handoff chasing every source's published buffer. +- First rework (`/home/ubuntu/s4-artifacts/rework/`): a per-layer slot-size table in `Evaluation.cpp`, and + row-written, column-read handoff views in `Construction.h`. Outputs are bitwise identical before and after + (8 fixtures, T = 1/2/4, both builds); the correctness and sanitizer matrix passes. Profiling-harness ratios + (not checkpoint samples): energy Heisenberg 1.36 -> 1.12, gradient Heisenberg 1.46 -> 1.22, propagate Pauli + 1.23 -> 1.02, propagate Hubbard 1.20 -> 1.13, build_graph control 0.89 -> 0.91. +- Checkpoint re-run (owner instruction, 2026-10-01; `/home/ubuntu/s4-artifacts/checkpoint2/report.md`): same + protocol, 134.5 of 170 minutes, 704 processes `ok`, all 32 numerics equal, preserved trees unchanged. off-1x96: + runtime 0.89–1.16 (8/16 <= 1.00), operation peaks 0.95–1.05 (7/16), construction peaks 0.96–1.005 (11/16); + 5/16 meet all five. mpi-1x1: 11/16 meet all five. The checkpoint still fails. +- Profiling (`/home/ubuntu/s4-artifacts/profile2/FINDINGS.md`): partner reads in replay finishes are latency-exposed + (the legacy runtime overlaps the same transfers in a bulk copy); every checkpoint costs two 96-thread barriers + (6.5 µs); the Pauli gradient's operation peak is allocator retention of geometric record growth, with live bytes + equal to the baseline's. +- Owner decision: **rework** with C (single-barrier checkpoint), D (overlapped partner reads) and E (gradient scratch + liveness). Second rework (`/home/ubuntu/s4-artifacts/rework2/REWORK2.md`): + - C: `TeamFailure::checkpoint` is one barrier; a recorded failure lowers a shared first-failed generation, and the + decision for checkpoint N is `first_failed > N`. + - D: finishes copy runs of consecutive partner blocks into owner-owned staging (`PartnerStaging`, reached by plain + reference: a thread-local there cost a TLS lookup per use), then apply them in the unchanged order. + - E: an exact per-layer cosine-set count (`CosCallbacks::count`) lets the gradient's frame phase reserve its + records once (`replay::reserve_records`). + - All 48 probe documents are bitwise identical to the first rework's; the correctness matrix, mutants and sanitizer + legs pass. Profiling-harness ratios (not checkpoint samples), off-1x96: evaluation 0.61–1.00, propagate and + build_graph 0.77–0.93; Pauli gradient whole-process peak 3.15 -> 2.05 GiB against the baseline. Hubbard cells keep + an allocator-level +0.3–1% whole-process peak, with live bytes equal. +- Checkpoint re-run on the second rework (owner-approved, 2026-10-02; `/home/ubuntu/s4-artifacts/checkpoint3/report.md`): + 134.0 of 170 minutes, 704 processes `ok`, all 32 numerics equal, preserved trees unchanged. off-1x96: runtime + 0.33–1.06 (15/16 <= 1.00; energy Heisenberg 1.06 on noisy first calls), operation peaks 0.65–1.007 (8/16), construction + peaks 0.80–1.007 (12/16); 7/16 meet all five. mpi-1x1: 9/16 meet all five (runtime 0.67–1.026). The checkpoint still + fails, narrowly and mostly on memory: evaluation operation peaks are +6–12 MiB over the baseline (D's per-owner staging + adds about 4 MiB to S4's pre-existing allocator-level excess). +- Owner decision (2026-10-02): **proceed**. The remaining deviations "appear entirely reasonable" and are accepted + for now; they are re-evaluated at a later stage. Accepted deviations (candidate median / baseline median - 1, from + `checkpoint3/comparison-vs-previous-runs.txt`): + - off-1x96 runtime: energy Heisenberg +5.8% (single cold calls of about 16 ms; the baseline's own samples span + 15.3–27.4 ms; the warm harness gave -5%). The other 15 cells are faster (0.33–0.96). + - off-1x96 operation peak: energy Hubbard +0.74%, energy Schrödinger +0.72%, build_graph Hubbard +0.55%, energy + Heisenberg +0.54%, gradient Hubbard +0.46%, gradient Heisenberg +0.42%, propagate Hubbard +0.35%, gradient + Schrödinger +0.31% (+6–12 MiB on 0.8–1.7 GiB peaks). + - off-1x96 construction peak: energy Hubbard +0.65%, build_graph Hubbard +0.46%, gradient Hubbard +0.33%, + propagate Schrödinger +0.07%. + - mpi-1x1 runtime: propagate Hubbard +2.6% (present since the first checkpoint), propagate Pauli +0.9%, gradient + Hubbard +0.6%, energy Schrödinger +0.5%, build_graph Heisenberg +0.2%, energy Pauli +0.1%; mpi-1x1 construction + peak: energy Hubbard +0.2%. +- What is known about them, for the later re-evaluation: + - The memory excess has two parts. About 4 MiB is the second rework's per-owner partner staging (one run of at + least 32 KiB plus positions and blocks, times 96 owners). The rest is a few-MiB excess S4 already had; on Hubbard, + an `LD_PRELOAD` allocation trace (`profile2/alloc/`) found live tracked bytes equal to the baseline's (0.670 + against 0.669 GiB), so it is allocator-level. + - Candidate remedies, none started: a smaller staging run or staging into an existing owner buffer (keep only if + the harness speed holds); attributing the pre-existing excess with the allocation tracer at a small threshold + inside the operation window; profiling energy Heisenberg's cold first call (thread-local scratch warm-up). + - These checkpoint results do not change S7's 250-cell campaign or its five per-cell gates; the deviations are + re-assessed there or earlier. +- Lessons recorded for later tasks: + - In the shared library, a `thread_local` object on a hot path costs a `__tls_get_addr` call wherever GCC + rematerializes its address, including inside loops; owner scratch is reached through the frame pointer instead. + - glibc keeps freed memory from geometric vector growth resident, so reserve exact sizes on peak paths; whole- + process RSS can differ between Release and RelWithDebInfo builds through allocation timing alone. + - Timing-shaped tests did not catch a non-generation (sticky) checkpoint flag; the forced-interleaving test does. + Small fixtures never reach multi-run staging; the test-only `EvaluationObserver::staging_run()` forces it. +- Next step: S5 under its own authorization, starting from this proceed decision. +- Pending: macOS, Linux aarch64, wheels, Nix, minimum-version compiler routes, the Python sanitizer legs and P>1 + candidate runs (S5). + +- [x] Add separate-process T=1/2/4 Python cases through the real public API. One concrete core fixture is: + +```python +from pathlib import Path + +import pytest + +from monoprop import MajoranaPropagator +from tests.cases import load_problem + + +def test_sharded_propagation_matches_graph() -> None: + problem = load_problem(Path(__file__).parent / "data" / "random_exact.msgpack") + circuit = problem.monomial_circuit.to_circuit() + graph_sim, direct_sim = [ + MajoranaPropagator( + problem.operator, + problem.monomial_circuit.initial_state, + cutoff=2 * problem.n_modes, + ) + for _ in range(2) + ] + graph_sim.build_graph(circuit) + direct_sim.propagate(circuit) + assert graph_sim.expectation_value(circuit) == pytest.approx( + direct_sim.expectation_value(), rel=0, abs=1e-12 + ) +``` + + This is preservation coverage; sharded ownership/participation needs its own genuine RED. Also compare decoded full + operators, unrounded energies and every gradient component. Across T, use subprocess output maps rather than changing + configuration mid-process. +- [x] Integrate rank-level state and validity, count/memory aggregation and direct shard iteration in + `evolved_operator_terms()`. Pair each shard index with its own evolved vector; do not use root `indexing()` at T>1. + Include retained zeros, identity de-duplication and count/export failure tests before timing any result. +- [x] Record a migration ledger for all old facade/one-store tests: kept at T=1, rewritten shard-aware, or retired with + equivalent coverage. Prove actual participation inside construction, publication, replay and derivative phases. + Audit trial selection from binary/build provenance, not `--runtime-shape openmp` or environment labels alone. +- [x] Run integrated R correctness, installed-consumer smoke and M-with-P=1 correctness. Public P>1 candidate support is + not claimed before S5. Update the candidate builder to use the new root constructor instead of `partitions=1`; + retain the legacy partition builder for the baseline. Keep `observe`, `validate`, `compare` and the + allocation-before-import fix. Record/synchronize any approved tool overlay without rebuilding baseline binaries or + rewriting formal records. If an adapter/schema change cannot consume the frozen evidence honestly, stop for review. +- [x] **Stop for experiment approval.** Propose matched baseline/candidate single-thread controls and actual MPI-off + full-machine reference cells spanning build_graph, propagate, energy and gradients, both bases/pictures. Use frozen + builders/parameters and operation/construction memory windows. Specify repetitions, task/per-trial/RAM caps and + first-touch/placement evidence before running; do not reuse Task 6's allowance. +- [x] If authorized, run through the existing driver against preserved partitions, not just candidate T=1. Report every + runtime and memory cell plus full numerical validation. Single-kernel speedup is not the checkpoint. Archive results + outside the checkout; retain natural first-touch effects as architectural evidence. +- [x] Obtain the owner's written proceed/rework/stop decision before S5. Failed parity needs a bounded proposed remedy, + not an automatic move to runtime deletion. The later frozen campaign is still mandatory after an early pass. + (2026-10-01: **rework** decided; first rework done. Re-run failed; 2026-10-02: second **rework** (C, D, E) done. + Its re-run (2026-10-02) left small deviations; the owner accepted them and decided **proceed**, to be re-evaluated + at a later stage.) + +## Task S5: Physical-MPI exchange and the MPI+OpenMP checkpoint + +**Deliverable:** supported P×T candidate with primary-only MPI, owner-parallel payload work and failure-safe requests. + +**Files:** create `detail/sharded/Exchange.h`, `Exchange.cpp`, `cpp/tests/sharded_exchange_tests.cpp`; extend S2/S3 +orchestration; adapt `detail/mpi/{Pairwise,Routing,MPIUtils,OperationFailure}.h`, `MPICompat.cpp`, +`cpp/tests/mpi_failure_driver.cpp`, `run_mpi_failure_scenario.cmake`, `mpi_fresh_insert_equivalence.cpp` and +`mpi_distributed_layer_equivalence.cpp`. + +**Interfaces:** a move-only physical exchange owns real-MPI request objects and stable staging buffers outside the team. +Expose physical post/wait operations to the primary and disjoint payload views to owners, not fake collective methods +on a shard. Reuse `begin_flat_exchange` / `wait_flat_exchange` and existing pending-request semantics when extracting +this code. The S2 owner-phase and S3 rank-evaluation signatures remain unchanged. + +**Outcome:** implemented and **accepted by the owner as done on 2026-10-02**. Start: `dc2e102`. Interface +plan, handoff, evidence and the measurement proposal (not run): `/home/ubuntu/s5-artifacts/` (`S5-PLAN.md`, `HANDOFF.md`, +`MEASUREMENT-PROPOSAL.md`), outside the checkout. + +- `detail/sharded/Exchange.{h,cpp}`: `PhysicalExchange`, one reusable physical round between this process's T + owners and those of a peer set. + - It is move-only and heap-stable. The destructor and move assignment drain. + - Owners write row-owned count rows and pack disjoint send slices; destinations read received blocks as const + views, with no scatter copy. + - Only the primary lays a round out and posts or completes it. Its MPI calls throw `std::logic_error` off + OpenMP thread 0 or off MPI's initializing thread. + - Wire layout as `HybridComm`: per peer, destination-shard major and source-shard minor. Counts, totals and + displacements are checked ints, refused before posting. + - Transports: pairwise over non-empty legs, or one `MPI_Ialltoallv`. The count round is pairwise. + - `PhysicalWorld::of()` reads rank, size and the agreed replay transport on the caller. +- Construction: a gate's window reaches another rank exactly when its peer plan does (the linear peer, or every + rank under splitmix), so every rank branches alike. + - Each such pass adds Q1-Q3 (lay out, count round, pack, post and wait) and R1-R3 (answers, laid out from rows + the resolvers write). + - Answer counts are query counts, not stream words, so they are known only after resolution; the first two-rank + run found this. + - Destinations resolve local and received blocks together, in ascending flat-slot order. +- Replay (energy, gradient, contraction, informed P5/P6, seed replay): every step at P > 1 adds a layout phase, a + pack phase and a post phase in which the callback part (record, cosine or reverse accumulation) runs while the + transfer is in flight. `EndpointBoard` reads remote partners from the round. P = 1 keeps the S4 phase sequence; + 48/48 probe documents are bitwise identical to S4's final record. +- Failure: after a failed join with live requests, a seam hands the error to `mpi::operation_failed()` before any + round is destroyed. +- Root: P ranks, router (P, T), flat owner `rank * T + t`, and `world_`. + - The multi-rank rejection is removed. Linear routing rejects a non-power-of-two P on every rank. + - Updates apply each rank's own share only. + - A thread-level failure goes through `operation_failed()`. + - `routing::Config` loses its partition field; the legacy facade keeps `PartitionCountMismatch`. +- Ledger rulings (`.superpowers/sdd/.../progress.md`): + - reuse of `begin_flat_exchange`'s semantics, not its code; + - trailing defaulted `PhysicalWorld` parameters on `evaluate_shards()` and `replay_shards()`, and + `ConstructionContext.world`; + - the answer round's legacy-equal count exposure; + - the local sanitizer settings for Open MPI 5.0.10. +- Results (frozen source `final-s5-source.diff` `3a3d17ec…`; the final matrix is in the handoff): + - Bitwise against the legacy runtime at P ranks x `partitions = T`: + - the seams, per shard, at P = 2/4 x T = 1/2/4 and P = 3 splitmix (`sharded_seams_mpi*`); + - the public API, per rank, at P = 2 x T = 1/2/4 for 8 fixtures (`test_multirank_matches_*`, 24/24). + - Within tolerance, against one process: + - the root at P = 2/4 x T = 1/2/4 and P = 3 splitmix (`sharded_root_mpi*`); + - the gathered public-API maps, energies and gradients over (1, 4), (2, 2), (4, 1) and splitmix (3, 2) against + (1, 6). + - Exchange rounds at P = 2/3 x T = 1/2/4. + - Failure scenarios (`mpi_failure_sharded_*`, 9/9) each end in the library's abort or fail-fast within 0.2-1.6 s. + - 11 mutants detected (one equivalent mutant replaced). + - ASan/UBSan (GCC + Open MPI) and TSan with Archer (Clang 18 + libomp + Open MPI, qualified under `mpiexec`): + 0 reports over the multi-rank launches. + - RED evidence: HEAD binaries plus only the new tests fail at P > 1 with "supports one MPI rank". +- MPI+OpenMP checkpoint, tier A (owner-approved, 2026-10-02; `/home/ubuntu/s5-artifacts/checkpoint/report.md`): + - 44 frozen reference cells, 5 alternating fresh-process pairs each, through the unchanged S4 overlay and runner + launch; 161.9 of 210 minutes; 968 processes `ok`. + - All 44 numerics equal the formal validations, and the preserved trees are unchanged. + - mpi-2x48: 13/16 meet all five. Runtime 0.49–0.96 apart from three evaluation cells: gradient Schrödinger 1.37, + energy Pauli 1.21, energy Schrödinger 1.20 (within the baseline's spread). Every memory peak is at or below the + baseline. + - mpi-3x32-splitmix, mpi-4x24-linear and mpi-4x24-splitmix: 12/12 meet all five (runtime 0.84–0.96). + - mpi-1x1: 8/16, with S4-sized deviations up to 1.031 (propagate Hubbard). + - Tier B and the MPI-only controls were not run. +- Owner decision (2026-10-02): **S5 done**. The three mpi-2x48 runtime misses and the mpi-1x1 deviations are recorded + here and re-evaluated with the S4 deviations at a later stage; S7's 250-cell campaign and its five per-cell gates are + unchanged. +- Potential optimizations flagged by S5 (none started; each needs its own authorization and must keep fixed-P/T + results bitwise): + - The replay's remote step costs four checkpoints instead of one (layout, pack, post/overlap, finish). Fold the pack + into the publishing phase (owners can pack once the layout exists) and lay out step p + 1 during step p's post + phase, bringing a remote step toward two checkpoints. Primary candidates: the three missed cells (gradient + Schrödinger 1.37, energy Pauli 1.21, energy Schrödinger 1.20 at 2 x 48). + - The primary lays every round out serially, O(P x T^2) per round. A layer's partner layout is static, so the replay + layout can be cached per layer for an evaluation (bounded memory: offsets only), or computed owner-parallel (each + owner its own destination column, then a prefix over peers). + - Construction adds up to six checkpoints per remote pass (Q1-Q3, R1-R3). Each owner could compute its own send + offsets from per-peer totals published in P1, merging Q1 into Q2; the answer layout could be planned in the same + primary phase as the query post. + - Profile gradient Schrödinger at 2 x 48 first to confirm where the extra ~18 ms per call goes (checkpoints, + layout, or MPI latency per layer). + - Robustness, not speed: the answer round trusts one answer per query (as the legacy runtime does); an optional + count round per remote pass would turn such a bug from a hang into an abort. + - Minor: each worker rebuilds its gate's peer vector per gate. +- Pending: + - the whole-suite multi-rank variants and the remaining suites at P > 1 in prototype builds (S6); + - macOS, Linux aarch64, wheels, Nix and minimum-version compilers; + - multi-node. + +- [x] Add two-process tests with T=1/2/4 for mixed local/remote destinations and canonical sender ordering. Check that + local sender blocks do not precede lower-numbered remote senders. Verify response source/query alignment, empty + messages, nonzero slot windows and exact pre-update endpoint values. +- [x] Define one staging layout with checked counts and prefix-sum offsets for `(source shard, destination shard)` + segments per physical peer. Workers write disjoint payload slices; the primary exchanges metadata/posts/waits and + publishes received extents; destination owners decode and mutate. Never serialize all payload processing on shard 0. +- [x] Preserve construction's linear peer window and replay's actual multi-peer summaries. Warm/use communicator-agreed + `routes_pairwise` on the caller/primary, including MPI attribute accesses. Keep mode/seed agreement and remove the + partition-count agreement field rather than replacing it with T agreement. No geometry-repair fallback. +- [x] Implement request lifetime explicitly. The owner of `pending` and every referenced buffer surrounds the team: + +```text +allocate operation frame and request owners +run_team: protected owner/primary phases, post -> independent work -> wait -> dependent work +inspect returned error while request owners are still alive +if error: invalidate after mutation; multi-rank abort, otherwise rethrow +only then permit normal request/buffer destruction +``` + + A primary post may fail after some requests became live; those requests still belong to the outer frame. Do not + return through a draining local destructor before reaching distributed failure handling. +- [x] Adapt failure scenarios for worker throw, before-exchange throw, active-ticket throw, malformed receive and + wrong-thread entry. Supervised two-process runs must finish in failure before the 30-second timeout; timeout fails the + test. Single-process rethrow/invalidation and independent-copy survival must also pass. Test actual MPI support, not + merely the requested level. Keep SERIALIZED until S8 removes executable legacy workers. +- [x] Run linear supported geometries and splitmix with three processes, numerical/full-map comparisons and qualified + MPI sanitizer runs. Assert MPI occurs only on the initializing primary; no arbitrary `omp single` winner may post. +- [x] **Stop for a separate bounded measurement approval.** (Proposal `/home/ubuntu/s5-artifacts/MEASUREMENT-PROPOSAL.md`; + tier A approved and run 2026-10-02, report `/home/ubuntu/s5-artifacts/checkpoint/report.md`.) Propose matched MPI-only, MPI+OpenMP and single-thread + controls plus relevant full-machine cells. Use the same partition baseline and all numerical/memory surfaces. Obtain + an owner proceed/rework/stop decision before API cutover; do not claim multi-node qualification from ordinary EC2 + links. + +## Task S6: Final public surface, exports and test migration + +**Deliverable:** candidate has the complete intended interface and coverage; no test depends on a one-store facade +at T>1. + +**Files:** `MonomialPropagator.h`, `MonomialPropagator.inl`, `src/monoprop/bindings/binder.h`, +`src/monoprop/monomial_propagator.py`, `detail/EnvConfig.h`, `cpp/tests/{unit_tests,partition_equivalence_tests, +openmp_runtime_tests,fused_cos_sweep_tests,mpi_fresh_insert_equivalence,mpi_failure_driver}.cpp`, +`cpp/tests/PropagatorTestAccess.h`, `cpp/tests/link_export_probe/link_export_probe.cpp`, `cpp/tests/CMakeLists.txt`, +`justfile`, `.github/workflows/test.yml`, `tests/test_openmp_config.py`; documentation below. + +**Interfaces:** no `partitions`, `child_factory`, `PartitionChildFactory` or `monoprop_PARTITIONS` reader in the +candidate. +No replacement constructor/setter. Single-shard raw accessor rejection uses launch-time T, not a user shard selector. +Mathematical operations, virtual clone/update hooks, aggregate reporting and decoded exports retain their contracts. + +**Outcome:** implemented and **accepted by the owner on 2026-10-03**. Start: `ef103e9`. Audit and migration ledger +(written first), handoff with identities, commands and evidence: `/home/ubuntu/s6-artifacts/` (`LEDGER.md`, +`HANDOFF.md`), outside the checkout. + +- Candidate C++ surface: the constructor ends with `basis_change, logical_num_modes, basis` (public and private + observer constructors); `partitions`, `child_factory`, `PartitionChildFactory`, the partition forward declaration, + `PartitionCountMismatch`, the `PartitionGroup.h` include and the `monoprop_PARTITIONS` check are gone from the + candidate, without aliases. Raw accessors throw `MultiShardUnsupported` (validity first). `EnvConfig.h`'s permissive, + cached thread parser is legacy-only. The legacy (OFF) surface and binaries are unchanged: its `_core`, `libmonoprop` + and failure driver hash as at S5. +- Python: the candidate `_core` constructor (callable, annotations, docstring and `_core.pyi`) has no `partitions`; + removed and never-added keywords and the old positional tail raise `TypeError` at every level. +- Tests: `sharded_api_tests.cpp` (compile-time probes asserted per build, so each must be true in the legacy build + and false in the candidate); the S1-S3 seam suites, `openmp_runtime_tests.cpp` and `fused_cos_sweep_tests.cpp` + compile in both builds with legacy-oracle cases OFF-only; a test-only `fused_records` observer hook audits real + per-shard fused records; extended virtual-hook, raw-access, ignored-variable and unset-budget cases. The runner no + longer supplies `monoprop_NUM_THREADS`; CTest registration sets T = 1; whole-suite MPI variants run in candidate + builds; `fused_env_t*`, `openmp_env_root_t4` and `sharded_root_env_openmp_default_t3` are new fixed launches. +- Consumers and CI: the probe and `find_package_smoke` check the constructor shape, derived hooks, raw access, the + inherited selector and MPI setting, and (MPI) a two-rank comparison with one process; CI gains `runtime: sharded` + legs with distinct cache and report identities and `just check-runtime`. +- Evidence (GCC 15.2/libgomp, Open MPI 5.0.10, this host): RED first (compile probes, two runtime cases, 24 Python + cases); 13 mutants and a TSan race mutant detected; final matrix r-off 492/492 + pytest 813, m-off 551/551 + + pytest 855 + 2 x 742 at two ranks, r-on 487/487 + pytest 874 (T = 1, with the legacy bitwise comparisons) and 813 + (T = 2, 4), m-on 550/550 + pytest 950/853/853 + 2 x 742; GCC ASan/UBSan and Clang 18/libomp/Archer TSan without + reports; Clang/libomp 487/492; installed consumers for all four packages. +- Pending: macOS, Linux aarch64, wheels, Nix, minimum-version compilers, Python sanitizer legs, multi-node. No + performance claim: the candidate MPI binaries changed (S5's checkpoint does not cover them); S7 is unchanged. + +- [x] Add compile/API tests proving old explicit partition/factory signatures are absent from the candidate, Python + construction has no new thread/shard argument, and budget parsing/capture/copy use the existing environment contract. +- [x] Implement single-shard access with an explicit T>1 exception and a T=1 remedy. Do not fabricate raw merged + storage. Update the old `MultiPartitionUnsupported` diagnostic/type as part of the documented C++ break. +- [x] Migrate the audit's real call sites and tests, preserving coverage. Export through shard indices/coefficient + vectors; fixed-geometry graph-layout tests inspect shards privately. Cross-geometry tests compare full retained maps. + Retain actual-participation, capacity, malformed-input, fresh-insertion, copy/clone/update and failure-lifetime tests. +- [x] Remove the test main's `monoprop_PARTITIONS=off` assignment; ordinary C++/Python tests launch at T=1. Register + dedicated fixed-T launches without `ScopedBudget` mutation in integration tests. Do not use `OMP_THREAD_LIMIT