diff --git a/.github/workflows/cleanup-images.yml b/.github/workflows/cleanup-images.yml new file mode 100644 index 000000000..fe91beb67 --- /dev/null +++ b/.github/workflows/cleanup-images.yml @@ -0,0 +1,47 @@ +name: Clean up container images + +on: + schedule: + - cron: "0 3 * * *" # daily at 03:00 UTC + workflow_dispatch: + inputs: + dry_run: + description: Only log the image versions that would be deleted + type: boolean + default: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: {} + +jobs: + cleanup: + name: Delete stale GHCR image versions + if: github.repository == 'Algorithmiq/monoprop' + runs-on: ubuntu-slim + permissions: + contents: read + packages: write + pull-requests: read + + steps: + - uses: actions/checkout@v7.0.1 + with: + sparse-checkout: tools/cleanup-ghcr.py + sparse-checkout-cone-mode: false + + # tools/cleanup-ghcr.py needs only the standard library of Python >= 3.11, which + # ubuntu-slim's system python3 provides. + - name: Delete stale image versions + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + run: | + if [[ "$DRY_RUN" == "true" ]]; then + mode=--dry-run + else + mode=--no-dry-run + fi + python3 tools/cleanup-ghcr.py "$mode" diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 000000000..724504ab6 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,293 @@ +name: Container images + +on: + workflow_dispatch: + release: + types: + - published + pull_request: + paths: + - .github/workflows/container.yml + - tools/container/** + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +env: + COLUMNS: 120 + FORCE_COLOR: 3 + HPCCM_VERSION: "26.5.0" + # -march target of the monoprop image, also the suffix of its tags; list it in + # CPU_TARGETS in tools/cleanup-ghcr.py so the suffixed tags stay pinned. + MARCH: sapphirerapids + REGISTRY: ghcr.io + BASE_IMAGE_NAME: ghcr.io/algorithmiq/monoprop-base + APP_IMAGE_NAME: ghcr.io/algorithmiq/monoprop + +jobs: + sdist: + name: Build source distribution + # Fork and Dependabot PRs get a read-only token and cannot push to GHCR, and the app build pulls the base image from it. + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' + runs-on: ubuntu-26.04 + permissions: + contents: read + outputs: + file: ${{ steps.sdist.outputs.file }} + version: ${{ steps.sdist.outputs.version }} + + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 # such that setuptools_scm can do its job correctly + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v10.3.0 + + - name: Build SDist + id: sdist + run: | + uv tool run --from build pyproject-build --verbose --sdist --installer uv + file="$(basename dist/monoprop-*.tar.gz)" + version="${file#monoprop-}" + version="${version%.tar.gz}" + echo "file=$file" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v7 + with: + name: container-sdist + path: dist/monoprop-*.tar.gz + + base: + name: Resolve or build the base image + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' + runs-on: ubuntu-26.04 + timeout-minutes: 180 + permissions: + contents: read + packages: write + outputs: + ref: ${{ steps.hash.outputs.ref }} + + steps: + - uses: actions/checkout@v7.0.1 + + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v10.3.0 + + - name: Render the base Dockerfile + run: | + mkdir -p build/container/base + uvx "hpccm==${HPCCM_VERSION}" --recipe tools/container/base.py --format docker \ + > build/container/base/Dockerfile + + - uses: actions/upload-artifact@v7 + with: + name: base-dockerfile + path: build/container/base/Dockerfile + + # The tag hashes the rendered Dockerfile, not the recipe, so an hpccm upgrade that + # changes the output also yields a new base image. + - name: Compute the content-addressed tag + id: hash + run: | + hash="$(sha256sum build/container/base/Dockerfile | cut -c1-12)" + echo "ref=${BASE_IMAGE_NAME}:${hash}" >> "$GITHUB_OUTPUT" + + - name: Log in to the container registry + uses: docker/login-action@v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.4.1 + + - name: Check whether the base image exists + id: exists + env: + REF: ${{ steps.hash.outputs.ref }} + run: | + if docker buildx imagetools inspect "$REF" > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Extract metadata for the base image + id: meta + uses: docker/metadata-action@v6.2.0 + with: + images: ${{ env.BASE_IMAGE_NAME }} + flavor: | + latest=false + tags: | + type=semver,pattern={{version}} + type=ref,event=tag + type=ref,event=branch + type=ref,event=pr + labels: | + org.opencontainers.image.vendor=Algorithmiq + org.opencontainers.image.description=MPI base image (UCX, PMIx, Open MPI) for monoprop + + - name: Build and push the base image + if: steps.exists.outputs.exists != 'true' + uses: docker/build-push-action@v7.4.0 + with: + context: build/container/base + file: build/container/base/Dockerfile + no-cache: true + push: true + tags: ${{ steps.hash.outputs.ref }} + labels: ${{ steps.meta.outputs.labels }} + + # Cleanup keeps versions by their release, main or pr-N tags, so the base image + # carries the tags of every event that used it. + - name: Tag the base image for this event + env: + REF: ${{ steps.hash.outputs.ref }} + TAGS: ${{ steps.meta.outputs.tags }} + run: | + args=() + while IFS= read -r tag; do + [[ -n "$tag" ]] && args+=(--tag "$tag") + done <<< "$TAGS" + if (( ${#args[@]} == 0 )); then + exit 0 + fi + docker buildx imagetools create "${args[@]}" "$REF" + # A retag must not create a new manifest: the hash tag would lose its pins. + expected="$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' | jq -r .digest)" + while IFS= read -r tag; do + [[ -z "$tag" ]] && continue + actual="$(docker buildx imagetools inspect "$tag" --format '{{json .Manifest}}' | jq -r .digest)" + if [[ "$actual" != "$expected" ]]; then + echo "::error::$tag points at $actual, expected $expected" + exit 1 + fi + done <<< "$TAGS" + + app: + name: Build, check and push the monoprop image + needs: + - sdist + - base + runs-on: ubuntu-26.04 + timeout-minutes: 90 + permissions: + contents: read + packages: write + pull-requests: write + + steps: + - uses: actions/checkout@v7.0.1 + + - uses: actions/download-artifact@v8 + with: + name: container-sdist + path: build/container/app + + - name: Log in to the container registry + uses: docker/login-action@v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4.4.1 + + - name: Extract metadata for the monoprop image + id: meta + uses: docker/metadata-action@v6.2.0 + with: + images: ${{ env.APP_IMAGE_NAME }} + flavor: | + latest=false + # The image runs only on MARCH CPUs, so every tag names it. + tags: | + type=semver,pattern={{version}},suffix=-${{ env.MARCH }} + type=ref,event=tag,suffix=-${{ env.MARCH }} + type=ref,event=branch,suffix=-${{ env.MARCH }} + type=ref,event=pr,suffix=-${{ env.MARCH }} + type=sha,prefix=sha-,format=short,suffix=-${{ env.MARCH }} + type=raw,value=latest,suffix=-${{ env.MARCH }},enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} + labels: | + org.opencontainers.image.vendor=Algorithmiq + org.opencontainers.image.base.name=${{ needs.base.outputs.ref }} + + - name: Build the image + uses: docker/build-push-action@v7.4.0 + with: + context: build/container/app + file: tools/container/Dockerfile + build-args: | + BASE_IMAGE=${{ needs.base.outputs.ref }} + SDIST=${{ needs.sdist.outputs.file }} + MARCH=${{ env.MARCH }} + no-cache: true + load: true + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Smoke-test the image + env: + IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }} + VERSION: ${{ needs.sdist.outputs.version }} + run: | + docker run --rm --shm-size=1g \ + -v "$PWD/tools/container/smoke-test.sh:/smoke-test.sh:ro" \ + "$IMAGE" bash /smoke-test.sh "$VERSION" + + # Same inputs as the build above, so Buildx serves every layer from its cache. + - name: Push the image + uses: docker/build-push-action@v7.4.0 + with: + context: build/container/app + file: tools/container/Dockerfile + build-args: | + BASE_IMAGE=${{ needs.base.outputs.ref }} + SDIST=${{ needs.sdist.outputs.file }} + MARCH=${{ env.MARCH }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Prepare pull instructions + if: github.event_name == 'pull_request' + env: + TAGS: ${{ steps.meta.outputs.tags }} + run: | + { + echo "Container image pushed to GHCR. Pull it with Docker or Singularity:" + echo + while IFS= read -r tag; do + [[ -z "$tag" ]] && continue + echo '```bash' + echo "docker pull $tag" + echo "singularity pull monoprop_${tag##*:}.sif docker://$tag" + echo '```' + echo + done <<< "$TAGS" + } > pull-instructions.md + + - name: Post pull instructions + if: github.event_name == 'pull_request' + uses: marocchino/sticky-pull-request-comment@v3.0.5 + with: + header: container-pull-instructions + path: pull-instructions.md diff --git a/AGENTS.md b/AGENTS.md index 34cb2b266..dee217fea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,7 +33,8 @@ shared engine; Majorana against Pauli is a runtime `Basis`, and Python exposes ` Public headers `cpp/include/monoprop/`, implementation `cpp/monoprop/`, Python API `src/monoprop/`, binding template `src/monoprop/bindings/binder.h`, generators -`tools/generate-*.py`, sibling distributions `packages/*`. The repository root is the `uv` +`tools/generate-*.py`, container recipes `tools/container/` (see `building.mdx`), sibling +distributions `packages/*`. The repository root is the `uv` workspace's `monoprop` package. `bindings.cpp` and `_dispatch.py` are generated -- never edit them. To change the C++ API: edit diff --git a/README.md b/README.md index 4656ad524..61096b0b0 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ pip install monoprop # or: uv add monoprop The prebuilt PyPI wheels are single-process (built **without** MPI). For multi-rank runs, or to build the C++ library and executables, build from source (see below). +For HPC systems without outbound network, an MPI-enabled container image is published to +`ghcr.io/algorithmiq/monoprop`; see the +[HPC clusters documentation](https://docs.monoprop.algorithmiq.tech/clusters). + ## Quick example Back-propagate a Majorana observable through a one-gate circuit: diff --git a/docs/content/docs/building.mdx b/docs/content/docs/building.mdx index 6225e225e..5303aa094 100644 --- a/docs/content/docs/building.mdx +++ b/docs/content/docs/building.mdx @@ -345,9 +345,67 @@ Python API under TSan would require a TSan-instrumented CPython. those files, so there is one list per platform to keep current. +## Container images + +`.github/workflows/container.yml` publishes two images to GHCR for +[HPC clusters](/clusters) without outbound network: + +- `ghcr.io/algorithmiq/monoprop-base`: the MPI stack. `tools/container/base.py` is an + [HPCCM](https://github.com/NVIDIA/hpc-container-maker) recipe: Ubuntu 26.04 (compilers, + Python, Boost and hwloc from the distribution), DOCA-OFED's InfiniBand user space, and + UCX, PMIx and Open MPI built from source. The versions are pinned at the top of the recipe. +- `ghcr.io/algorithmiq/monoprop`: `tools/container/Dockerfile` installs the sdist on top, + with MPI and the `mpi` and `qiskit` extras, compiled with `-march=$MARCH`. The workflow + sets `MARCH` (currently `sapphirerapids`, also the Dockerfile's default). + +The base image is tagged with the first 12 hex digits of the SHA-256 of the rendered +Dockerfile, so it is rebuilt only when the recipe's output changes, including through an +hpccm upgrade (`HPCCM_VERSION` in the workflow). Each run also tags the base image it used +with the run's tag (`pr-N`, `main` or the release version). The base image is built without +`-march`, so its tags carry no CPU target. The monoprop image gets the same tag and +`sha-`, each followed by `-$MARCH` (e.g. `main-sapphirerapids`), because it runs only +on that CPU; non-pre-release releases also move `latest-$MARCH`. + +The workflow runs on pushes to `main`, on releases, on manual dispatch, and on pull +requests that touch `tools/container/` or the workflow. `tools/container/smoke-test.sh` +checks the monoprop image before it is pushed. It never imports monoprop, because the CI +machine's CPU may lack the instructions `-march` enables, so check a new image on the +target machine as well (see [MareNostrum5](/clusters/marenostrum5#check-the-image)). + +To reproduce the build locally: + +```bash +mkdir -p build/container/base build/container/app +uvx hpccm==26.5.0 --recipe tools/container/base.py --format docker > build/container/base/Dockerfile +docker build -t monoprop-base:local build/container/base +uv tool run --from build pyproject-build --sdist --installer uv --outdir build/container/app +sdist="$(basename build/container/app/monoprop-*.tar.gz)" +docker build -f tools/container/Dockerfile \ + --build-arg BASE_IMAGE=monoprop-base:local --build-arg SDIST="$sdist" \ + -t monoprop:local build/container/app +version="${sdist#monoprop-}"; version="${version%.tar.gz}" +docker run --rm --shm-size=1g -v "$PWD/tools/container/smoke-test.sh:/smoke-test.sh:ro" \ + monoprop:local bash /smoke-test.sh "$version" +``` + +### Image cleanup + +`.github/workflows/cleanup-images.yml` runs `tools/cleanup-ghcr.py` daily. It keeps the +versions tagged with a release, `latest`, `main` or the `pr-N` of an open pull request, +with or without a CPU-target suffix listed in the script's `CPU_TARGETS` (building for a +new `MARCH` means adding it there), the platform and attestation manifests those reference, and anything less than 24 hours +old, and deletes the rest of both packages. A manual dispatch is a dry run by default, +which only logs what would be deleted. The script authenticates with the workflow's +`GITHUB_TOKEN`, which can manage the packages this repository published. + +A package created by a workflow can start out private on GHCR. If so, an organisation +owner makes it public once, in the package settings, so that cluster users can pull +without credentials. + ## See also - [Getting Started](/getting-started) — installing a prebuilt release from PyPI. +- [HPC clusters](/clusters) — running the published container image on HPC systems. - [Parallelism and distribution](/features/parallelism) — running across MPI ranks and shared-memory threads. - [Testing](/testing) — the full Python and C++ test workflow. - [How to Contribute](/how-to-contribute) — contributor workflow and documentation checks. diff --git a/docs/content/docs/clusters/index.mdx b/docs/content/docs/clusters/index.mdx new file mode 100644 index 000000000..acb66b839 --- /dev/null +++ b/docs/content/docs/clusters/index.mdx @@ -0,0 +1,30 @@ +--- +title: HPC clusters +description: Running monoprop on HPC systems from its MPI-enabled container image. +--- + +Some HPC systems give their login nodes no outbound network, so monoprop cannot be +installed there with `pip`. For those, every release, the `main` branch and pull requests that +change the container recipes publish an MPI-enabled container image to the GitHub Container Registry: + +```text +ghcr.io/algorithmiq/monoprop: +``` + +Every tag ends with the CPU target the image is compiled for, currently `sapphirerapids` +(Intel Sapphire Rapids, as in MareNostrum5 and Leonardo). The image does not run on CPUs +without that instruction set. + +| Tag | Contents | +| --- | --- | +| `X.Y.Z-sapphirerapids` | A monoprop release. `latest-sapphirerapids` points at the newest one. | +| `main-sapphirerapids` | The tip of the `main` branch. | +| `pr-N-sapphirerapids` | Pull request `N`, when it changes the container recipes, while it is open. | + +The image holds monoprop built with MPI, with its `mpi` and `qiskit` extras, on top of an +MPI stack built for InfiniBand clusters that launch jobs with Slurm: Open MPI 5 with PMIx +and UCX, and NVIDIA's DOCA-OFED user-space drivers. To build it yourself, see +[Container images](/building#container-images). + +- [MareNostrum5](/clusters/marenostrum5) — pulling the image, running it with + Singularity under Slurm, and checking the CPU placement. diff --git a/docs/content/docs/clusters/marenostrum5.mdx b/docs/content/docs/clusters/marenostrum5.mdx new file mode 100644 index 000000000..4c77ecbb1 --- /dev/null +++ b/docs/content/docs/clusters/marenostrum5.mdx @@ -0,0 +1,210 @@ +--- +title: MareNostrum5 +description: Running the monoprop container image on MareNostrum5 with Singularity and Slurm. +--- + +[MareNostrum5](https://www.bsc.es/supportkc/docs/MareNostrum5/intro) (MN5) at BSC has no +outbound network on its login nodes, so run monoprop from the +[container image](/clusters): pull it on a machine with internet access, copy it over, and +launch it with Singularity under Slurm. The image targets the Sapphire Rapids CPUs of both +the general-purpose (GPP) and accelerated (ACC) partitions. + +## Get the image + +The image is public, so no registry login is needed. On a machine with internet access +and Singularity (or Apptainer) installed: + +```bash +singularity pull monoprop_.sif docker://ghcr.io/algorithmiq/monoprop: +``` + +`` is one of the [published tags](/clusters), for example `main-sapphirerapids`. + +Copy the `.sif` file (the image is about 1.3 GB unpacked; the `.sif` is compressed and +smaller) to your project directory on MN5 with `scp` or `rsync` through a login or +data-transfer node, for example: + +```bash +rsync -P monoprop_.sif @:/ +``` + +## Check the image + +Get an interactive allocation on a compute node, then check that monoprop imports and sees +MPI on two ranks: + +```bash +salloc --nodes=1 --account= -q gp_debug --exclusive --time=00:10:00 +module load singularity +srun -n 2 --mpi=pmix singularity exec monoprop_.sif \ + python3 -c 'import monoprop; print(monoprop.__version__, monoprop.has_mpi)' +``` + +Both ranks should print the image's version and `True`. This is the check CI cannot run: +the CI machines may lack the Sapphire Rapids instructions the image is compiled for. + +MN5's Slurm offers munge authentication to PMIx, which the container's PMIx is built +without. The image sets `PMIX_MCA_psec=native`, so the ranks use PMIx's native +authentication directly instead of warning that the `munge` component is missing. + +## Run a job + +Adapt this script and submit it with `sbatch`. The account and QoS can also be given on +the command line. + +```bash +#!/usr/bin/env bash + +#SBATCH --job-name=monoprop # <-- CHANGE ME +#SBATCH --chdir=. +#SBATCH --exclusive +#SBATCH --ntasks-per-node=8 +#SBATCH --cpus-per-task=14 +#SBATCH --threads-per-core=1 +#SBATCH --output=%x-%j.out +#SBATCH --error=%x-%j.err + +# GIVEN THROUGH THE COMMAND LINE +##SBATCH --partition=gpp +##SBATCH --account= +##SBATCH --nodes=2 +##SBATCH --time=06:00:00 +##SBATCH --qos= + +module purge +module load singularity/4.1.5 +module list + +#---- +# Printing the status of the given allocation +echo -e "\n\n#------------------------#" +echo "Printing from scontrol:" +scontrol show job "${SLURM_JOBID}" + +export SRUN_CPUS_PER_TASK=$SLURM_CPUS_PER_TASK + +echo -e "\n\n#------------------------#" +echo "Diagnostic for task distribution (with task and taskset):" +srun \ + --label \ + --verbose \ + --mpi=pmix \ + --nodes="$SLURM_NNODES" \ + --ntasks="$(( SLURM_NTASKS_PER_NODE * SLURM_NNODES))" \ + --ntasks-per-node="$SLURM_NTASKS_PER_NODE" \ + --threads-per-core=1 \ + --cpu-bind="verbose,threads" \ + --distribution=block:block \ + bash -c 'echo -n "task $SLURM_PROCID (node name: $(hostname), node id: $SLURM_NODEID): "; taskset -cp $$' | sort -n + +# An MPI rank keeps a single partition unless monoprop_NUM_THREADS is set. +export monoprop_NUM_THREADS="$SLURM_CPUS_PER_TASK" +# Directories the container may read and write besides $HOME and the working directory. +export SINGULARITY_BIND="/gpfs/projects,/gpfs/scratch,$TMPDIR" + +####### HERE we actually run monoprop ####### +srun \ + --label \ + --verbose \ + --mpi=pmix \ + --nodes="$SLURM_NNODES" \ + --ntasks="$(( SLURM_NTASKS_PER_NODE * SLURM_NNODES))" \ + --ntasks-per-node="$SLURM_NTASKS_PER_NODE" \ + --threads-per-core=1 \ + --cpu-bind="verbose,threads" \ + --distribution=block:block \ + --kill-on-bad-exit=1 \ + singularity exec monoprop_.sif \ + python3 -u run.py # <-- CHANGE ME +status=$? +########################################### + +#---- +# Printing information of finished job steps: +echo -e "\n\n#------------------------#" +echo "Task and CPU usage stats:" +sacct --format=JobID,JobName,NCPUS,NNodes,NTasks,AveCPU,MinCPU,MinCPUNode,MinCPUTask,Elapsed,ExitCode --jobs="$SLURM_JOBID" + +echo "Memory usage stats:" +sacct --format=JobID,JobName,AveRSS%-50,MaxRSS%-50,MaxRSSNode,MaxRSSTask,AvePages,MaxPages,MaxPagesNode,MaxPagesTask --units=M --jobs="$SLURM_JOBID" + +echo "Disk usage stats:" +sacct --format=JobID,JobName,AveDiskRead,MaxDiskRead,MaxDiskReadNode,MaxDiskReadTask,AveDiskWrite,MaxDiskWrite,MaxDiskWriteNode,MaxDiskWriteTask --units=G --jobs="$SLURM_JOBID" + +echo "Trackable resources usage stats:" +sacct --format=JobID,JobName,AllocNodes,AllocCPUS,AllocTRES%-100 --units=G --jobs="$SLURM_JOBID" + +echo "Trackable resources (ingress) usage stats:" +# ingress resources are disk reads and network incoming transfers +sacct --format=JobID,JobName,TRESUsageInTot%-70,TRESUsageInAve%-70,TRESUsageInMax%-70,TRESUsageInMaxNode%-70,TRESUsageInMaxTask%-70 --units=G --jobs="$SLURM_JOBID" + +echo "Trackable resources (egress) usage stats:" +# egress resources are disk writes and network outgoing transfers +sacct --format=JobID,JobName,TRESUsageOutTot%-70,TRESUsageOutAve%-70,TRESUsageOutMax%-70,TRESUsageOutMaxNode%-70,TRESUsageOutMaxTask%-70 --units=G --jobs="$SLURM_JOBID" + +#---- +# Done +echo -e "\n\n#------------------------#" +echo "Done" +exit "$status" +``` + +- `--exclusive`: MN5 only honours CPU affinity on full nodes. +- `--ntasks-per-node` × `--cpus-per-task` should fill a GPP node's 112 cores (2 sockets of + 56) without splitting a rank across sockets: 8 ranks × 14 cores puts ranks 0–3 on the + first socket and ranks 4–7 on the second. On ACC nodes (2 sockets of 40), use for + example 8 ranks × 10 cores. +- `--threads-per-core=1` and `--cpu-bind=verbose,threads` bind each rank to one hardware + thread on each of its cores, and `--distribution=block:block` hands each rank consecutive + cores. monoprop pins one partition per core inside the mask it is given, so + `monoprop_NUM_THREADS="$SLURM_CPUS_PER_TASK"` runs one partition per core of the rank; + see [Parallelism and distribution](/features/parallelism). +- `SRUN_CPUS_PER_TASK` is needed because `srun` does not inherit `--cpus-per-task` from + `sbatch`. +- The first `srun` prints every rank's binding before monoprop runs. The job exits with the + status of the monoprop step, after the `sacct` reports. +- The script loads `singularity/4.1.5`; check `module avail singularity` for the installed + versions. + +### Check the placement + +Every rank prints a `COMMPLACE` line to stderr when it builds a propagator. With the script +above it reads `masks=private cpus=14`: the ranks on a node have disjoint masks of 14 CPUs. +To check the binding before running monoprop, from an allocation with the job's geometry: + +```bash +srun --ntasks-per-node=8 --cpus-per-task=14 --threads-per-core=1 \ + --cpu-bind=verbose,threads --distribution=block:block \ + bash -c 'echo "rank $SLURM_PROCID: $(grep Cpus_allowed_list /proc/self/status)"' +``` + +Each rank should list 14 consecutive CPUs, one per physical core: `0-13`, `14-27`, `28-41` +and `42-55` for ranks 0–3 on the first socket, `56-69` to `98-111` for ranks 4–7 on the +second. CPUs 112–223 are the second hardware threads of cores 0–111, so a list such as +`0,112` means one core with both of its hardware threads, not two cores. + +### Tune UCX + +The container brings its own UCX, so `module load ucx` on the host has no effect; set UCX +through environment variables, which Singularity passes into the container. With many +ranks per node, select the dynamically connected transport, since `rc_mlx5` can run out +of network-device memory at scale: + +```bash +export UCX_TLS=self,sm,dc_mlx5 +``` + +To list the transports UCX sees on a compute node: + +```bash +srun -n 1 singularity exec monoprop_.sif ucx_info -d | grep Transport -A2 +``` + +Launch with `srun` as above rather than `mpirun`; `mpirun` inside an allocation also needs +`SLURM_CPU_BIND=none`. + +## Further reading + +- [Best practices: getting your code working in the large scale](https://eurohpcsupport.eu/best-practice/best-practices-getting-your-code-working-in-the-large-scale/) + (EuroHPC), on affinity, UCX and MPI tuning on MN5. +- [MareNostrum5 performance guide](https://www.bsc.es/supportkc/docs/MareNostrum5/performance) (BSC). diff --git a/docs/content/docs/clusters/meta.json b/docs/content/docs/clusters/meta.json new file mode 100644 index 000000000..0ccd4fc0d --- /dev/null +++ b/docs/content/docs/clusters/meta.json @@ -0,0 +1,4 @@ +{ + "title": "HPC clusters", + "pages": ["marenostrum5"] +} diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index 66e879a29..436235849 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -1,3 +1,3 @@ { - "pages": ["index", "getting-started", "building", "concepts", "features", "tutorials", "api", "benchmarks", "references", "how-to-contribute"] + "pages": ["index", "getting-started", "building", "clusters", "concepts", "features", "tutorials", "api", "benchmarks", "references", "how-to-contribute"] } diff --git a/tests/test_cleanup_ghcr.py b/tests/test_cleanup_ghcr.py new file mode 100644 index 000000000..5cdf702b6 --- /dev/null +++ b/tests/test_cleanup_ghcr.py @@ -0,0 +1,309 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the GHCR cleanup policy in tools/cleanup-ghcr.py.""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import sys +from pathlib import Path + +import pytest + +_MODULE_PATH = Path(__file__).parents[1] / "tools" / "cleanup-ghcr.py" +_spec = importlib.util.spec_from_file_location("cleanup_ghcr", _MODULE_PATH) +assert _spec is not None +assert _spec.loader is not None +_module = importlib.util.module_from_spec(_spec) +# dataclasses resolves the defining module through sys.modules. +sys.modules[_spec.name] = _module +_spec.loader.exec_module(_module) + +Version = _module.Version +is_pinned = _module.is_pinned +select_deletions = _module.select_deletions + +NOW = dt.datetime(2026, 10, 21, 12, 0, tzinfo=dt.UTC) +MIN_AGE = dt.timedelta(hours=24) + + +def _version(version_id, digest, *tags, age=dt.timedelta(days=3)): + return Version( + version_id=version_id, digest=digest, tags=tags, created_at=NOW - age + ) + + +def _deleted(versions, *, open_prs=(), children_of=None): + selected = select_deletions( + versions, set(open_prs), children_of or {}, now=NOW, min_age=MIN_AGE + ) + return {version.digest for version in selected} + + +@pytest.mark.parametrize( + "tag", ["1.2.3", "v1.2.3", "0.5.0rc1", "v0.5.0-rc.1", "latest", "main"] +) +def test_release_and_branch_tags_are_pinned(tag): + assert is_pinned((tag,), set()) + + +@pytest.mark.parametrize( + "tag", ["sha-1a2b3c4", "3f9c0a1b2d4e", "pr-7", "feature-x", "1.2"] +) +def test_other_tags_are_not_pinned(tag): + assert not is_pinned((tag,), set()) + + +def test_open_pr_tag_is_pinned(): + assert is_pinned(("pr-7",), {7}) + + +@pytest.mark.parametrize( + "tag", + [ + "1.2.3-sapphirerapids", + "v0.5.0-rc.1-sapphirerapids", + "latest-sapphirerapids", + "main-sapphirerapids", + ], +) +def test_cpu_target_suffix_keeps_release_and_branch_tags_pinned(tag): + assert is_pinned((tag,), set()) + + +def test_cpu_target_suffix_keeps_open_pr_tags_pinned(): + assert is_pinned(("pr-7-sapphirerapids",), {7}) + assert not is_pinned(("pr-7-sapphirerapids",), set()) + + +@pytest.mark.parametrize( + "tag", + [ + "sha-1a2b3c4-sapphirerapids", + # a dispatch on a branch named main-feature + "main-feature-sapphirerapids", + # an unknown suffix is not a CPU target + "main-feature", + ], +) +def test_other_suffixed_tags_are_not_pinned(tag): + assert not is_pinned((tag,), set()) + + +def test_release_keeps_its_platform_and_attestation_manifests(): + versions = [ + _version(1, "sha256:index", "1.2.3"), + _version(2, "sha256:amd64"), + _version(3, "sha256:attestation"), + ] + children = {"sha256:index": ["sha256:amd64", "sha256:attestation"]} + assert _deleted(versions, children_of=children) == set() + + +def test_superseded_main_is_deleted_with_its_children(): + versions = [ + _version(1, "sha256:new", "main", "sha-bbbbbbb"), + _version(2, "sha256:new-amd64"), + _version(3, "sha256:old", "sha-aaaaaaa"), + _version(4, "sha256:old-amd64"), + ] + children = {"sha256:new": ["sha256:new-amd64"]} + assert _deleted(versions, children_of=children) == { + "sha256:old", + "sha256:old-amd64", + } + + +def test_closed_pr_is_deleted_and_open_pr_is_kept(): + versions = [ + _version(1, "sha256:open", "pr-9"), + _version(2, "sha256:closed", "pr-7"), + ] + assert _deleted(versions, open_prs={9}) == {"sha256:closed"} + + +def test_young_versions_are_kept(): + versions = [ + _version(1, "sha256:pushing", age=dt.timedelta(hours=1)), + _version(2, "sha256:stale", age=dt.timedelta(hours=25)), + ] + assert _deleted(versions) == {"sha256:stale"} + + +def test_young_index_keeps_its_older_children(): + # A retag can point a new index at platform manifests pushed long before it. + versions = [ + _version(1, "sha256:young-index", age=dt.timedelta(hours=1)), + _version(2, "sha256:old-amd64"), + ] + children = {"sha256:young-index": ["sha256:old-amd64"]} + assert _deleted(versions, children_of=children) == set() + + +def test_children_are_fetched_for_young_indexes(monkeypatch): + versions = [_version(1, "sha256:young-index", age=dt.timedelta(hours=1))] + _fake_registry(monkeypatch, versions) + fetched = [] + monkeypatch.setattr( + _module, + "child_digests", + lambda _package, digest, _token: fetched.append(digest) or [], + ) + _module.clean_package( + "monoprop", "token", set(), now=NOW, min_age=MIN_AGE, dry_run=True + ) + assert fetched == ["sha256:young-index"] + + +def test_base_shared_by_two_prs_is_kept_while_either_is_open(): + versions = [_version(1, "sha256:base", "3f9c0a1b2d4e", "pr-7", "pr-9")] + assert _deleted(versions, open_prs={9}) == set() + assert _deleted(versions, open_prs=set()) == {"sha256:base"} + + +def test_sha_only_versions_are_deleted(): + versions = [_version(1, "sha256:commit", "sha-1a2b3c4")] + assert _deleted(versions) == {"sha256:commit"} + + +def _fake_registry(monkeypatch, versions, *, fail_on=()): + deleted = [] + + def delete_version(_package, version, _token): + if version.digest in fail_on: + raise _module.urllib.error.HTTPError( + "https://api.github.com", 403, "Forbidden", {}, None + ) + deleted.append(version.digest) + + monkeypatch.setattr(_module, "list_versions", lambda *_args: versions) + monkeypatch.setattr(_module, "registry_token", lambda *_args: "pull-token") + monkeypatch.setattr(_module, "child_digests", lambda *_args: []) + monkeypatch.setattr(_module, "delete_version", delete_version) + return deleted + + +def test_dry_run_deletes_nothing(monkeypatch): + deleted = _fake_registry(monkeypatch, [_version(1, "sha256:stale", "sha-1a2b3c4")]) + failures = _module.clean_package( + "monoprop", "token", set(), now=NOW, min_age=MIN_AGE, dry_run=True + ) + assert failures == 0 + assert deleted == [] + + +def test_real_run_deletes_selected_and_counts_failures(monkeypatch): + versions = [ + _version(1, "sha256:stale", "sha-1a2b3c4"), + _version(2, "sha256:protected", "sha-5d6e7f8"), + _version(3, "sha256:release", "1.2.3"), + ] + deleted = _fake_registry(monkeypatch, versions, fail_on={"sha256:protected"}) + failures = _module.clean_package( + "monoprop", "token", set(), now=NOW, min_age=MIN_AGE, dry_run=False + ) + assert failures == 1 + assert deleted == ["sha256:stale"] + + +def test_missing_package_is_skipped(monkeypatch): + def list_versions(*_args): + raise _module.urllib.error.HTTPError( + "https://api.github.com", 404, "Not Found", {}, None + ) + + monkeypatch.setattr(_module, "list_versions", list_versions) + failures = _module.clean_package( + "monoprop-base", "token", set(), now=NOW, min_age=MIN_AGE, dry_run=False + ) + assert failures == 0 + + +def test_main_requires_a_token(monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + with pytest.raises(SystemExit) as excinfo: + _module.main(["--dry-run"]) + assert excinfo.value.code == 2 + + +def test_main_rejects_a_negative_minimum_age(monkeypatch): + monkeypatch.setenv("GITHUB_TOKEN", "token") + monkeypatch.setattr( + _module, + "open_pull_requests", + lambda _token: pytest.fail("must not reach the API"), + ) + with pytest.raises(SystemExit) as excinfo: + _module.main(["--dry-run", "--min-age-hours", "-1"]) + assert excinfo.value.code == 2 + + +def test_paginate_follows_link_headers(monkeypatch): + pages = { + "https://api.github.com/first": ( + b'[{"number": 7}]', + '; rel="next", ; rel="last"', + ), + "https://api.github.com/second": (b'[{"number": 9}]', ""), + } + requested = [] + + def request(url, **_options): + requested.append(url) + return pages[url] + + monkeypatch.setattr(_module, "_request", request) + assert _module._paginate("https://api.github.com/first", "token") == [ + {"number": 7}, + {"number": 9}, + ] + assert requested == [ + "https://api.github.com/first", + "https://api.github.com/second", + ] + + +def test_child_digests_lists_index_members_only(monkeypatch): + bodies = { + "sha256:index": ( + b'{"mediaType": "application/vnd.oci.image.index.v1+json", ' + b'"manifests": [{"digest": "sha256:amd64"}, {"digest": "sha256:attestation"}]}' + ), + "sha256:plain": ( + b'{"mediaType": "application/vnd.oci.image.manifest.v1+json", "layers": []}' + ), + } + + def request(url, **_options): + return bodies[url.rsplit("/", 1)[1]], "" + + monkeypatch.setattr(_module, "_request", request) + assert _module.child_digests("monoprop", "sha256:index", "pull-token") == [ + "sha256:amd64", + "sha256:attestation", + ] + assert _module.child_digests("monoprop", "sha256:plain", "pull-token") == [] + + +def test_child_digests_recognises_an_index_without_a_media_type(monkeypatch): + body = ( + b'{"manifests": [{"digest": "sha256:amd64"}, {"digest": "sha256:attestation"}]}' + ) + monkeypatch.setattr(_module, "_request", lambda *_args, **_options: (body, "")) + assert _module.child_digests("monoprop", "sha256:index", "pull-token") == [ + "sha256:amd64", + "sha256:attestation", + ] diff --git a/tools/check-workflow-commands.py b/tools/check-workflow-commands.py index aea749bc9..df0a7eabb 100755 --- a/tools/check-workflow-commands.py +++ b/tools/check-workflow-commands.py @@ -22,7 +22,8 @@ perfectly well until it drifts from the recipe it duplicates. The exceptions in ``ALLOWED`` are keyed by step name, so renaming an exempt step fails -here and forces the exception to be reconsidered rather than inherited silently. +here and forces the exception to be reconsidered rather than inherited silently. Whole +workflows listed in ``EXEMPT_WORKFLOWS`` are not scanned at all. """ from __future__ import annotations @@ -82,6 +83,13 @@ ): "the benchmarking ladder is only meaningful in CI, so it lives in the workflow", } +# Workflow file name -> why every step in it may keep its commands. +EXEMPT_WORKFLOWS: dict[str, str] = { + "container.yml": ( + "container builds and their checks only run in CI against GHCR, so they have no recipe" + ), +} + # Version and help queries are not builds. BENIGN = re.compile(r"\b(uv|cmake|ctest|lychee)\s+--(version|help)\b") @@ -107,6 +115,9 @@ def steps_of(document: object) -> list[tuple[str, str]]: def check(path: Path) -> list[str]: """Return one message per forbidden command found in ``path``.""" + if path.name in EXEMPT_WORKFLOWS: + return [] + document = yaml.safe_load(path.read_text()) steps = steps_of(document) problems = [] diff --git a/tools/cleanup-ghcr.py b/tools/cleanup-ghcr.py new file mode 100755 index 000000000..2a4575bb0 --- /dev/null +++ b/tools/cleanup-ghcr.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 + +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Delete stale versions of monoprop's container images from GHCR. + +``container.yml`` pushes ``ghcr.io/algorithmiq/monoprop`` and +``ghcr.io/algorithmiq/monoprop-base`` for pull requests, ``main`` and releases. This +keeps what someone may still pull and deletes the rest. A version is kept when it is + +- tagged with a release (``1.2.3``, ``v1.2.3``, pre-releases), ``latest`` or ``main``; +- tagged ``pr-N`` while pull request ``N`` is open; +- tagged as above with a CPU-target suffix, e.g. ``main-sapphirerapids``; +- younger than ``--min-age-hours``, so a push in progress is never cut short; +- a platform or attestation manifest of a version kept above. GHCR lists those as + separate untagged versions, and deleting one breaks the image that references it. + +Only the standard library is used, so the workflow runs it with a bare ``python3``. +The token is read from ``GITHUB_TOKEN``. +""" + +from __future__ import annotations + +import argparse +import base64 +import datetime as dt +import json +import logging +import os +import re +import sys +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Collection, Mapping, Sequence + +logger = logging.getLogger("cleanup-ghcr") + +OWNER = "algorithmiq" +REPOSITORY = "Algorithmiq/monoprop" +PACKAGES = ("monoprop", "monoprop-base") + +API_URL = "https://api.github.com" +REGISTRY_URL = "https://ghcr.io" +INDEX_MEDIA_TYPES = frozenset( + { + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + } +) +MANIFEST_ACCEPT = ", ".join( + [ + *sorted(INDEX_MEDIA_TYPES), + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + ] +) +_NEXT_LINK = re.compile(r'<([^>]+)>;\s*rel="next"') + +RELEASE_TAG = re.compile(r"^v?\d+\.\d+\.\d+([.+-]?[0-9A-Za-z.]+)?$") +PR_TAG = re.compile(r"^pr-(\d+)$") +BRANCH_TAGS = frozenset({"latest", "main"}) +# container.yml suffixes every monoprop tag with the image's -march target (its MARCH); +# list each target it builds, or its suffixed tags lose their pins. +CPU_TARGETS = frozenset({"sapphirerapids"}) + + +@dataclass(frozen=True) +class Version: + """One container package version as GHCR lists it.""" + + version_id: int + """GitHub's package version id, used to delete the version.""" + digest: str + """Manifest digest, ``sha256:...``.""" + tags: tuple[str, ...] + """Tags pointing at the manifest; empty for untagged versions.""" + created_at: dt.datetime + """Creation time, timezone-aware.""" + + +def _strip_cpu_target(tag: str) -> str: + for target in CPU_TARGETS: + if tag.endswith(f"-{target}"): + return tag.removesuffix(f"-{target}") + return tag + + +def is_pinned(tags: Collection[str], open_prs: Collection[int]) -> bool: + """Return whether a version must be kept because of its own tags. + + Args: + tags: The version's tags. + open_prs: Numbers of the repository's open pull requests. + + Returns: + True for a release, ``latest`` or ``main`` tag, or the ``pr-N`` tag of an open + pull request, with or without a ``-`` suffix from ``CPU_TARGETS``. + """ + for suffixed in tags: + tag = _strip_cpu_target(suffixed) + if tag in BRANCH_TAGS or RELEASE_TAG.match(tag): + return True + pr = PR_TAG.match(tag) + if pr and int(pr.group(1)) in open_prs: + return True + return False + + +def is_retained( + version: Version, + open_prs: Collection[int], + *, + now: dt.datetime, + min_age: dt.timedelta, +) -> bool: + """Return whether a version is kept for its own sake, by its tags or its age. + + Args: + version: The version to classify. + open_prs: Numbers of the repository's open pull requests. + now: The current time. + min_age: Versions younger than this are always kept. + + Returns: + True if the version is pinned or younger than ``min_age``. + """ + return is_pinned(version.tags, open_prs) or now - version.created_at < min_age + + +def select_deletions( + versions: Sequence[Version], + open_prs: Collection[int], + children_of: Mapping[str, Sequence[str]], + *, + now: dt.datetime, + min_age: dt.timedelta, +) -> list[Version]: + """Return the versions of one package that the policy deletes. + + Args: + versions: Every version of the package. + open_prs: Numbers of the repository's open pull requests. + children_of: For each retained image index, the digests of the platform and + attestation manifests it references. + now: The current time. + min_age: Versions younger than this are always kept. + + Returns: + The versions to delete, in input order. + """ + kept: set[str] = set() + for version in versions: + if is_retained(version, open_prs, now=now, min_age=min_age): + kept.add(version.digest) + # A child can be older than its index, e.g. after a registry-side retag. + kept.update(children_of.get(version.digest, ())) + return [version for version in versions if version.digest not in kept] + + +def _request( + url: str, *, headers: Mapping[str, str], method: str = "GET" +) -> tuple[bytes, str]: + """Send one request and return its body and ``Link`` header.""" + # The URLs are built from the fixed https bases above. + request = urllib.request.Request(url, headers=dict(headers), method=method) # noqa: S310 + with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 + return response.read(), response.headers.get("Link", "") + + +def _api_headers(token: str) -> dict[str, str]: + return { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + } + + +def _paginate(url: str, token: str) -> list[dict]: + items: list[dict] = [] + next_url: str | None = url + while next_url: + body, link = _request(next_url, headers=_api_headers(token)) + items.extend(json.loads(body)) + match = _NEXT_LINK.search(link) + next_url = match.group(1) if match else None + return items + + +def list_versions(package: str, token: str) -> list[Version]: + """List every version of one of the organisation's container packages. + + Args: + package: Package name, e.g. ``monoprop``. + token: GitHub token with ``packages: read``. + + Returns: + The package's versions. + """ + url = f"{API_URL}/orgs/{OWNER}/packages/container/{package}/versions?per_page=100" + return [ + Version( + version_id=item["id"], + digest=item["name"], + tags=tuple(item["metadata"]["container"]["tags"]), + created_at=dt.datetime.fromisoformat(item["created_at"]), + ) + for item in _paginate(url, token) + ] + + +def open_pull_requests(token: str) -> set[int]: + """Return the numbers of the repository's open pull requests. + + Args: + token: GitHub token with ``pull-requests: read``. + + Returns: + The open pull request numbers. + """ + url = f"{API_URL}/repos/{REPOSITORY}/pulls?state=open&per_page=100" + return {item["number"] for item in _paginate(url, token)} + + +def registry_token(package: str, token: str) -> str: + """Exchange a GitHub token for a GHCR pull token for one package. + + Args: + package: Package name. + token: GitHub token; authenticating also covers packages that are still private. + + Returns: + A bearer token for the registry API. + """ + basic = base64.b64encode(f"token:{token}".encode()).decode() + url = ( + f"{REGISTRY_URL}/token?service=ghcr.io&scope=repository:{OWNER}/{package}:pull" + ) + body, _ = _request(url, headers={"Authorization": f"Basic {basic}"}) + return json.loads(body)["token"] + + +def child_digests(package: str, digest: str, pull_token: str) -> list[str]: + """Return the manifests an image index references, or nothing for a plain manifest. + + Args: + package: Package name. + digest: Digest of the version to inspect. + pull_token: Registry token from ``registry_token``. + + Returns: + The referenced platform and attestation manifest digests. + """ + url = f"{REGISTRY_URL}/v2/{OWNER}/{package}/manifests/{digest}" + body, _ = _request( + url, + headers={"Accept": MANIFEST_ACCEPT, "Authorization": f"Bearer {pull_token}"}, + ) + manifest = json.loads(body) + # mediaType is optional in OCI, so a body with a manifests list is an index too. + if ( + manifest.get("mediaType") not in INDEX_MEDIA_TYPES + and "manifests" not in manifest + ): + return [] + return [entry["digest"] for entry in manifest.get("manifests", [])] + + +def delete_version(package: str, version: Version, token: str) -> None: + """Delete one package version. + + Args: + package: Package name. + version: The version to delete. + token: GitHub token with ``packages: write`` and admin access to the package. + """ + url = f"{API_URL}/orgs/{OWNER}/packages/container/{package}/versions/{version.version_id}" + _request(url, headers=_api_headers(token), method="DELETE") + + +def clean_package( + package: str, + token: str, + open_prs: Collection[int], + *, + now: dt.datetime, + min_age: dt.timedelta, + dry_run: bool, +) -> int: + """Apply the retention policy to one package. + + Args: + package: Package name. + token: GitHub token. + open_prs: Numbers of the repository's open pull requests. + now: The current time. + min_age: Versions younger than this are always kept. + dry_run: Only log what would be deleted. + + Returns: + The number of deletions that failed. + """ + try: + versions = list_versions(package, token) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + # An unclosed HTTPError leaks its response file (ResourceWarning on 3.14). + error.close() + logger.info("%s: no such package yet, skipping", package) + return 0 + pull_token = registry_token(package, token) + children_of = { + version.digest: child_digests(package, version.digest, pull_token) + for version in versions + if is_retained(version, open_prs, now=now, min_age=min_age) + } + deletions = select_deletions( + versions, open_prs, children_of, now=now, min_age=min_age + ) + logger.info("%s: %d versions, %d to delete", package, len(versions), len(deletions)) + failures = 0 + for version in deletions: + tags = ", ".join(version.tags) or "untagged" + if dry_run: + logger.info(" would delete %s (%s)", version.digest, tags) + continue + try: + delete_version(package, version, token) + except urllib.error.HTTPError as error: + error.close() + failures += 1 + logger.warning( + " failed to delete %s (%s): HTTP %d", version.digest, tags, error.code + ) + else: + logger.info(" deleted %s (%s)", version.digest, tags) + return failures + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the cleanup from the command line.""" + parser = argparse.ArgumentParser( + description="Delete stale monoprop container images from GHCR." + ) + parser.add_argument( + "--dry-run", + action=argparse.BooleanOptionalAction, + default=True, + help="only log what would be deleted (default: on)", + ) + parser.add_argument( + "--min-age-hours", + type=float, + default=24.0, + help="always keep versions younger than this (default: 24)", + ) + parser.add_argument( + "--package", + action="append", + choices=PACKAGES, + help="package to clean; repeatable (default: all)", + ) + args = parser.parse_args(argv) + if args.min_age_hours < 0: + parser.error("--min-age-hours must not be negative") + token = os.environ.get("GITHUB_TOKEN") + if not token: + parser.error("GITHUB_TOKEN is not set") + + logging.basicConfig(level=logging.INFO, format="%(message)s") + open_prs = open_pull_requests(token) + logger.info( + "open pull requests: %s", ", ".join(map(str, sorted(open_prs))) or "none" + ) + now = dt.datetime.now(dt.UTC) + min_age = dt.timedelta(hours=args.min_age_hours) + failures = sum( + clean_package( + package, token, open_prs, now=now, min_age=min_age, dry_run=args.dry_run + ) + for package in args.package or PACKAGES + ) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/container/Dockerfile b/tools/container/Dockerfile new file mode 100644 index 000000000..bb0672446 --- /dev/null +++ b/tools/container/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 + +# monoprop on top of the MPI base image rendered from tools/container/base.py. The build +# context holds only the sdist; container.yml passes BASE_IMAGE (content-addressed) and SDIST. + +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +ARG SDIST +# MareNostrum5's GPP and ACC partitions are both Intel Sapphire Rapids. +ARG MARCH=sapphirerapids + +ENV VIRTUAL_ENV=/opt/monoprop +ENV PATH=${VIRTUAL_ENV}/bin:${PATH} + +# monoprop_ENABLE_MPI must come from the environment: it is what activates the +# scikit-build override that adds mpi4py to the build requirements. +# PIP_ONLY_BINARY makes pip, including its isolated build environment, fail instead +# of compiling mpi4py when its PyPI wheel is missing. +RUN --mount=type=bind,source=${SDIST},target=/tmp/${SDIST} \ + python3 -m venv "${VIRTUAL_ENV}" && \ + PIP_ONLY_BINARY=mpi4py monoprop_ENABLE_MPI=ON pip install --no-cache-dir --verbose \ + --config-settings=cmake.build-type=Release \ + --config-settings=cmake.define.monoprop_ENABLE_ARCH_FLAGS=OFF \ + --config-settings=cmake.define.monoprop_ENABLE_CXX_UNIT_TESTS=OFF \ + "--config-settings=cmake.define.EXTRA_CXXFLAGS=-march=${MARCH}" \ + "--config-settings=build.tool-args=-j$(nproc)" \ + "monoprop[mpi,qiskit] @ file:///tmp/${SDIST}" diff --git a/tools/container/base.py b/tools/container/base.py new file mode 100644 index 000000000..a056c07fb --- /dev/null +++ b/tools/container/base.py @@ -0,0 +1,112 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HPCCM recipe for monoprop's MPI base image. + +Render it with ``uvx hpccm==26.5.0 --recipe tools/container/base.py --format docker``. +container.yml tags the image with a hash of that output, so changing anything here (or +the hpccm version) produces a new base image. + +The compilers, Python, Boost and hwloc come from Ubuntu. The communication path is +pinned: DOCA-OFED's InfiniBand user space, and UCX, PMIx and Open MPI built from source. +Ranks are launched by Slurm with ``srun --mpi=pmix``, so no PMI2 library is built. +""" + +# hpccm runs this file with ``Stage0`` predefined. +# ruff: noqa: INP001, F821 + +from __future__ import annotations + +from hpccm.building_blocks import gnu, openmpi, packages, pmix, python, ucx +from hpccm.primitives import baseimage, comment, environment, label, shell + +UBUNTU_IMAGE = "ubuntu:26.04" +DOCA_OFED_VERSION = "3.5.0" +UCX_VERSION = "1.22.0" +PMIX_VERSION = "5.0.11" +OPENMPI_VERSION = "5.0.11" + +UCX_PREFIX = "/usr/local/ucx" +PMIX_PREFIX = "/usr/local/pmix" + +Stage0 += baseimage(image=UBUNTU_IMAGE, _distro="ubuntu26") +Stage0 += label( + metadata={ + "org.opencontainers.image.source": "https://github.com/Algorithmiq/monoprop", + "org.opencontainers.image.description": '"MPI base image (UCX, PMIx, Open MPI) for monoprop"', + } +) + +compiler = gnu(fortran=False) +Stage0 += compiler +Stage0 += python(python2=False, devel=True) +Stage0 += packages( + apt=[ + "libboost-dev", + "libevent-dev", + "libhwloc-dev", + "libnuma-dev", + "pkg-config", + "python3-venv", + ] +) + +# DOCA-OFED's InfiniBand user space. hpccm's doca_ofed block cannot be used on Ubuntu 26.04: +# it trusts the legacy Mellanox key, which does not sign the 3.5.0 repository (the +# repository ships its own doca_keyring.gpg), and apt 3 on 26.04 aborts the TLS download +# of the repository because linux.mellanox.com closes connections without a close_notify. +# The repository stays signed, so apt fetches it over plain HTTP. +DOCA_REPOSITORY = ( + f"linux.mellanox.com/public/repo/doca/{DOCA_OFED_VERSION}/ubuntu26.04/x86_64" +) +DOCA_KEYRING = "/usr/share/keyrings/doca_keyring.gpg" +DOCA_PACKAGES = ( + "ibverbs-providers ibverbs-utils libibmad-dev libibmad5 libibumad-dev libibumad3" + " libibverbs-dev libibverbs1 librdmacm-dev librdmacm1" +) +Stage0 += comment(f"DOCA OFED version {DOCA_OFED_VERSION}") +Stage0 += packages(ospackages=["ca-certificates", "wget"]) +Stage0 += shell( + commands=[ + f"wget -qO {DOCA_KEYRING} https://{DOCA_REPOSITORY}/doca_keyring.gpg", + f'echo "deb [signed-by={DOCA_KEYRING}] http://{DOCA_REPOSITORY}/ ./" > /etc/apt/sources.list.d/doca.list', + "apt-get update -y", + f"DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends {DOCA_PACKAGES}", + "rm -rf /var/lib/apt/lists/*", + ] +) + +Stage0 += ucx( + version=UCX_VERSION, + prefix=UCX_PREFIX, + ofed=True, + enable_mt=True, + cuda=False, + gdrcopy=False, + knem=False, + xpmem=False, + toolchain=compiler.toolchain, +) +Stage0 += pmix(version=PMIX_VERSION, prefix=PMIX_PREFIX, toolchain=compiler.toolchain) +# Slurm may offer munge authentication to the PMIx client, which is built without it; the +# client then warns before falling back to native. Ask for native directly. +Stage0 += environment(variables={"PMIX_MCA_psec": "native"}) +Stage0 += openmpi( + version=OPENMPI_VERSION, + pmix=PMIX_PREFIX, + ucx=UCX_PREFIX, + cuda=False, + with_slurm=True, + toolchain=compiler.toolchain, +) diff --git a/tools/container/smoke-test.sh b/tools/container/smoke-test.sh new file mode 100755 index 000000000..13d5841bb --- /dev/null +++ b/tools/container/smoke-test.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Checks a built monoprop image without executing monoprop code: the image is compiled +# for -march=$MARCH, which the machine running the check (a CI runner) may not implement. +# +# Usage, inside the image: smoke-test.sh + +set -euo pipefail + +if (( $# != 1 )); then + echo "usage: $0 " >&2 + exit 2 +fi +expected_version="$1" + +# The checks run as root in a throwaway container. +export OMPI_ALLOW_RUN_AS_ROOT=1 +export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 + +fail() { + echo "smoke test failed: $*" >&2 + exit 1 +} + +echo "== monoprop distribution" +installed_version="$(python3 -c 'import importlib.metadata as m; print(m.version("monoprop"))')" +[[ "$installed_version" == "$expected_version" ]] \ + || fail "expected monoprop $expected_version, found $installed_version" +python3 -m pip check + +echo "== shared libraries" +# find_spec locates the package without importing it. +package_dir="$(python3 -c 'import importlib.util, pathlib; print(pathlib.Path(importlib.util.find_spec("monoprop").origin).parent)')" +mapfile -t libraries < <(find "$package_dir" -type f -name '*.so*') +(( ${#libraries[@]} > 0 )) || fail "no shared libraries under $package_dir" +all_dependencies="" +for library in "${libraries[@]}"; do + dependencies="$(ldd "$library")" + if grep -q 'not found' <<< "$dependencies"; then + echo "$dependencies" >&2 + fail "unresolved dependency in $library" + fi + # Collected in a variable: grep -q on a pipe makes ldd die of SIGPIPE, failing pipefail. + all_dependencies+="$dependencies"$'\n' +done +grep -q 'libmpi\.so' <<< "$all_dependencies" || fail "monoprop is not linked against libmpi" + +echo "== MPI library loaded by mpi4py" +ompi_details="$(ompi_info)" +ompi_version="$(awk '/^ *Open MPI:/ {print $3; exit}' <<< "$ompi_details")" +[[ -n "$ompi_version" ]] || fail "ompi_info reported no version" +mpi4py_library="$(python3 -c 'import mpi4py; mpi4py.rc.initialize = False; from mpi4py import MPI; print(MPI.Get_library_version().replace(chr(0), ""))')" +echo "$mpi4py_library" +[[ "$mpi4py_library" == *"Open MPI v${ompi_version}"* ]] \ + || fail "mpi4py did not load the image's Open MPI v$ompi_version" + +echo "== Open MPI build" +grep -q 'MCA pml: ucx' <<< "$ompi_details" || fail "Open MPI was built without the UCX PML" +# The base image builds PMIx into /usr/local/pmix; srun --mpi=pmix talks to it. +mpi_dependencies="$(ldd "$(dirname "$(command -v ompi_info)")/../lib/libmpi.so")" +grep -q '/usr/local/pmix/' <<< "$mpi_dependencies" \ + || fail "Open MPI is not linked against the base image's PMIx" + +echo "== two-rank MPI job" +mpiexec -n 2 --map-by :OVERSUBSCRIBE python3 -c ' +from mpi4py import MPI +comm = MPI.COMM_WORLD +assert comm.Get_size() == 2, comm.Get_size() +assert comm.allreduce(1) == 2 +' + +echo "smoke test passed"