Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .config/.copier-answers.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Changes here will be overwritten by Copier
_commit: v0.0.152
_commit: v0.0.156
_src_path: gh:LabAutomationAndScreening/copier-base-template.git
copier_answers_directory: .config
description: Copier template for creating Python libraries and executables
Expand Down
7 changes: 4 additions & 3 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@
"coderabbit.coderabbit-vscode@0.21.3",
"ms-vscode.live-server@0.5.2025051301",
"MS-vsliveshare.vsliveshare@1.0.5905",
"anthropic.claude-code@2.1.220",
"anthropic.claude-code@2.1.238",

// Python
"-ms-python.vscode-pylance", // pyrefly is the language server for this project; ms-python.python auto-installs Pylance via its extension pack, and Pylance then warns on every boot that it's disabled
"ms-python.python@2026.5.2026052901",
"meta.pyrefly@1.1.9003",
"meta.pyrefly@1.2.9002",
"ms-vscode-remote.remote-containers@0.414.0",
"charliermarsh.ruff@2026.68.0",

Expand Down Expand Up @@ -60,5 +61,5 @@
"initializeCommand": "sh .devcontainer/initialize-command.sh",
"onCreateCommand": "sh .devcontainer/on-create-command.sh",
"postStartCommand": "sh .devcontainer/post-start-command.sh"
// Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): ad6af682 # spellchecker:disable-line
// Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): 6a68e568 # spellchecker:disable-line
}
8 changes: 4 additions & 4 deletions .devcontainer/install-ci-tooling.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
import tempfile
from pathlib import Path

UV_VERSION = "0.12.3"
PNPM_VERSION = "11.21.0"
UV_VERSION = "0.12.5"
PNPM_VERSION = "11.22.0"
COPIER_VERSION = "==9.17.1"
COPIER_TEMPLATE_EXTENSIONS_VERSION = "==0.3.3"
PRE_COMMIT_VERSION = "4.5.1"
PRE_COMMIT_VERSION = "4.6.2"
GITHUB_WINDOWS_RUNNER_BIN_PATH = r"C:\Users\runneradmin\.local\bin"
INSTALL_SSM_PLUGIN_BY_DEFAULT = False
parser = argparse.ArgumentParser(description="Install CI tooling for the repo")
Expand Down Expand Up @@ -61,7 +61,7 @@ def main():
# invoke installer in a pwsh process
_ = subprocess.run( # noqa: S603 # this is all our own input
[
pwsh, # type: ignore[reportPossiblyUnboundVariable] # this matches the conditional above that defines pwsh
pwsh,
"-NoProfile",
"-NonInteractive",
"-Command",
Expand Down
43 changes: 27 additions & 16 deletions .github/actions/install_deps/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,50 +9,44 @@ name: Install requirements
description: Setup Python and/or Node, and install dependencies based on the devcontainer specification
inputs:
python-version:
type: string
description: Python version to install
required: false # if this is set, then assume you want python dependencies installed
default: 'notUsing'
node-version:
type: string
description: Node version to install
required: false # if this is set, then assume you want node dependencies installed
default: 'notUsing'
package-json-file:
description: Relative path to a package.json to read the pnpm version from (its `packageManager` or `devEngines.packageManager` field). When unset, the pnpm version pinned by the template is used instead.
required: false
default: 'notUsing'
install-deps:
required: false
default: true
type: boolean
description: Whether to run the setup-deps script, or just to setup basic CI tooling
skip-installing-ssm-plugin-manager:
required: false
default: false
type: boolean
description: Whether to explicitly skip installing the SSM Plugin manager when setting up basic CI tooling
project-dir:
type: string
description: What's the relative path to the project?
required: false
default: ./
code-artifact-auth-role-name:
type: string
description: What's the role name to use for CodeArtifact authentication?
required: false
default: no-code-artifact
code-artifact-auth-role-account-id:
type: string
description: What's the AWS Account ID that the role is in?
required: false
code-artifact-auth-region:
type: string
description: What region should the role use?
required: false
skip-updating-devcontainer-hash:
type: boolean
description: Whether to skip updating the hash when running manual-setup-deps.py
default: true
required: false
skip-installing-pulumi-cli:
type: boolean
description: Whether to skip installing the Pulumi CLI even if the lock file references it
default: false
required: false
Expand All @@ -70,24 +64,41 @@ runs:

- name: Setup python
if: ${{ inputs.python-version != 'notUsing' }}
uses: actions/setup-python@v6.2.0
uses: actions/setup-python@v7.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}

- name: Setup node
- name: Check for a pnpm lock file # pnpm/setup can only cache the store when there's a lock file at the repo root to key the cache off of
id: pnpm-lock-probe
if: ${{ inputs.node-version != 'notUsing' }}
run: |
if [ -f pnpm-lock.yaml ]; then
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
shell: bash

- name: Setup pnpm and node # this action installs both pnpm and the node runtime, so actions/setup-node is not needed
if: ${{ inputs.node-version != 'notUsing' }}
uses: actions/setup-node@v6.4.0
uses: pnpm/setup@v2.0.2
with:
node-version: ${{ inputs.node-version }}
# an empty version makes pnpm/setup read it from package-json-file instead. Passing both a version and a package.json with a conflicting `packageManager` field is a hard error in pnpm/setup.
version: ${{ inputs.package-json-file == 'notUsing' && '11.22.0' || '' }}
package-json-file: ${{ inputs.package-json-file == 'notUsing' && 'package.json' || inputs.package-json-file }}
runtime: node@${{ inputs.node-version }}
cache: ${{ steps.pnpm-lock-probe.outputs.exists }}
install: false # dependency installation is handled by manual-setup-deps.py, which knows about every environment in the repo

- name: Install tooling
# the funky syntax is github action ternary
run: python .devcontainer/install-ci-tooling.py ${{ inputs.python-version == 'notUsing' && '--no-python' || '' }} ${{ inputs.node-version == 'notUsing' && '--no-node' || '' }} ${{ inputs.skip-installing-ssm-plugin-manager && '--skip-installing-ssm-plugin' || '' }}
# --no-node is always passed because the only node tooling this script installs is pnpm, which pnpm/setup above already installed
run: python .devcontainer/install-ci-tooling.py ${{ inputs.python-version == 'notUsing' && '--no-python' || '' }} --no-node ${{ inputs.skip-installing-ssm-plugin-manager && '--skip-installing-ssm-plugin' || '' }}
shell: pwsh

- name: OIDC Auth for CodeArtifact
if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }}
uses: aws-actions/configure-aws-credentials@v6.1.0
uses: aws-actions/configure-aws-credentials@v6.2.3
with:
role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }}
aws-region: ${{ inputs.code-artifact-auth-region }}
Expand Down
2 changes: 1 addition & 1 deletion .github/actions/update-devcontainer-hash/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ runs:
shell: bash

- name: Checkout code
uses: actions/checkout@v6.0.2
uses: actions/checkout@v7.0.1
with:
persist-credentials: true
fetch-depth: 1
Expand Down
62 changes: 52 additions & 10 deletions .github/reusable_workflows/build-docker-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ jobs:
shell: bash

- name: Checkout code
uses: actions/checkout@v6.0.2
uses: actions/checkout@v7.0.1
with:
persist-credentials: false

Expand Down Expand Up @@ -127,16 +127,30 @@ jobs:
echo "Image name without slashes: ${IMAGE_NAME_NO_SLASHES}"
echo "full_image_tag=${{ inputs.repository }}/${{ inputs.image_name }}:context-${BUILD_HASH}" >> "$GITHUB_OUTPUT"

- name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658
- name: Test if docker image exists # deliberately before the mutex; describe-images is a read, so it is safe to run concurrently
if: ${{ inputs.push-role-name != 'no-push' }}
id: check-if-exists
run: |
BUILD_HASH=${{ steps.calculate-build-context-hash.outputs.build_context_tag }}
echo Checking for : $BUILD_HASH
if aws ecr describe-images --region ${{ steps.parse_ecr_url.outputs.aws_region }} --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-ids=imageTag=$BUILD_HASH; then \
echo "Image was found in ECR"; \
echo "status=found" >> $GITHUB_OUTPUT
else \
echo "Image was not found in ECR"; \
echo "status=notfound" >> $GITHUB_OUTPUT
fi

- name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658
if: ${{ inputs.push-role-name != 'no-push' && steps.check-if-exists.outputs.status == 'notfound' }} # only serialize when this job may have to build and push a new image
uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10
with:
branch: mutex-${{ inputs.repository }}-${{ inputs.image_name }}
timeout-minutes: 8 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it

- name: Test if docker image exists
if: ${{ inputs.push-role-name != 'no-push' }}
id: check-if-exists
- name: Re-test if docker image exists # another job may have pushed this exact build context while we waited on the mutex
if: ${{ inputs.push-role-name != 'no-push' && steps.check-if-exists.outputs.status == 'notfound' }}
id: recheck-if-exists
run: |
BUILD_HASH=${{ steps.calculate-build-context-hash.outputs.build_context_tag }}
echo Checking for : $BUILD_HASH
Expand All @@ -148,24 +162,34 @@ jobs:
echo "status=notfound" >> $GITHUB_OUTPUT
fi

- name: Resolve whether the image needs building # collapses the two probes into the single status the rest of the job keys off
if: ${{ inputs.push-role-name != 'no-push' }}
id: image-status
run: |
if [ "${{ steps.check-if-exists.outputs.status }}" = "found" ]; then
echo "status=found" >> $GITHUB_OUTPUT
else
echo "status=${{ steps.recheck-if-exists.outputs.status }}" >> $GITHUB_OUTPUT
fi

- name: Pull existing image to package as artifact
if: ${{ inputs.save-as-artifact && steps.check-if-exists.outputs.status == 'found' }}
if: ${{ inputs.save-as-artifact && steps.image-status.outputs.status == 'found' }}
run: |
docker pull ${{ steps.calculate-build-context-hash.outputs.full_image_tag }}

- name: Set up Docker Buildx
if: ${{ (inputs.save-as-artifact && inputs.push-role-name == 'no-push') || steps.check-if-exists.outputs.status == 'notfound' }}
if: ${{ (inputs.save-as-artifact && inputs.push-role-name == 'no-push') || steps.image-status.outputs.status == 'notfound' }}
uses: docker/setup-buildx-action@v4.0.0
with:
version: v0.33.0

- name: Build Docker Image
if: ${{ (inputs.save-as-artifact && inputs.push-role-name == 'no-push') || steps.check-if-exists.outputs.status == 'notfound' }}
if: ${{ (inputs.save-as-artifact && inputs.push-role-name == 'no-push') || steps.image-status.outputs.status == 'notfound' }}
uses: docker/build-push-action@v7.1.0
with:
context: ${{ inputs.context }}
build-contexts: ${{ inputs.build-contexts }}
push: ${{ inputs.push-role-name != 'no-push' && steps.check-if-exists.outputs.status == 'notfound' }}
push: ${{ inputs.push-role-name != 'no-push' && steps.image-status.outputs.status == 'notfound' }}
load: ${{ inputs.save-as-artifact }} # make the image available later for the `docker save` step
tags: ${{ steps.calculate-build-context-hash.outputs.full_image_tag }}

Expand All @@ -177,12 +201,27 @@ jobs:
echo "Tag $GIT_SHA_TAG already exists in ECR, skipping"
else
aws ecr batch-get-image --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-ids imageTag=${{ steps.calculate-build-context-hash.outputs.build_context_tag }} --query 'images[].imageManifest' --output text > manifest.json
aws ecr put-image --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-tag "$GIT_SHA_TAG" --image-manifest file://manifest.json
# Every job on the push path tags the git sha, and jobs that find their image already in ECR no longer serialize on the mutex,
# so another job can apply this same tag between the check above and the put below. ECR reports that as
# ImageAlreadyExistsException, which by definition means the tag already points at this exact manifest, so it is safe to ignore.
# ImageTagAlreadyExistsException (a different manifest on an immutable repository) is a real conflict and still fails the step.
if ! aws ecr put-image --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-tag "$GIT_SHA_TAG" --image-manifest file://manifest.json 2> put-image-error.txt; then
cat put-image-error.txt
if ! grep -q 'ImageAlreadyExistsException' put-image-error.txt; then
exit 1
fi
echo "Tag $GIT_SHA_TAG was applied concurrently by another job, nothing to do"
fi
fi

- name: Add tag for Production
if: ${{ inputs.push-role-name != 'no-push' && inputs.tag-for-production }}
run: |
# Unlike the git sha tag above, a concurrent ImageAlreadyExistsException is deliberately left fatal here.
# This step only runs from deploy workflows, which are not expected to overlap for the same image, so a
# duplicate put-image points at something unexpected about how the deploy was triggered and should be loud
# rather than swallowed. Note that PRODUCTION_CONTEXT_TAG keys on the build context hash rather than the
# commit, so two commits that leave the build context unchanged do share a tag.
aws ecr batch-get-image --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-ids imageTag=${{ steps.calculate-build-context-hash.outputs.build_context_tag }} --query 'images[].imageManifest' --output text > manifest.json
PRODUCTION_CONTEXT_TAG="production--${{ steps.calculate-build-context-hash.outputs.build_context_tag }}"
if aws ecr describe-images --region ${{ steps.parse_ecr_url.outputs.aws_region }} --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-ids imageTag="$PRODUCTION_CONTEXT_TAG" > /dev/null 2>&1; then
Expand All @@ -205,6 +244,9 @@ jobs:
echo "Invalid release-tag: '$RELEASE_TAG'. Must start with alphanumeric or underscore and contain only alphanumerics, dots, hyphens, underscores (max 128 chars)."
exit 1
fi
# This check is an intentional immutability guard rather than a concurrency one: reusing a version number is a
# human mistake worth failing on. A concurrent ImageAlreadyExistsException below is likewise left fatal, since
# release workflows are not expected to run twice at once for the same tag.
if aws ecr describe-images --region ${{ steps.parse_ecr_url.outputs.aws_region }} --registry-id=${{ steps.parse_ecr_url.outputs.aws_account_id }} --repository-name=${{ inputs.image_name }} --image-ids imageTag="$RELEASE_TAG" > /dev/null 2>&1; then
echo "Release tag '$RELEASE_TAG' already exists in ECR repository '${{ inputs.image_name }}'. Release tags are immutable; pick a new version or delete the existing tag if this was unintended."
exit 1
Expand Down
Loading