Skip to content

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ericdog wants to merge 1 commit into
base: main
Choose a base branch
from
+19 −11
Draft

[CIEXE-2681] Use published Docker CI images with internal Buildx #882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions ci/input_files/build.yaml.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ default:
# Retry when the runner fails to start
- runner_system_failure

# The Noble Python image uses managed system Python and does not include AWS CLI.
.install-python-tools: &install-python-tools
- apt-get update
- apt-get install -y --no-install-recommends awscli python3-venv python-is-python3

# This is for serverless framework
.install-node: &install-node
- apt-get update
Expand All @@ -38,7 +43,7 @@ default:
build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
artifacts:
expire_in: 1 day # Unsigned zips expire in 1 day
paths:
Expand All @@ -53,7 +58,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
needs:
- build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
dependencies:
Expand Down Expand Up @@ -99,7 +104,7 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: test
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand All @@ -112,6 +117,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
variables:
CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
before_script:
- *install-python-tools
- *install-node
- EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
- yarn global add serverless@^3.38.0 --prefix /usr/local
Expand All @@ -124,7 +130,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: sign
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
when: manual
Expand All @@ -141,6 +147,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
paths:
- .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip
before_script:
- *install-python-tools
- apt-get update
- apt-get install -y uuid-runtime
{{ with $environment := (ds "environments").environments.prod }}
Expand All @@ -155,7 +162,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
rules:
- if: '$SKIP_E2E_TESTS == "true"'
when: never
Expand Down Expand Up @@ -190,6 +197,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
- {{ .code }}
{{- end}}
before_script:
- *install-python-tools
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
script:
- STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
Expand All @@ -201,8 +209,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
publish-pypi-package:
stage: publish
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
before_script:
- *install-python-tools
- ./scripts/setup_python_env.sh
cache: []
rules:
Expand All @@ -217,7 +226,7 @@ publish-pypi-package:
layer bundle:
stage: build
tags: ["arch:amd64"]
image: registry.ddbuild.io/images/docker:20.10
image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
needs:
{{ range (ds "runtimes").runtimes }}
- build-layer ({{ .name }}-{{ .arch }})
Expand All @@ -238,7 +247,7 @@ layer bundle:

signed layer bundle:
stage: sign
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
tags: ["arch:amd64"]
rules:
- if: '$CI_COMMIT_TAG =~ /^v.*/'
Expand Down Expand Up @@ -286,7 +295,7 @@ e2e-test:

e2e-test-status:
stage: e2e
image: registry.ddbuild.io/images/docker:20.10-py3
image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
tags: ["arch:amd64"]
timeout: 3h
rules:
Expand Down
3 changes: 1 addition & 2 deletions scripts/setup_python_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ]
spec_ddtrace_dep
fi

pip install virtualenv
virtualenv venv
python3 -m venv venv
source venv/bin/activate
pip install .[dev]
pip install poetry
Expand Down
Loading