[CIEXE-2681] Use published internal-Buildx Docker CI images

Author: ericdog

ci/input_files/build.yaml.tpl

@@ -14,6 +14,11 @@ default:
       # Retry when the runner fails to start
       - runner_system_failure
 
+# The Noble Python image uses managed system Python and does not include AWS CLI.
+.install-python-tools: &install-python-tools
+  - apt-get update
+  - apt-get install -y --no-install-recommends awscli python3-venv python-is-python3
+
 # This is for serverless framework
 .install-node: &install-node
   - apt-get update
@@ -38,7 +43,7 @@ default:
 build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: build
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
+  image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
   artifacts:
     expire_in: 1 day # Unsigned zips expire in 1 day
     paths:
@@ -53,7 +58,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
 check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: test
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
+  image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
   needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
@@ -99,7 +104,7 @@ unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
 integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: test
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   rules:
     - if: '$SKIP_E2E_TESTS == "true"'
       when: never
@@ -112,6 +117,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
   variables:
     CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
   before_script:
+    - *install-python-tools
     - *install-node
     - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
     - yarn global add serverless@^3.38.0 --prefix /usr/local
@@ -124,7 +130,7 @@ integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
 sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: sign
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
       when: manual
@@ -141,6 +147,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
     paths:
       - .layers/datadog_lambda_py-{{ $runtime.arch }}-{{ $runtime.python_version }}.zip
   before_script:
+    - *install-python-tools
     - apt-get update
     - apt-get install -y uuid-runtime
     {{ with $environment := (ds "environments").environments.prod }}
@@ -155,7 +162,7 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
 publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: publish
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   rules:
     - if: '$SKIP_E2E_TESTS == "true"'
       when: never
@@ -190,6 +197,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
           - {{ .code }}
         {{- end}}
   before_script:
+    - *install-python-tools
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
@@ -201,8 +209,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
 publish-pypi-package:
   stage: publish
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   before_script:
+    - *install-python-tools
     - ./scripts/setup_python_env.sh
   cache: []
   rules:
@@ -217,7 +226,7 @@ publish-pypi-package:
 layer bundle:
   stage: build
   tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
+  image: registry.ddbuild.io/images/docker:27.3.1@sha256:966b6cd18c53e3730e7aaea2c0ca6b6b9877dcff11d9446f25ee5bc2293585d0
   needs:
     {{ range (ds "runtimes").runtimes }}
     - build-layer ({{ .name }}-{{ .arch }})
@@ -238,7 +247,7 @@ layer bundle:
 
 signed layer bundle:
   stage: sign
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   tags: ["arch:amd64"]
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
@@ -286,7 +295,7 @@ e2e-test:
 
 e2e-test-status:
   stage: e2e
-  image: registry.ddbuild.io/images/docker:20.10-py3
+  image: registry.ddbuild.io/images/docker:29.4.0-noble-py3@sha256:a1e09b5d90ac6c8576313064f1b0e5809ba718355715fb43a832ea7aa1ec7b70
   tags: ["arch:amd64"]
   timeout: 3h
   rules:

scripts/setup_python_env.sh

@@ -65,8 +65,7 @@ if [ -z "${IS_LINT:-}" ] || [ "${IS_LINT}" = "0" ] || [ "${IS_LINT}" = "false" ]
     spec_ddtrace_dep
 fi
 
-pip install virtualenv
-virtualenv venv
+python3 -m venv venv
 source venv/bin/activate
 pip install .[dev]
 pip install poetry