From 226aa8948a392cd7329180e328412d8e7ea9c60a Mon Sep 17 00:00:00 2001 From: Roberto Montero Date: Thu, 5 Nov 2026 14:54:29 +0100 Subject: [PATCH 1/3] Retry FPD, JUnit, and S3 publishes in the GitLab after_script. A stalled publish could hit the runner timeout and fail a job whose tests had passed. Co-authored-by: Cursor --- .../test_the_test/test_gitlab_finalize_job.py | 47 ++++++++++++++++++- utils/scripts/gitlab_finalize_job.py | 41 ++++++++++++---- 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/tests/test_the_test/test_gitlab_finalize_job.py b/tests/test_the_test/test_gitlab_finalize_job.py index 9a331157867..728c1dbbfb7 100644 --- a/tests/test_the_test/test_gitlab_finalize_job.py +++ b/tests/test_the_test/test_gitlab_finalize_job.py @@ -22,7 +22,7 @@ def commands(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: """Record the commands run by the steps instead of running them.""" recorded: list[list[str]] = [] - def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001 + def run(command: list[str], **_kwargs: object) -> None: recorded.append(command) monkeypatch.setattr(finalizer, "run", run) @@ -135,11 +135,54 @@ def test_log_folders_with_sockets_are_collected(tmp_path: Path, commands: list[l assert commands[-1][0] == "bash" +@scenarios.test_the_test +def test_publish_failures_are_retried_and_then_ignored(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + calls: list[str] = [] + + def run(command: list[str], **_kwargs: object) -> None: + calls.append(command[0]) + if command[0] in {"curl", "aws"} or (command[0] == "datadog-ci" and "junit" in command): + raise subprocess.CalledProcessError(56, command) + + monkeypatch.setattr(finalizer, "run", run) + monkeypatch.setattr(finalizer, "read_secret", lambda *path: "/".join(path)) + monkeypatch.setattr(finalizer.time, "sleep", lambda _delay: None) + monkeypatch.setenv("HOME", str(tmp_path / "home")) + + logs = tmp_path / "system-tests" / "logs_docker_ssi" + _write(logs / "feature_parity.json", "{}") + _write(logs / "reportJunit.xml", "") + docker_job = _job( + "docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="system-tests", CI_COMMIT_BRANCH="main" + ) + + assert finalizer.finalize(docker_job) == 0 + assert calls.count("curl") == finalizer.PUBLISH_ATTEMPTS + assert calls.count("datadog-ci") == 1 + finalizer.PUBLISH_ATTEMPTS + + calls.clear() + _write(tmp_path / "system-tests" / "logs_simple_installer_auto_injection" / "vm_feature_parity.json", "{}") + aws_job = _job( + "aws", + tmp_path, + SCENARIO="SIMPLE_INSTALLER_AUTO_INJECTION", + VM="Ubuntu_22_amd64", + WEBLOG="test-app-java", + CI_PIPELINE_ID="42", + CI_PROJECT_NAME="system-tests", + CI_COMMIT_BRANCH="main", + ) + + assert finalizer.finalize(aws_job) == 0 + assert calls.count("aws") == finalizer.PUBLISH_ATTEMPTS + assert aws_job.aws_console_url == "" + + @scenarios.test_the_test def test_a_failing_step_does_not_stop_the_next_ones( tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch ): - def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001 + def run(command: list[str], **_kwargs: object) -> None: if command[0] == "datadog-ci": raise subprocess.CalledProcessError(1, command) commands.append(command) diff --git a/utils/scripts/gitlab_finalize_job.py b/utils/scripts/gitlab_finalize_job.py index 0054d7049ee..86a59ec4273 100644 --- a/utils/scripts/gitlab_finalize_job.py +++ b/utils/scripts/gitlab_finalize_job.py @@ -23,11 +23,15 @@ import shutil import subprocess import sys +import time SYSTEM_TESTS_DIR = Path(__file__).resolve().parents[2] AWS_LOGS_BUCKET = "system-tests-aws-ssi-apm" MAX_ARTIFACT_FILE_SIZE = 50 * 1024 * 1024 +PUBLISH_ATTEMPTS = 3 +PUBLISH_TIMEOUT_SECONDS = 30 +PUBLISH_RETRY_DELAY_SECONDS = 5 @dataclass @@ -70,8 +74,28 @@ def datadog_env(self) -> dict[str, str]: Step = Callable[[Job], None] -def run(command: list[str], env: Mapping[str, str] | None = None, cwd: Path | None = None) -> None: - subprocess.run(command, check=True, cwd=cwd, env=None if env is None else {**os.environ, **env}) +def run( + command: list[str], + env: Mapping[str, str] | None = None, + cwd: Path | None = None, + timeout: float | None = None, +) -> None: + subprocess.run(command, check=True, cwd=cwd, timeout=timeout, env=None if env is None else {**os.environ, **env}) + + +def run_publish(command: list[str], env: Mapping[str, str] | None = None) -> bool: + """Retry a best-effort publish. A remaining failure is logged and ignored.""" + for attempt in range(1, PUBLISH_ATTEMPTS + 1): + try: + run(command, env=env, timeout=PUBLISH_TIMEOUT_SECONDS) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + print(f"⚠️ {command[0]} attempt {attempt}/{PUBLISH_ATTEMPTS} failed: {error}", flush=True) + if attempt < PUBLISH_ATTEMPTS: + time.sleep(PUBLISH_RETRY_DELAY_SECONDS) + continue + return True + print(f"⚠️ {command[0]} failed after {PUBLISH_ATTEMPTS} attempts, ignored", flush=True) + return False @cache @@ -100,13 +124,14 @@ def upload_logs_to_s3(job: Job) -> None: for folder in job.log_folders: pipeline_id = job.env.get("CI_PIPELINE_ID", "datadog-local") prefix = f"{pipeline_id}/{folder.name}/{job.env['VM']}_{job.env['WEBLOG']}/" - run( + uploaded = run_publish( ["aws", "s3", "cp", str(folder), f"s3://{AWS_LOGS_BUCKET}/{prefix}", "--recursive", "--quiet"], env={"AWS_PROFILE": "apm-ecosystems-reliability-ci"}, ) - job.aws_console_url = ( - f"https://s3.console.aws.amazon.com/s3/buckets/{AWS_LOGS_BUCKET}?region=us-east-1&prefix={prefix}" - ) + if uploaded: + job.aws_console_url = ( + f"https://s3.console.aws.amazon.com/s3/buckets/{AWS_LOGS_BUCKET}?region=us-east-1&prefix={prefix}" + ) def delete_large_log_files(job: Job) -> None: @@ -138,7 +163,7 @@ def publish_feature_parity(job: Job) -> None: api_key = read_secret("fpd", "fp_api_key") for report in sorted(job.reports_dir.glob(f"logs*/{job.profile.feature_parity_pattern}")): print(f"Processing report: {report}") - run( + run_publish( [ "curl", "-X", @@ -161,7 +186,7 @@ def upload_junit_reports(job: Job) -> None: if not reports: print("No reports/logs*/reportJunit.xml found; skipping Test Optimization upload") return - run( + run_publish( [ "datadog-ci", "junit", From 17f0391ca9e68a35685a9270c286ff54d0baa4bb Mon Sep 17 00:00:00 2001 From: Roberto Montero Date: Thu, 5 Nov 2026 15:02:50 +0100 Subject: [PATCH 2/3] Fail the after_script when FPD or JUnit retries are exhausted. Only the S3 log upload stays exit 0 after three failed attempts. Co-authored-by: Cursor --- .../test_the_test/test_gitlab_finalize_job.py | 43 ------------------- utils/scripts/gitlab_finalize_job.py | 23 +++++++--- 2 files changed, 17 insertions(+), 49 deletions(-) diff --git a/tests/test_the_test/test_gitlab_finalize_job.py b/tests/test_the_test/test_gitlab_finalize_job.py index 728c1dbbfb7..8bf8245842e 100644 --- a/tests/test_the_test/test_gitlab_finalize_job.py +++ b/tests/test_the_test/test_gitlab_finalize_job.py @@ -135,49 +135,6 @@ def test_log_folders_with_sockets_are_collected(tmp_path: Path, commands: list[l assert commands[-1][0] == "bash" -@scenarios.test_the_test -def test_publish_failures_are_retried_and_then_ignored(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): - calls: list[str] = [] - - def run(command: list[str], **_kwargs: object) -> None: - calls.append(command[0]) - if command[0] in {"curl", "aws"} or (command[0] == "datadog-ci" and "junit" in command): - raise subprocess.CalledProcessError(56, command) - - monkeypatch.setattr(finalizer, "run", run) - monkeypatch.setattr(finalizer, "read_secret", lambda *path: "/".join(path)) - monkeypatch.setattr(finalizer.time, "sleep", lambda _delay: None) - monkeypatch.setenv("HOME", str(tmp_path / "home")) - - logs = tmp_path / "system-tests" / "logs_docker_ssi" - _write(logs / "feature_parity.json", "{}") - _write(logs / "reportJunit.xml", "") - docker_job = _job( - "docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="system-tests", CI_COMMIT_BRANCH="main" - ) - - assert finalizer.finalize(docker_job) == 0 - assert calls.count("curl") == finalizer.PUBLISH_ATTEMPTS - assert calls.count("datadog-ci") == 1 + finalizer.PUBLISH_ATTEMPTS - - calls.clear() - _write(tmp_path / "system-tests" / "logs_simple_installer_auto_injection" / "vm_feature_parity.json", "{}") - aws_job = _job( - "aws", - tmp_path, - SCENARIO="SIMPLE_INSTALLER_AUTO_INJECTION", - VM="Ubuntu_22_amd64", - WEBLOG="test-app-java", - CI_PIPELINE_ID="42", - CI_PROJECT_NAME="system-tests", - CI_COMMIT_BRANCH="main", - ) - - assert finalizer.finalize(aws_job) == 0 - assert calls.count("aws") == finalizer.PUBLISH_ATTEMPTS - assert aws_job.aws_console_url == "" - - @scenarios.test_the_test def test_a_failing_step_does_not_stop_the_next_ones( tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch diff --git a/utils/scripts/gitlab_finalize_job.py b/utils/scripts/gitlab_finalize_job.py index 86a59ec4273..e1a9eebebfe 100644 --- a/utils/scripts/gitlab_finalize_job.py +++ b/utils/scripts/gitlab_finalize_job.py @@ -83,19 +83,29 @@ def run( subprocess.run(command, check=True, cwd=cwd, timeout=timeout, env=None if env is None else {**os.environ, **env}) -def run_publish(command: list[str], env: Mapping[str, str] | None = None) -> bool: - """Retry a best-effort publish. A remaining failure is logged and ignored.""" +def run_publish(command: list[str], env: Mapping[str, str] | None = None, *, ignore_failure: bool = False) -> bool: + """Retry a publish command. + + Raise the last error when every attempt fails. The S3 log upload sets ignore_failure + so a remaining failure is logged and the after_script still exits 0. + """ + error: subprocess.CalledProcessError | subprocess.TimeoutExpired | None = None for attempt in range(1, PUBLISH_ATTEMPTS + 1): try: run(command, env=env, timeout=PUBLISH_TIMEOUT_SECONDS) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: - print(f"⚠️ {command[0]} attempt {attempt}/{PUBLISH_ATTEMPTS} failed: {error}", flush=True) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as attempt_error: + error = attempt_error + print(f"⚠️ {command[0]} attempt {attempt}/{PUBLISH_ATTEMPTS} failed: {attempt_error}", flush=True) if attempt < PUBLISH_ATTEMPTS: time.sleep(PUBLISH_RETRY_DELAY_SECONDS) continue return True - print(f"⚠️ {command[0]} failed after {PUBLISH_ATTEMPTS} attempts, ignored", flush=True) - return False + if error is None: + raise RuntimeError(f"{command[0]} failed without an error") + if ignore_failure: + print(f"⚠️ {command[0]} failed after {PUBLISH_ATTEMPTS} attempts, ignored", flush=True) + return False + raise error @cache @@ -127,6 +137,7 @@ def upload_logs_to_s3(job: Job) -> None: uploaded = run_publish( ["aws", "s3", "cp", str(folder), f"s3://{AWS_LOGS_BUCKET}/{prefix}", "--recursive", "--quiet"], env={"AWS_PROFILE": "apm-ecosystems-reliability-ci"}, + ignore_failure=True, ) if uploaded: job.aws_console_url = ( From 5b6a8d983cddae50a0d8d4766dc84c67837c6cd5 Mon Sep 17 00:00:00 2001 From: Roberto Montero Date: Fri, 6 Nov 2026 15:06:52 +0100 Subject: [PATCH 3/3] Keep publish failures out of the job log and give S3 a longer timeout. The feature-parity curl passes FP_API_KEY on the command line, so retry warnings now record only the program and the status. S3 gets two 60-second attempts because the raw log folder is uploaded before large files are removed. Co-authored-by: Cursor --- utils/scripts/gitlab_finalize_job.py | 44 ++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/utils/scripts/gitlab_finalize_job.py b/utils/scripts/gitlab_finalize_job.py index e1a9eebebfe..88388947c37 100644 --- a/utils/scripts/gitlab_finalize_job.py +++ b/utils/scripts/gitlab_finalize_job.py @@ -32,6 +32,10 @@ PUBLISH_ATTEMPTS = 3 PUBLISH_TIMEOUT_SECONDS = 30 PUBLISH_RETRY_DELAY_SECONDS = 5 +# The raw log folder is uploaded before large files are removed. Two attempts of 60s, plus a +# full feature-parity retry, stay inside the runner's 5-minute after_script timeout. +S3_PUBLISH_ATTEMPTS = 2 +S3_PUBLISH_TIMEOUT_SECONDS = 60 @dataclass @@ -83,29 +87,41 @@ def run( subprocess.run(command, check=True, cwd=cwd, timeout=timeout, env=None if env is None else {**os.environ, **env}) -def run_publish(command: list[str], env: Mapping[str, str] | None = None, *, ignore_failure: bool = False) -> bool: +def _publish_failure(command_name: str, error: subprocess.CalledProcessError | subprocess.TimeoutExpired) -> str: + """Status only. CalledProcessError stringifies the whole argument list.""" + if isinstance(error, subprocess.TimeoutExpired): + return f"{command_name} timed out after {error.timeout} seconds" + return f"{command_name} exited with status {error.returncode}" + + +def run_publish( + command: list[str], + env: Mapping[str, str] | None = None, + *, + attempts: int = PUBLISH_ATTEMPTS, + timeout: float = PUBLISH_TIMEOUT_SECONDS, + ignore_failure: bool = False, +) -> bool: """Retry a publish command. - Raise the last error when every attempt fails. The S3 log upload sets ignore_failure - so a remaining failure is logged and the after_script still exits 0. + Raise when every attempt fails. The S3 log upload sets ignore_failure so a remaining + failure is logged and the after_script still exits 0. """ - error: subprocess.CalledProcessError | subprocess.TimeoutExpired | None = None - for attempt in range(1, PUBLISH_ATTEMPTS + 1): + failure = "" + for attempt in range(1, attempts + 1): try: - run(command, env=env, timeout=PUBLISH_TIMEOUT_SECONDS) + run(command, env=env, timeout=timeout) except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as attempt_error: - error = attempt_error - print(f"⚠️ {command[0]} attempt {attempt}/{PUBLISH_ATTEMPTS} failed: {attempt_error}", flush=True) - if attempt < PUBLISH_ATTEMPTS: + failure = _publish_failure(command[0], attempt_error) + print(f"⚠️ {failure} (attempt {attempt}/{attempts})", flush=True) + if attempt < attempts: time.sleep(PUBLISH_RETRY_DELAY_SECONDS) continue return True - if error is None: - raise RuntimeError(f"{command[0]} failed without an error") if ignore_failure: - print(f"⚠️ {command[0]} failed after {PUBLISH_ATTEMPTS} attempts, ignored", flush=True) + print(f"⚠️ {failure} after {attempts} attempts, ignored", flush=True) return False - raise error + raise RuntimeError(f"{failure} after {attempts} attempts") @cache @@ -137,6 +153,8 @@ def upload_logs_to_s3(job: Job) -> None: uploaded = run_publish( ["aws", "s3", "cp", str(folder), f"s3://{AWS_LOGS_BUCKET}/{prefix}", "--recursive", "--quiet"], env={"AWS_PROFILE": "apm-ecosystems-reliability-ci"}, + attempts=S3_PUBLISH_ATTEMPTS, + timeout=S3_PUBLISH_TIMEOUT_SECONDS, ignore_failure=True, ) if uploaded: