From 3ab9a29f96382cb9efb5b9e9c9a7c1b494981019 Mon Sep 17 00:00:00 2001 From: Ji Jia Jia Date: Wed, 30 Sep 2026 08:46:55 +0200 Subject: [PATCH 1/4] Restart the messenger worker after install and keep container logs on failure The supervisord container is started in step 1, before Pimcore is installed, and nothing restarts it afterwards. Its messenger:consume worker (time-limit 3600) therefore keeps whatever state it booted with. Since 2026-09-17 every execution-engine job in the API tests stays "running" forever, which is what fails the three ZIP downloads and HideCompletedJobSucceeds. - restart supervisord at the end of 02-install-pimcore.sh - add 07-collect-logs.sh and upload container logs + var/log on failure, so the worker's output is no longer discarded by 04-shutdown.sh; secret values are redacted because the artifact is public Co-Authored-By: Claude Opus 5.5 --- .github/scripts/02-install-pimcore.sh | 6 +++ .github/scripts/07-collect-logs.sh | 55 +++++++++++++++++++++++++++ .github/workflows/api-tests.yml | 14 +++++++ .gitignore | 3 ++ doc/04_API_Testing/01_API_TESTS.md | 5 ++- 5 files changed, 81 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/07-collect-logs.sh diff --git a/.github/scripts/02-install-pimcore.sh b/.github/scripts/02-install-pimcore.sh index 364ee0b..db41846 100755 --- a/.github/scripts/02-install-pimcore.sh +++ b/.github/scripts/02-install-pimcore.sh @@ -164,6 +164,12 @@ docker compose exec -T php bin/console generic-data-index:update:index -r 2>/dev echo " [sudo] Setting ownership to match container user (DOCKER_UID:DOCKER_GID)" sudo chown -R "${DOCKER_UID}:${DOCKER_GID}" . +# The supervisord container was started in step 1, before Pimcore was installed. Its long-running +# messenger:consume worker would otherwise keep whatever state it booted with (no schema, stale +# container cache), so restart it now that installation, cache:clear and chown are done. +echo ">>> Restarting supervisord so the messenger worker boots against the installed app..." +docker compose restart supervisord + echo "" echo ">>> Installation complete." echo " Pimcore UI: http://localhost:${NGINX_PORT}/pimcore-studio" diff --git a/.github/scripts/07-collect-logs.sh b/.github/scripts/07-collect-logs.sh new file mode 100755 index 0000000..ca12746 --- /dev/null +++ b/.github/scripts/07-collect-logs.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# 07-collect-logs.sh — Save container and Pimcore logs before shutdown +# +# Used by CI (on failure, before 04-shutdown.sh) and local. +# Writes everything to /container-logs/. +# +set -euo pipefail + +SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPTS_DIR}/../.." && pwd)" +FILES_DIR="${SCRIPTS_DIR}/../files" +PROJECT_PATH="${REPO_ROOT}/../test-project" +LOGS_DIR="${REPO_ROOT}/container-logs" + +if [[ ! -d "$PROJECT_PATH" ]]; then + echo ">>> No project directory found at ${PROJECT_PATH}, no logs to collect." + exit 0 +fi + +source "${FILES_DIR}/.env" +[[ -f "${SCRIPTS_DIR}/.env.local" ]] && source "${SCRIPTS_DIR}/.env.local" + +export NGINX_PORT OPENSEARCH_DASHBOARDS_PORT MAILPIT_PORT MERCURE_PORT DB_PORT +export PHP_IMAGE PHP_SUPERVISORD_IMAGE +export MYSQL_ROOT_PASSWORD MYSQL_DATABASE MYSQL_USER MYSQL_PASSWORD +export OPENSEARCH_INITIAL_ADMIN_PASSWORD +export DOCKER_UID="${DOCKER_UID:-$(id -u)}" +export DOCKER_GID="${DOCKER_GID:-$(id -g)}" + +mkdir -p "$LOGS_DIR" +cd "$PROJECT_PATH" + +echo ">>> Collecting container logs into ${LOGS_DIR}..." +docker compose ps --all > "${LOGS_DIR}/compose-ps.txt" 2>&1 || true +for service in $(docker compose config --services); do + docker compose logs --no-color --timestamps "$service" > "${LOGS_DIR}/${service}.log" 2>&1 || true +done + +echo ">>> Collecting Pimcore var/log..." +if [[ -d var/log ]]; then + cp -r var/log "${LOGS_DIR}/var-log" || true +fi + +# The artifact is downloadable from a public repository, so scrub the secret values the job +# exports before it is uploaded — GitHub only masks them in the step log, not in artifacts. +echo ">>> Redacting secrets..." +for secret_var in PIMCORE_PRODUCT_KEY PIMCORE_ENCRYPTION_SECRET PIMCORE_INSTANCE_IDENTIFIER COMPOSER_TOKEN; do + secret="${!secret_var:-}" + [[ -z "$secret" ]] && continue + SECRET="$secret" find "$LOGS_DIR" -type f -exec \ + perl -pi -e 's/\Q$ENV{SECRET}\E/[REDACTED]/g' {} + +done + +echo ">>> Log collection complete." diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index d0be3b1..9a03fe9 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -111,6 +111,20 @@ jobs: path: tests/test-results/ retention-days: 30 + - name: Collect container logs + if: failure() + env: + COMPOSER_TOKEN: ${{ secrets.COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN }} + run: bash .github/scripts/07-collect-logs.sh + + - name: Upload container logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: container-logs-${{ matrix.platform }} + path: container-logs/ + retention-days: 30 + - name: Shutdown if: always() run: bash .github/scripts/04-shutdown.sh diff --git a/.gitignore b/.gitignore index 648dda5..be34ce0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ Thumbs.db /tests/playwright-report /tests/test-results /tests/.env + +# API test log dumps (.github/scripts/07-collect-logs.sh) +/container-logs/ diff --git a/doc/04_API_Testing/01_API_TESTS.md b/doc/04_API_Testing/01_API_TESTS.md index 6ed835f..339079f 100644 --- a/doc/04_API_Testing/01_API_TESTS.md +++ b/doc/04_API_Testing/01_API_TESTS.md @@ -68,8 +68,9 @@ Admin credentials: `admin` / `admin` ## Reset / shutdown ```bash -.github/scripts/05-reset.sh # fast DB reset, keeps containers -.github/scripts/06-teardown.sh # full shutdown +.github/scripts/05-reset.sh # fast DB reset, keeps containers +.github/scripts/06-teardown.sh # full shutdown +.github/scripts/07-collect-logs.sh # dump container logs + var/log to container-logs/ ``` ## Testing a Platform Version Before Its Release From 784ae2827f6d1ce81b98a82ae6602c0782a3142c Mon Sep 17 00:00:00 2001 From: Ji Jia Jia Date: Wed, 30 Sep 2026 09:27:06 +0200 Subject: [PATCH 2/4] Pin the Mercure hub to v0.24 in the API test stack The untagged dunglas/mercure image moved to v1.0.0 on 2026-09-16. Mercure 1.0 only accepts OAuth 2.0 access tokens (typ at+jwt, a trusted iss, aud, exp, authorization_details) and rejects the legacy mercure-claim JWT that Pimcore publishes with: the hub logs "invalid JWT: untrusted issuer" and answers 401. The execution-engine handlers publish progress to Mercure, so every job's message fails and, with max_retries 0, is dropped, leaving the job running forever. That is what fails the three ZIP downloads and HideCompletedJobSucceeds since 2026-09-17. Co-Authored-By: Claude Opus 5.5 --- .github/files/docker-compose.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/files/docker-compose.yaml b/.github/files/docker-compose.yaml index 2e5ba8c..071023e 100644 --- a/.github/files/docker-compose.yaml +++ b/.github/files/docker-compose.yaml @@ -64,7 +64,11 @@ services: image: gotenberg/gotenberg:8 mercure: - image: dunglas/mercure + # Pinned to the 0.x protocol: Mercure 1.0 (2026-09-16) only accepts OAuth 2.0 access tokens + # (typ at+jwt, iss, aud, authorization_details) and rejects the legacy `mercure`-claim JWT + # Pimcore publishes with ("invalid JWT: untrusted issuer"), which leaves every + # execution-engine job stuck in "running". + image: dunglas/mercure:v0.24 restart: unless-stopped environment: SERVER_NAME: ':80' From 3f38af2c7c6a6e985bd8b2e6dd095f559ce44874 Mon Sep 17 00:00:00 2001 From: Ji Jia Jia Date: Wed, 30 Sep 2026 09:29:13 +0200 Subject: [PATCH 3/4] Fail log collection when a secret survives redaction, and skip the upload then perl -pi only warns and exits 0 when it cannot open a file, so a skipped file kept the secret while the collector reported success. Verify with grep afterwards and drop the logs if any secret is left. Upload the public artifact only when the collector step succeeded. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/07-collect-logs.sh | 9 +++++++++ .github/workflows/api-tests.yml | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/scripts/07-collect-logs.sh b/.github/scripts/07-collect-logs.sh index ca12746..08fcd90 100755 --- a/.github/scripts/07-collect-logs.sh +++ b/.github/scripts/07-collect-logs.sh @@ -50,6 +50,15 @@ for secret_var in PIMCORE_PRODUCT_KEY PIMCORE_ENCRYPTION_SECRET PIMCORE_INSTANCE [[ -z "$secret" ]] && continue SECRET="$secret" find "$LOGS_DIR" -type f -exec \ perl -pi -e 's/\Q$ENV{SECRET}\E/[REDACTED]/g' {} + + # perl -i only warns and still exits 0 when it cannot open a file, so verify the result + # instead: grep exits 1 only if every file was read and none still holds the secret. + grep_status=0 + grep -rqF -- "$secret" "$LOGS_DIR" || grep_status=$? + if [[ "$grep_status" -ne 1 ]]; then + echo ">>> ${secret_var} could not be redacted from every file, refusing to keep the logs." >&2 + rm -rf "$LOGS_DIR" + exit 1 + fi done echo ">>> Log collection complete." diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 9a03fe9..0f3238b 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -112,13 +112,15 @@ jobs: retention-days: 30 - name: Collect container logs + id: collect-logs if: failure() env: COMPOSER_TOKEN: ${{ secrets.COMPOSER_PIMCORE_REPO_PACKAGIST_TOKEN }} run: bash .github/scripts/07-collect-logs.sh - name: Upload container logs - if: failure() + # The artifact is public: only upload once the collector has finished redacting. + if: failure() && steps.collect-logs.outcome == 'success' uses: actions/upload-artifact@v7 with: name: container-logs-${{ matrix.platform }} From 0dc6ba9dd1a2cb9194bc19b910b41df12b6bd642 Mon Sep 17 00:00:00 2001 From: Ji Jia Jia Date: Wed, 30 Sep 2026 10:16:49 +0200 Subject: [PATCH 4/4] Also redact the JSON-escaped form of each secret in collected logs JSON log lines (the Mercure hub, Monolog context) write "/" as "\/", so a secret containing a slash survived both the literal redaction and the post-redaction grep check. Scrub and verify that form too. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/07-collect-logs.sh | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/scripts/07-collect-logs.sh b/.github/scripts/07-collect-logs.sh index 08fcd90..a2bb752 100755 --- a/.github/scripts/07-collect-logs.sh +++ b/.github/scripts/07-collect-logs.sh @@ -48,17 +48,23 @@ echo ">>> Redacting secrets..." for secret_var in PIMCORE_PRODUCT_KEY PIMCORE_ENCRYPTION_SECRET PIMCORE_INSTANCE_IDENTIFIER COMPOSER_TOKEN; do secret="${!secret_var:-}" [[ -z "$secret" ]] && continue - SECRET="$secret" find "$LOGS_DIR" -type f -exec \ - perl -pi -e 's/\Q$ENV{SECRET}\E/[REDACTED]/g' {} + - # perl -i only warns and still exits 0 when it cannot open a file, so verify the result - # instead: grep exits 1 only if every file was read and none still holds the secret. - grep_status=0 - grep -rqF -- "$secret" "$LOGS_DIR" || grep_status=$? - if [[ "$grep_status" -ne 1 ]]; then - echo ">>> ${secret_var} could not be redacted from every file, refusing to keep the logs." >&2 - rm -rf "$LOGS_DIR" - exit 1 - fi + # JSON log lines (Mercure, Monolog context) escape "/" as "\/", so scrub that form as well. + variants=("$secret") + json_escaped="${secret//\//\\/}" + [[ "$json_escaped" != "$secret" ]] && variants+=("$json_escaped") + for variant in "${variants[@]}"; do + SECRET="$variant" find "$LOGS_DIR" -type f -exec \ + perl -pi -e 's/\Q$ENV{SECRET}\E/[REDACTED]/g' {} + + # perl -i only warns and still exits 0 when it cannot open a file, so verify the result + # instead: grep exits 1 only if every file was read and none still holds the secret. + grep_status=0 + grep -rqF -- "$variant" "$LOGS_DIR" || grep_status=$? + if [[ "$grep_status" -ne 1 ]]; then + echo ">>> ${secret_var} could not be redacted from every file, refusing to keep the logs." >&2 + rm -rf "$LOGS_DIR" + exit 1 + fi + done done echo ">>> Log collection complete."