diff --git a/.github/workflows/regression_reusable.yaml b/.github/workflows/regression_reusable.yaml index 2f9318347..7808b8bbb 100644 --- a/.github/workflows/regression_reusable.yaml +++ b/.github/workflows/regression_reusable.yaml @@ -225,6 +225,7 @@ jobs: run_workdir/requirements_coverage.json run_workdir/monitor.log run_workdir/failure_analysis.md + run_workdir/cm-status-*.db* - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/.github/workflows/upgrade_reusable.yaml b/.github/workflows/upgrade_reusable.yaml index d6db71ff2..5e510f463 100644 --- a/.github/workflows/upgrade_reusable.yaml +++ b/.github/workflows/upgrade_reusable.yaml @@ -172,6 +172,7 @@ jobs: run_workdir/scheduling.log run_workdir/errors_all.log run_workdir/failure_analysis.md + run_workdir/cm-status-*.db* - name: ↟ Upload CLI coverage uses: actions/upload-artifact@v7 if: success() || failure() diff --git a/cardano_node_tests/cluster_management/status_db.py b/cardano_node_tests/cluster_management/status_db.py index e3a7b9f91..0b62fb202 100644 --- a/cardano_node_tests/cluster_management/status_db.py +++ b/cardano_node_tests/cluster_management/status_db.py @@ -22,7 +22,7 @@ The current status can be inspected with the stock `sqlite3` CLI. The `overview` view combines all status records into one human-readable table: - db=/tmp/pytest-of-$USER/pytest-0/status.db + db=/tmp/pytest-of-$USER/pytest-0/cm-status.db sqlite3 -readonly -header "$db" 'SELECT * FROM overview ORDER BY instance_num, kind' The underlying tables (`test_running`, `resources`, `flags`) can be queried directly diff --git a/runner/node_upgrade.sh b/runner/node_upgrade.sh index 12f7c9abd..b654fabfa 100755 --- a/runner/node_upgrade.sh +++ b/runner/node_upgrade.sh @@ -190,6 +190,9 @@ fi _last_cleanup +# Copy cluster status databases to workdir +./runner/status_dbs.sh "$ARTIFACTS_DIR" "$WORKDIR" || : + # prepare artifacts for upload in GitHub Actions if [ -n "${GITHUB_ACTIONS:-}" ]; then # save testing artifacts diff --git a/runner/regression.sh b/runner/regression.sh index 0130b4f90..13d73400e 100755 --- a/runner/regression.sh +++ b/runner/regression.sh @@ -434,6 +434,9 @@ fi # Create results archive ./runner/create_results.sh "$REPORTS_DIR" "$WORKDIR" || : +# Copy cluster status databases to workdir +./runner/status_dbs.sh "$ARTIFACTS_DIR" "$WORKDIR" || : + # Save testing artifacts ./runner/save_artifacts.sh "$ARTIFACTS_DIR" "$WORKDIR" || : diff --git a/runner/status_dbs.sh b/runner/status_dbs.sh new file mode 100755 index 000000000..43dc26f26 --- /dev/null +++ b/runner/status_dbs.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +artifacts_dir="$1" +output_dir="$2" + +mkdir -p "$output_dir" || { echo "Cannot create $output_dir" >&2; exit 1; } + +# The status database was copied from the pytest temp dir to the artifacts dir together +# with the other testing artifacts. Copy the database of each pytest run to the output +# dir. The output databases are numbered in the run order (e.g. one per node upgrade +# step) - the `pytest-N` component of the artifacts subdir names cannot be used directly, +# as the number comes from pytest's numbered basetemp counter that is shared by all +# pytest invocations in a testrun, including those that don't collect artifacts. +# Copy also the WAL sidecar, which is present when database connections were still open +# during artifacts collection - without it the database copy would miss the +# not-yet-checkpointed writes. The `-shm` file is skipped on purpose, it is transient +# and not needed for an offline copy. +found=0 +num=0 +while read -r db; do + # Skip the unexpanded pattern when the glob didn't match anything + [ -f "$db" ] || continue + num=$((num + 1)) + cp "$db" "${output_dir}/cm-status-${num}.db" \ + || { echo "Failed to copy $db" >&2; continue; } + found=1 + if [ -e "${db}-wal" ]; then + cp "${db}-wal" "${output_dir}/cm-status-${num}.db-wal" \ + || echo "Failed to copy ${db}-wal" >&2 + fi +done < <(printf '%s\n' "$artifacts_dir"/pytest-*/cm-status.db | sort -V) + +if [ "$found" -eq 0 ]; then + echo "No status database copied from $artifacts_dir" >&2 + if [ -n "${GITHUB_ACTIONS:-}" ]; then + echo "::warning::No cluster status database was copied from the testing artifacts, none uploaded." + fi +fi