diff --git a/dist/index.js b/dist/index.js index 4af9a575..d5b66f92 100644 --- a/dist/index.js +++ b/dist/index.js @@ -104838,6 +104838,36 @@ const CYCLES_TAG_KEY = 'ec2-github-runner:cycles'; // Shared bootstrap shell helpers, emitted into any shell that phones home: // derive instance identity from IMDSv2 and tag the bootstrap phase (best- // effort — needs ec2:CreateTags, degrades silently otherwise). +// +// Failure detail capture: every shell that sources these helpers also +// wires its own stderr through `tee` into a scratch file (GH_RUNNER_ERRLOG), +// so gh_runner_phone_home_failed() can attach a short tail of whatever the +// failing command actually printed to the "failed:" tag — instead of +// just the step name. Value shape: `failed:` (unchanged, no detail +// captured) or `failed::`. Step names never contain ':', so +// src/wait.js splits on the FIRST ':' after the failed: prefix to recover +// step vs. detail — safe even if itself contains colons (e.g. a +// config.sh "error: ..." line). is sanitized to a single printable +// line and the whole value is hard-capped at 256 chars, the EC2 tag-value +// limit, regardless of how much garbage the failing command emitted. +// gh_runner_phone_home() truncates the scratch file on every *successful* +// (non-"failed:...") call — i.e. right as each new phase begins, before its +// risky commands run — so a step's detail only ever reflects stderr from +// that step's own commands, not noise left behind by an earlier `|| true` +// -tolerated one (e.g. `mount ... || true` still writes to stderr even +// though its exit code is swallowed). +// +// `tee`'s copy into the scratch file happens in a background process +// substitution, so it can lag behind the failing command by a scheduling +// tick — reading the file immediately in the ERR trap can race and see it +// still empty. gh_runner_phone_home_failed() closes this shell's end of +// that pipe (fd 2, about to go away anyway — the script is terminating) +// and polls (bounded: at most ~200ms) for the tee process to exit, which it +// only does once it has drained and flushed everything written to the +// pipe. This is deliberately not a plain `wait` on the tee PID: `wait` +// tracking a process-substitution PID needs bash >= 4.4, and an unbounded +// wait risks hanging the trap if tee's write end ever blocks; `kill -0` for +// liveness works on any bash and the loop always terminates. const PHONE_HOME_HELPERS = [ 'gh_runner_imds() {', ' local token', @@ -104846,10 +104876,39 @@ const PHONE_HOME_HELPERS = [ '}', 'GH_RUNNER_IID=$(gh_runner_imds instance-id)', 'GH_RUNNER_REGION=$(gh_runner_imds placement/region)', + '# Tee this shell\'s stderr into a scratch file (best-effort) so a failing', + '# command\'s own error text can be recovered by gh_runner_phone_home_failed.', + '# Guarded end-to-end: an empty GH_RUNNER_ERRLOG (mktemp unavailable) just', + '# skips detail capture, same as any other best-effort tagging failure here.', + 'GH_RUNNER_ERRLOG=$(mktemp /tmp/gh-runner-err.XXXXXX 2>/dev/null || true)', + 'GH_RUNNER_ERRTEE_PID=""', + 'if [ -n "$GH_RUNNER_ERRLOG" ]; then exec 2> >(tee -a "$GH_RUNNER_ERRLOG" >&2) || true; GH_RUNNER_ERRTEE_PID=$!; fi', 'gh_runner_phone_home() {', + ' case "$1" in', + ' failed:*) : ;;', + ' *) [ -n "${GH_RUNNER_ERRLOG:-}" ] && : > "$GH_RUNNER_ERRLOG" 2>/dev/null || true ;;', + ' esac', ' [ -n "$GH_RUNNER_IID" ] && [ -n "$GH_RUNNER_REGION" ] || return 0', ` aws ec2 create-tags --region "$GH_RUNNER_REGION" --resources "$GH_RUNNER_IID" --tags "Key=${BOOTSTRAP_TAG_KEY},Value=$1" >/dev/null 2>&1 || true`, '}', + 'gh_runner_phone_home_failed() {', + ' local step="$1" detail="" value waited=0', + ' if [ -n "${GH_RUNNER_ERRTEE_PID:-}" ]; then', + ' exec 2>/dev/null', + ' while [ "$waited" -lt 10 ] && kill -0 "$GH_RUNNER_ERRTEE_PID" 2>/dev/null; do', + ' sleep 0.02', + ' waited=$((waited + 1))', + ' done', + ' fi', + ' if [ -n "${GH_RUNNER_ERRLOG:-}" ] && [ -s "$GH_RUNNER_ERRLOG" ]; then', + ' detail=$(tr "\\n\\r\\t" " " < "$GH_RUNNER_ERRLOG" 2>/dev/null | tr -c "[:print:]" " " | tail -c 150 | tr -s " ")', + ' detail="${detail# }"', + ' detail="${detail% }"', + ' fi', + ' value="failed:${step}"', + ' [ -n "$detail" ] && value="failed:${step}:${detail}"', + ' gh_runner_phone_home "${value:0:256}"', + '}', ]; // Console-output capture caps: keep the printed tail useful but bounded so a @@ -105013,7 +105072,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=configuring', - 'trap \'gh_runner_phone_home "failed:${GH_RUNNER_STEP}"\' ERR', + 'trap \'gh_runner_phone_home_failed "${GH_RUNNER_STEP}"\' ERR', 'IMDS_TOKEN=$(curl -fsS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 120" 2>/dev/null || true)', 'UD=$(curl -fsS -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" "http://169.254.169.254/latest/user-data" 2>/dev/null || true)', 'GH_TOKEN=$(printf \'%s\\n\' "$UD" | sed -n "s/^GH_TOKEN=\'\\(.*\\)\'$/\\1/p" | head -n1)', @@ -105047,7 +105106,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist '# --- ec2-github-runner: warm-pool bootstrap (reuse: stop) -----------', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=preparing', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", '', ...ttlLines, '# One-time setup (idempotent across warm restarts).', @@ -105072,7 +105131,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=downloading', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", 'gh_runner_phone_home downloading', 'cd "$HOME"', 'mkdir -p actions-runner && cd actions-runner', @@ -105146,6 +105205,14 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist // registration timeout. Tagging is best-effort: it needs // `ec2:CreateTags` on the instance profile and degrades to // timeout-only detection when absent (every write is `|| true`). +// +// - Failure detail capture: the `failed:` tag can carry a +// trailing `:` — a sanitized, single-line, ≤150-char tail of +// whatever the failing command wrote to stderr (see PHONE_HOME_HELPERS +// and gh_runner_phone_home_failed()) — so a bootstrap failure surfaces +// the actual error (e.g. config.sh's stderr) without needing live +// SSH/SSM access. src/wait.js splits on the first ':' after the +// prefix and stays backward compatible with the no-detail shape. function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationToken, shaX64, shaArm64, maxLifetimeMinutes, preRunnerScript, reuse }) { // User-supplied pre-runner-script (#46): injected verbatim into the outer // (root) shell before runner configuration, under the same set -euo @@ -105191,7 +105258,7 @@ function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationTo '# --- ec2-github-runner: bootstrap diagnostics (phone-home) ----------', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=preparing', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", '', ...ttlLines, '# Root-required setup.', @@ -105218,7 +105285,7 @@ function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationTo 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=downloading', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", 'gh_runner_phone_home downloading', 'cd "$HOME"', 'mkdir -p actions-runner && cd actions-runner', @@ -106650,8 +106717,13 @@ module.exports = { const core = __nccwpck_require__(7484); const log = __nccwpck_require__(7223); -// Terminal bootstrap phone-home states carry a `failed:` value; see -// src/aws.js buildUserData() for the producing side. +// Terminal bootstrap phone-home states carry a `failed:` value, or +// `failed::` when the failing command's stderr was captured +// (see PHONE_HOME_HELPERS / gh_runner_phone_home_failed() in src/aws.js). +// Step names never contain ':', so the FIRST ':' after this prefix is the +// step/detail boundary — safe even when itself contains colons +// (e.g. a captured "config.sh: error: ..." line). A value with no second +// ':' is the old/no-detail shape and is fully backward compatible. const FAILED_PREFIX = 'failed:'; // Wait for the EC2 runner to come online, watching two signals in lockstep: @@ -106690,11 +106762,18 @@ async function waitForRunnerReady(deps, opts = {}) { // 1. Fast-fail on a phoned-home bootstrap failure. const status = await getBootstrapStatus(); if (typeof status === 'string' && status.startsWith(FAILED_PREFIX)) { - const step = status.slice(FAILED_PREFIX.length); - log.error('wait_for_runner', { outcome: 'bootstrap_failed', step }); - core.error(`EC2 runner bootstrap failed during the "${step}" step`); - const error = new Error(`EC2 runner bootstrap failed during the "${step}" step. See the captured console output below.`); + const rest = status.slice(FAILED_PREFIX.length); + const sepIndex = rest.indexOf(':'); + const step = sepIndex === -1 ? rest : rest.slice(0, sepIndex); + const detail = sepIndex === -1 ? '' : rest.slice(sepIndex + 1); + const detailSuffix = detail ? ` — ${detail}` : ''; + log.error('wait_for_runner', { outcome: 'bootstrap_failed', step, detail: detail || null }); + core.error(`EC2 runner bootstrap failed during the "${step}" step${detailSuffix}`); + const error = new Error(`EC2 runner bootstrap failed during the "${step}" step${detailSuffix}. See the captured console output below.`); error.bootstrapStep = step; + if (detail) { + error.bootstrapDetail = detail; + } throw error; } diff --git a/src/aws.js b/src/aws.js index 683af1df..d6bb414f 100644 --- a/src/aws.js +++ b/src/aws.js @@ -188,6 +188,36 @@ const CYCLES_TAG_KEY = 'ec2-github-runner:cycles'; // Shared bootstrap shell helpers, emitted into any shell that phones home: // derive instance identity from IMDSv2 and tag the bootstrap phase (best- // effort — needs ec2:CreateTags, degrades silently otherwise). +// +// Failure detail capture: every shell that sources these helpers also +// wires its own stderr through `tee` into a scratch file (GH_RUNNER_ERRLOG), +// so gh_runner_phone_home_failed() can attach a short tail of whatever the +// failing command actually printed to the "failed:" tag — instead of +// just the step name. Value shape: `failed:` (unchanged, no detail +// captured) or `failed::`. Step names never contain ':', so +// src/wait.js splits on the FIRST ':' after the failed: prefix to recover +// step vs. detail — safe even if itself contains colons (e.g. a +// config.sh "error: ..." line). is sanitized to a single printable +// line and the whole value is hard-capped at 256 chars, the EC2 tag-value +// limit, regardless of how much garbage the failing command emitted. +// gh_runner_phone_home() truncates the scratch file on every *successful* +// (non-"failed:...") call — i.e. right as each new phase begins, before its +// risky commands run — so a step's detail only ever reflects stderr from +// that step's own commands, not noise left behind by an earlier `|| true` +// -tolerated one (e.g. `mount ... || true` still writes to stderr even +// though its exit code is swallowed). +// +// `tee`'s copy into the scratch file happens in a background process +// substitution, so it can lag behind the failing command by a scheduling +// tick — reading the file immediately in the ERR trap can race and see it +// still empty. gh_runner_phone_home_failed() closes this shell's end of +// that pipe (fd 2, about to go away anyway — the script is terminating) +// and polls (bounded: at most ~200ms) for the tee process to exit, which it +// only does once it has drained and flushed everything written to the +// pipe. This is deliberately not a plain `wait` on the tee PID: `wait` +// tracking a process-substitution PID needs bash >= 4.4, and an unbounded +// wait risks hanging the trap if tee's write end ever blocks; `kill -0` for +// liveness works on any bash and the loop always terminates. const PHONE_HOME_HELPERS = [ 'gh_runner_imds() {', ' local token', @@ -196,10 +226,39 @@ const PHONE_HOME_HELPERS = [ '}', 'GH_RUNNER_IID=$(gh_runner_imds instance-id)', 'GH_RUNNER_REGION=$(gh_runner_imds placement/region)', + '# Tee this shell\'s stderr into a scratch file (best-effort) so a failing', + '# command\'s own error text can be recovered by gh_runner_phone_home_failed.', + '# Guarded end-to-end: an empty GH_RUNNER_ERRLOG (mktemp unavailable) just', + '# skips detail capture, same as any other best-effort tagging failure here.', + 'GH_RUNNER_ERRLOG=$(mktemp /tmp/gh-runner-err.XXXXXX 2>/dev/null || true)', + 'GH_RUNNER_ERRTEE_PID=""', + 'if [ -n "$GH_RUNNER_ERRLOG" ]; then exec 2> >(tee -a "$GH_RUNNER_ERRLOG" >&2) || true; GH_RUNNER_ERRTEE_PID=$!; fi', 'gh_runner_phone_home() {', + ' case "$1" in', + ' failed:*) : ;;', + ' *) [ -n "${GH_RUNNER_ERRLOG:-}" ] && : > "$GH_RUNNER_ERRLOG" 2>/dev/null || true ;;', + ' esac', ' [ -n "$GH_RUNNER_IID" ] && [ -n "$GH_RUNNER_REGION" ] || return 0', ` aws ec2 create-tags --region "$GH_RUNNER_REGION" --resources "$GH_RUNNER_IID" --tags "Key=${BOOTSTRAP_TAG_KEY},Value=$1" >/dev/null 2>&1 || true`, '}', + 'gh_runner_phone_home_failed() {', + ' local step="$1" detail="" value waited=0', + ' if [ -n "${GH_RUNNER_ERRTEE_PID:-}" ]; then', + ' exec 2>/dev/null', + ' while [ "$waited" -lt 10 ] && kill -0 "$GH_RUNNER_ERRTEE_PID" 2>/dev/null; do', + ' sleep 0.02', + ' waited=$((waited + 1))', + ' done', + ' fi', + ' if [ -n "${GH_RUNNER_ERRLOG:-}" ] && [ -s "$GH_RUNNER_ERRLOG" ]; then', + ' detail=$(tr "\\n\\r\\t" " " < "$GH_RUNNER_ERRLOG" 2>/dev/null | tr -c "[:print:]" " " | tail -c 150 | tr -s " ")', + ' detail="${detail# }"', + ' detail="${detail% }"', + ' fi', + ' value="failed:${step}"', + ' [ -n "$detail" ] && value="failed:${step}:${detail}"', + ' gh_runner_phone_home "${value:0:256}"', + '}', ]; // Console-output capture caps: keep the printed tail useful but bounded so a @@ -363,7 +422,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=configuring', - 'trap \'gh_runner_phone_home "failed:${GH_RUNNER_STEP}"\' ERR', + 'trap \'gh_runner_phone_home_failed "${GH_RUNNER_STEP}"\' ERR', 'IMDS_TOKEN=$(curl -fsS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 120" 2>/dev/null || true)', 'UD=$(curl -fsS -H "X-aws-ec2-metadata-token: $IMDS_TOKEN" "http://169.254.169.254/latest/user-data" 2>/dev/null || true)', 'GH_TOKEN=$(printf \'%s\\n\' "$UD" | sed -n "s/^GH_TOKEN=\'\\(.*\\)\'$/\\1/p" | head -n1)', @@ -397,7 +456,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist '# --- ec2-github-runner: warm-pool bootstrap (reuse: stop) -----------', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=preparing', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", '', ...ttlLines, '# One-time setup (idempotent across warm restarts).', @@ -422,7 +481,7 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=downloading', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", 'gh_runner_phone_home downloading', 'cd "$HOME"', 'mkdir -p actions-runner && cd actions-runner', @@ -496,6 +555,14 @@ function buildReusableUserData({ runnerVersion, owner, repo, label, githubRegist // registration timeout. Tagging is best-effort: it needs // `ec2:CreateTags` on the instance profile and degrades to // timeout-only detection when absent (every write is `|| true`). +// +// - Failure detail capture: the `failed:` tag can carry a +// trailing `:` — a sanitized, single-line, ≤150-char tail of +// whatever the failing command wrote to stderr (see PHONE_HOME_HELPERS +// and gh_runner_phone_home_failed()) — so a bootstrap failure surfaces +// the actual error (e.g. config.sh's stderr) without needing live +// SSH/SSM access. src/wait.js splits on the first ':' after the +// prefix and stays backward compatible with the no-detail shape. function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationToken, shaX64, shaArm64, maxLifetimeMinutes, preRunnerScript, reuse }) { // User-supplied pre-runner-script (#46): injected verbatim into the outer // (root) shell before runner configuration, under the same set -euo @@ -541,7 +608,7 @@ function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationTo '# --- ec2-github-runner: bootstrap diagnostics (phone-home) ----------', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=preparing', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", '', ...ttlLines, '# Root-required setup.', @@ -568,7 +635,7 @@ function buildUserData({ runnerVersion, owner, repo, label, githubRegistrationTo 'set -euo pipefail', ...PHONE_HOME_HELPERS, 'GH_RUNNER_STEP=downloading', - "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR", + "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR", 'gh_runner_phone_home downloading', 'cd "$HOME"', 'mkdir -p actions-runner && cd actions-runner', diff --git a/src/wait.js b/src/wait.js index b2a0b4ea..4a4cbfe6 100644 --- a/src/wait.js +++ b/src/wait.js @@ -1,8 +1,13 @@ const core = require('@actions/core'); const log = require('./log'); -// Terminal bootstrap phone-home states carry a `failed:` value; see -// src/aws.js buildUserData() for the producing side. +// Terminal bootstrap phone-home states carry a `failed:` value, or +// `failed::` when the failing command's stderr was captured +// (see PHONE_HOME_HELPERS / gh_runner_phone_home_failed() in src/aws.js). +// Step names never contain ':', so the FIRST ':' after this prefix is the +// step/detail boundary — safe even when itself contains colons +// (e.g. a captured "config.sh: error: ..." line). A value with no second +// ':' is the old/no-detail shape and is fully backward compatible. const FAILED_PREFIX = 'failed:'; // Wait for the EC2 runner to come online, watching two signals in lockstep: @@ -41,11 +46,18 @@ async function waitForRunnerReady(deps, opts = {}) { // 1. Fast-fail on a phoned-home bootstrap failure. const status = await getBootstrapStatus(); if (typeof status === 'string' && status.startsWith(FAILED_PREFIX)) { - const step = status.slice(FAILED_PREFIX.length); - log.error('wait_for_runner', { outcome: 'bootstrap_failed', step }); - core.error(`EC2 runner bootstrap failed during the "${step}" step`); - const error = new Error(`EC2 runner bootstrap failed during the "${step}" step. See the captured console output below.`); + const rest = status.slice(FAILED_PREFIX.length); + const sepIndex = rest.indexOf(':'); + const step = sepIndex === -1 ? rest : rest.slice(0, sepIndex); + const detail = sepIndex === -1 ? '' : rest.slice(sepIndex + 1); + const detailSuffix = detail ? ` — ${detail}` : ''; + log.error('wait_for_runner', { outcome: 'bootstrap_failed', step, detail: detail || null }); + core.error(`EC2 runner bootstrap failed during the "${step}" step${detailSuffix}`); + const error = new Error(`EC2 runner bootstrap failed during the "${step}" step${detailSuffix}. See the captured console output below.`); error.bootstrapStep = step; + if (detail) { + error.bootstrapDetail = detail; + } throw error; } diff --git a/tests/phone-home-detail.test.js b/tests/phone-home-detail.test.js new file mode 100644 index 00000000..b1c50e89 --- /dev/null +++ b/tests/phone-home-detail.test.js @@ -0,0 +1,242 @@ +// End-to-end tests for the failure-detail capture added to the phone-home +// mechanism (PHONE_HOME_HELPERS / gh_runner_phone_home_failed in src/aws.js). +// +// These tests do not re-implement the generated shell in JS and check +// strings against it — they pull the ACTUAL lines buildUserData() / +// buildReusableUserData() produce, splice in a fabricated failing command in +// place of a real one, and execute the result as a genuine bash script file +// (never a hand-typed command line — this codebase has a documented history +// of subtle shell-quoting bugs, e.g. #63). A fake `curl` supplies IMDS +// identity and a fake `aws` records the create-tags Value it's invoked +// with, so the real gh_runner_phone_home_failed()/gh_runner_phone_home() +// pipeline runs unmodified and we observe exactly what it phones home. +jest.mock('../src/config', () => ({ + input: { mode: 'start', debug: 'false' }, + githubContext: { owner: 'o', repo: 'r' }, +})); +jest.mock('@actions/core', () => ({ + info: jest.fn(), warning: jest.fn(), error: jest.fn(), setFailed: jest.fn(), getInput: jest.fn(), + startGroup: jest.fn(), endGroup: jest.fn(), debug: jest.fn(), +})); + +const { execFileSync } = require('child_process'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { buildUserData } = require('../src/aws'); + +const args = { + runnerVersion: '2.335.1', owner: 'o', repo: 'r', label: 'l', + githubRegistrationToken: 'TOK', shaX64: 'x', shaArm64: 'a', +}; + +// Fake IMDS (so gh_runner_imds resolves without a real network call) and a +// fake `aws` CLI that records the create-tags Value it receives. Defining +// these as shell FUNCTIONS (not PATH executables) is enough: bash resolves +// a bare command name against functions before PATH. +const HARNESS_PREFIX = [ + 'curl() {', + ' case "$*" in', + ' *api/token*) echo FAKETOKEN ;;', + ' *meta-data/instance-id*) echo i-fake123 ;;', + ' *meta-data/placement/region*) echo us-east-1 ;;', + ' *) return 1 ;;', + ' esac', + '}', + // Real `mount`/`uname` behave differently (or fail outright) on the + // non-Linux/non-EC2 machine running this test; stub them so the parts of + // the script we are NOT testing behave the way they do on a real Amazon + // Linux instance instead of tripping over host differences. + 'mount() { return 0; }', + 'uname() { echo x86_64; }', + 'aws() {', + ' for arg in "$@"; do', + ' case "$arg" in', + ' Key=*,Value=*) printf \'%s\' "${arg#*,Value=}" > "$AWS_TAG_CAPTURE_FILE" ;;', + ' esac', + ' done', + ' return 0', + '}', +].join('\n'); + +function extractBetween(text, startMarker, endMarker) { + const start = text.indexOf(startMarker); + if (start === -1) throw new Error(`start marker not found: ${startMarker}`); + const bodyStart = start + startMarker.length; + const end = text.indexOf(endMarker, bodyStart); + if (end === -1) throw new Error(`end marker not found: ${endMarker}`); + return text.slice(bodyStart, end); +} + +// Runs a real generated script body (always starting with its own +// `set -euo pipefail`, exactly as buildUserData()/buildReusableUserData() +// emit it) after substituting one real command for a fabricated failing +// one, and returns whatever the fake `aws` captured as the create-tags +// Value — '' if it was never invoked. +function runAndCapturePhoneHome(scriptBody, { marker, fakeCommand, fakeCommandDef, neutralize = [] }) { + if (!scriptBody.includes(marker)) { + throw new Error(`replacement marker not found in generated script: ${marker}`); + } + let body = scriptBody.replace(marker, fakeCommand); + // /home/runner only exists on the real instance; give the script a + // writable stand-in so unrelated lines don't fail before our injected one. + body = body.replace('cd /home/runner/actions-runner', 'mkdir -p "$PWD/fake-runner-home" && cd "$PWD/fake-runner-home"'); + // No-op out other real commands (e.g. an earlier download/verify step) + // that are irrelevant to the failure under test but would otherwise run + // for real (and fail for host/environment reasons unrelated to this test). + for (const line of neutralize) { + if (!body.includes(line)) { + throw new Error(`neutralize target not found in generated script: ${line}`); + } + body = body.replace(line, ': # neutralized for test'); + } + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ec2gh-phonehome-')); + const captureFile = path.join(dir, 'capture'); + fs.writeFileSync(captureFile, ''); + const scriptPath = path.join(dir, 'script.sh'); + fs.writeFileSync(scriptPath, ['#!/bin/bash', HARNESS_PREFIX, fakeCommandDef || '', body].join('\n')); + + try { + execFileSync('bash', [scriptPath], { + env: { ...process.env, AWS_TAG_CAPTURE_FILE: captureFile }, + stdio: ['ignore', 'ignore', 'ignore'], + }); + } catch { + // Expected: the fabricated command fails and the script exits non-zero. + } + return fs.readFileSync(captureFile, 'utf8'); +} + +// A genuinely adversarial failing command: >3KB of output across 60 lines, +// with tabs, NUL-adjacent control bytes, and embedded ANSI color escapes, +// ending in the one line that actually matters. Proves truncation grabs the +// *tail* (where real errors land) and sanitization survives non-ASCII junk. +const LONG_MULTILINE_FAKE_CMD = { + def: [ + 'gh_fake_failing_command() {', + ' printf \'%s\\n\' "BEGIN-ERROR-MARKER-SHOULD-BE-TRUNCATED-AWAY" >&2', + ' local i', + ' for i in $(seq 1 60); do', + ' printf "garbage line %02d: \\x01\\x02\\tsome\\ttabs\\tand\\x1b[31mANSI\\x1b[0m colors and padding to make this line long enough to matter\\n" "$i" >&2', + ' done', + ' printf \'%s\\n\' "REAL_ERROR: config.sh failed: Not configured. Run config.sh to configure a runner." >&2', + ' return 1', + '}', + ].join('\n'), + call: 'gh_fake_failing_command', +}; + +describe('phone-home failure-detail capture (real bash execution)', () => { + test('registerScript (reuse: stop): configuring-step failure surfaces a truncated, sanitized snippet', () => { + const ud = buildUserData({ ...args, reuse: 'stop' }); + const registerScriptBody = extractBetween( + ud, + "cat >/opt/gh-runner-register.sh <<'GH_REGISTER_SCRIPT'\n", + '\nGH_REGISTER_SCRIPT', + ).replace(/^#!\/bin\/bash\n/, ''); + + const marker = 'sudo -u runner -H env DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 ./config.sh --url "$GH_REPO_URL" --token "$GH_TOKEN" --labels "$GH_LABEL" --ephemeral --unattended --disableupdate'; + const value = runAndCapturePhoneHome(registerScriptBody, { + marker, + fakeCommand: LONG_MULTILINE_FAKE_CMD.call, + fakeCommandDef: LONG_MULTILINE_FAKE_CMD.def, + }); + + expect(value).toMatch(/^failed:configuring:/); + expect(value).toContain('REAL_ERROR: config.sh failed: Not configured. Run config.sh to configure a runner.'); + // Truncation actually happened: the start of the (much longer) captured + // output must not have survived into the tag value. + expect(value).not.toContain('BEGIN-ERROR-MARKER'); + expect(Buffer.byteLength(value, 'utf8')).toBeLessThanOrEqual(256); + expect(value).not.toMatch(/[\n\r]/); + // eslint-disable-next-line no-control-regex + expect(value).not.toMatch(/[\x00-\x08\x0e-\x1f]/); + }); + + test('RUNNER_DOWNLOAD heredoc (reuse: stop): downloading-step failure with a pathological huge, newline-filled error stays under 256 chars with no embedded newlines', () => { + const ud = buildUserData({ ...args, reuse: 'stop' }); + const heredocBody = extractBetween(ud, "sudo -u runner -H bash <<'RUNNER_DOWNLOAD'\n", '\nRUNNER_DOWNLOAD'); + + // Pathological: ~50KB of newline-separated garbage with embedded CRs and + // control bytes, no realistic "error line" at all — worst case for the + // sanitizer/truncator, not just an adversarial-but-tidy example. + const hugeFakeCmdDef = [ + 'gh_fake_huge_failure() {', + ' local i', + ' for i in $(seq 1 2000); do', + ' printf "junk-%d\\r\\n\\ttabbed\\x07bell\\x08backspace-more-filler-text-here\\n" "$i" >&2', + ' done', + ' return 1', + '}', + ].join('\n'); + + const value = runAndCapturePhoneHome(heredocBody, { + marker: 'curl -fsSLo "$TARBALL" "$BASE/$TARBALL"', + fakeCommand: 'gh_fake_huge_failure', + fakeCommandDef: hugeFakeCmdDef, + }); + + expect(value).toMatch(/^failed:downloading:/); + expect(Buffer.byteLength(value, 'utf8')).toBeLessThanOrEqual(256); + expect(value).not.toMatch(/[\n\r]/); + }); + + test('plain (reuse: terminate) RUNNER_BOOTSTRAP heredoc: configuring-step failure carries a detail snippet', () => { + const ud = buildUserData({ ...args, reuse: 'terminate' }); + const heredocBody = extractBetween(ud, "sudo -u runner -H bash <<'RUNNER_BOOTSTRAP'\n", '\nRUNNER_BOOTSTRAP'); + + const marker = './config.sh --url "https://github.com/o/r" --token "TOK" --labels "l" --ephemeral --unattended --disableupdate'; + const value = runAndCapturePhoneHome(heredocBody, { + marker, + fakeCommand: LONG_MULTILINE_FAKE_CMD.call, + fakeCommandDef: LONG_MULTILINE_FAKE_CMD.def, + // The download/verify/extract steps run for real before config.sh in + // this heredoc; neutralize them so this test isolates the configuring + // step (they have their own dedicated coverage in the download test). + neutralize: [ + 'curl -fsSLo "$TARBALL" "$BASE/$TARBALL"', + 'echo "$EXPECTED_SHA $TARBALL" | sha256sum -c -', + 'tar xzf "$TARBALL"', + ], + }); + + expect(value).toMatch(/^failed:configuring:/); + expect(value).toContain('REAL_ERROR: config.sh failed'); + expect(Buffer.byteLength(value, 'utf8')).toBeLessThanOrEqual(256); + expect(value).not.toMatch(/[\n\r]/); + }); + + test('plain (reuse: terminate) outer root shell: installing-step failure with NO stderr output stays backward compatible ("failed:" only)', () => { + const ud = buildUserData({ ...args, reuse: 'terminate' }); + const outerBody = ud.slice(0, ud.indexOf("sudo -u runner -H bash <<'RUNNER_BOOTSTRAP'")); + + const value = runAndCapturePhoneHome(outerBody, { + marker: 'yum install -y libicu make sudo', + fakeCommand: 'gh_fake_silent_failure', + fakeCommandDef: 'gh_fake_silent_failure() { return 1; }', + }); + + // No detail was ever written to stderr, so no scratch file content — the + // value must be exactly the old, no-detail shape (no trailing ':'). + expect(value).toBe('failed:installing'); + }); + + test('reuse: stop outer root shell (one-time install): installing-step failure carries a detail snippet', () => { + const ud = buildUserData({ ...args, reuse: 'stop' }); + const outerBody = ud.slice(0, ud.indexOf("sudo -u runner -H bash <<'RUNNER_DOWNLOAD'")); + + const value = runAndCapturePhoneHome(outerBody, { + marker: 'yum install -y libicu make sudo', + fakeCommand: 'gh_fake_yum_failure', + fakeCommandDef: [ + 'gh_fake_yum_failure() {', + ' printf \'%s\\n\' "Error: Unable to find a match: libicu" >&2', + ' return 1', + '}', + ].join('\n'), + }); + + expect(value).toBe('failed:installing:Error: Unable to find a match: libicu'); + }); +}); diff --git a/tests/userdata.test.js b/tests/userdata.test.js index 0366793f..262b9759 100644 --- a/tests/userdata.test.js +++ b/tests/userdata.test.js @@ -44,14 +44,22 @@ describe('buildUserData', () => { } }); - test('installs an ERR trap that phones home failed: in both shells', () => { + test('installs an ERR trap that phones home failed: (+ detail) in both shells', () => { const ud = buildUserData(args); - const trapLine = "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR"; + const trapLine = "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR"; // Once in the outer (root) shell, once in the inner (runner-user) shell. const occurrences = ud.split(trapLine).length - 1; expect(occurrences).toBe(2); }); + test('wires stderr through tee into a scratch file for failure-detail capture', () => { + const ud = buildUserData(args); + // Once in the outer (root) shell, once in the inner (runner-user) shell. + expect(ud.split('GH_RUNNER_ERRLOG=$(mktemp').length - 1).toBe(2); + expect(ud.split('exec 2> >(tee -a "$GH_RUNNER_ERRLOG" >&2)').length - 1).toBe(2); + expect(ud.split('gh_runner_phone_home_failed() {').length - 1).toBe(2); + }); + test('writes the bootstrap tag via ec2 create-tags, best-effort', () => { const ud = buildUserData(args); expect(ud).toContain(`--tags "Key=${BOOTSTRAP_TAG_KEY},Value=$1"`); diff --git a/tests/wait.test.js b/tests/wait.test.js index 34852883..3482a56a 100644 --- a/tests/wait.test.js +++ b/tests/wait.test.js @@ -4,10 +4,15 @@ jest.mock('@actions/core', () => ({ info: jest.fn(), warning: jest.fn(), error: jest.fn(), debug: jest.fn() })); jest.mock('../src/config', () => ({ input: { mode: 'start', debug: 'false' } })); +const core = require('@actions/core'); const { waitForRunnerReady } = require('../src/wait'); const noSleep = () => Promise.resolve(); +beforeEach(() => { + core.error.mockClear(); +}); + describe('waitForRunnerReady', () => { test('resolves as soon as the runner is online', async () => { const getBootstrapStatus = jest.fn().mockResolvedValue(null); @@ -76,4 +81,74 @@ describe('waitForRunnerReady', () => { waitForRunnerReady({ getBootstrapStatus, isRunnerOnline, sleep: noSleep }, { quietSeconds: 0, intervalSeconds: 1 }), ).resolves.toBeUndefined(); }); + + // Failure-detail capture: a phoned-home value can now be + // `failed::` (see PHONE_HOME_HELPERS in src/aws.js). These + // cover the JS-side split/surface logic; tests/phone-home-detail.test.js + // covers the shell side that produces these values for real. + describe('failure-detail capture (failed::)', () => { + test('surfaces the detail in both the thrown error and error.bootstrapDetail', async () => { + const getBootstrapStatus = jest.fn().mockResolvedValue('failed:configuring:config.sh failed: Not configured.'); + const isRunnerOnline = jest.fn().mockResolvedValue(false); + + let caught; + try { + await waitForRunnerReady({ getBootstrapStatus, isRunnerOnline, sleep: noSleep }, { quietSeconds: 0 }); + } catch (error) { + caught = error; + } + + expect(caught).toBeDefined(); + expect(caught.bootstrapStep).toBe('configuring'); + expect(caught.bootstrapDetail).toBe('config.sh failed: Not configured.'); + expect(caught.message).toContain('"configuring"'); + expect(caught.message).toContain('config.sh failed: Not configured.'); + // The workflow log (core.error) must also carry the detail, not just + // the thrown error. + expect(core.error).toHaveBeenCalledWith(expect.stringContaining('config.sh failed: Not configured.')); + }); + + test('a detail snippet containing colons is preserved whole (only the FIRST colon after the prefix is the step/detail boundary)', async () => { + const getBootstrapStatus = jest.fn().mockResolvedValue('failed:configuring:config.sh: error: Not configured: run config.sh'); + const isRunnerOnline = jest.fn().mockResolvedValue(false); + + await expect( + waitForRunnerReady({ getBootstrapStatus, isRunnerOnline, sleep: noSleep }, { quietSeconds: 0 }), + ).rejects.toMatchObject({ + bootstrapStep: 'configuring', + bootstrapDetail: 'config.sh: error: Not configured: run config.sh', + }); + }); + + test('surfaces whatever detail it is given, even if the tag-value length/newline constraint were somehow violated upstream', async () => { + // The 256-char/no-newline guarantee is enforced shell-side (see + // tests/phone-home-detail.test.js for the real end-to-end proof); + // this only checks that src/wait.js itself never chokes on a + // pathological value if that guarantee were ever violated. + const hugeDetail = 'x'.repeat(5000); + const getBootstrapStatus = jest.fn().mockResolvedValue(`failed:configuring:${hugeDetail}`); + const isRunnerOnline = jest.fn().mockResolvedValue(false); + + await expect( + waitForRunnerReady({ getBootstrapStatus, isRunnerOnline, sleep: noSleep }, { quietSeconds: 0 }), + ).rejects.toMatchObject({ bootstrapStep: 'configuring', bootstrapDetail: hugeDetail }); + }); + + test('backward compatible: an old-style "failed:" value with no detail is unaffected', async () => { + const getBootstrapStatus = jest.fn().mockResolvedValue('failed:downloading'); + const isRunnerOnline = jest.fn().mockResolvedValue(false); + + let caught; + try { + await waitForRunnerReady({ getBootstrapStatus, isRunnerOnline, sleep: noSleep }, { quietSeconds: 0 }); + } catch (error) { + caught = error; + } + + expect(caught.bootstrapStep).toBe('downloading'); + expect(caught.bootstrapDetail).toBeUndefined(); + // No " — " detail suffix leaks into the message when there is no detail. + expect(caught.message).toBe('EC2 runner bootstrap failed during the "downloading" step. See the captured console output below.'); + }); + }); }); diff --git a/tests/warmpool.test.js b/tests/warmpool.test.js index dd169642..ade0fb70 100644 --- a/tests/warmpool.test.js +++ b/tests/warmpool.test.js @@ -158,7 +158,7 @@ describe('buildUserData — reuse: stop variant', () => { test('arms an ERR trap for the downloading phase inside the RUNNER_DOWNLOAD heredoc (regression, #61)', () => { const ud = aws.buildUserData({ ...args, reuse: 'stop' }); const heredoc = extractRunnerDownloadHeredoc(ud); - const trapLine = "trap 'gh_runner_phone_home \"failed:${GH_RUNNER_STEP}\"' ERR"; + const trapLine = "trap 'gh_runner_phone_home_failed \"${GH_RUNNER_STEP}\"' ERR"; // Scoped to this heredoc alone: the outer shell's 'preparing' trap and // the register script's 'configuring' trap live elsewhere in the script // and must not be able to satisfy this assertion.