From 2233e6982335e1a794e3fe8c384977c372613706 Mon Sep 17 00:00:00 2001 From: wallstop Date: Fri, 2 Oct 2026 04:13:09 +0000 Subject: [PATCH] Document release credentials and pin them to the workflows (T12) The release chain is fully built, but nothing recorded the credentials an owner must configure, and the roadmap named two that do not exist. ## Behavior - `.llm/references/RELEASING.md` records the chain (prepare -> tag -> publish), the `RELEASE_TOKEN` secret and its scope, why the default `GITHUB_TOKEN` cannot complete a release, the npm Trusted Publisher fields, the rerun and recovery matrix, and the fail-closed guards. - `scripts/lint-release-secrets.js` cross-checks the documented credential set against the secrets the release workflows actually read, in both directions, and fails closed on a missing or empty documented block. Wired into `lint:llm:full`, llm-lint CI, and pre-commit. ## Validation - 10 new self-tests, both drift directions, clean/empty/malformed cases, and a check against this repository. Harness 18/18 files green. - Red on the real defect: documenting `AUTO_COMMIT_APP_ID` and `AUTO_COMMIT_APP_PRIVATE_KEY` fails as unread, and adding a secret to a release workflow without documenting it fails as undocumented. - Lint ladder green; `npm pack` payload unchanged at 172 files. ## Risk / Rollback - Docs and one new check; no workflow, script, or production behavior changes. Rollback by reverting this commit. Refs T12. --- .github/workflows/llm-lint.yml | 9 + .llm/references/RELEASING.md | 135 ++++++++ .llm/skills/bump-version-release/SKILL.md | 5 + .pre-commit-config.yaml | 7 + package.json | 3 +- scripts/lint-release-secrets.js | 217 +++++++++++++ scripts/lint-release-secrets.js.meta | 7 + scripts/tests/test-lint-release-secrets.ps1 | 292 ++++++++++++++++++ .../tests/test-lint-release-secrets.ps1.meta | 7 + 9 files changed, 681 insertions(+), 1 deletion(-) create mode 100644 .llm/references/RELEASING.md create mode 100644 scripts/lint-release-secrets.js create mode 100644 scripts/lint-release-secrets.js.meta create mode 100644 scripts/tests/test-lint-release-secrets.ps1 create mode 100644 scripts/tests/test-lint-release-secrets.ps1.meta diff --git a/.github/workflows/llm-lint.yml b/.github/workflows/llm-lint.yml index e7957d6..22674c4 100644 --- a/.github/workflows/llm-lint.yml +++ b/.github/workflows/llm-lint.yml @@ -6,6 +6,9 @@ on: paths: - ".llm/**" - "scripts/**" + - ".github/workflows/release-prep.yml" + - ".github/workflows/release-tag.yml" + - ".github/workflows/npm-publish.yml" - "AGENTS.md" - "CLAUDE.md" - "GEMINI.md" @@ -26,6 +29,9 @@ on: paths: - ".llm/**" - "scripts/**" + - ".github/workflows/release-prep.yml" + - ".github/workflows/release-tag.yml" + - ".github/workflows/npm-publish.yml" - "AGENTS.md" - "CLAUDE.md" - "GEMINI.md" @@ -79,6 +85,9 @@ jobs: - name: Enforce namespace-scoped usings run: node scripts/lint-csharp-usings.js --verbose + - name: Enforce the release credential contract + run: node scripts/lint-release-secrets.js --verbose + - name: Enforce assembly warning policy run: pwsh -NoProfile -File scripts/lint-assembly-warnings.ps1 diff --git a/.llm/references/RELEASING.md b/.llm/references/RELEASING.md new file mode 100644 index 0000000..d4788b8 --- /dev/null +++ b/.llm/references/RELEASING.md @@ -0,0 +1,135 @@ +# Releasing com.wallstop-studios.data-visualizer + +This reference records what an owner must configure before the first release +and how to recover a release that stopped halfway. The agent-facing procedure +lives in [bump-version-release](../skills/bump-version-release/SKILL.md). + +The release chain never runs Unity. EditMode and PlayMode suites stay a local +release gate (see `.llm/context.md`). + +## Release chain + +Three workflows run in order. Each one consumes the previous step's output. + +| Step | Workflow | Trigger | Output | +| --- | --- | --- | --- | +| 1. Prepare | `release-prep.yml` | manual dispatch | `release/vX.Y.Z` pull request | +| 2. Tag | `release-tag.yml` | the release pull request merges into `main` | annotated `vX.Y.Z` tag | +| 3. Publish | `npm-publish.yml` | the tag is pushed | npm version + GitHub Release | + +`release-prep.yml` takes `bump` (`patch`, `minor`, or `major`), an optional +`explicit_version`, and `dry_run`. It bumps `package.json`, syncs the version +line in `.llm/context.md`, rotates the `Unreleased` changelog section into +`## [X.Y.Z] - YYYY-MM-DD`, validates the tree, and opens the pull request. + +`release-tag.yml` also runs by dispatch, where `branch` is required and +`dry_run` verifies without tagging. + +`npm-publish.yml` also runs by dispatch, where `tag` is the recovery entry +point (see [Rerun and recovery](#rerun-and-recovery)). + +## Required secrets + + + +| Secret | Required | Scope | Read by | +| --- | --- | --- | --- | +| `RELEASE_TOKEN` | yes, for the automatic chain | GitHub App installation token or PAT with `contents: write`; `release-prep.yml` also needs `pull-requests: write` | `release-prep.yml`, `release-tag.yml` | + + + +npm publishing uses no token. `npm-publish.yml` authenticates through npm +Trusted Publishing over OIDC, which needs the `id-token: write` permission and +a matching entry on npmjs.com (see below). + +`GITHUB_TOKEN` needs no configuration. It is the built-in per-run token, and +`npm-publish.yml` uses it to create the GitHub Release. + +### Why `RELEASE_TOKEN` is required + +GitHub does not start workflow runs for events created with the default +`GITHUB_TOKEN`. Both workflows read `secrets.RELEASE_TOKEN || github.token`, so +they stay runnable before the secret exists. That fallback has two silent +consequences: + +- `release-prep.yml` pushes the release branch and opens the pull request, but + the pull request gets no CI run. +- `release-tag.yml` creates and pushes the tag, but `npm-publish.yml` never + starts. The release stops after tagging with no npm publish and no GitHub + Release, and every workflow reports success. + +The tag push is the step that cannot recover on its own, so configure +`RELEASE_TOKEN` before the first release. + +## npm Trusted Publisher setup + +Configure the package once on npmjs.com, under the package settings for +`com.wallstop-studios.data-visualizer` → Trusted Publisher → GitHub Actions. + +| Field | Value | +| --- | --- | +| Organization or user | `wallstop` | +| Repository | `DataVisualizer` | +| Workflow filename | `npm-publish.yml` | +| Environment name | leave empty | +| Allowed actions | allow `npm publish` | + +Points that decide whether a release succeeds: + +- npm does not verify the configuration when it is saved. A wrong field appears + only at publish time, as an `ENEEDAUTH` error. +- Allowed actions default matters. Trusted publisher configurations created + after 2026-09-03 allow `npm stage publish` only. `npm-publish.yml` calls + `npm publish` directly, so that action must be enabled explicitly. +- The workflow filename is the file that contains the publish command, given as + a bare filename with its extension. The manual rerun dispatches + `npm-publish.yml` itself, so the same entry covers both the tag push and the + rerun. Do not add a second publisher for a dispatch workflow. +- Every field is case-sensitive and must match exactly. +- Trusted publishing needs npm CLI 11.5.1 or later and Node 22.14.0 or later. + `npm-publish.yml` pins Node 24 and fails closed when npm is older than + 11.5.1. +- Only GitHub-hosted runners are supported. Self-hosted runners cannot + publish. +- `package.json` `repository.url` must match the GitHub repository, which it + already does. + +After the first successful publish, set the package's publishing access to +require two-factor authentication and disallow tokens, and add a tag +protection rule. Those are recommended hardening, not prerequisites. + +## Rerun and recovery + +| State | Action | +| --- | --- | +| The release pull request was not opened | Re-dispatch `release-prep.yml` with the same inputs. Use `dry_run` first to check the computed version. | +| `release-tag.yml` failed | Fix the cause, then dispatch it with `branch` set to `release/vX.Y.Z`. Use `dry_run` to verify before tagging. | +| The tag already exists | Never retargeted. Delete the tag deliberately, or cut a new version. | +| npm published, the GitHub Release failed | Dispatch `npm-publish.yml` with the `tag` input. The publish step is skipped and the release is edited and re-uploaded. | +| Nothing published yet | Dispatch `npm-publish.yml` with the `tag` input to run the whole step again. | + +Two rules make reruns safe: + +- `tag-release.mjs` refuses to move an existing tag, so a rerun cannot repoint + a published version at different commits. +- `npm-publish.yml` skips `npm publish` when `name@version` is already on the + registry, and `gh release upload --clobber` replaces artifacts. A rerun + therefore finishes a partially completed release instead of duplicating it. + +npm publication is otherwise irreversible. There is no reliable unpublish. +Recover from a bad release with a new patch version. + +## Fail-closed guards + +The chain refuses to continue rather than publish something wrong. + +| Script | Rejects | +| --- | --- | +| `prepare-release.mjs` | an unsupported version, a dirty tree, changelog rotation that cannot be applied | +| `tag-release.mjs` | a branch that is not `release/v`, a missing, duplicated, or empty changelog section, an existing tag, a dirty tree, an unresolvable commit | +| `verify-release.mjs` | a tag that is not `v`, packed files outside the allowlist, allowlisted files missing from the tarball | +| `build-unitypackage.mjs` | unsafe paths, missing or malformed `.meta` companions, duplicate or malformed GUIDs | +| `validate-unitypackage.mjs` | a corrupt or empty archive, members outside the GUID-directory layout, duplicate members, unsafe paths, checksum problems, and any disagreement with the tracked tree | + +`npm-publish.yml` also checks that npm is new enough for trusted publishing +before it publishes. diff --git a/.llm/skills/bump-version-release/SKILL.md b/.llm/skills/bump-version-release/SKILL.md index 4eef004..da27912 100644 --- a/.llm/skills/bump-version-release/SKILL.md +++ b/.llm/skills/bump-version-release/SKILL.md @@ -7,6 +7,11 @@ metadata: # Skill: Bump Version / Release +Owner setup for the release chain (required secrets, npm Trusted Publisher +fields, rerun and recovery) is in +[releasing](../../references/RELEASING.md). Read it before the first release of +a clone or after a partial failure. + ## Steps 1. Edit `package.json` `version` (semver; append `-rc`/`-alpha`/`-beta`/`-preview` diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48ffb65..969619b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,6 +41,13 @@ repos: pass_filenames: false files: ^(Editor|Runtime|Tests|Generator~)/.*\.cs$ description: Moves `using` directives inside the namespace; file-level usings stay sanctioned for assembly-attribute preambles. + - id: lint-release-secrets + name: Enforce the release credential contract + entry: node scripts/lint-release-secrets.js + language: system + pass_filenames: false + files: ^(\.llm/references/RELEASING\.md|\.github/workflows/(release-prep|release-tag|npm-publish)\.yml)$ + description: Fails when a release workflow reads an undocumented secret, or a documented secret is read by no release workflow. - id: lint-assembly-warnings name: Enforce assembly warning policy entry: pwsh -NoProfile -File scripts/lint-assembly-warnings.ps1 diff --git a/package.json b/package.json index c760b3d..9e285da 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,8 @@ "lint:csharp-null-assertions": "node scripts/lint-csharp-null-assertions.js", "lint:csharp-usings": "node scripts/lint-csharp-usings.js", "lint:csharp-usings:fix": "node scripts/lint-csharp-usings.js --fix", - "lint:llm:full": "pwsh -NoProfile -File scripts/generate-skills-index.ps1 && pwsh -NoProfile -File scripts/lint-llm-instructions.ps1 -VerboseOutput && pwsh -NoProfile -File scripts/lint-file-lengths.ps1 -VerboseOutput && node scripts/lint-csharp-member-order.js --verbose && pwsh -NoProfile -File scripts/tests/run-all.ps1", + "lint:release-secrets": "node scripts/lint-release-secrets.js", + "lint:llm:full": "pwsh -NoProfile -File scripts/generate-skills-index.ps1 && pwsh -NoProfile -File scripts/lint-llm-instructions.ps1 -VerboseOutput && pwsh -NoProfile -File scripts/lint-file-lengths.ps1 -VerboseOutput && node scripts/lint-csharp-member-order.js --verbose && node scripts/lint-release-secrets.js --verbose && pwsh -NoProfile -File scripts/tests/run-all.ps1", "mcp:sync": "bash .llm/mcp/sync-mcp.sh", "unity:mcp:host": "node .llm/mcp/unity-mcp-host.mjs", "tools:install": "bash .devcontainer/install-npm-tools.sh", diff --git a/scripts/lint-release-secrets.js b/scripts/lint-release-secrets.js new file mode 100644 index 0000000..f33c4ca --- /dev/null +++ b/scripts/lint-release-secrets.js @@ -0,0 +1,217 @@ +#!/usr/bin/env node +/** + * Release credential contract: every secret the release workflows read must be + * documented, and every credential the release reference documents must be read + * by a release workflow. + * + * The release chain (prepare -> tag -> publish) is driven by workflow YAML that + * no test executes, so its credential requirements drift silently. A roadmap + * item once named `AUTO_COMMIT_APP_ID` and `AUTO_COMMIT_APP_PRIVATE_KEY` as the + * credentials to configure; neither exists in the repository, the workflows read + * a single `RELEASE_TOKEN`, and the resulting failure mode is invisible because + * a `GITHUB_TOKEN`-pushed tag never starts the publish workflow while every job + * still reports success. This check fails closed on that class of drift. + * + * The documented set is read from the `release-secrets` block in + * `.llm/references/RELEASING.md`, so prose elsewhere in the reference is free to + * mention credentials that are not required (for example "no NPM_TOKEN is + * involved") without failing the scan. + * + * The release workflow set is the three workflows the reference documents. A + * new release workflow must be added to RELEASE_WORKFLOWS and to the reference. + * + * `--verbose` prints the compared sets when clean. Exit codes: 0 = in sync, + * 1 = drift, a missing or malformed documented block, or an unreadable input. + * + * `RELEASE_SECRETS_WORKFLOW_DIR` and `RELEASE_SECRETS_DOC` override the inputs + * so the self-test can point at a fixture tree. Nothing in CI sets them. + */ + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const REPO_ROOT = path.resolve(__dirname, ".."); + +// The workflows `.llm/references/RELEASING.md` documents. Kept explicit so the +// scan does not demand release documentation for unrelated workflows. +const RELEASE_WORKFLOWS = ["release-prep.yml", "release-tag.yml", "npm-publish.yml"]; + +// Provided by GitHub for every run; configuring it is not an owner task, so it +// is never part of the documented contract. +const BUILT_IN_SECRETS = new Set(["GITHUB_TOKEN"]); + +const BLOCK_START = ""; +const BLOCK_END = ""; + +// Both documented GitHub Actions secret forms: `secrets.NAME` and +// `secrets['NAME']`. A computed index such as `secrets[format(...)]` cannot be +// resolved statically and is outside this check's coverage. +const SECRET_REFERENCE = + /\bsecrets(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\])/g; +const DOCUMENTED_SECRET = /`([A-Z][A-Z0-9_]*)`/g; + +const WORKFLOW_DIR = process.env.RELEASE_SECRETS_WORKFLOW_DIR + ? path.resolve(REPO_ROOT, process.env.RELEASE_SECRETS_WORKFLOW_DIR) + : path.join(REPO_ROOT, ".github", "workflows"); +const REFERENCE_DOC = process.env.RELEASE_SECRETS_DOC + ? path.resolve(REPO_ROOT, process.env.RELEASE_SECRETS_DOC) + : path.join(REPO_ROOT, ".llm", "references", "RELEASING.md"); + +function readFileOrThrow(filePath, purpose) { + try { + return fs.readFileSync(filePath, "utf8"); + } catch { + throw new Error(`Cannot ${purpose}: '${filePath}' is missing or unreadable.`); + } +} + +/* + Blanks YAML comments so a commented-out secret reference cannot create a + documented-set requirement. Quoted regions are preserved, because a '#' + inside a string is data rather than a comment. +*/ +function maskYamlComments(text) { + const characters = text.split(""); + const blank = (start, end) => { + for (let index = start; index < end; index++) { + if (characters[index] !== "\n") { + characters[index] = " "; + } + } + }; + let index = 0; + while (index < text.length) { + const character = text[index]; + if (character === "'" || character === '"') { + const quote = character; + index++; + while (index < text.length) { + if (text[index] === "\\" && quote === '"') { + index += 2; + continue; + } + if (text[index] === quote) { + if (text[index + 1] === quote) { + index += 2; + continue; + } + index++; + break; + } + index++; + } + continue; + } + if (character === "#") { + const end = text.indexOf("\n", index); + const stop = end < 0 ? text.length : end; + blank(index, stop); + index = stop; + continue; + } + index++; + } + return characters.join(""); +} + +function readWorkflowSecrets(filePath) { + const masked = maskYamlComments(readFileOrThrow(filePath, "read the release workflow")); + const secrets = new Set(); + for (const match of masked.matchAll(SECRET_REFERENCE)) { + // Alternation group 1 is the dot form, group 2 the bracket form. + const name = match[1] ?? match[2]; + if (!BUILT_IN_SECRETS.has(name)) { + secrets.add(name); + } + } + return secrets; +} + +function readDocumentedSecrets(filePath) { + const text = readFileOrThrow(filePath, "read the release reference"); + const start = text.indexOf(BLOCK_START); + const end = text.indexOf(BLOCK_END); + if (start < 0 || end < 0 || end < start) { + throw new Error( + `'${filePath}' must carry a '${BLOCK_START}' / '${BLOCK_END}' block listing the ` + + "secrets the release workflows read; the documented set is parsed from it.", + ); + } + const block = text.slice(start + BLOCK_START.length, end); + const secrets = new Set(); + for (const match of block.matchAll(DOCUMENTED_SECRET)) { + secrets.add(match[1]); + } + if (secrets.size === 0) { + throw new Error( + `'${filePath}' has an empty release-secrets block; name each secret in ` + + 'backticks, for example `RELEASE_TOKEN`.', + ); + } + return secrets; +} + +function sorted(set) { + return [...set].sort(); +} + +/* + A repo-relative path when the file lives inside the repository, otherwise the + absolute path, so a message never shows a '../../..' chain. +*/ +function displayPath(filePath) { + const relative = path.relative(REPO_ROOT, filePath).split(path.sep).join("/"); + return relative.startsWith("../") ? filePath : relative; +} + +function main() { + const verbose = process.argv.includes("--verbose"); + const read = new Set(); + for (const workflow of RELEASE_WORKFLOWS) { + for (const secret of readWorkflowSecrets(path.join(WORKFLOW_DIR, workflow))) { + read.add(secret); + } + } + const documented = readDocumentedSecrets(REFERENCE_DOC); + + const undocumented = sorted(read).filter((secret) => !documented.has(secret)); + const unused = sorted(documented).filter((secret) => !read.has(secret)); + + if (undocumented.length > 0 || unused.length > 0) { + console.error( + "The release credential contract is out of sync. The release workflows read " + + `${sorted(read).join(", ") || "no secret"}; ` + + `'${displayPath(REFERENCE_DOC)}' documents ${sorted(documented).join(", ")}.`, + ); + for (const secret of undocumented) { + console.error( + ` ${secret}: read by a release workflow but not documented. Add it to the ` + + "release-secrets block with the scope an owner must grant.", + ); + } + for (const secret of unused) { + console.error( + ` ${secret}: documented but read by no release workflow. Remove it or point ` + + "the workflow at the credential the owner actually configures.", + ); + } + process.exit(1); + } + + if (verbose) { + console.log( + `Release credential contract in sync: ${sorted(read).length} secret(s) documented ` + + `across ${RELEASE_WORKFLOWS.length} release workflow(s).`, + ); + } + process.exit(0); +} + +try { + main(); +} catch (error) { + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/scripts/lint-release-secrets.js.meta b/scripts/lint-release-secrets.js.meta new file mode 100644 index 0000000..069e68c --- /dev/null +++ b/scripts/lint-release-secrets.js.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 1bb407ab0651444e8c56846dec25ed6f +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/scripts/tests/test-lint-release-secrets.ps1 b/scripts/tests/test-lint-release-secrets.ps1 new file mode 100644 index 0000000..f58f217 --- /dev/null +++ b/scripts/tests/test-lint-release-secrets.ps1 @@ -0,0 +1,292 @@ +Set-StrictMode -Version 2.0 + +$script:TestFailureCount = 0 +. (Join-Path $PSScriptRoot 'TestHelpers.ps1') + +$lintScript = Join-Path (Split-Path -Parent $PSScriptRoot) 'lint-release-secrets.js' +$repoRoot = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +Write-Host '== Release credential self-tests ==' + +function Invoke-ReleaseLint { + param( + [string]$WorkflowDir, + [string]$Document, + [string[]]$LintArguments = @() + ) + + $hadWorkflowDir = Test-Path Env:RELEASE_SECRETS_WORKFLOW_DIR + $previousWorkflowDir = $env:RELEASE_SECRETS_WORKFLOW_DIR + $hadDocument = Test-Path Env:RELEASE_SECRETS_DOC + $previousDocument = $env:RELEASE_SECRETS_DOC + try { + $env:RELEASE_SECRETS_WORKFLOW_DIR = $WorkflowDir + $env:RELEASE_SECRETS_DOC = $Document + $output = & node $lintScript @LintArguments 2>&1 | Out-String + return [pscustomobject]@{ ExitCode = $LASTEXITCODE; Output = $output } + } finally { + if ($hadWorkflowDir) { + $env:RELEASE_SECRETS_WORKFLOW_DIR = $previousWorkflowDir + } else { + Remove-Item Env:RELEASE_SECRETS_WORKFLOW_DIR -ErrorAction SilentlyContinue + } + if ($hadDocument) { + $env:RELEASE_SECRETS_DOC = $previousDocument + } else { + Remove-Item Env:RELEASE_SECRETS_DOC -ErrorAction SilentlyContinue + } + } +} + +# The three release workflows the guard reads. Every fixture writes all three so +# a case exercises the real file set rather than a reduced one. +function Write-ReleaseWorkflows { + param([string]$Root, [string]$Prep, [string]$Tag, [string]$Publish) + + Write-FixtureFile -Root $Root -RelativePath 'workflows/release-prep.yml' -Content $Prep + Write-FixtureFile -Root $Root -RelativePath 'workflows/release-tag.yml' -Content $Tag + Write-FixtureFile -Root $Root -RelativePath 'workflows/npm-publish.yml' -Content $Publish +} + +# The lint resolves its overrides against the repository root, so fixtures pass +# absolute paths. +function Invoke-FixtureLint { + param([string]$Root, [string[]]$LintArguments = @()) + + return Invoke-ReleaseLint ` + -WorkflowDir (Join-Path $Root 'workflows') ` + -Document (Join-Path $Root 'RELEASING.md') ` + -LintArguments $LintArguments +} + +$inSyncPrep = @' +name: Prepare release +jobs: + prepare: + steps: + - uses: actions/checkout@v7 + with: + token: ${{ secrets.RELEASE_TOKEN || github.token }} +'@ + +$inSyncTag = @' +name: Tag release +jobs: + tag: + steps: + - uses: actions/checkout@v7 + with: + token: ${{ secrets.RELEASE_TOKEN || github.token }} +'@ + +$inSyncPublish = @' +name: Publish to NPM +jobs: + publish: + steps: + - run: gh release view "$TAG" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +'@ + +$inSyncDocument = @' +# Releasing + +Prose may mention NPM_TOKEN because no token is involved. + + + +| Secret | Scope | +| --- | --- | +| `RELEASE_TOKEN` | `contents: write` | + + + +More prose. +'@ + +Invoke-TestCase 'Passes_WhenWorkflowSecretsAreDocumented' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $inSyncDocument + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 0) "in-sync fixture should pass: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Fails_WhenWorkflowSecretIsUndocumented' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + $prep = $inSyncPrep + "`n - run: echo `"`${{ secrets.DRY_RUN_SIGNING_KEY }}`"`n" + Write-ReleaseWorkflows -Root $root -Prep $prep -Tag $inSyncTag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $inSyncDocument + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "undocumented secret must fail: $($result.Output)" + Assert-True ( + $result.Output -match 'DRY_RUN_SIGNING_KEY' + ) "must name the undocumented secret: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Fails_WhenDocumentedSecretIsReadByNoWorkflow' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + $document = $inSyncDocument -replace '`RELEASE_TOKEN`', '`RELEASE_TOKEN` / `AUTO_COMMIT_APP_ID`' + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $document + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "unused documented secret must fail: $($result.Output)" + Assert-True ( + $result.Output -match 'AUTO_COMMIT_APP_ID' + ) "must name the phantom credential: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Ignores_CommentedSecretsAndBuiltInTokens' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + $tag = @' +name: Tag release +# A commented-out reference: ${{ secrets.REMOVED_SIGNING_KEY }} +jobs: + tag: + steps: + - run: echo "# ${{ secrets.RELEASE_TOKEN }} stays data, not a comment" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +'@ + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $tag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $inSyncDocument + $result = Invoke-FixtureLint -Root $root + Assert-True ( + $result.ExitCode -eq 0 + ) "comments must be masked and built-ins excluded: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Detects_BracketFormSecretReferences' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + # GitHub Actions accepts secrets['NAME'] and secrets["NAME"]; a guard that + # only matched the dot form would pass a workflow holding an undocumented + # credential, which is the exact failure this check exists to prevent. + $publish = @' +name: Publish to NPM +jobs: + publish: + steps: + - env: + SINGLE: ${{ secrets['SINGLE_QUOTED_SECRET'] }} + DOUBLE: ${{ secrets["DOUBLE_QUOTED_SECRET"] }} +'@ + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $publish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $inSyncDocument + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "bracket-form secrets must be reported: $($result.Output)" + Assert-True ( + $result.Output -match 'SINGLE_QUOTED_SECRET' + ) "must name the single-quoted secret: $($result.Output)" + Assert-True ( + $result.Output -match 'DOUBLE_QUOTED_SECRET' + ) "must name the double-quoted secret: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Finds_SecretsInAnyReleaseWorkflow' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + # Guards against per-file matcher state leaking between workflows: a secret + # reachable only from the last workflow read must still be found. + $noSecretPrep = "name: Prepare release`njobs:`n prepare:`n steps:`n - run: echo ok`n" + $noSecretTag = "name: Tag release`njobs:`n tag:`n steps:`n - run: echo ok`n" + $publish = @' +name: Publish to NPM +jobs: + publish: + steps: + - env: + K: ${{ secrets.LAST_WORKFLOW_ONLY }} +'@ + Write-ReleaseWorkflows -Root $root -Prep $noSecretPrep -Tag $noSecretTag -Publish $publish + $document = $inSyncDocument -replace '`RELEASE_TOKEN`', '`LAST_WORKFLOW_ONLY`' + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $document + $result = Invoke-FixtureLint -Root $root + Assert-True ( + $result.ExitCode -eq 0 + ) "the last workflow's secrets must be read: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Fails_WhenDocumentedBlockIsMissing' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content "# Releasing`n`nNo block here.`n" + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "a missing block must fail closed: $($result.Output)" + Assert-True ( + $result.Output -match 'release-secrets:begin' + ) "must explain the required block: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Fails_WhenDocumentedBlockIsEmpty' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + $document = "# Releasing`n`n`n`n`n" + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $inSyncPublish + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $document + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "an empty block must fail closed: $($result.Output)" + Assert-True ( + $result.Output -match 'empty release-secrets block' + ) "must explain the empty block: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Fails_WhenWorkflowIsMissing' { + $root = New-TempRoot -Prefix 'release-secrets-' + try { + Write-ReleaseWorkflows -Root $root -Prep $inSyncPrep -Tag $inSyncTag -Publish $inSyncPublish + Remove-Item -LiteralPath (Join-Path $root 'workflows/release-tag.yml') -Force + Write-FixtureFile -Root $root -RelativePath 'RELEASING.md' -Content $inSyncDocument + $result = Invoke-FixtureLint -Root $root + Assert-True ($result.ExitCode -eq 1) "a missing release workflow must fail closed: $($result.Output)" + Assert-True ( + $result.Output -match 'release-tag\.yml' + ) "must name the missing workflow: $($result.Output)" + } finally { + Remove-TempRoot $root + } +} + +Invoke-TestCase 'Passes_AgainstTheRepositoryItself' { + $result = Invoke-ReleaseLint -WorkflowDir (Join-Path $repoRoot '.github/workflows') -Document (Join-Path $repoRoot '.llm/references/RELEASING.md') -LintArguments @('--verbose') + Assert-True ($result.ExitCode -eq 0) "the repository must satisfy its own contract: $($result.Output)" + Assert-True ( + $result.Output -match 'Release credential contract in sync: 1 secret' + ) "verbose output must report the compared set: $($result.Output)" +} + +if ($script:TestFailureCount -gt 0) { + Write-Host "release credential self-tests: $($script:TestFailureCount) failed" + exit 1 +} +Write-Host 'release credential self-tests: all passed' +exit 0 diff --git a/scripts/tests/test-lint-release-secrets.ps1.meta b/scripts/tests/test-lint-release-secrets.ps1.meta new file mode 100644 index 0000000..72b2970 --- /dev/null +++ b/scripts/tests/test-lint-release-secrets.ps1.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 4b04103409b146aab9e275a807eb1f4e +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: