From a7ae06873f630fb280d5576f2c18e90a508a7cb3 Mon Sep 17 00:00:00 2001 From: Qianhao Dong Date: Mon, 21 Sep 2026 09:58:41 +0800 Subject: [PATCH 1/2] test: devpack smoke --- .github/scripts/devpack_smoke_schedule.mjs | 212 ++++++++++++++++++ .../scripts/devpack_smoke_schedule.test.mjs | 189 ++++++++++++++++ .github/workflows/verify-devpack-scripts.yml | 164 +++++++++++++- 3 files changed, 554 insertions(+), 11 deletions(-) create mode 100644 .github/scripts/devpack_smoke_schedule.mjs create mode 100644 .github/scripts/devpack_smoke_schedule.test.mjs diff --git a/.github/scripts/devpack_smoke_schedule.mjs b/.github/scripts/devpack_smoke_schedule.mjs new file mode 100644 index 0000000..f2f7649 --- /dev/null +++ b/.github/scripts/devpack_smoke_schedule.mjs @@ -0,0 +1,212 @@ +import { spawnSync } from "node:child_process"; +import { appendFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const issueTitle = "DevPack daily smoke failures"; +export const issueMarker = ""; + +export function selectRelease(releases) { + const candidates = releases.filter(release => + !release.draft && release.published_at && + /^devpack-installer-\d+\.\d+\.\d+$/.test(release.tag_name)); + candidates.sort((a, b) => { + const left = a.tag_name.slice("devpack-installer-".length).split(".").map(Number); + const right = b.tag_name.slice("devpack-installer-".length).split(".").map(Number); + for (let index = 0; index < 3; index++) { + if (left[index] !== right[index]) return right[index] - left[index]; + } + return 0; + }); + if (!candidates.length) throw new Error("No published DevPack release found (including GitHub prereleases)"); + return candidates[0].tag_name; +} + +export function sourceMatrices(source) { + if (!["all", "release", "aka", "winget", "brew"].includes(source)) { + throw new Error(`Invalid installation source: ${source}`); + } + const supported = { + windows: ["release", "winget", "aka"], + linux: ["release", "aka"], + macos: ["release", "brew", "aka"], + }; + return Object.fromEntries(Object.entries(supported).map(([os, sources]) => + [os, source === "all" ? sources : sources.filter(item => item === source)])); +} + +export function prepareInputs(event, inputSource, inputTag, releaseTag) { + const source = event === "schedule" ? "all" : event === "release" ? "release" : inputSource; + const tag = event === "release" ? releaseTag : inputTag; + if (tag && !/^devpack-installer-\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(tag)) { + throw new Error(`Invalid DevPack release tag: ${tag}`); + } + return { source, tag, matrices: sourceMatrices(source) }; +} + +async function api(path, method = "GET", body) { + const response = await fetch(`https://api.github.com/${path}`, { + method, + headers: { + Authorization: `Bearer ${process.env.GH_TOKEN}`, + Accept: "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(60_000), + }); + if (!response.ok) throw new Error(`GitHub ${method} ${path}: HTTP ${response.status}`); + return response.json(); +} + +async function paginate(path, property) { + const items = []; + for (let page = 1; ; page++) { + const data = await api(`${path}${path.includes("?") ? "&" : "?"}per_page=100&page=${page}`); + const batch = property ? data[property] : data; + if (!Array.isArray(batch)) throw new Error(`Invalid paginated response: ${path}`); + items.push(...batch); + if (batch.length < 100) return items; + } +} + +export async function prepare() { + const { source, tag, matrices } = prepareInputs( + process.env.GITHUB_EVENT_NAME, process.env.INPUT_SOURCE || "release", + process.env.INPUT_TAG || "", process.env.EVENT_RELEASE_TAG || "", + ); + const selected = tag || selectRelease(await paginate(`repos/${process.env.GITHUB_REPOSITORY}/releases`)); + const outputs = { release_tag: selected }; + for (const [os, sources] of Object.entries(matrices)) { + outputs[`${os}_sources`] = JSON.stringify(sources); + } + for (const [key, value] of Object.entries(outputs)) { + await appendFile(process.env.GITHUB_OUTPUT, `${key}=${value}\n`); + } + await appendFile(process.env.GITHUB_STEP_SUMMARY, + `### DevPack smoke target\n- Release: ${selected}\n- Sources: ${source}\n` + + "- Copilot: stable from the normal distribution sources\n" + + "- Distribution lag is reported as a failure, not silently downgraded to an older DevPack.\n"); + console.log(JSON.stringify(outputs, null, 2)); +} + +async function collect() { + const directory = join(process.env.RUNNER_TEMP, "devpack-diagnostics"); + await mkdir(directory, { recursive: true }); + const inventory = { + releaseTag: process.env.RELEASE_TAG, source: process.env.INSTALL_SOURCE, + scenario: process.env.SCENARIO, platform: process.platform, arch: process.arch, + jobStatus: process.env.SMOKE_JOB_STATUS, + runnerImage: `${process.env.ImageOS || "unknown"} ${process.env.ImageVersion || "unknown"}`, + tools: {}, + }; + const tools = [ + ["azureCli", "az", ["version", "-o", "json"]], + ["azd", "azd", ["version"]], + ["extensions", "azd", ["ext", "list", "--installed", "-o", "json"]], + ["copilot", "copilot", ["--version"]], + ["plugins", "copilot", ["--no-color", "plugin", "list"]], + ["vscode", "code", ["--list-extensions", "--show-versions"]], + ]; + for (const [name, file, args] of tools) { + // Only fixed commands are passed to cmd.exe, for Windows .cmd shims. + const result = process.platform === "win32" + ? spawnSync("cmd.exe", ["/d", "/s", "/c", `${file} ${args.join(" ")}`], + { encoding: "utf8", timeout: 30_000, maxBuffer: 4 * 1024 * 1024, windowsHide: true }) + : spawnSync(file, args, { encoding: "utf8", timeout: 30_000, maxBuffer: 4 * 1024 * 1024 }); + inventory.tools[name] = { + exitCode: result.status, output: result.stdout || "", error: result.error?.message || result.stderr || "", + }; + } + await writeFile(join(directory, "inventory.json"), JSON.stringify(inventory, null, 2)); + await appendFile(process.env.GITHUB_STEP_SUMMARY, + `### ${inventory.source} / ${inventory.platform}-${inventory.arch} / ${inventory.scenario}\n` + + `- Release: ${inventory.releaseTag}\n- Result before cleanup: ${inventory.jobStatus}\n` + + `- Runner: ${inventory.runnerImage}\n- Version inventory and installer logs are retained in artifacts.\n`); +} + +export function reportAction(results, issue) { + const complete = ["prepare", "windows", "linux", "macos"].every(name => results[name]?.result === "success"); + if (complete) return issue ? "close" : "none"; + return issue ? "update" : "create"; +} + +function cell(value) { + return String(value ?? "unknown").replace(/[\r\n|]/g, " ").replace(/ + `| ${name} | ${cell(results[name]?.result)} |`), + "", "### Failed or incomplete jobs", + ]; + const incomplete = jobs.filter(job => job.conclusion !== "success" && job.name !== "Report daily smoke"); + lines.push(...incomplete.map(job => `- [${cell(job.name)}](${job.html_url}): ${cell(job.conclusion || job.status)}`)); + lines.push("", "### Resolved versions", "| Source / platform / scenario | DevPack | Copilot | azd | Runner image |", + "|---|---|---|---|---|"); + for (const item of inventories) { + lines.push(`| ${cell(`${item.source} / ${item.platform}-${item.arch} / ${item.scenario}`)} | ` + + `${cell(item.releaseTag)} | ${cell(item.tools?.copilot?.output?.trim())} | ` + + `${cell(item.tools?.azd?.output?.trim())} | ${cell(item.runnerImage)} |`); + } + lines.push("", "Download the run artifacts for installer logs and the complete dependency inventory.", + "A channel still serving an older DevPack is reported rather than silently skipped."); + return lines.join("\n"); +} + +export async function report() { + const repo = process.env.GITHUB_REPOSITORY; + const runUrl = `https://github.com/${repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const results = JSON.parse(process.env.SMOKE_RESULTS); + const jobs = await paginate(`repos/${repo}/actions/runs/${process.env.GITHUB_RUN_ID}/jobs?filter=latest`, "jobs"); + const inventories = []; + const directory = process.env.SMOKE_ARTIFACTS; + if (directory) { + const entries = await readdir(directory, { recursive: true }).catch(error => { + if (error.code === "ENOENT") { + console.warn("No diagnostic artifacts available; setup may have failed."); + return []; + } + throw error; + }); + for (const entry of entries.filter(path => path.endsWith("inventory.json"))) { + inventories.push(JSON.parse(await readFile(join(directory, entry), "utf8"))); + } + } + const body = buildReport(results, jobs, inventories, runUrl); + await appendFile(process.env.GITHUB_STEP_SUMMARY, body); + if (process.env.REPORT_ISSUE !== "true") { + console.log("Manual run: issue updates disabled."); + return; + } + const issues = await paginate(`repos/${repo}/issues?state=open`); + const issue = issues.find(item => !item.pull_request && item.title === issueTitle && item.body?.includes(issueMarker)); + const action = reportAction(results, issue); + if (action === "create") { + await api(`repos/${repo}/issues`, "POST", { title: issueTitle, body }); + } else if (action === "update") { + await api(`repos/${repo}/issues/${issue.number}`, "PATCH", { body }); + } else if (action === "close") { + await api(`repos/${repo}/issues/${issue.number}/comments`, "POST", { body: `Daily smoke recovered: ${runUrl}` }); + await api(`repos/${repo}/issues/${issue.number}`, "PATCH", { state: "closed", state_reason: "completed" }); + } + console.log(`Daily smoke issue action: ${action}`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + const commands = { prepare, collect, report }; + const command = commands[process.argv[2]]; + Promise.resolve().then(() => { + if (!command) throw new Error("Usage: devpack_smoke_schedule.mjs "); + return command(); + }).catch(error => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/.github/scripts/devpack_smoke_schedule.test.mjs b/.github/scripts/devpack_smoke_schedule.test.mjs new file mode 100644 index 0000000..2bc9613 --- /dev/null +++ b/.github/scripts/devpack_smoke_schedule.test.mjs @@ -0,0 +1,189 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { buildReport, issueMarker, issueTitle, prepare, prepareInputs, report, reportAction, selectRelease, sourceMatrices } from "./devpack_smoke_schedule.mjs"; + +const release = (tag, extras = {}) => ({ tag_name: tag, published_at: "2026-09-21T00:00:00Z", draft: false, ...extras }); + +test("selects newest numeric DevPack version, including GitHub prereleases", () => { + assert.equal(selectRelease([ + release("other-product-99.0.0"), + release("devpack-installer-0.1.9"), + release("devpack-installer-0.1.10", { prerelease: true }), + release("devpack-installer-0.2.0", { draft: true }), + release("devpack-installer-0.3.0", { published_at: null }), + release("devpack-installer-0.4.0-rc.1"), + ]), "devpack-installer-0.1.10"); +}); + +test("missing published DevPack is an error, not a fallback tag", () => { + assert.throws(() => selectRelease([]), /No published DevPack/); + assert.throws(() => selectRelease([release("other-1.0.0")]), /No published DevPack/); +}); + +test("daily matrix has 48 valid jobs across all four sources", () => { + const matrix = sourceMatrices("all"); + assert.deepEqual(matrix, { + windows: ["release", "winget", "aka"], + linux: ["release", "aka"], + macos: ["release", "brew", "aka"], + }); + assert.equal(Object.values(matrix).reduce((count, sources) => count + sources.length * 2 * 3, 0), 48); +}); + +test("existing release/manual single-source matrices are unchanged", () => { + for (const source of ["release", "aka"]) { + const matrix = sourceMatrices(source); + assert.equal(Object.values(matrix).reduce((count, sources) => count + sources.length * 6, 0), 18); + } + assert.deepEqual(sourceMatrices("winget"), { windows: ["winget"], linux: [], macos: [] }); + assert.deepEqual(sourceMatrices("brew"), { windows: [], linux: [], macos: ["brew"] }); + assert.throws(() => sourceMatrices("homebrew"), /Invalid installation source/); +}); + +test("schedule resolves latest while a release event uses its exact tag", () => { + assert.equal(prepareInputs("schedule", "", "", "").source, "all"); + assert.equal(prepareInputs("schedule", "", "", "").tag, ""); + const event = prepareInputs("release", "all", "", "devpack-installer-0.1.4"); + assert.equal(event.source, "release"); + assert.equal(event.tag, "devpack-installer-0.1.4"); +}); + +test("manual inputs preserve pinned tags or request latest, including all-source dry runs", () => { + const pinned = prepareInputs("workflow_dispatch", "winget", "devpack-installer-0.1.5", ""); + assert.equal(pinned.tag, "devpack-installer-0.1.5"); + assert.deepEqual(pinned.matrices.windows, ["winget"]); + assert.equal(prepareInputs("workflow_dispatch", "all", "", "").tag, ""); + assert.throws(() => prepareInputs("workflow_dispatch", "all", "malformed\nvalue", ""), /Invalid DevPack/); +}); + +const success = () => Object.fromEntries(["prepare", "windows", "linux", "macos"].map(name => [name, { result: "success" }])); + +test("first failure creates one issue and subsequent failures update it", () => { + const results = success(); + results.windows.result = "failure"; + assert.equal(reportAction(results, undefined), "create"); + assert.equal(reportAction(results, { number: 1 }), "update"); +}); + +test("only full recovery closes the issue; cancelled, skipped and missing jobs are not green", () => { + assert.equal(reportAction(success(), { number: 1 }), "close"); + assert.equal(reportAction(success(), undefined), "none"); + for (const result of ["failure", "cancelled", "skipped", undefined]) { + const results = success(); + results.prepare = result ? { result } : undefined; + assert.equal(reportAction(results, { number: 1 }), "update"); + } +}); + +test("report identifies failures and records executed versions rather than assuming latest", () => { + const text = buildReport(success(), + [{ name: "windows | x64", conclusion: "failure", html_url: "https://github.com/job/1" }], + [{ + source: "winget", platform: "win32", arch: "x64", scenario: "with-code", + releaseTag: "devpack-installer-0.1.5", runnerImage: "windows2025 20260920", + tools: { copilot: { output: "GitHub Copilot CLI 1.0.86.\n" }, azd: { output: "azd version 1.34.1\n" } }, + }], "https://github.com/run/1"); + assert.ok(text.includes(issueMarker)); + assert.ok(text.includes("windows x64")); + assert.ok(text.includes("GitHub Copilot CLI 1.0.86.")); + assert.ok(text.includes("azd version 1.34.1")); + assert.ok(text.includes("https://github.com/job/1")); + assert.ok(!text.includes("undefined")); +}); + +test("report still describes setup failure without inventories", () => { + const text = buildReport({ prepare: { result: "failure" } }, [], [], "https://github.com/run/1"); + assert.ok(text.includes("| prepare | failure |")); + assert.ok(text.includes("| windows | unknown |")); +}); + +test("preparation paginates releases and emits the full matrix to Actions outputs", async t => { + const root = await mkdtemp(join(tmpdir(), "devpack-schedule-")); + const previousEnv = { ...process.env }; + t.after(async () => { + process.env = previousEnv; + await rm(root, { recursive: true, force: true }); + }); + Object.assign(process.env, { + GITHUB_REPOSITORY: "test/repo", GITHUB_EVENT_NAME: "schedule", + INPUT_TAG: "", INPUT_SOURCE: "", EVENT_RELEASE_TAG: "", + GITHUB_OUTPUT: join(root, "outputs.txt"), GITHUB_STEP_SUMMARY: join(root, "summary.md"), + }); + const requests = []; + t.mock.method(globalThis, "fetch", async url => { + requests.push(url); + return { + ok: true, + json: async () => url.endsWith("page=1") + ? Array.from({ length: 100 }, () => release("other-product-1.0.0")) + : [release("devpack-installer-0.1.5", { prerelease: true })], + }; + }); + await prepare(); + const output = await readFile(process.env.GITHUB_OUTPUT, "utf8"); + assert.equal(requests.length, 2); + assert.ok(output.includes("release_tag=devpack-installer-0.1.5")); + assert.ok(output.includes('windows_sources=["release","winget","aka"]')); + assert.ok(output.includes('linux_sources=["release","aka"]')); + assert.ok(output.includes('macos_sources=["release","brew","aka"]')); +}); + +test("reporting creates, updates, and closes only the tracking issue; manual all-source runs never write", async t => { + const root = await mkdtemp(join(tmpdir(), "devpack-report-")); + const previousEnv = { ...process.env }; + t.after(async () => { + process.env = previousEnv; + await rm(root, { recursive: true, force: true }); + }); + Object.assign(process.env, { + GITHUB_REPOSITORY: "test/repo", GITHUB_RUN_ID: "123", + GITHUB_STEP_SUMMARY: join(root, "summary.md"), SMOKE_ARTIFACTS: root, + }); + const failed = success(); + failed.windows.result = "failure"; + let issues = []; + const requests = []; + t.mock.method(globalThis, "fetch", async (url, options) => { + requests.push({ url, method: options.method, body: options.body && JSON.parse(options.body) }); + return { + ok: true, + json: async () => url.includes("/jobs?") + ? { jobs: [{ name: "windows x64", conclusion: "failure", html_url: "https://github.com/job/1" }] } + : url.includes("/issues?") ? issues : {}, + }; + }); + for (const [existing, enabled, results, expected] of [ + [false, false, failed, []], + [false, true, failed, ["POST"]], + [true, true, failed, ["PATCH"]], + [true, true, success(), ["POST", "PATCH"]], + [false, true, success(), []], + ]) { + // An unrelated issue with the same title but no marker must not be overwritten. + issues = [{ number: 7, title: issueTitle, body: "User-authored issue" }]; + if (existing) issues.push({ number: 42, title: issueTitle, body: issueMarker }); + process.env.REPORT_ISSUE = String(enabled); + process.env.SMOKE_RESULTS = JSON.stringify(results); + requests.length = 0; + await report(); + const writes = requests.filter(request => request.method !== "GET"); + assert.deepEqual(writes.map(request => request.method), expected); + for (const request of writes) { + assert.ok(!request.url.includes("/issues/7")); + } + if (existing && results.windows.result === "success") { + assert.deepEqual(writes[1].body, { state: "closed", state_reason: "completed" }); + } + } +}); + +test("API failures are surfaced instead of pretending resolution succeeded", async t => { + const previousEnv = { ...process.env }; + t.after(() => { process.env = previousEnv; }); + Object.assign(process.env, { GITHUB_EVENT_NAME: "schedule", INPUT_TAG: "", INPUT_SOURCE: "" }); + t.mock.method(globalThis, "fetch", async () => ({ ok: false, status: 403 })); + await assert.rejects(prepare(), /HTTP 403/); +}); diff --git a/.github/workflows/verify-devpack-scripts.yml b/.github/workflows/verify-devpack-scripts.yml index 5a86c1c..4479a9e 100644 --- a/.github/workflows/verify-devpack-scripts.yml +++ b/.github/workflows/verify-devpack-scripts.yml @@ -1,13 +1,16 @@ name: Verify DevPack scripts on: + schedule: + # 07:17 UTC+8 daily, away from the top-of-hour queue. + - cron: "17 23 * * *" release: types: [published] workflow_dispatch: inputs: release_tag: - description: "DevPack release tag to verify" - required: true + description: "DevPack release tag to verify (blank selects newest published DevPack)" + required: false type: string install_source: description: "Installation source" @@ -19,6 +22,7 @@ on: - aka - brew - winget + - all telemetry: description: "Send installer telemetry during this validation run" required: true @@ -29,32 +33,58 @@ permissions: contents: read concurrency: - group: verify-devpack-scripts-${{ github.event.release.tag_name || inputs.release_tag }}-${{ inputs.install_source || 'release' }}-telemetry-${{ inputs.telemetry || false }} + group: verify-devpack-scripts-${{ github.event_name == 'schedule' && 'daily' || github.event.release.tag_name || inputs.release_tag || 'latest' }}-${{ inputs.install_source || 'release' }}-telemetry-${{ inputs.telemetry || false }} cancel-in-progress: false env: - RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }} - INSTALL_SOURCE: ${{ inputs.install_source || 'release' }} TELEMETRY_ENABLED: ${{ inputs.telemetry || false }} FOUNDRY_DEVPACK_COLLECT_TELEMETRY: ${{ inputs.telemetry && '1' || '0' }} GITHUB_TOKEN: ${{ github.token }} GH_TOKEN: ${{ github.token }} jobs: + prepare: + if: ${{ github.event_name != 'release' || startsWith(github.event.release.tag_name, 'devpack-installer-') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + release_tag: ${{ steps.target.outputs.release_tag }} + windows_sources: ${{ steps.target.outputs.windows_sources }} + linux_sources: ${{ steps.target.outputs.linux_sources }} + macos_sources: ${{ steps.target.outputs.macos_sources }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'release' && github.event.repository.default_branch || github.ref }} + persist-credentials: false + - name: Test smoke assertions and scheduling + run: node --test .github/scripts/verify_devpack_smoke.test.mjs .github/scripts/devpack_smoke_schedule.test.mjs + - name: Select release and source matrices + id: target + env: + INPUT_TAG: ${{ inputs.release_tag }} + INPUT_SOURCE: ${{ inputs.install_source || 'release' }} + EVENT_RELEASE_TAG: ${{ github.event.release.tag_name }} + run: node .github/scripts/devpack_smoke_schedule.mjs prepare + windows: - if: ${{ (startsWith(github.event.release.tag_name, 'devpack-installer-') || startsWith(inputs.release_tag, 'devpack-installer-')) && (github.event_name == 'release' || inputs.install_source != 'brew') }} - name: windows ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ inputs.install_source || 'release' }}, telemetry=${{ inputs.telemetry || false }}) + needs: prepare + if: ${{ needs.prepare.outputs.windows_sources != '[]' }} + name: windows ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ matrix.source }}, telemetry=${{ inputs.telemetry || false }}) runs-on: ${{ matrix.platform.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: + source: ${{ fromJSON(needs.prepare.outputs.windows_sources) }} scenario: [baseline, no-az, with-code] platform: - { os: windows-latest, arch: x64 } - { os: windows-11-arm, arch: arm64 } env: + RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }} + INSTALL_SOURCE: ${{ matrix.source }} SCENARIO: ${{ matrix.scenario }} steps: @@ -294,6 +324,17 @@ jobs: throw 'Claude Code skill unexpectedly used the copy fallback' } + - name: Collect dependency inventory + if: always() + shell: pwsh + env: + SMOKE_JOB_STATUS: ${{ job.status }} + run: | + $env:Path = [Environment]::GetEnvironmentVariable('Path', 'Machine') + ';' + + [Environment]::GetEnvironmentVariable('Path', 'User') + ';' + $env:Path + node .github/scripts/devpack_smoke_schedule.mjs collect + if ($LASTEXITCODE -ne 0) { throw 'Diagnostic collection failed' } + - name: Verify ARP and uninstall shell: pwsh run: | @@ -351,20 +392,38 @@ jobs: } Remove-Item $env:DEVPACK_SCRIPT -Force -ErrorAction SilentlyContinue + - name: Upload smoke diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: devpack-smoke-windows-${{ matrix.platform.arch }}-${{ matrix.scenario }}-${{ matrix.source }} + path: | + ${{ runner.temp }}/devpack-diagnostics/ + ${{ runner.temp }}/foundry-devpack*.log + ${{ runner.temp }}/foundry-devpack-output.txt + ${{ runner.temp }}/azd-version.txt + ${{ runner.temp }}/azd-extensions.json + retention-days: 7 + if-no-files-found: warn + linux: - if: ${{ (startsWith(github.event.release.tag_name, 'devpack-installer-') || startsWith(inputs.release_tag, 'devpack-installer-')) && (github.event_name == 'release' || (inputs.install_source != 'brew' && inputs.install_source != 'winget')) }} - name: linux ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ inputs.install_source || 'release' }}, telemetry=${{ inputs.telemetry || false }}) + needs: prepare + if: ${{ needs.prepare.outputs.linux_sources != '[]' }} + name: linux ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ matrix.source }}, telemetry=${{ inputs.telemetry || false }}) runs-on: ${{ matrix.platform.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: + source: ${{ fromJSON(needs.prepare.outputs.linux_sources) }} scenario: [baseline, no-az, with-code] platform: - { os: ubuntu-latest, arch: x64 } - { os: ubuntu-24.04-arm, arch: arm64 } env: + RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }} + INSTALL_SOURCE: ${{ matrix.source }} SCENARIO: ${{ matrix.scenario }} steps: @@ -486,25 +545,53 @@ jobs: test "$(readlink "$HOME/.claude/skills/microsoft-foundry")" = "$HOME/.agents/skills/microsoft-foundry" test ! -e "$HOME/.claude/skills/microsoft-foundry/.foundry-devpack-copy" + - name: Collect dependency inventory + if: always() + shell: bash + env: + SMOKE_JOB_STATUS: ${{ job.status }} + run: | + export PATH="$HOME/.local/bin:$PATH" + node .github/scripts/devpack_smoke_schedule.mjs collect + - name: Remove bootstrap if: always() shell: bash run: rm -f "$DEVPACK_SCRIPT" + - name: Upload smoke diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: devpack-smoke-linux-${{ matrix.platform.arch }}-${{ matrix.scenario }}-${{ matrix.source }} + path: | + ${{ runner.temp }}/devpack-diagnostics/ + ${{ runner.temp }}/foundry-devpack*.log + ${{ runner.temp }}/foundry-devpack-output.txt + ${{ runner.temp }}/azd-version.txt + ${{ runner.temp }}/azd-extensions.json + ${{ runner.temp }}/copilot-plugins.txt + retention-days: 7 + if-no-files-found: warn + macos: - if: ${{ (startsWith(github.event.release.tag_name, 'devpack-installer-') || startsWith(inputs.release_tag, 'devpack-installer-')) && (github.event_name == 'release' || inputs.install_source != 'winget') }} - name: macos ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ inputs.install_source || 'release' }}, telemetry=${{ inputs.telemetry || false }}) + needs: prepare + if: ${{ needs.prepare.outputs.macos_sources != '[]' }} + name: macos ${{ matrix.platform.arch }} ${{ matrix.scenario }} (${{ matrix.source }}, telemetry=${{ inputs.telemetry || false }}) runs-on: ${{ matrix.platform.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: + source: ${{ fromJSON(needs.prepare.outputs.macos_sources) }} scenario: [baseline, no-az, with-code] platform: - { os: macos-15-intel, arch: x64 } - { os: macos-latest, arch: arm64 } env: + RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }} + INSTALL_SOURCE: ${{ matrix.source }} SCENARIO: ${{ matrix.scenario }} HOMEBREW_NO_AUTO_UPDATE: "1" HOMEBREW_NO_ANALYTICS: "1" @@ -529,6 +616,11 @@ jobs: [ "$FOUNDRY_DEVPACK_COLLECT_TELEMETRY" = "$expected" ] echo "Testing $RELEASE_TAG from $INSTALL_SOURCE on macos ${{ matrix.platform.arch }} (${{ matrix.scenario }}), telemetry=$TELEMETRY_ENABLED" + - name: Refresh Homebrew metadata for daily smoke + if: github.event_name == 'schedule' || inputs.install_source == 'all' + shell: bash + run: brew update + - name: Remove Azure CLI if: matrix.scenario == 'no-az' shell: bash @@ -640,6 +732,15 @@ jobs: test "$(readlink "$HOME/.claude/skills/microsoft-foundry")" = "$HOME/.agents/skills/microsoft-foundry" test ! -e "$HOME/.claude/skills/microsoft-foundry/.foundry-devpack-copy" + - name: Collect dependency inventory + if: always() + shell: bash + env: + SMOKE_JOB_STATUS: ${{ job.status }} + run: | + export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + node .github/scripts/devpack_smoke_schedule.mjs collect + - name: Uninstall Homebrew cask if: env.INSTALL_SOURCE == 'brew' shell: bash @@ -653,3 +754,44 @@ jobs: if: always() && env.INSTALL_SOURCE != 'brew' shell: bash run: rm -f "$DEVPACK_SCRIPT" + + - name: Upload smoke diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: devpack-smoke-macos-${{ matrix.platform.arch }}-${{ matrix.scenario }}-${{ matrix.source }} + path: | + ${{ runner.temp }}/devpack-diagnostics/ + ${{ runner.temp }}/foundry-devpack*.log + ${{ runner.temp }}/foundry-devpack-output.txt + ${{ runner.temp }}/azd-version.txt + ${{ runner.temp }}/azd-extensions.json + ${{ runner.temp }}/copilot-plugins.txt + retention-days: 7 + if-no-files-found: warn + + report: + name: Report daily smoke + needs: [prepare, windows, linux, macos] + if: ${{ always() && (github.event_name == 'schedule' || inputs.install_source == 'all') }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + issues: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + pattern: devpack-smoke-* + path: ${{ runner.temp }}/smoke-artifacts + - name: Summarize results and update tracking issue + if: always() + env: + SMOKE_RESULTS: ${{ toJSON(needs) }} + SMOKE_ARTIFACTS: ${{ runner.temp }}/smoke-artifacts + REPORT_ISSUE: ${{ github.event_name == 'schedule' }} + run: node .github/scripts/devpack_smoke_schedule.mjs report From b6825019289ee3ae9d75c852e312c29032ed1001 Mon Sep 17 00:00:00 2001 From: Qianhao Dong Date: Mon, 21 Sep 2026 10:39:08 +0800 Subject: [PATCH 2/2] test: devpack smoke --- .../scripts/devpack_smoke_schedule.test.mjs | 18 +++++++++++++----- .github/workflows/verify-devpack-scripts.yml | 5 +++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/scripts/devpack_smoke_schedule.test.mjs b/.github/scripts/devpack_smoke_schedule.test.mjs index 2bc9613..c2b2611 100644 --- a/.github/scripts/devpack_smoke_schedule.test.mjs +++ b/.github/scripts/devpack_smoke_schedule.test.mjs @@ -6,6 +6,9 @@ import test from "node:test"; import { buildReport, issueMarker, issueTitle, prepare, prepareInputs, report, reportAction, selectRelease, sourceMatrices } from "./devpack_smoke_schedule.mjs"; const release = (tag, extras = {}) => ({ tag_name: tag, published_at: "2026-09-21T00:00:00Z", draft: false, ...extras }); +// The workflow excludes Intel/no-az on macOS: five jobs per source instead of six. +const jobCount = matrix => Object.entries(matrix) + .reduce((count, [os, sources]) => count + sources.length * (os === "macos" ? 5 : 6), 0); test("selects newest numeric DevPack version, including GitHub prereleases", () => { assert.equal(selectRelease([ @@ -23,23 +26,25 @@ test("missing published DevPack is an error, not a fallback tag", () => { assert.throws(() => selectRelease([release("other-1.0.0")]), /No published DevPack/); }); -test("daily matrix has 48 valid jobs across all four sources", () => { +test("daily matrix has 45 valid jobs across all four sources", () => { const matrix = sourceMatrices("all"); assert.deepEqual(matrix, { windows: ["release", "winget", "aka"], linux: ["release", "aka"], macos: ["release", "brew", "aka"], }); - assert.equal(Object.values(matrix).reduce((count, sources) => count + sources.length * 2 * 3, 0), 48); + assert.equal(jobCount(matrix), 45); }); -test("existing release/manual single-source matrices are unchanged", () => { +test("single-source matrices retain OS coverage except macOS Intel/no-az", () => { for (const source of ["release", "aka"]) { const matrix = sourceMatrices(source); - assert.equal(Object.values(matrix).reduce((count, sources) => count + sources.length * 6, 0), 18); + assert.equal(jobCount(matrix), 17); } assert.deepEqual(sourceMatrices("winget"), { windows: ["winget"], linux: [], macos: [] }); assert.deepEqual(sourceMatrices("brew"), { windows: [], linux: [], macos: ["brew"] }); + assert.equal(jobCount(sourceMatrices("winget")), 6); + assert.equal(jobCount(sourceMatrices("brew")), 5); assert.throws(() => sourceMatrices("homebrew"), /Invalid installation source/); }); @@ -90,7 +95,10 @@ test("report identifies failures and records executed versions rather than assum assert.ok(text.includes("windows x64")); assert.ok(text.includes("GitHub Copilot CLI 1.0.86.")); assert.ok(text.includes("azd version 1.34.1")); - assert.ok(text.includes("https://github.com/job/1")); + assert.equal( + text.split("\n").find(line => line.startsWith("- [")), + "- [windows x64](https://github.com/job/1): failure", + ); assert.ok(!text.includes("undefined")); }); diff --git a/.github/workflows/verify-devpack-scripts.yml b/.github/workflows/verify-devpack-scripts.yml index 4479a9e..84ee51b 100644 --- a/.github/workflows/verify-devpack-scripts.yml +++ b/.github/workflows/verify-devpack-scripts.yml @@ -588,6 +588,11 @@ jobs: platform: - { os: macos-15-intel, arch: x64 } - { os: macos-latest, arch: arm64 } + # Fresh Azure CLI installs on Intel now build dependencies from source + # without Homebrew bottle support. Retain baseline/with-code coverage. + exclude: + - platform: { os: macos-15-intel, arch: x64 } + scenario: no-az env: RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}