From e1a513915120e36b00925ee662c105a78c7a92b1 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 24 Sep 2026 18:39:54 +0200 Subject: [PATCH 01/11] chore(ci): Preview weekly flaky test report Co-Authored-By: GPT-6 --- .github/workflows/weekly-flaky-tests.yml | 41 ++++ scripts/report-ci-failures.mjs | 4 +- scripts/weekly-flaky-tests.mjs | 196 +++++++++++++++++++ scripts/weekly-flaky-tests.test.ts | 227 +++++++++++++++++++++++ 4 files changed, 466 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/weekly-flaky-tests.yml create mode 100644 scripts/weekly-flaky-tests.mjs create mode 100644 scripts/weekly-flaky-tests.test.ts diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml new file mode 100644 index 000000000000..675507cb2a1a --- /dev/null +++ b/.github/workflows/weekly-flaky-tests.yml @@ -0,0 +1,41 @@ +name: 'CI: Weekly flaky tests (preview)' + +on: + pull_request: + branches: [develop] + paths: + - '.github/workflows/weekly-flaky-tests.yml' + - 'scripts/weekly-flaky-tests*' + - 'scripts/report-ci-failures.mjs' + workflow_dispatch: + # Enable after validating the preview on PRs. + # schedule: + # - cron: '0 7 * * 1' + +permissions: + contents: read + actions: read + checks: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + report: + name: Weekly flaky test report + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Analyze the last seven days of develop CI + uses: actions/github-script@v9 + with: + retries: 3 + script: | + const { default: run } = await import( + `${process.env.GITHUB_WORKSPACE}/scripts/weekly-flaky-tests.mjs` + ); + await run({ github, context, core }); diff --git a/scripts/report-ci-failures.mjs b/scripts/report-ci-failures.mjs index 58278b4b5a8f..4978a14a1c6b 100644 --- a/scripts/report-ci-failures.mjs +++ b/scripts/report-ci-failures.mjs @@ -31,7 +31,7 @@ import { readFileSync } from 'node:fs'; * "Playwright esm (1/4) Tests" -> "Playwright Tests" * "E2E some-app-node-20-18 Test" -> "E2E some-app Test" */ -function normalizeJobName(name) { +export function normalizeJobName(name) { return name .replace(/\(\s*(?:\d+|TS\s+[\d.]+|Node\s+\d+|\d+\/\d+)\s*\)/gi, ' ') .replace(/Playwright\s+(?:bundle\w*|esm|cjs)\s+Tests/gi, 'Playwright Tests') @@ -50,7 +50,7 @@ function normalizeJobName(name) { * - bare esm/cjs describe block: "... > esm/cjs > x" -> "... > x" * - trailing module suffix: "... should send [esm]" -> "... should send" */ -function normalizeTestName(name) { +export function normalizeTestName(name) { return name .replace(/^\[(?:chromium|firefox|webkit)\]\s*›\s*/i, '') .replace(/(\.[cm]?[jt]sx?):\d+:\d+/gi, '$1') diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs new file mode 100644 index 000000000000..98a2bd65bb71 --- /dev/null +++ b/scripts/weekly-flaky-tests.mjs @@ -0,0 +1,196 @@ +import { normalizeJobName, normalizeTestName } from './report-ci-failures.mjs'; + +const LOOKBACK_DAYS = 7; +const MAX_TESTS = 10; + +function jobFamily(name) { + return normalizeJobName( + name + .replace(/\(\d+(?:\.\d+)+\)/g, '') + .replace(/(Playwright\s+\S+)\s+(?:chromium|firefox|webkit)(?=\s+Tests)/g, '$1'), + ); +} + +function markdownCell(value) { + return String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/[\\`*_[\]|]/g, '\\$&') + .replace(/[\r\n]+/g, ' '); +} + +export async function collectReport({ github, context, now = new Date() }) { + const until = now.toISOString(); + const since = new Date(now.getTime() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000).toISOString(); + const repo = context.repo; + const listedRuns = await github.paginate(github.rest.actions.listWorkflowRuns, { + ...repo, + workflow_id: 'build.yml', + branch: 'develop', + created: `${since}..${until}`, + per_page: 100, + }); + const warnings = []; + if (listedRuns.length >= 1000) { + warnings.push('GitHub limits this query to 1,000 runs; this report may be incomplete.'); + } + const runs = listedRuns.filter(run => ['push', 'schedule', 'workflow_dispatch'].includes(run.event)); + const tests = new Map(); + let failedJobsWithoutTests = 0; + let failedJobs = 0; + + for (const run of runs) { + if (run.conclusion === 'success' && run.run_attempt === 1) { + continue; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...repo, + run_id: run.id, + filter: 'all', + per_page: 100, + }); + for (const job of jobs) { + if (job.conclusion !== 'failure' || job.name.includes('(optional)')) { + continue; + } + failedJobs++; + + let annotations; + try { + annotations = await github.paginate(github.rest.checks.listAnnotations, { + ...repo, + check_run_id: Number(job.check_run_url.split('/').pop()), + per_page: 100, + }); + } catch (error) { + if (error.status !== 404 && error.status !== 410) { + throw error; + } + warnings.push(`Annotations unavailable for job ${job.id} in run ${run.id}.`); + continue; + } + + const failures = annotations.filter( + annotation => + annotation.annotation_level === 'failure' && + annotation.title && + /(?:^|\/)(?:tests?|__tests__|suites)\/|(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/i.test(annotation.path), + ); + if (failures.length === 0) { + failedJobsWithoutTests++; + } + + // Match the exact matrix job, not its normalized family, when checking recovery. + const recovered = jobs.some( + later => + later.name === job.name && + later.head_sha === job.head_sha && + later.run_attempt > job.run_attempt && + later.conclusion === 'success', + ); + for (const annotation of failures) { + const family = jobFamily(job.name); + const name = normalizeTestName(annotation.title); + const key = JSON.stringify([family, annotation.path, name]); + let test = tests.get(key); + if (!test) { + test = { + family, + path: annotation.path, + name, + runs: new Set(), + recoveredRuns: new Set(), + examples: new Map(), + }; + tests.set(key, test); + } + test.runs.add(run.id); + if (recovered) { + test.recoveredRuns.add(run.id); + } + if (test.examples.size < 3 && !test.examples.has(run.id)) { + test.examples.set(run.id, job.html_url); + } + } + } + } + + const ranked = [...tests.values()].sort( + (a, b) => + b.runs.size - a.runs.size || + b.recoveredRuns.size - a.recoveredRuns.size || + a.name.localeCompare(b.name) || + a.family.localeCompare(b.family) || + a.path.localeCompare(b.path), + ); + + return { + since, + until, + runs: runs.length, + failedJobs, + failedJobsWithoutTests, + warnings, + totalTests: ranked.length, + tests: ranked.slice(0, MAX_TESTS).map(test => ({ + family: test.family, + path: test.path, + name: test.name, + runs: test.runs.size, + recoveredRuns: test.recoveredRuns.size, + examples: [...test.examples.values()], + })), + }; +} + +export function renderReport(report) { + const lines = [ + '## Top 10 failing tests this week', + '', + `Develop CI runs created between ${report.since} and ${report.until}.`, + '', + `${report.runs} runs examined, including earlier attempts of runs that eventually passed.`, + `Showing the ${report.tests.length} most frequent failures out of ${report.totalTests} failing tests.`, + '', + 'Each test counts once per workflow run, regardless of matrix variants or reruns.', + 'Recovery means the same job passed in a later attempt on the same commit; it does not prove the individual test reran. Recurring failures can also be regressions.', + '', + ]; + + if (report.tests.length === 0) { + lines.push('No test failures found in the available annotations.', ''); + } else { + lines.push( + '| Test | Job family | Affected runs | Runs with job recovery | Examples |', + '| --- | --- | ---: | ---: | --- |', + ); + for (const test of report.tests) { + const links = test.examples.map((url, index) => `[${index + 1}](${url})`).join(', '); + lines.push( + `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs} | ${test.recoveredRuns} | ${links} |`, + ); + } + } + + lines.push( + '', + '### Coverage', + '', + `${report.failedJobs} failed job attempts inspected; ${report.failedJobsWithoutTests} had no recognizable test failure annotations.`, + 'Optional jobs are excluded. Setup failures and tests without annotations are not ranked. Counts are not per-test failure rates.', + '', + ...report.warnings.map(warning => `- ${markdownCell(warning)}`), + ); + return `${lines.join('\n')}\n`; +} + +export default async function run({ github, context, core }) { + const report = await collectReport({ github, context }); + for (const warning of report.warnings) { + core.warning(warning); + } + await core.summary.addRaw(renderReport(report)).write(); + return report; +} diff --git a/scripts/weekly-flaky-tests.test.ts b/scripts/weekly-flaky-tests.test.ts new file mode 100644 index 000000000000..23941fb69cd2 --- /dev/null +++ b/scripts/weekly-flaky-tests.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it, vi } from 'vitest'; +import { collectReport, renderReport } from './weekly-flaky-tests.mjs'; + +const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' } }; +const now = new Date('2026-09-24T12:00:00Z'); +const testPath = 'suites/tracing/http-timings/test.ts'; +const testTitle = `${testPath} › adds HTTP timing`; + +function workflowRun(id: number, overrides = {}) { + return { id, event: 'push', conclusion: 'failure', run_attempt: 1, ...overrides }; +} + +function job(id: number, overrides = {}) { + return { + id, + name: 'Playwright esm (1/4) Tests', + conclusion: 'failure', + head_sha: `commit-${id}`, + run_attempt: 1, + check_run_url: `https://api.github.com/repos/getsentry/sentry-javascript/check-runs/${id + 1000}`, + html_url: `https://github.com/getsentry/sentry-javascript/actions/runs/1/job/${id}`, + ...overrides, + }; +} + +function annotation(overrides = {}) { + return { + annotation_level: 'failure', + path: testPath, + title: `[chromium] › ${testPath}:6:11 › adds HTTP timing`, + ...overrides, + }; +} + +function githubFixture({ + runs = [workflowRun(1), workflowRun(2), workflowRun(3)], + jobs = { 1: [job(1)], 2: [job(2)], 3: [job(3)] }, + annotations = {}, +}: { + runs?: ReturnType[]; + jobs?: Record[]>; + annotations?: Record[] | Error>; +} = {}) { + const listWorkflowRuns = vi.fn(); + const listJobsForWorkflowRun = vi.fn(); + const listAnnotations = vi.fn(); + const paginate = vi.fn(async (endpoint, params) => { + if (endpoint === listWorkflowRuns) { + return runs; + } + if (endpoint === listJobsForWorkflowRun) { + return jobs[params.run_id] || []; + } + if (endpoint === listAnnotations) { + const result = annotations[params.check_run_id] || [annotation()]; + if (result instanceof Error) { + throw result; + } + return result; + } + throw new Error('Unexpected endpoint'); + }); + return { paginate, rest: { actions: { listWorkflowRuns, listJobsForWorkflowRun }, checks: { listAnnotations } } }; +} + +describe('collectReport', () => { + it('counts a test once per run across matrix variants, duplicate annotations and attempts', async () => { + const github = githubFixture({ + jobs: { + 1: [ + job(1), + job(11, { name: 'Playwright bundle_tracing_replay webkit Tests', head_sha: 'commit-1' }), + job(12, { run_attempt: 2, head_sha: 'commit-1' }), + ], + 2: [job(2)], + 3: [job(3)], + }, + annotations: { + 1001: [annotation(), annotation()], + 1011: [annotation({ title: `[webkit] › ${testPath}:90:11 › adds HTTP timing` })], + }, + }); + + const report = await collectReport({ github, context, now }); + + expect(report.tests).toEqual([ + { + family: 'Playwright Tests', + path: testPath, + name: testTitle, + runs: 3, + recoveredRuns: 0, + examples: [job(1).html_url, job(2).html_url, job(3).html_url], + }, + ]); + expect(github.paginate).toHaveBeenCalledWith(github.rest.actions.listWorkflowRuns, { + ...context.repo, + workflow_id: 'build.yml', + branch: 'develop', + created: '2026-09-17T12:00:00.000Z..2026-09-24T12:00:00.000Z', + per_page: 100, + }); + expect(github.paginate).toHaveBeenCalledWith(github.rest.checks.listAnnotations, { + ...context.repo, + check_run_id: 1001, + per_page: 100, + }); + }); + + it('includes failures hidden by successful reruns and only credits recovery of the same matrix job and SHA', async () => { + const github = githubFixture({ + runs: [workflowRun(1, { conclusion: 'success', run_attempt: 2 }), workflowRun(2), workflowRun(3)], + jobs: { + 1: [job(1), job(11, { conclusion: 'success', run_attempt: 2, head_sha: 'commit-1' })], + 2: [job(2), job(22, { conclusion: 'success', run_attempt: 2, head_sha: 'different-commit' })], + 3: [ + job(3), + job(33, { conclusion: 'success', run_attempt: 2, head_sha: 'commit-3', name: 'Playwright bundle Tests' }), + ], + }, + }); + + const report = await collectReport({ github, context, now }); + + expect(report.tests).toHaveLength(1); + expect(report.tests[0].runs).toBe(3); + expect(report.tests[0].recoveredRuns).toBe(1); + expect(github.paginate).toHaveBeenCalledWith(github.rest.actions.listJobsForWorkflowRun, { + ...context.repo, + run_id: 1, + filter: 'all', + per_page: 100, + }); + }); + + it.each([ + { label: 'only two runs', runs: [workflowRun(1), workflowRun(2)], jobs: { 1: [job(1)], 2: [job(2)] } }, + { + label: 'only one commit', + runs: [workflowRun(1), workflowRun(2), workflowRun(3)], + jobs: { 1: [job(1)], 2: [job(2, { head_sha: 'commit-1' })], 3: [job(3, { head_sha: 'commit-1' })] }, + }, + ])('includes failures affecting $label without minimum thresholds', async ({ runs, jobs }) => { + const github = githubFixture({ runs, jobs }); + + const report = await collectReport({ github, context, now }); + + expect(report.tests).toHaveLength(1); + expect(report.tests[0].runs).toBe(runs.length); + }); + + it('excludes PR runs, optional jobs and setup errors while reporting missing test annotations', async () => { + const github = githubFixture({ + runs: [workflowRun(1, { event: 'pull_request' }), workflowRun(2), workflowRun(3)], + jobs: { 2: [job(2, { name: 'E2E nuxt Test (optional)' })], 3: [job(3)] }, + annotations: { 1003: [annotation({ path: '.github', title: 'Process completed with exit code 1.' })] }, + }); + + const report = await collectReport({ github, context, now }); + + expect(report.tests).toEqual([]); + expect(report.runs).toBe(2); + expect(report.failedJobs).toBe(1); + expect(report.failedJobsWithoutTests).toBe(1); + expect(github.paginate).toHaveBeenCalledTimes(4); + }); + + it('ranks by affected runs and limits the digest to ten tests', async () => { + const failures = Array.from({ length: 11 }, (_, index) => annotation({ title: `test ${index}` })); + const github = githubFixture({ + runs: [workflowRun(1), workflowRun(2), workflowRun(3), workflowRun(4)], + jobs: { 1: [job(1)], 2: [job(2)], 3: [job(3)], 4: [job(4)] }, + annotations: { 1001: failures, 1002: failures, 1003: failures, 1004: [failures[9]] }, + }); + + const report = await collectReport({ github, context, now }); + + expect(report.tests.map(test => [test.name, test.runs])).toEqual([ + ['test 9', 4], + ['test 0', 3], + ['test 1', 3], + ['test 10', 3], + ['test 2', 3], + ['test 3', 3], + ['test 4', 3], + ['test 5', 3], + ['test 6', 3], + ['test 7', 3], + ]); + expect(report.totalTests).toBe(11); + }); + + it('reports unavailable annotations instead of silently treating the run as clean', async () => { + const github = githubFixture({ annotations: { 1001: Object.assign(new Error('Not Found'), { status: 404 }) } }); + + const report = await collectReport({ github, context, now }); + + expect(report.warnings).toEqual(['Annotations unavailable for job 1 in run 1.']); + expect(report.tests).toHaveLength(1); + expect(report.tests[0].runs).toBe(2); + }); + + it('fails on API authorization or rate limit errors', async () => { + const github = githubFixture({ + annotations: { 1001: Object.assign(new Error('API rate limit'), { status: 403 }) }, + }); + + await expect(collectReport({ github, context, now })).rejects.toThrow('API rate limit'); + }); +}); + +describe('renderReport', () => { + it('escapes annotation text so it cannot break the digest table or inject HTML', async () => { + const failure = annotation({ title: 'handles | [links]\ncorrectly' }); + const github = githubFixture({ annotations: { 1001: [failure], 1002: [failure], 1003: [failure] } }); + const report = await collectReport({ github, context, now }); + + const tableRows = renderReport(report) + .split('\n') + .filter(line => line.startsWith('| handles')); + + expect(tableRows).toEqual([ + '| handles <img> \\| \\[links\\] correctly | Playwright Tests | 3 | 0 | ' + + `[1](${job(1).html_url}), [2](${job(2).html_url}), [3](${job(3).html_url}) |`, + ]); + }); +}); From 24c693147da2d6b64ccc93692fccf9f93c910d39 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 24 Sep 2026 18:43:29 +0200 Subject: [PATCH 02/11] chore(ci): Validate flaky report through preview workflow Co-Authored-By: GPT-6 --- scripts/weekly-flaky-tests.test.ts | 227 ----------------------------- 1 file changed, 227 deletions(-) delete mode 100644 scripts/weekly-flaky-tests.test.ts diff --git a/scripts/weekly-flaky-tests.test.ts b/scripts/weekly-flaky-tests.test.ts deleted file mode 100644 index 23941fb69cd2..000000000000 --- a/scripts/weekly-flaky-tests.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { collectReport, renderReport } from './weekly-flaky-tests.mjs'; - -const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' } }; -const now = new Date('2026-09-24T12:00:00Z'); -const testPath = 'suites/tracing/http-timings/test.ts'; -const testTitle = `${testPath} › adds HTTP timing`; - -function workflowRun(id: number, overrides = {}) { - return { id, event: 'push', conclusion: 'failure', run_attempt: 1, ...overrides }; -} - -function job(id: number, overrides = {}) { - return { - id, - name: 'Playwright esm (1/4) Tests', - conclusion: 'failure', - head_sha: `commit-${id}`, - run_attempt: 1, - check_run_url: `https://api.github.com/repos/getsentry/sentry-javascript/check-runs/${id + 1000}`, - html_url: `https://github.com/getsentry/sentry-javascript/actions/runs/1/job/${id}`, - ...overrides, - }; -} - -function annotation(overrides = {}) { - return { - annotation_level: 'failure', - path: testPath, - title: `[chromium] › ${testPath}:6:11 › adds HTTP timing`, - ...overrides, - }; -} - -function githubFixture({ - runs = [workflowRun(1), workflowRun(2), workflowRun(3)], - jobs = { 1: [job(1)], 2: [job(2)], 3: [job(3)] }, - annotations = {}, -}: { - runs?: ReturnType[]; - jobs?: Record[]>; - annotations?: Record[] | Error>; -} = {}) { - const listWorkflowRuns = vi.fn(); - const listJobsForWorkflowRun = vi.fn(); - const listAnnotations = vi.fn(); - const paginate = vi.fn(async (endpoint, params) => { - if (endpoint === listWorkflowRuns) { - return runs; - } - if (endpoint === listJobsForWorkflowRun) { - return jobs[params.run_id] || []; - } - if (endpoint === listAnnotations) { - const result = annotations[params.check_run_id] || [annotation()]; - if (result instanceof Error) { - throw result; - } - return result; - } - throw new Error('Unexpected endpoint'); - }); - return { paginate, rest: { actions: { listWorkflowRuns, listJobsForWorkflowRun }, checks: { listAnnotations } } }; -} - -describe('collectReport', () => { - it('counts a test once per run across matrix variants, duplicate annotations and attempts', async () => { - const github = githubFixture({ - jobs: { - 1: [ - job(1), - job(11, { name: 'Playwright bundle_tracing_replay webkit Tests', head_sha: 'commit-1' }), - job(12, { run_attempt: 2, head_sha: 'commit-1' }), - ], - 2: [job(2)], - 3: [job(3)], - }, - annotations: { - 1001: [annotation(), annotation()], - 1011: [annotation({ title: `[webkit] › ${testPath}:90:11 › adds HTTP timing` })], - }, - }); - - const report = await collectReport({ github, context, now }); - - expect(report.tests).toEqual([ - { - family: 'Playwright Tests', - path: testPath, - name: testTitle, - runs: 3, - recoveredRuns: 0, - examples: [job(1).html_url, job(2).html_url, job(3).html_url], - }, - ]); - expect(github.paginate).toHaveBeenCalledWith(github.rest.actions.listWorkflowRuns, { - ...context.repo, - workflow_id: 'build.yml', - branch: 'develop', - created: '2026-09-17T12:00:00.000Z..2026-09-24T12:00:00.000Z', - per_page: 100, - }); - expect(github.paginate).toHaveBeenCalledWith(github.rest.checks.listAnnotations, { - ...context.repo, - check_run_id: 1001, - per_page: 100, - }); - }); - - it('includes failures hidden by successful reruns and only credits recovery of the same matrix job and SHA', async () => { - const github = githubFixture({ - runs: [workflowRun(1, { conclusion: 'success', run_attempt: 2 }), workflowRun(2), workflowRun(3)], - jobs: { - 1: [job(1), job(11, { conclusion: 'success', run_attempt: 2, head_sha: 'commit-1' })], - 2: [job(2), job(22, { conclusion: 'success', run_attempt: 2, head_sha: 'different-commit' })], - 3: [ - job(3), - job(33, { conclusion: 'success', run_attempt: 2, head_sha: 'commit-3', name: 'Playwright bundle Tests' }), - ], - }, - }); - - const report = await collectReport({ github, context, now }); - - expect(report.tests).toHaveLength(1); - expect(report.tests[0].runs).toBe(3); - expect(report.tests[0].recoveredRuns).toBe(1); - expect(github.paginate).toHaveBeenCalledWith(github.rest.actions.listJobsForWorkflowRun, { - ...context.repo, - run_id: 1, - filter: 'all', - per_page: 100, - }); - }); - - it.each([ - { label: 'only two runs', runs: [workflowRun(1), workflowRun(2)], jobs: { 1: [job(1)], 2: [job(2)] } }, - { - label: 'only one commit', - runs: [workflowRun(1), workflowRun(2), workflowRun(3)], - jobs: { 1: [job(1)], 2: [job(2, { head_sha: 'commit-1' })], 3: [job(3, { head_sha: 'commit-1' })] }, - }, - ])('includes failures affecting $label without minimum thresholds', async ({ runs, jobs }) => { - const github = githubFixture({ runs, jobs }); - - const report = await collectReport({ github, context, now }); - - expect(report.tests).toHaveLength(1); - expect(report.tests[0].runs).toBe(runs.length); - }); - - it('excludes PR runs, optional jobs and setup errors while reporting missing test annotations', async () => { - const github = githubFixture({ - runs: [workflowRun(1, { event: 'pull_request' }), workflowRun(2), workflowRun(3)], - jobs: { 2: [job(2, { name: 'E2E nuxt Test (optional)' })], 3: [job(3)] }, - annotations: { 1003: [annotation({ path: '.github', title: 'Process completed with exit code 1.' })] }, - }); - - const report = await collectReport({ github, context, now }); - - expect(report.tests).toEqual([]); - expect(report.runs).toBe(2); - expect(report.failedJobs).toBe(1); - expect(report.failedJobsWithoutTests).toBe(1); - expect(github.paginate).toHaveBeenCalledTimes(4); - }); - - it('ranks by affected runs and limits the digest to ten tests', async () => { - const failures = Array.from({ length: 11 }, (_, index) => annotation({ title: `test ${index}` })); - const github = githubFixture({ - runs: [workflowRun(1), workflowRun(2), workflowRun(3), workflowRun(4)], - jobs: { 1: [job(1)], 2: [job(2)], 3: [job(3)], 4: [job(4)] }, - annotations: { 1001: failures, 1002: failures, 1003: failures, 1004: [failures[9]] }, - }); - - const report = await collectReport({ github, context, now }); - - expect(report.tests.map(test => [test.name, test.runs])).toEqual([ - ['test 9', 4], - ['test 0', 3], - ['test 1', 3], - ['test 10', 3], - ['test 2', 3], - ['test 3', 3], - ['test 4', 3], - ['test 5', 3], - ['test 6', 3], - ['test 7', 3], - ]); - expect(report.totalTests).toBe(11); - }); - - it('reports unavailable annotations instead of silently treating the run as clean', async () => { - const github = githubFixture({ annotations: { 1001: Object.assign(new Error('Not Found'), { status: 404 }) } }); - - const report = await collectReport({ github, context, now }); - - expect(report.warnings).toEqual(['Annotations unavailable for job 1 in run 1.']); - expect(report.tests).toHaveLength(1); - expect(report.tests[0].runs).toBe(2); - }); - - it('fails on API authorization or rate limit errors', async () => { - const github = githubFixture({ - annotations: { 1001: Object.assign(new Error('API rate limit'), { status: 403 }) }, - }); - - await expect(collectReport({ github, context, now })).rejects.toThrow('API rate limit'); - }); -}); - -describe('renderReport', () => { - it('escapes annotation text so it cannot break the digest table or inject HTML', async () => { - const failure = annotation({ title: 'handles | [links]\ncorrectly' }); - const github = githubFixture({ annotations: { 1001: [failure], 1002: [failure], 1003: [failure] } }); - const report = await collectReport({ github, context, now }); - - const tableRows = renderReport(report) - .split('\n') - .filter(line => line.startsWith('| handles')); - - expect(tableRows).toEqual([ - '| handles <img> \\| \\[links\\] correctly | Playwright Tests | 3 | 0 | ' + - `[1](${job(1).html_url}), [2](${job(2).html_url}), [3](${job(3).html_url}) |`, - ]); - }); -}); From 0b3954cefcb279a68620ca557d865eade6cb0050 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Thu, 24 Sep 2026 19:11:07 +0200 Subject: [PATCH 03/11] fix(ci): Parallelize flaky report scans and log progress Co-Authored-By: GPT-6 --- scripts/weekly-flaky-tests.mjs | 151 ++++++++++++++++++--------------- 1 file changed, 82 insertions(+), 69 deletions(-) diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index 98a2bd65bb71..359351a63a27 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -2,6 +2,7 @@ import { normalizeJobName, normalizeTestName } from './report-ci-failures.mjs'; const LOOKBACK_DAYS = 7; const MAX_TESTS = 10; +const CONCURRENT_RUNS = 4; function jobFamily(name) { return normalizeJobName( @@ -20,10 +21,11 @@ function markdownCell(value) { .replace(/[\r\n]+/g, ' '); } -export async function collectReport({ github, context, now = new Date() }) { +export async function collectReport({ github, context, core, now = new Date() }) { const until = now.toISOString(); const since = new Date(now.getTime() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000).toISOString(); const repo = context.repo; + core.info(`Listing develop CI runs from ${since} to ${until}.`); const listedRuns = await github.paginate(github.rest.actions.listWorkflowRuns, { ...repo, workflow_id: 'build.yml', @@ -39,83 +41,94 @@ export async function collectReport({ github, context, now = new Date() }) { const tests = new Map(); let failedJobsWithoutTests = 0; let failedJobs = 0; + let completedRuns = 0; + const pendingRuns = runs.filter(run => run.conclusion !== 'success' || run.run_attempt > 1); + const runsToInspect = pendingRuns.length; + core.info( + `Found ${runs.length} runs; ${runsToInspect} need inspection, ${runs.length - runsToInspect} passed on the first attempt.`, + ); - for (const run of runs) { - if (run.conclusion === 'success' && run.run_attempt === 1) { - continue; - } + async function worker() { + while (pendingRuns.length > 0) { + const run = pendingRuns.shift(); + core.info(`Run ${run.id}: fetching jobs across ${run.run_attempt} attempt(s).`); - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - ...repo, - run_id: run.id, - filter: 'all', - per_page: 100, - }); - for (const job of jobs) { - if (job.conclusion !== 'failure' || job.name.includes('(optional)')) { - continue; - } - failedJobs++; + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...repo, + run_id: run.id, + filter: 'all', + per_page: 100, + }); + const jobsToInspect = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); + core.info(`Run ${run.id}: ${jobs.length} job attempts, ${jobsToInspect.length} failures to inspect.`); + for (const job of jobsToInspect) { + failedJobs++; - let annotations; - try { - annotations = await github.paginate(github.rest.checks.listAnnotations, { - ...repo, - check_run_id: Number(job.check_run_url.split('/').pop()), - per_page: 100, - }); - } catch (error) { - if (error.status !== 404 && error.status !== 410) { - throw error; + let annotations; + try { + annotations = await github.paginate(github.rest.checks.listAnnotations, { + ...repo, + check_run_id: Number(job.check_run_url.split('/').pop()), + per_page: 100, + }); + } catch (error) { + if (error.status !== 404 && error.status !== 410) { + throw error; + } + warnings.push(`Annotations unavailable for job ${job.id} in run ${run.id}.`); + continue; } - warnings.push(`Annotations unavailable for job ${job.id} in run ${run.id}.`); - continue; - } - const failures = annotations.filter( - annotation => - annotation.annotation_level === 'failure' && - annotation.title && - /(?:^|\/)(?:tests?|__tests__|suites)\/|(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/i.test(annotation.path), - ); - if (failures.length === 0) { - failedJobsWithoutTests++; - } - - // Match the exact matrix job, not its normalized family, when checking recovery. - const recovered = jobs.some( - later => - later.name === job.name && - later.head_sha === job.head_sha && - later.run_attempt > job.run_attempt && - later.conclusion === 'success', - ); - for (const annotation of failures) { - const family = jobFamily(job.name); - const name = normalizeTestName(annotation.title); - const key = JSON.stringify([family, annotation.path, name]); - let test = tests.get(key); - if (!test) { - test = { - family, - path: annotation.path, - name, - runs: new Set(), - recoveredRuns: new Set(), - examples: new Map(), - }; - tests.set(key, test); - } - test.runs.add(run.id); - if (recovered) { - test.recoveredRuns.add(run.id); + const failures = annotations.filter( + annotation => + annotation.annotation_level === 'failure' && + annotation.title && + /(?:^|\/)(?:tests?|__tests__|suites)\/|(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/i.test(annotation.path), + ); + if (failures.length === 0) { + failedJobsWithoutTests++; } - if (test.examples.size < 3 && !test.examples.has(run.id)) { - test.examples.set(run.id, job.html_url); + + // Match the exact matrix job, not its normalized family, when checking recovery. + const recovered = jobs.some( + later => + later.name === job.name && + later.head_sha === job.head_sha && + later.run_attempt > job.run_attempt && + later.conclusion === 'success', + ); + for (const annotation of failures) { + const family = jobFamily(job.name); + const name = normalizeTestName(annotation.title); + const key = JSON.stringify([family, annotation.path, name]); + let test = tests.get(key); + if (!test) { + test = { + family, + path: annotation.path, + name, + runs: new Set(), + recoveredRuns: new Set(), + examples: new Map(), + }; + tests.set(key, test); + } + test.runs.add(run.id); + if (recovered) { + test.recoveredRuns.add(run.id); + } + if (test.examples.size < 3 && !test.examples.has(run.id)) { + test.examples.set(run.id, job.html_url); + } } } + completedRuns++; + core.info( + `Completed ${completedRuns}/${runsToInspect} runs; ${failedJobs} failed jobs inspected, ${tests.size} failing tests found.`, + ); } } + await Promise.all(Array.from({ length: CONCURRENT_RUNS }, () => worker())); const ranked = [...tests.values()].sort( (a, b) => @@ -187,7 +200,7 @@ export function renderReport(report) { } export default async function run({ github, context, core }) { - const report = await collectReport({ github, context }); + const report = await collectReport({ github, context, core }); for (const warning of report.warnings) { core.warning(warning); } From 583669797a368d3b671c6b35751fb6beb7c67d5b Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 10:41:34 +0200 Subject: [PATCH 04/11] ref(ci): Simplify weekly test failure report Co-Authored-By: GPT-6 --- scripts/weekly-flaky-tests.mjs | 82 +++++++++------------------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index 359351a63a27..6d5b9473cba3 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -1,7 +1,6 @@ import { normalizeJobName, normalizeTestName } from './report-ci-failures.mjs'; const LOOKBACK_DAYS = 7; -const MAX_TESTS = 10; const CONCURRENT_RUNS = 4; function jobFamily(name) { @@ -89,36 +88,17 @@ export async function collectReport({ github, context, core, now = new Date() }) failedJobsWithoutTests++; } - // Match the exact matrix job, not its normalized family, when checking recovery. - const recovered = jobs.some( - later => - later.name === job.name && - later.head_sha === job.head_sha && - later.run_attempt > job.run_attempt && - later.conclusion === 'success', - ); for (const annotation of failures) { const family = jobFamily(job.name); const name = normalizeTestName(annotation.title); const key = JSON.stringify([family, annotation.path, name]); let test = tests.get(key); if (!test) { - test = { - family, - path: annotation.path, - name, - runs: new Set(), - recoveredRuns: new Set(), - examples: new Map(), - }; + test = { family, path: annotation.path, name, runs: new Map() }; tests.set(key, test); } - test.runs.add(run.id); - if (recovered) { - test.recoveredRuns.add(run.id); - } - if (test.examples.size < 3 && !test.examples.has(run.id)) { - test.examples.set(run.id, job.html_url); + if (!test.runs.has(run.id)) { + test.runs.set(run.id, job.html_url); } } } @@ -130,15 +110,6 @@ export async function collectReport({ github, context, core, now = new Date() }) } await Promise.all(Array.from({ length: CONCURRENT_RUNS }, () => worker())); - const ranked = [...tests.values()].sort( - (a, b) => - b.runs.size - a.runs.size || - b.recoveredRuns.size - a.recoveredRuns.size || - a.name.localeCompare(b.name) || - a.family.localeCompare(b.family) || - a.path.localeCompare(b.path), - ); - return { since, until, @@ -146,53 +117,40 @@ export async function collectReport({ github, context, core, now = new Date() }) failedJobs, failedJobsWithoutTests, warnings, - totalTests: ranked.length, - tests: ranked.slice(0, MAX_TESTS).map(test => ({ - family: test.family, - path: test.path, - name: test.name, - runs: test.runs.size, - recoveredRuns: test.recoveredRuns.size, - examples: [...test.examples.values()], - })), + tests: [...tests.values()].sort( + (a, b) => + b.runs.size - a.runs.size || + a.name.localeCompare(b.name) || + a.family.localeCompare(b.family) || + a.path.localeCompare(b.path), + ), }; } export function renderReport(report) { const lines = [ - '## Top 10 failing tests this week', + '## Weekly test failures', '', - `Develop CI runs created between ${report.since} and ${report.until}.`, - '', - `${report.runs} runs examined, including earlier attempts of runs that eventually passed.`, - `Showing the ${report.tests.length} most frequent failures out of ${report.totalTests} failing tests.`, - '', - 'Each test counts once per workflow run, regardless of matrix variants or reruns.', - 'Recovery means the same job passed in a later attempt on the same commit; it does not prove the individual test reran. Recurring failures can also be regressions.', + `Develop · ${report.since.slice(0, 10)}–${report.until.slice(0, 10)} · ${report.runs} CI runs. Each test counts once per run.`, '', ]; if (report.tests.length === 0) { - lines.push('No test failures found in the available annotations.', ''); + lines.push('No test failures found.'); } else { - lines.push( - '| Test | Job family | Affected runs | Runs with job recovery | Examples |', - '| --- | --- | ---: | ---: | --- |', - ); + lines.push('| Test | Job | Affected runs | Example runs |', '| --- | --- | ---: | --- |'); for (const test of report.tests) { - const links = test.examples.map((url, index) => `[${index + 1}](${url})`).join(', '); - lines.push( - `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs} | ${test.recoveredRuns} | ${links} |`, - ); + const links = [...test.runs] + .sort(([a], [b]) => b - a) + .map(([id, url]) => `[${id}](${url})`) + .join(', '); + lines.push(`| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} |`); } } lines.push( '', - '### Coverage', - '', - `${report.failedJobs} failed job attempts inspected; ${report.failedJobsWithoutTests} had no recognizable test failure annotations.`, - 'Optional jobs are excluded. Setup failures and tests without annotations are not ranked. Counts are not per-test failure rates.', + `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs excluded.`, '', ...report.warnings.map(warning => `- ${markdownCell(warning)}`), ); From fe7550630c3c4492fd6572c3731bc418579f67d1 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 10:51:35 +0200 Subject: [PATCH 05/11] ref(ci): Replace per-run flaky issues with weekly reporting Co-Authored-By: GPT-6 --- .github/FLAKY_CI_FAILURE_TEMPLATE.md | 24 ---- .github/workflows/build.yml | 22 +-- .github/workflows/weekly-flaky-tests.yml | 9 +- scripts/report-ci-failures.mjs | 168 ----------------------- scripts/weekly-flaky-tests.mjs | 28 ++-- 5 files changed, 24 insertions(+), 227 deletions(-) delete mode 100644 .github/FLAKY_CI_FAILURE_TEMPLATE.md delete mode 100644 scripts/report-ci-failures.mjs diff --git a/.github/FLAKY_CI_FAILURE_TEMPLATE.md b/.github/FLAKY_CI_FAILURE_TEMPLATE.md deleted file mode 100644 index 23843c2c507f..000000000000 --- a/.github/FLAKY_CI_FAILURE_TEMPLATE.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: '[Flaky CI]: {{ env.JOB_NAME }} - {{ env.TEST_NAME }}' -labels: Tests, "Flaky Test" ---- - -### Flakiness Type - -Other / Unknown - -### Name of Job - -{{ env.JOB_NAME }} - -### Name of Test - -{{ env.TEST_NAME }} - -### Link to Test Run - -{{ env.RUN_LINK }} - ---- - -_This issue was automatically created._ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45e298794f10..1091965d89c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1261,28 +1261,8 @@ jobs: # Always run this, even if a dependent job failed if: always() runs-on: ubuntu-24.04 - permissions: - issues: write - checks: read + permissions: {} steps: - - name: Check out current commit - if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') - uses: actions/checkout@v7 - with: - sparse-checkout: | - .github - scripts - - - name: Create issues for failed jobs - if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') - uses: actions/github-script@v9 - with: - script: | - const { default: run } = await import( - `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` - ); - await run({ github, context, core }); - - name: Check for failures if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: | diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml index 675507cb2a1a..0d3ec3f93993 100644 --- a/.github/workflows/weekly-flaky-tests.yml +++ b/.github/workflows/weekly-flaky-tests.yml @@ -1,4 +1,4 @@ -name: 'CI: Weekly flaky tests (preview)' +name: 'CI: Weekly flaky tests' on: pull_request: @@ -6,11 +6,10 @@ on: paths: - '.github/workflows/weekly-flaky-tests.yml' - 'scripts/weekly-flaky-tests*' - - 'scripts/report-ci-failures.mjs' workflow_dispatch: - # Enable after validating the preview on PRs. - # schedule: - # - cron: '0 7 * * 1' + schedule: + - cron: '0 7 * * 1' + timezone: 'Europe/Vienna' permissions: contents: read diff --git a/scripts/report-ci-failures.mjs b/scripts/report-ci-failures.mjs deleted file mode 100644 index 4978a14a1c6b..000000000000 --- a/scripts/report-ci-failures.mjs +++ /dev/null @@ -1,168 +0,0 @@ -/** - * CI Failure Reporter script. - * - * Creates GitHub issues for tests that fail on the develop branch. - * For each failed job in the workflow run, it fetches check run annotations - * to identify individual failing tests, then creates one issue per failing - * test using the FLAKY_CI_FAILURE_TEMPLATE.md template. Existing open issues - * with matching titles are skipped to avoid duplicates. - * - * Intended to be called from a GitHub Actions workflow via actions/github-script: - * - * const { default: run } = await import( - * `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` - * ); - * await run({ github, context, core }); - */ - -import { readFileSync } from 'node:fs'; - -/** - * Collapse matrix variants of a job name so the same test failing across the matrix dedupes to a - * single issue instead of one per variant. We strip version-like parenthetical groups — a bare - * number (node version), a `TS x.y` bracket, a `Node xx` bracket, or a `n/m` shard — and fold the - * Playwright bundle/esm build configs and E2E `-node-xx` app suffixes into a single bucket, leaving - * other parentheticals (e.g. `(nextjs-app, 20)`) intact: - * - * "Node (22) Integration Tests" -> "Node Integration Tests" - * "Node (24) (TS 5.0) Integration Tests" -> "Node Integration Tests" - * "aws-serverless-layer (Node 22) Test" -> "aws-serverless-layer Test" - * "Playwright bundle_tracing_replay Tests" -> "Playwright Tests" - * "Playwright esm (1/4) Tests" -> "Playwright Tests" - * "E2E some-app-node-20-18 Test" -> "E2E some-app Test" - */ -export function normalizeJobName(name) { - return name - .replace(/\(\s*(?:\d+|TS\s+[\d.]+|Node\s+\d+|\d+\/\d+)\s*\)/gi, ' ') - .replace(/Playwright\s+(?:bundle\w*|esm|cjs)\s+Tests/gi, 'Playwright Tests') - .replace(/-node-\d+(?:-\d+)*/gi, '') - .replace(/\s+/g, ' ') - .trim(); -} - -/** - * Collapse variants of a test name so the same test failing under different module formats, browser - * projects, or (for Playwright) at a different source line dedupes to a single issue. We strip: - * - * - Playwright browser prefix: "[chromium] › suites/... " -> "suites/... " - * - Playwright file line/column: "test.ts:33:11 › ..." -> "test.ts › ..." (drifts on edits) - * - old esm/cjs describe block: "... > esm/cjs > esm > x" -> "... > x" - * - bare esm/cjs describe block: "... > esm/cjs > x" -> "... > x" - * - trailing module suffix: "... should send [esm]" -> "... should send" - */ -export function normalizeTestName(name) { - return name - .replace(/^\[(?:chromium|firefox|webkit)\]\s*›\s*/i, '') - .replace(/(\.[cm]?[jt]sx?):\d+:\d+/gi, '$1') - .replace(/esm\/cjs\s*>\s*(?:esm|cjs)\b/gi, 'esm/cjs') - .replace(/\besm\/cjs\s*>\s*/gi, '') - .replace(/\s*\[(?:esm|cjs)\]/gi, '') - .replace(/\s+/g, ' ') - .trim(); -} - -function applyVars(text, vars) { - let result = text; - for (const [key, value] of Object.entries(vars)) { - result = result.replace(new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g'), value); - } - return result; -} - -export default async function run({ github, context, core }) { - const { owner, repo } = context.repo; - - // Fetch actual job details from the API to get descriptive names - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner, - repo, - run_id: context.runId, - per_page: 100, - }); - - const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); - - if (failedJobs.length === 0) { - core.info('No failed jobs found'); - return; - } - - // Read and parse template - const template = readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8'); - const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); - const titleTemplate = frontmatter.match(/title:\s*'(.*)'/)[1]; - - // Titles we've already created or matched in this run, so the same flaky test failing on - // multiple matrix jobs within a single run doesn't open duplicate issues (the `existing` list - // below is fetched once and won't include issues created earlier in this same run). - const handledTitles = new Set(); - - // Get existing open issues with Tests label - const existing = await github.paginate(github.rest.issues.listForRepo, { - owner, - repo, - state: 'open', - labels: 'Tests', - per_page: 100, - }); - - for (const job of failedJobs) { - const jobName = job.name; - const normalizedJobName = normalizeJobName(jobName); - const jobUrl = job.html_url; - - // Fetch annotations from the check run to extract failed test names - let testNames = []; - try { - const annotations = await github.paginate(github.rest.checks.listAnnotations, { - owner, - repo, - check_run_id: job.id, - per_page: 100, - }); - - const testAnnotations = annotations.filter(a => a.annotation_level === 'failure' && a.path !== '.github'); - testNames = [...new Set(testAnnotations.map(a => a.title || a.path))]; - } catch (e) { - core.info(`Could not fetch annotations for ${jobName}: ${e.message}`); - } - - // If no test names found, abort - this could mean something else, e.g. cache restoration or similar fails - // and also the issue is not super helpful in this case - if (testNames.length === 0) { - continue; - } - - // Create one issue per failing test for proper deduplication - for (const testName of testNames) { - const normalizedTestName = normalizeTestName(testName); - - // The title is keyed on the *normalized* job name + test name so the same test failing across - // matrix variants (different node / TS versions) or module formats (esm / cjs) dedupes to a - // single issue. - const title = applyVars(titleTemplate, { JOB_NAME: normalizedJobName, TEST_NAME: normalizedTestName }); - // The body keeps the concrete job name + run link of the variant that actually failed. - const issueBody = applyVars(bodyTemplate, { JOB_NAME: jobName, RUN_LINK: jobUrl, TEST_NAME: testName }); - - if (handledTitles.has(title)) { - continue; - } - handledTitles.add(title); - - const existingIssue = existing.find(i => i.title === title); - if (existingIssue) { - core.info(`Issue already exists for "${normalizedTestName}" in ${normalizedJobName}: #${existingIssue.number}`); - continue; - } - - const newIssue = await github.rest.issues.create({ - owner, - repo, - title, - body: issueBody.trim(), - labels: ['Tests', 'Flaky Test'], - }); - core.info(`Created issue #${newIssue.data.number} for "${normalizedTestName}" in ${normalizedJobName}`); - } - } -} diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index 6d5b9473cba3..93ea05811570 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -1,14 +1,24 @@ -import { normalizeJobName, normalizeTestName } from './report-ci-failures.mjs'; - const LOOKBACK_DAYS = 7; const CONCURRENT_RUNS = 4; -function jobFamily(name) { - return normalizeJobName( - name - .replace(/\(\d+(?:\.\d+)+\)/g, '') - .replace(/(Playwright\s+\S+)\s+(?:chromium|firefox|webkit)(?=\s+Tests)/g, '$1'), - ); +function normalizeJobName(name) { + return name + .replace(/\(\s*(?:(?:(?:Node|TS)\s+)?\d+(?:\.\d+)*|\d+\/\d+)\s*\)/gi, ' ') + .replace(/Playwright\s+(?:bundle\w*|esm|cjs)(?:\s+(?:chromium|firefox|webkit))?\s+Tests/gi, 'Playwright Tests') + .replace(/-node-\d+(?:-\d+)*/gi, '') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeTestName(name) { + return name + .replace(/^\[(?:chromium|firefox|webkit)\]\s*›\s*/i, '') + .replace(/(\.[cm]?[jt]sx?):\d+:\d+/gi, '$1') + .replace(/esm\/cjs\s*>\s*(?:esm|cjs)\b/gi, 'esm/cjs') + .replace(/\besm\/cjs\s*>\s*/gi, '') + .replace(/\s*\[(?:esm|cjs)\]/gi, '') + .replace(/\s+/g, ' ') + .trim(); } function markdownCell(value) { @@ -89,7 +99,7 @@ export async function collectReport({ github, context, core, now = new Date() }) } for (const annotation of failures) { - const family = jobFamily(job.name); + const family = normalizeJobName(job.name); const name = normalizeTestName(annotation.title); const key = JSON.stringify([family, annotation.path, name]); let test = tests.get(key); From bb8b332f247d6305abc0c3cec1dbd080612584ea Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 11:02:14 +0200 Subject: [PATCH 06/11] chore(ci): Rename report run links column Co-Authored-By: GPT-6 --- scripts/weekly-flaky-tests.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index 93ea05811570..b0c410a72139 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -148,7 +148,7 @@ export function renderReport(report) { if (report.tests.length === 0) { lines.push('No test failures found.'); } else { - lines.push('| Test | Job | Affected runs | Example runs |', '| --- | --- | ---: | --- |'); + lines.push('| Test | Job | Affected runs | Links to runs |', '| --- | --- | ---: | --- |'); for (const test of report.tests) { const links = [...test.runs] .sort(([a], [b]) => b - a) From 7fdf535ae522e781921b84de4b2c14ce8fc1f07b Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 11:02:54 +0200 Subject: [PATCH 07/11] chore(ci): Shorten weekly report description Co-Authored-By: GPT-6 --- scripts/weekly-flaky-tests.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index b0c410a72139..2217bb3f8d6b 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -141,7 +141,7 @@ export function renderReport(report) { const lines = [ '## Weekly test failures', '', - `Develop · ${report.since.slice(0, 10)}–${report.until.slice(0, 10)} · ${report.runs} CI runs. Each test counts once per run.`, + `Develop · ${report.since.slice(0, 10)}–${report.until.slice(0, 10)} · ${report.runs} CI runs.`, '', ]; From a41f2629f625a10363cc6e0c2c4851fd8eb48cfb Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 11:29:30 +0200 Subject: [PATCH 08/11] chore(ci): Remove weekly report PR trigger Co-Authored-By: GPT-6 --- .github/workflows/weekly-flaky-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml index 0d3ec3f93993..5c99a4b5f93f 100644 --- a/.github/workflows/weekly-flaky-tests.yml +++ b/.github/workflows/weekly-flaky-tests.yml @@ -1,11 +1,6 @@ name: 'CI: Weekly flaky tests' on: - pull_request: - branches: [develop] - paths: - - '.github/workflows/weekly-flaky-tests.yml' - - 'scripts/weekly-flaky-tests*' workflow_dispatch: schedule: - cron: '0 7 * * 1' From 0f4b8b07601be449f9660a22ca068554ce26ad2a Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 11:50:48 +0200 Subject: [PATCH 09/11] chore(ci): Link weekly failures to the flaky issue template Co-Authored-By: GPT-6 --- .github/workflows/weekly-flaky-tests.yml | 5 +++++ scripts/weekly-flaky-tests.mjs | 27 +++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml index 5c99a4b5f93f..0d3ec3f93993 100644 --- a/.github/workflows/weekly-flaky-tests.yml +++ b/.github/workflows/weekly-flaky-tests.yml @@ -1,6 +1,11 @@ name: 'CI: Weekly flaky tests' on: + pull_request: + branches: [develop] + paths: + - '.github/workflows/weekly-flaky-tests.yml' + - 'scripts/weekly-flaky-tests*' workflow_dispatch: schedule: - cron: '0 7 * * 1' diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index 2217bb3f8d6b..f3ab63d8016a 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -137,7 +137,9 @@ export async function collectReport({ github, context, core, now = new Date() }) }; } -export function renderReport(report) { +export function renderReport(report, context) { + const repositoryUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}`; + const reportUrl = `${repositoryUrl}/actions/runs/${context.runId}`; const lines = [ '## Weekly test failures', '', @@ -148,13 +150,22 @@ export function renderReport(report) { if (report.tests.length === 0) { lines.push('No test failures found.'); } else { - lines.push('| Test | Job | Affected runs | Links to runs |', '| --- | --- | ---: | --- |'); + lines.push('| Test | Job | Affected runs | Links to runs | Issue |', '| --- | --- | ---: | --- | --- |'); for (const test of report.tests) { - const links = [...test.runs] - .sort(([a], [b]) => b - a) - .map(([id, url]) => `[${id}](${url})`) - .join(', '); - lines.push(`| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} |`); + const runs = [...test.runs].sort(([a], [b]) => b - a); + const links = runs.map(([id, url]) => `[${id}](${url})`).join(', '); + const issueParams = new URLSearchParams({ + template: 'flaky.yml', + title: `[Flaky CI]: ${test.name}`, + 'job-name': test.family, + 'test-name': test.name, + 'test-run-link': runs[0][1], + details: `${markdownCell(test.path)}\n\nFailed in ${test.runs.size} CI runs on develop during ${report.since.slice(0, 10)}–${report.until.slice(0, 10)}.\n\n[Weekly report](${reportUrl})`, + }); + const issueUrl = `${repositoryUrl}/issues/new?${issueParams}`; + lines.push( + `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} | [Create issue](${issueUrl}) |`, + ); } } @@ -172,6 +183,6 @@ export default async function run({ github, context, core }) { for (const warning of report.warnings) { core.warning(warning); } - await core.summary.addRaw(renderReport(report)).write(); + await core.summary.addRaw(renderReport(report, context)).write(); return report; } From 29d66d8f37cdd9c7f72bc780ec5f0517a72344df Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 12:02:02 +0200 Subject: [PATCH 10/11] chore(ci): Default flaky issues to assertion failure Co-Authored-By: GPT-6 --- .github/ISSUE_TEMPLATE/flaky.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/ISSUE_TEMPLATE/flaky.yml b/.github/ISSUE_TEMPLATE/flaky.yml index 22b035ec1145..91b467a8b4dd 100644 --- a/.github/ISSUE_TEMPLATE/flaky.yml +++ b/.github/ISSUE_TEMPLATE/flaky.yml @@ -12,6 +12,7 @@ body: - Timeout - Assertion failure - Other / Unknown + default: 1 validations: required: true - type: input From 45f2d47824e72435bb72410fafb48c827fac6930 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Fri, 25 Sep 2026 12:10:33 +0200 Subject: [PATCH 11/11] chore(ci): Remove temporary weekly report PR trigger Co-Authored-By: GPT-6 --- .github/workflows/weekly-flaky-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml index 0d3ec3f93993..5c99a4b5f93f 100644 --- a/.github/workflows/weekly-flaky-tests.yml +++ b/.github/workflows/weekly-flaky-tests.yml @@ -1,11 +1,6 @@ name: 'CI: Weekly flaky tests' on: - pull_request: - branches: [develop] - paths: - - '.github/workflows/weekly-flaky-tests.yml' - - 'scripts/weekly-flaky-tests*' workflow_dispatch: schedule: - cron: '0 7 * * 1'