diff --git a/.github/FLAKY_CI_FAILURE_TEMPLATE.md b/.github/FLAKY_CI_FAILURE_TEMPLATE.md deleted file mode 100644 index 23843c2c507f..000000000000 --- a/.github/FLAKY_CI_FAILURE_TEMPLATE.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: '[Flaky CI]: {{ env.JOB_NAME }} - {{ env.TEST_NAME }}' -labels: Tests, "Flaky Test" ---- - -### Flakiness Type - -Other / Unknown - -### Name of Job - -{{ env.JOB_NAME }} - -### Name of Test - -{{ env.TEST_NAME }} - -### Link to Test Run - -{{ env.RUN_LINK }} - ---- - -_This issue was automatically created._ diff --git a/.github/ISSUE_TEMPLATE/flaky.yml b/.github/ISSUE_TEMPLATE/flaky.yml index 22b035ec1145..91b467a8b4dd 100644 --- a/.github/ISSUE_TEMPLATE/flaky.yml +++ b/.github/ISSUE_TEMPLATE/flaky.yml @@ -12,6 +12,7 @@ body: - Timeout - Assertion failure - Other / Unknown + default: 1 validations: required: true - type: input diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45e298794f10..1091965d89c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1261,28 +1261,8 @@ jobs: # Always run this, even if a dependent job failed if: always() runs-on: ubuntu-24.04 - permissions: - issues: write - checks: read + permissions: {} steps: - - name: Check out current commit - if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') - uses: actions/checkout@v7 - with: - sparse-checkout: | - .github - scripts - - - name: Create issues for failed jobs - if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') - uses: actions/github-script@v9 - with: - script: | - const { default: run } = await import( - `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` - ); - await run({ github, context, core }); - - name: Check for failures if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') run: | diff --git a/.github/workflows/weekly-flaky-tests.yml b/.github/workflows/weekly-flaky-tests.yml new file mode 100644 index 000000000000..5c99a4b5f93f --- /dev/null +++ b/.github/workflows/weekly-flaky-tests.yml @@ -0,0 +1,35 @@ +name: 'CI: Weekly flaky tests' + +on: + workflow_dispatch: + schedule: + - cron: '0 7 * * 1' + timezone: 'Europe/Vienna' + +permissions: + contents: read + actions: read + checks: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + report: + name: Weekly flaky test report + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Analyze the last seven days of develop CI + uses: actions/github-script@v9 + with: + retries: 3 + script: | + const { default: run } = await import( + `${process.env.GITHUB_WORKSPACE}/scripts/weekly-flaky-tests.mjs` + ); + await run({ github, context, core }); diff --git a/scripts/report-ci-failures.mjs b/scripts/report-ci-failures.mjs deleted file mode 100644 index 58278b4b5a8f..000000000000 --- a/scripts/report-ci-failures.mjs +++ /dev/null @@ -1,168 +0,0 @@ -/** - * CI Failure Reporter script. - * - * Creates GitHub issues for tests that fail on the develop branch. - * For each failed job in the workflow run, it fetches check run annotations - * to identify individual failing tests, then creates one issue per failing - * test using the FLAKY_CI_FAILURE_TEMPLATE.md template. Existing open issues - * with matching titles are skipped to avoid duplicates. - * - * Intended to be called from a GitHub Actions workflow via actions/github-script: - * - * const { default: run } = await import( - * `${process.env.GITHUB_WORKSPACE}/scripts/report-ci-failures.mjs` - * ); - * await run({ github, context, core }); - */ - -import { readFileSync } from 'node:fs'; - -/** - * Collapse matrix variants of a job name so the same test failing across the matrix dedupes to a - * single issue instead of one per variant. We strip version-like parenthetical groups — a bare - * number (node version), a `TS x.y` bracket, a `Node xx` bracket, or a `n/m` shard — and fold the - * Playwright bundle/esm build configs and E2E `-node-xx` app suffixes into a single bucket, leaving - * other parentheticals (e.g. `(nextjs-app, 20)`) intact: - * - * "Node (22) Integration Tests" -> "Node Integration Tests" - * "Node (24) (TS 5.0) Integration Tests" -> "Node Integration Tests" - * "aws-serverless-layer (Node 22) Test" -> "aws-serverless-layer Test" - * "Playwright bundle_tracing_replay Tests" -> "Playwright Tests" - * "Playwright esm (1/4) Tests" -> "Playwright Tests" - * "E2E some-app-node-20-18 Test" -> "E2E some-app Test" - */ -function normalizeJobName(name) { - return name - .replace(/\(\s*(?:\d+|TS\s+[\d.]+|Node\s+\d+|\d+\/\d+)\s*\)/gi, ' ') - .replace(/Playwright\s+(?:bundle\w*|esm|cjs)\s+Tests/gi, 'Playwright Tests') - .replace(/-node-\d+(?:-\d+)*/gi, '') - .replace(/\s+/g, ' ') - .trim(); -} - -/** - * Collapse variants of a test name so the same test failing under different module formats, browser - * projects, or (for Playwright) at a different source line dedupes to a single issue. We strip: - * - * - Playwright browser prefix: "[chromium] › suites/... " -> "suites/... " - * - Playwright file line/column: "test.ts:33:11 › ..." -> "test.ts › ..." (drifts on edits) - * - old esm/cjs describe block: "... > esm/cjs > esm > x" -> "... > x" - * - bare esm/cjs describe block: "... > esm/cjs > x" -> "... > x" - * - trailing module suffix: "... should send [esm]" -> "... should send" - */ -function normalizeTestName(name) { - return name - .replace(/^\[(?:chromium|firefox|webkit)\]\s*›\s*/i, '') - .replace(/(\.[cm]?[jt]sx?):\d+:\d+/gi, '$1') - .replace(/esm\/cjs\s*>\s*(?:esm|cjs)\b/gi, 'esm/cjs') - .replace(/\besm\/cjs\s*>\s*/gi, '') - .replace(/\s*\[(?:esm|cjs)\]/gi, '') - .replace(/\s+/g, ' ') - .trim(); -} - -function applyVars(text, vars) { - let result = text; - for (const [key, value] of Object.entries(vars)) { - result = result.replace(new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g'), value); - } - return result; -} - -export default async function run({ github, context, core }) { - const { owner, repo } = context.repo; - - // Fetch actual job details from the API to get descriptive names - const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { - owner, - repo, - run_id: context.runId, - per_page: 100, - }); - - const failedJobs = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); - - if (failedJobs.length === 0) { - core.info('No failed jobs found'); - return; - } - - // Read and parse template - const template = readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8'); - const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); - const titleTemplate = frontmatter.match(/title:\s*'(.*)'/)[1]; - - // Titles we've already created or matched in this run, so the same flaky test failing on - // multiple matrix jobs within a single run doesn't open duplicate issues (the `existing` list - // below is fetched once and won't include issues created earlier in this same run). - const handledTitles = new Set(); - - // Get existing open issues with Tests label - const existing = await github.paginate(github.rest.issues.listForRepo, { - owner, - repo, - state: 'open', - labels: 'Tests', - per_page: 100, - }); - - for (const job of failedJobs) { - const jobName = job.name; - const normalizedJobName = normalizeJobName(jobName); - const jobUrl = job.html_url; - - // Fetch annotations from the check run to extract failed test names - let testNames = []; - try { - const annotations = await github.paginate(github.rest.checks.listAnnotations, { - owner, - repo, - check_run_id: job.id, - per_page: 100, - }); - - const testAnnotations = annotations.filter(a => a.annotation_level === 'failure' && a.path !== '.github'); - testNames = [...new Set(testAnnotations.map(a => a.title || a.path))]; - } catch (e) { - core.info(`Could not fetch annotations for ${jobName}: ${e.message}`); - } - - // If no test names found, abort - this could mean something else, e.g. cache restoration or similar fails - // and also the issue is not super helpful in this case - if (testNames.length === 0) { - continue; - } - - // Create one issue per failing test for proper deduplication - for (const testName of testNames) { - const normalizedTestName = normalizeTestName(testName); - - // The title is keyed on the *normalized* job name + test name so the same test failing across - // matrix variants (different node / TS versions) or module formats (esm / cjs) dedupes to a - // single issue. - const title = applyVars(titleTemplate, { JOB_NAME: normalizedJobName, TEST_NAME: normalizedTestName }); - // The body keeps the concrete job name + run link of the variant that actually failed. - const issueBody = applyVars(bodyTemplate, { JOB_NAME: jobName, RUN_LINK: jobUrl, TEST_NAME: testName }); - - if (handledTitles.has(title)) { - continue; - } - handledTitles.add(title); - - const existingIssue = existing.find(i => i.title === title); - if (existingIssue) { - core.info(`Issue already exists for "${normalizedTestName}" in ${normalizedJobName}: #${existingIssue.number}`); - continue; - } - - const newIssue = await github.rest.issues.create({ - owner, - repo, - title, - body: issueBody.trim(), - labels: ['Tests', 'Flaky Test'], - }); - core.info(`Created issue #${newIssue.data.number} for "${normalizedTestName}" in ${normalizedJobName}`); - } - } -} diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs new file mode 100644 index 000000000000..f3ab63d8016a --- /dev/null +++ b/scripts/weekly-flaky-tests.mjs @@ -0,0 +1,188 @@ +const LOOKBACK_DAYS = 7; +const CONCURRENT_RUNS = 4; + +function normalizeJobName(name) { + return name + .replace(/\(\s*(?:(?:(?:Node|TS)\s+)?\d+(?:\.\d+)*|\d+\/\d+)\s*\)/gi, ' ') + .replace(/Playwright\s+(?:bundle\w*|esm|cjs)(?:\s+(?:chromium|firefox|webkit))?\s+Tests/gi, 'Playwright Tests') + .replace(/-node-\d+(?:-\d+)*/gi, '') + .replace(/\s+/g, ' ') + .trim(); +} + +function normalizeTestName(name) { + return name + .replace(/^\[(?:chromium|firefox|webkit)\]\s*›\s*/i, '') + .replace(/(\.[cm]?[jt]sx?):\d+:\d+/gi, '$1') + .replace(/esm\/cjs\s*>\s*(?:esm|cjs)\b/gi, 'esm/cjs') + .replace(/\besm\/cjs\s*>\s*/gi, '') + .replace(/\s*\[(?:esm|cjs)\]/gi, '') + .replace(/\s+/g, ' ') + .trim(); +} + +function markdownCell(value) { + return String(value) + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/[\\`*_[\]|]/g, '\\$&') + .replace(/[\r\n]+/g, ' '); +} + +export async function collectReport({ github, context, core, now = new Date() }) { + const until = now.toISOString(); + const since = new Date(now.getTime() - LOOKBACK_DAYS * 24 * 60 * 60 * 1000).toISOString(); + const repo = context.repo; + core.info(`Listing develop CI runs from ${since} to ${until}.`); + const listedRuns = await github.paginate(github.rest.actions.listWorkflowRuns, { + ...repo, + workflow_id: 'build.yml', + branch: 'develop', + created: `${since}..${until}`, + per_page: 100, + }); + const warnings = []; + if (listedRuns.length >= 1000) { + warnings.push('GitHub limits this query to 1,000 runs; this report may be incomplete.'); + } + const runs = listedRuns.filter(run => ['push', 'schedule', 'workflow_dispatch'].includes(run.event)); + const tests = new Map(); + let failedJobsWithoutTests = 0; + let failedJobs = 0; + let completedRuns = 0; + const pendingRuns = runs.filter(run => run.conclusion !== 'success' || run.run_attempt > 1); + const runsToInspect = pendingRuns.length; + core.info( + `Found ${runs.length} runs; ${runsToInspect} need inspection, ${runs.length - runsToInspect} passed on the first attempt.`, + ); + + async function worker() { + while (pendingRuns.length > 0) { + const run = pendingRuns.shift(); + core.info(`Run ${run.id}: fetching jobs across ${run.run_attempt} attempt(s).`); + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + ...repo, + run_id: run.id, + filter: 'all', + per_page: 100, + }); + const jobsToInspect = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); + core.info(`Run ${run.id}: ${jobs.length} job attempts, ${jobsToInspect.length} failures to inspect.`); + for (const job of jobsToInspect) { + failedJobs++; + + let annotations; + try { + annotations = await github.paginate(github.rest.checks.listAnnotations, { + ...repo, + check_run_id: Number(job.check_run_url.split('/').pop()), + per_page: 100, + }); + } catch (error) { + if (error.status !== 404 && error.status !== 410) { + throw error; + } + warnings.push(`Annotations unavailable for job ${job.id} in run ${run.id}.`); + continue; + } + + const failures = annotations.filter( + annotation => + annotation.annotation_level === 'failure' && + annotation.title && + /(?:^|\/)(?:tests?|__tests__|suites)\/|(?:^|[/.])(?:test|spec)\.[cm]?[jt]sx?$/i.test(annotation.path), + ); + if (failures.length === 0) { + failedJobsWithoutTests++; + } + + for (const annotation of failures) { + const family = normalizeJobName(job.name); + const name = normalizeTestName(annotation.title); + const key = JSON.stringify([family, annotation.path, name]); + let test = tests.get(key); + if (!test) { + test = { family, path: annotation.path, name, runs: new Map() }; + tests.set(key, test); + } + if (!test.runs.has(run.id)) { + test.runs.set(run.id, job.html_url); + } + } + } + completedRuns++; + core.info( + `Completed ${completedRuns}/${runsToInspect} runs; ${failedJobs} failed jobs inspected, ${tests.size} failing tests found.`, + ); + } + } + await Promise.all(Array.from({ length: CONCURRENT_RUNS }, () => worker())); + + return { + since, + until, + runs: runs.length, + failedJobs, + failedJobsWithoutTests, + warnings, + tests: [...tests.values()].sort( + (a, b) => + b.runs.size - a.runs.size || + a.name.localeCompare(b.name) || + a.family.localeCompare(b.family) || + a.path.localeCompare(b.path), + ), + }; +} + +export function renderReport(report, context) { + const repositoryUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}`; + const reportUrl = `${repositoryUrl}/actions/runs/${context.runId}`; + const lines = [ + '## Weekly test failures', + '', + `Develop · ${report.since.slice(0, 10)}–${report.until.slice(0, 10)} · ${report.runs} CI runs.`, + '', + ]; + + if (report.tests.length === 0) { + lines.push('No test failures found.'); + } else { + lines.push('| Test | Job | Affected runs | Links to runs | Issue |', '| --- | --- | ---: | --- | --- |'); + for (const test of report.tests) { + const runs = [...test.runs].sort(([a], [b]) => b - a); + const links = runs.map(([id, url]) => `[${id}](${url})`).join(', '); + const issueParams = new URLSearchParams({ + template: 'flaky.yml', + title: `[Flaky CI]: ${test.name}`, + 'job-name': test.family, + 'test-name': test.name, + 'test-run-link': runs[0][1], + details: `${markdownCell(test.path)}\n\nFailed in ${test.runs.size} CI runs on develop during ${report.since.slice(0, 10)}–${report.until.slice(0, 10)}.\n\n[Weekly report](${reportUrl})`, + }); + const issueUrl = `${repositoryUrl}/issues/new?${issueParams}`; + lines.push( + `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} | [Create issue](${issueUrl}) |`, + ); + } + } + + lines.push( + '', + `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs excluded.`, + '', + ...report.warnings.map(warning => `- ${markdownCell(warning)}`), + ); + return `${lines.join('\n')}\n`; +} + +export default async function run({ github, context, core }) { + const report = await collectReport({ github, context, core }); + for (const warning of report.warnings) { + core.warning(warning); + } + await core.summary.addRaw(renderReport(report, context)).write(); + return report; +}