diff --git a/.github/scripts/agents_pr_meta_update_body.js b/.github/scripts/agents_pr_meta_update_body.js index d4b6bc4b..d2d6efd1 100644 --- a/.github/scripts/agents_pr_meta_update_body.js +++ b/.github/scripts/agents_pr_meta_update_body.js @@ -10,6 +10,7 @@ * - Building and updating PR body with preamble and status blocks */ +const { visibleChecklistContent, stripPrTemplateControls } = require('./issue_scope_parser'); const path = require('path'); const fs = require('fs'); const os = require('os'); @@ -728,8 +729,13 @@ function stripPrTemplateContent(body) { firstMarkerIndex = statusStart; } - // If we found a marker and there's content before it, strip that content + // A checkbox-bearing prefix may be reviewer-added work, not a template. + // Preserve it with its context and continuation lines across regeneration. if (firstMarkerIndex > 0) { + const prefix = stripPrTemplateControls(body.slice(0, firstMarkerIndex)); + if (/^\s*(?:[-*+]|\d+[.)])\s*\[[ xX]\]/m.test(visibleChecklistContent(prefix))) { + return prefix + body.slice(firstMarkerIndex); + } return body.slice(firstMarkerIndex); } @@ -921,24 +927,107 @@ function selectLatestWorkflows(runs) { const SELF_OBSERVING_WORKFLOW_NAMES = new Set([ 'agents pr meta manager', + 'agents pr event hub', + 'pr 46 dependency repair contract', +]); + +const SELF_OBSERVING_WORKFLOW_PATHS = new Set([ + '.github/workflows/agents-pr-meta-v4.yml', + '.github/workflows/agents-80-pr-event-hub.yml', + '.github/workflows/pr-46-dependency-repair-contract.yml', ]); +const DEPENDENCY_CONTRACT_NAME = 'PR 46 Dependency Repair Contract'; + +function isDependencyContractRun(run) { + return String(run?.name || '').trim().toLowerCase() === DEPENDENCY_CONTRACT_NAME.toLowerCase() + || String(run?.path || '').trim().split('@')[0].toLowerCase() + === '.github/workflows/pr-46-dependency-repair-contract.yml'; +} + function isSelfObservingWorkflowRun(run) { const name = String(run?.name || '').trim().toLowerCase(); - return Boolean(name && SELF_OBSERVING_WORKFLOW_NAMES.has(name)); + const workflowPath = String(run?.path || '').trim().split('@')[0].toLowerCase(); + return SELF_OBSERVING_WORKFLOW_NAMES.has(name) + || SELF_OBSERVING_WORKFLOW_PATHS.has(workflowPath); } function filterWorkflowRunsForStatus(workflowRuns) { const filtered = new Map(); for (const [key, run] of workflowRuns || new Map()) { - if (isSelfObservingWorkflowRun(run)) { - continue; + if (isDependencyContractRun(run)) { + // PR 46 validates provenance as well as observing body edits. Keep its + // last completed result, but never echo the run URL that caused an edit. + if (run.status === 'completed' && run.conclusion) { + filtered.set(DEPENDENCY_CONTRACT_NAME.toLowerCase(), { + ...run, name: DEPENDENCY_CONTRACT_NAME, html_url: '', + }); + } + } else if (!isSelfObservingWorkflowRun(run)) { + filtered.set(key, run); } - filtered.set(key, run); } return filtered; } +async function collectStatusWorkflowRuns({github, owner, repo, headSha, core}) { + // Without an implementation head, neither endpoint can provide exact-head evidence. + if (!normalise(headSha)) return new Map(); + const response = await withRetries( + () => github.rest.actions.listWorkflowRunsForRepo({ + owner, repo, head_sha: headSha, per_page: 100, + }), + {description: 'list workflow runs', core}, + ); + const exactRuns = (response.data.workflow_runs || []).filter(run => run.head_sha === headSha); + const runs = filterWorkflowRunsForStatus(selectLatestWorkflows(exactRuns.filter(run => + !isDependencyContractRun(run) || (run.status === 'completed' && run.conclusion), + ).map(run => isDependencyContractRun(run) ? {...run, name: DEPENDENCY_CONTRACT_NAME} : run))); + if (exactRuns.some(isDependencyContractRun) && !runs.has(DEPENDENCY_CONTRACT_NAME.toLowerCase())) { + // A new body edit starts PR 46 again. Resolve only the last completed + // exact-head result so its pending/completed cycle cannot retrigger itself. + try { + const contractResponse = await withRetries( + () => github.rest.actions.listWorkflowRuns({ + owner, repo, workflow_id: 'pr-46-dependency-repair-contract.yml', + head_sha: headSha, status: 'completed', per_page: 1, + }), + {description: 'recover completed dependency contract', core, attempts: 1}, + ); + const completed = filterWorkflowRunsForStatus(selectLatestWorkflows( + (contractResponse.data.workflow_runs || []).filter(run => + run.head_sha === headSha && isDependencyContractRun(run), + ), + )); + for (const [key, run] of completed) runs.set(key, run); + } catch (error) { + if (error?.status !== 404) throw error; + core?.warning('Dependency contract workflow unavailable; consult PR checks.'); + } + } + if (!runs.has('gate')) { + // Metadata-only edited events can fill the latest page. Recover the real + // exact-head Gate directly rather than paging through thousands of observers. + try { + const gateResponse = await withRetries( + () => github.rest.actions.listWorkflowRuns({ + owner, repo, workflow_id: 'pr-00-gate.yml', head_sha: headSha, per_page: 1, + }), + {description: 'recover exact-head Gate', core, attempts: 1}, + ); + for (const run of gateResponse.data.workflow_runs || []) { + if (run.head_sha === headSha && String(run.name || '').toLowerCase() === 'gate') { + runs.set('gate', run); + } + } + } catch (error) { + if (error?.status !== 404) throw error; + core?.warning('Gate workflow unavailable; leaving its status unknown.'); + } + } + return runs; +} + function fallbackChecklist(message) { return `- [ ] ${message}`; } @@ -1199,7 +1288,7 @@ function buildStatusBlock({scope, contextSection, tasks, acceptance, headSha, wo if (!isCliAgent) { statusLines.push(`**Head SHA:** ${headSha}`); - const latestRuns = Array.from(statusWorkflowRuns.values()).sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + const latestRuns = Array.from(statusWorkflowRuns.values()).filter(run => !isDependencyContractRun(run)).sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); let latestLine = '—'; if (latestRuns.length > 0) { const gate = latestRuns.find((run) => (run.name || '').toLowerCase() === 'gate'); @@ -1213,10 +1302,12 @@ function buildStatusBlock({scope, contextSection, tasks, acceptance, headSha, wo for (const name of requiredChecks) { const run = Array.from(statusWorkflowRuns.values()).find((item) => (item.name || '').toLowerCase() === name.toLowerCase()); if (!run) { - requiredParts.push(`${name}: ⏸️ not started`); + requiredParts.push(SELF_OBSERVING_WORKFLOW_NAMES.has(name.toLowerCase()) + ? `${name}: reported separately in PR checks` + : `${name}: ⏸️ not started`); } else { const status = combineStatus(run); - requiredParts.push(`${name}: ${status.icon} ${status.label}`); + requiredParts.push(`${name}: ${status.icon} ${status.label}${isDependencyContractRun(run) ? ' (last completed; current run in PR checks)' : ''}`); } } statusLines.push(`**Required:** ${requiredParts.length > 0 ? requiredParts.join(', ') : '—'}`); @@ -1230,7 +1321,9 @@ function buildStatusBlock({scope, contextSection, tasks, acceptance, headSha, wo } else { for (const run of runs) { const status = combineStatus(run); - const link = run.html_url ? `[View run](${run.html_url})` : '—'; + const link = isDependencyContractRun(run) + ? 'Last completed result; current run in PR checks' + : run.html_url ? `[View run](${run.html_url})` : '—'; table.push(`| ${run.name || 'Unnamed workflow'} | ${status.icon} ${status.label} | ${link} |`); } } @@ -1617,16 +1710,9 @@ async function run({github: rawGithub, context, core, inputs}) { sourceIssue: issueResponse.data, }); - const workflowRunResponse = await withRetries( - () => github.rest.actions.listWorkflowRunsForRepo({ - owner, - repo, - head_sha: pr.head.sha, - per_page: 100, - }), - {description: 'list workflow runs', core}, - ); - const workflowRuns = selectLatestWorkflows(workflowRunResponse.data.workflow_runs || []); + const workflowRuns = await collectStatusWorkflowRuns({ + github, owner, repo, headSha: pr.head.sha, core, + }); const requiredChecksRaw = await fetchRequiredChecks(github, owner, repo, pr.base.ref, core); // Avoid mutating the returned array - create a new one with 'gate' appended if needed @@ -1725,6 +1811,7 @@ module.exports = { stripPrTemplateContent, upsertBlock, filterWorkflowRunsForStatus, + collectStatusWorkflowRuns, buildContextBlock, buildPreamble, buildSourceContextRepairCommentBody, diff --git a/.github/scripts/agents_verifier_context.js b/.github/scripts/agents_verifier_context.js index 8748fa31..392132a7 100644 --- a/.github/scripts/agents_verifier_context.js +++ b/.github/scripts/agents_verifier_context.js @@ -650,6 +650,7 @@ async function buildVerifierContext({ github, context, core, ciWorkflows }) { fs.writeFileSync(diffPath, diffText + '\n', 'utf8'); } + core?.setOutput?.('pr_head_sha', pull.head?.sha || ''); core?.setOutput?.('should_run', 'true'); core?.setOutput?.('skip_reason', ''); core?.setOutput?.('pr_number', String(pull.number || '')); diff --git a/.github/scripts/issue_scope_parser.js b/.github/scripts/issue_scope_parser.js index c1ce66ff..2f91fd1c 100644 --- a/.github/scripts/issue_scope_parser.js +++ b/.github/scripts/issue_scope_parser.js @@ -2,7 +2,7 @@ const normalizeNewlines = (value) => String(value || '').replace(/\r\n/g, '\n'); const stripBlockquotePrefixes = (value) => - String(value || '').replace(/^[ \t]*>+[ \t]?/gm, ''); + String(value || '').replace(/^(?:[ \t]*>[ \t]?)+/gm, ''); /** * Check if a line is a code fence delimiter (``` or ~~~). @@ -10,6 +10,89 @@ const stripBlockquotePrefixes = (value) => */ const isCodeFenceLine = (line) => /^(`{3,}|~{3,})/.test(line.trim()); + +// Shared by PR metadata preservation and keepalive's visible-work scan. +function visibleChecklistContent(markdown) { + let fence = null; + let comment = false; + return stripBlockquotePrefixes(normalizeNewlines(markdown)) + .split('\n').map((line) => { + // Fenced examples are literal: comment markers inside them must never + // consume real checklist lines after the closing fence. + const delimiter = !comment && line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/); + if (delimiter) { + const token = delimiter[1]; + if (!fence) fence = token; + else if (token[0] === fence[0] && token.length >= fence.length && !delimiter[2].trim()) fence = null; + return ''; + } + if (fence) return ''; + let visible = ''; + let offset = 0; + while (offset < line.length) { + if (comment) { + const end = line.indexOf('-->', offset); + if (end < 0) break; + comment = false; + offset = end + 3; + } else { + const start = line.indexOf('[\s\S]*?/g, '', + ); + const itemKey = (item) => `${item.checked}:${normaliseTaskKey(item.text)}`; + const represented = outside === String(body || '') + ? new Set(extractChecklistItems([sections.tasks, sections.acceptance].join('\n')).map(itemKey)) + : new Set(); + const additional = []; + let capturing = false; + for (const line of visibleChecklistContent(outside).split('\n')) { + const item = extractChecklistItems(line)[0]; + if (item) { + capturing = isActionableChecklistItemText(item.text) && !represented.has(itemKey(item)); + if (capturing) additional.push(line); + } else if (capturing && /^\s+\S/.test(line)) { + additional.push(line); + } else { + capturing = false; + } + } + if (additional.length) { + sections.tasks = [sections.tasks, '### Additional PR tasks', additional.join('\n')] + .filter(Boolean).join('\n\n'); + } + return sections; +} + function toActionableChecklistCounts(markdown) { const actionable = extractChecklistItems(markdown).filter((item) => isActionableChecklistItemText(item.text)); const checked = actionable.filter((item) => item.checked).length; @@ -2625,7 +2660,7 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload const runCapZero = labels.includes('agents:max-runs:0'); const sections = parseScopeTasksAcceptanceSections(pr.body || ''); - const normalisedSections = normaliseChecklistSections(sections); + const normalisedSections = parseKeepaliveChecklistSections(pr.body || ''); const combinedChecklist = [normalisedSections?.tasks, normalisedSections?.acceptance] .filter(Boolean) .join('\n'); @@ -3222,8 +3257,8 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in } const gateConclusion = normalise(inputs.gateConclusion || inputs.gate_conclusion); - const action = normalise(inputs.action); - const reason = normalise(inputs.reason); + let action = normalise(inputs.action); + let reason = normalise(inputs.reason); const tasksTotalInput = inputs.tasksTotal ?? inputs.tasks_total; const tasksUncheckedInput = inputs.tasksUnchecked ?? inputs.tasks_unchecked; const keepaliveEnabledInput = inputs.keepaliveEnabled ?? inputs.keepalive_enabled; @@ -3397,7 +3432,7 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in const previousFailure = previousState?.failure || {}; const prBody = await fetchPrBody({ github, context, prNumber, core }); - const focusSections = prBody ? normaliseChecklistSections(parseScopeTasksAcceptanceSections(prBody)) : {}; + const focusSections = prBody ? parseKeepaliveChecklistSections(prBody) : {}; const focusItems = extractChecklistItems(focusSections.tasks || focusSections.acceptance || ''); const focusUnchecked = focusItems.filter((item) => !item.checked); const currentFocus = normaliseTaskText(previousState?.current_focus || ''); @@ -3424,6 +3459,30 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in } } + // A reviewer can add work after evaluation but before this live re-count. + // Do not publish a stale success or add automerge while that work is visible. + if (action === 'stop' && reason === 'tasks-complete' && tasksUnchecked > 0) { + action = 'wait'; + reason = 'tasks-changed'; + core?.info?.('Visible tasks changed after evaluation; keepalive must re-evaluate.'); + } + + // The root merger selects by this label, so a previous completion must not + // authorize a merge after visible work reopens, regardless of current action. + if (prBody && tasksUnchecked > 0 && labels.some((label) => label.toLowerCase() === 'automerge')) { + try { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + name: 'automerge', + }); + core?.info?.('Removed stale automerge authorization while visible tasks remain.'); + } catch (error) { + if (error?.status !== 404) throw error; + } + } + // Recalculate rounds_without_task_completion using live checkbox counts. // The evaluate step calculated this counter before autoReconcile ran, so it // may incorrectly show "no progress" even though autoReconcile just checked @@ -4867,7 +4926,7 @@ async function markAgentRunning({ github: rawGithub, context, core, inputs }) { ); } const prBody = await fetchPrBody({ github, context, prNumber, core }); - const focusSections = prBody ? normaliseChecklistSections(parseScopeTasksAcceptanceSections(prBody)) : {}; + const focusSections = prBody ? parseKeepaliveChecklistSections(prBody) : {}; const focusItems = extractChecklistItems(focusSections.tasks || focusSections.acceptance || ''); const focusUnchecked = focusItems.filter((item) => !item.checked); const attemptedTasks = normaliseAttemptedTasks(previousState?.attempted_tasks); diff --git a/.github/workflows/agents-81-gate-followups.yml b/.github/workflows/agents-81-gate-followups.yml index c9f65dbc..de4ede9e 100644 --- a/.github/workflows/agents-81-gate-followups.yml +++ b/.github/workflows/agents-81-gate-followups.yml @@ -1532,8 +1532,10 @@ jobs: return stop('missing workflow_run payload'); } - if ((run.conclusion || '').toLowerCase() === 'success') { - return stop('upstream Gate succeeded'); + const isFailure = (value) => + ['failure', 'timed_out'].includes(String(value || '').toLowerCase()); + if (!isFailure(run.conclusion)) { + return stop('upstream Gate has no counted failure', 'gate_not_failed'); } if ((run.event || '').toLowerCase() !== 'pull_request') { @@ -1623,43 +1625,6 @@ jobs: ? configMatch[1].toLowerCase() === 'true' : hasAgentLabel; - // Auto-escalation: Escalate to agent CLI when Gate fails - // Triggers if: (1) basic autofix ran but insufficient, OR (2) no basic autofix applied - // Note: We do NOT add agent:* label here because that triggers external agent UI - // which would conflict with our internal agent CLI run. Only add autofix:escalated. - if (!autofixEnabled && !configMatch) { - const hasAutofixLabel = - labels.includes('autofix:applied') || labels.includes('autofix'); - const hasEscalatedLabel = labels.includes('autofix:escalated'); - const gateConclusion = (run.conclusion || '').toLowerCase(); - const gateFailed = gateConclusion === 'failure'; - - // Escalate if Gate failed and we haven't already escalated - if (gateFailed && !hasEscalatedLabel) { - const reason = hasAutofixLabel - ? 'Basic autofix ran but Gate still failing' - : 'No basic autofix (non-Python PR?) and Gate failing'; - core.info(`🔄 Auto-escalation: ${reason}. Escalating to agent CLI...`); - try { - await withRetry((client) => client.rest.issues.addLabels({ - owner, - repo, - issue_number: prNumber, - labels: ['autofix:escalated'], - })); - core.info( - '✅ Added autofix:escalated label - agent CLI will run in this workflow' - ); - autofixEnabled = true; - } catch (error) { - core.warning(`Failed to add escalation labels: ${error.message}`); - } - } - } - if (!autofixEnabled) { - return stop('autofix disabled for this pull request'); - } - const jobs = await paginateWithRetry(github.rest.actions.listJobsForWorkflowRun, { owner, repo, @@ -1667,7 +1632,8 @@ jobs: per_page: 100, }); - const workflowFile = 'agents-81-gate-followups.yml'; + // Count failed Gate executions, not unrelated follow-up workflow runs. + const workflowFile = run.workflow_id; // Reduce attempts for auto-escalated PRs (they weren't agent-initiated) const isEscalated = labels.includes('autofix:escalated'); const maxAttempts = isEscalated @@ -1682,7 +1648,77 @@ jobs: status: 'completed', }); - const attemptCount = previousRuns.length + 1; + const listJobsForGateAttempt = async (runId, attemptNumber, latestJobs = null) => { + const currentAttempt = Math.max(1, Number(run.run_attempt) || 1); + if (runId === run.id && attemptNumber === currentAttempt && latestJobs) { + return latestJobs; + } + if (typeof github['rest']['actions']['listJobsForWorkflowRunAttempt'] === 'function') { + return await paginateWithRetry(github.rest.actions.listJobsForWorkflowRunAttempt, { + owner, + repo, + run_id: runId, + attempt_number: attemptNumber, + per_page: 100, + }); + } + if (attemptNumber === 1) { + return await paginateWithRetry(github.rest.actions.listJobsForWorkflowRun, { + owner, + repo, + run_id: runId, + per_page: 100, + }); + } + return []; + }; + + const countFailingAttemptsForRun = async (gateRun, latestJobs = null) => { + const maxAttempt = Math.max(1, Number(gateRun.run_attempt) || 1); + let count = 0; + for (let attempt = 1; attempt <= maxAttempt; attempt += 1) { + const attemptJobs = await listJobsForGateAttempt( + gateRun.id, + attempt, + gateRun.id === run.id ? latestJobs : null + ); + if (attemptJobs.some((job) => isFailure(job.conclusion))) { + count += 1; + } + } + return count; + }; + + const gateRunsById = new Map(); + for (const candidate of previousRuns) { + if (candidate.head_sha !== run.head_sha || !isFailure(candidate.conclusion)) { + continue; + } + const existing = gateRunsById.get(candidate.id); + const candidateAttempt = Number(candidate.run_attempt) || 1; + const existingAttempt = existing ? Number(existing.run_attempt) || 1 : 0; + if (!existing || candidateAttempt >= existingAttempt) { + gateRunsById.set(candidate.id, candidate); + } + } + gateRunsById.set(run.id, { + ...run, + run_attempt: Math.max( + Number(run.run_attempt) || 1, + Number(gateRunsById.get(run.id)?.run_attempt) || 0 + ), + }); + + let attemptsWithFailingJobs = 0; + let gateAttemptsExamined = 0; + for (const gateRun of gateRunsById.values()) { + gateAttemptsExamined += Math.max(1, Number(gateRun.run_attempt) || 1); + attemptsWithFailingJobs += await countFailingAttemptsForRun( + gateRun, + gateRun.id === run.id ? jobs : null + ); + } + const attemptCount = attemptsWithFailingJobs; outputs.attempts = String(attemptCount); outputs.max_attempts = String(maxAttempts); @@ -1691,7 +1727,7 @@ jobs: let triggerStep = null; for (const job of jobs) { const conclusion = (job.conclusion || job.status || '').toLowerCase(); - if (!conclusion || ['success', 'skipped'].includes(conclusion)) { + if (!isFailure(conclusion)) { continue; } @@ -1700,7 +1736,7 @@ jobs: ? job.steps .filter((step) => { const stepConclusion = (step.conclusion || step.status || '').toLowerCase(); - return stepConclusion && !['success', 'skipped'].includes(stepConclusion); + return isFailure(stepConclusion); }) .map((step) => `${step.name} (${step.conclusion || step.status || 'unknown'})` @@ -1718,7 +1754,7 @@ jobs: const failingStep = Array.isArray(job.steps) ? job.steps.find((step) => { const stepConclusion = (step.conclusion || step.status || '').toLowerCase(); - return stepConclusion && !['success', 'skipped'].includes(stepConclusion); + return isFailure(stepConclusion); }) : null; triggerStep = failingStep || null; @@ -1754,6 +1790,7 @@ jobs: `PR: #${prNumber}`, `Head SHA: ${headSha}`, `Autofix attempts for this head: ${attemptCount} / ${maxAttempts}`, + `Gate runs examined: ${gateRunsById.size}; gate attempts examined: ${gateAttemptsExamined}; attempts with failing jobs: ${attemptsWithFailingJobs}`, 'Fix scope: src/, tests/, tools/, scripts/, agents/, templates/, .github/', ]; @@ -1765,6 +1802,44 @@ jobs: outputs.appendix = appendixLines.join('\n'); + if (failingJobs.length === 0) { + return stop('Gate has no failing jobs', 'no_failing_jobs'); + } + + // Auto-escalation: only after confirming real failed/timed-out Gate jobs. + // Note: We do NOT add agent:* label here because that triggers external agent UI + // which would conflict with our internal agent CLI run. Only add autofix:escalated. + if (!autofixEnabled && !configMatch) { + const hasAutofixLabel = + labels.includes('autofix:applied') || labels.includes('autofix'); + const hasEscalatedLabel = labels.includes('autofix:escalated'); + const gateFailed = isFailure((run.conclusion || '').toLowerCase()); + + if (gateFailed && !hasEscalatedLabel) { + const reason = hasAutofixLabel + ? 'Basic autofix ran but Gate still failing' + : 'No basic autofix (non-Python PR?) and Gate failing'; + core.info(`🔄 Auto-escalation: ${reason}. Escalating to agent CLI...`); + try { + await withRetry((client) => client.rest.issues.addLabels({ + owner, + repo, + issue_number: prNumber, + labels: ['autofix:escalated'], + })); + core.info( + '✅ Added autofix:escalated label - agent CLI will run in this workflow' + ); + autofixEnabled = true; + } catch (error) { + core.warning(`Failed to add escalation labels: ${error.message}`); + } + } + } + if (!autofixEnabled) { + return stop('autofix disabled for this pull request'); + } + if (attemptCount > maxAttempts) { return stop( `autofix attempt limit reached (${attemptCount} > ${maxAttempts})`, diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 47c2d3bf..e4598229 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -86,7 +86,7 @@ jobs: custom-predicate: >- (event_name == 'pull_request_target' && action == 'labeled' && (label.name == 'autofix' || label.name == 'autofix:clean')) || - (event_name == 'workflow_run' && workflow_run.conclusion == 'failure' && + (event_name == 'workflow_run' && (workflow_run.conclusion == 'failure' || workflow_run.conclusion == 'timed_out') && length(workflow_run.pull_requests) > 0) - name: Checkout for API helpers @@ -154,7 +154,7 @@ jobs: ); const failedChecks = (data.check_runs || []).filter( - (cr) => (cr.conclusion || '').toLowerCase() === 'failure' + (cr) => ['failure', 'timed_out'].includes((cr.conclusion || '').toLowerCase()) ); const relevantFailures = failedChecks.filter((cr) => { @@ -189,7 +189,7 @@ jobs: { maxRetries: 3 } ); const failedJobs = jobs.filter( - (job) => (job.conclusion || '').toLowerCase() === 'failure' + (job) => ['failure', 'timed_out'].includes((job.conclusion || '').toLowerCase()) ); const relevantFailures = failedJobs.filter((job) => { const name = String(job.name || '').toLowerCase(); @@ -255,7 +255,7 @@ jobs: const runId = Number(run?.id || 0); // Only proceed when the upstream workflow failed - if (run.conclusion !== 'failure') { + if (!['failure', 'timed_out'].includes(run.conclusion)) { core.info( `${workflowName} concluded '${run.conclusion}' — ` + 'no autofix needed' diff --git a/docs/MODEL_SELECTION_POLICY.md b/docs/MODEL_SELECTION_POLICY.md index 208bf9cc..3c9c45b3 100644 --- a/docs/MODEL_SELECTION_POLICY.md +++ b/docs/MODEL_SELECTION_POLICY.md @@ -102,13 +102,17 @@ quality gate and an explicit approval update. ### Prepared promotions and rollbacks `tools/prepare_model_promotion.py` (run by `maint-86`) can *prepare* a selection -change from a passing benchmark, but never applies one on its own. It only -prepares a candidate that is the **same family** as the incumbent (e.g. openai -`gpt-5.x`, anthropic `claude-`), **passed every quality gate** (including -paired non-inferiority), and costs **≤** the incumbent per accepted review. -Cross-family swaps are never auto-prepared. It writes the registry mutation -(recording the prior selection in `selection_history`) and opens a PR; merging -that PR is the human approval this policy requires — `human_approval_required` +change from a passing benchmark, but never applies one on its own. Candidates +must pass every benchmark quality gate and have known, finite, nonnegative costs. +Same-family candidates (e.g. openai `gpt-5.x`, anthropic `claude-`) costing +**≤** the incumbent receive `preparation_mode=bounded`. Cross-family or pricier +candidates receive `preparation_mode=approval-required` and explicit +`approval_reasons`. The tool selects at most one candidate per provider, preferring +bounded changes, then lower cost and latency. Both modes retain +`human_approval_required=true`; preparation metadata does not authorize auto-merge. +The tool writes the registry mutation (recording the prior selection in +`selection_history`) for the workflow to open as a PR; merging that PR is the +human approval this policy requires — `human_approval_required` stays true. The inverse path prepares a rollback to the prior selection when the active model shows a failed workload-benchmark (a quality-gate breach). @@ -141,3 +145,23 @@ Review at least every 30 days and immediately after any of: Update the facts and catalog baseline first, run the paired benchmark, attach evidence, then update the explicit selection. Maint-68 propagates the registry; consumer slot provider preferences remain intact. + +### Replayable corpus evidence + +`maint-79` harvests only PR outcomes joined to a bot-published +`verifier-corpus-decision/v1` record. The comparison verifier records the PR head, +evaluated merge SHA, repository/PR, run ID and attempt beside the durable report. +A candidate retains that decision and its comment URL. A stable merge without a +matching decision is excluded; a NON_PASS decision cannot become a clean PASS +just because the PR merged. Provider errors and unavailable reviews are not +benchmark verdicts. A failed merge CI check floors the structured verdict to +NON_PASS even when every provider says PASS. Missing or invalid CI-gate context +suppresses publication rather than creating unverifiable benchmark evidence. +Historical reports without these fields are not backfilled +from merge metadata. They can enter future harvests after fresh verification. + +Case identity includes repository, PR, head and verifier run/attempt. Replaying +the same evidence does not duplicate a case. Existing adjudicated corpus entries +keep their historical identifiers. The staging file is FYI-only; a staging-only +PR does not grow approval metrics. Only additions to `model_eval_pilot.json` count +as promotions, and existing category/size caps and model approval policy remain.