diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 4085ffb28..b1ebbb9a5 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -284,6 +284,7 @@ jobs: UNMAPPED_ENTITIES_OUTPUT: ${{ github.workspace }}/unmapped-entities.json POWER_PUBLICATION_MANIFEST: ${{ github.workspace }}/power-publication.json DATABASE_WRITE_RECEIPT: ${{ github.workspace }}/database-writes.json + INGEST_REPORT_OUTPUT: ${{ github.workspace }}/ingest-report.json run: | : "${INGEST_REPO:?BENCHMARK_REPO repository variable is not set}" bun run admin:db:ingest:ci @@ -376,3 +377,47 @@ jobs: { "text": ":rotating_light: *Ingest results workflow failed*\nRun ID: ${{ steps.artifacts.outputs.source-run-id || github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }}\n${{ steps.powerx.outcome == 'failure' && 'PowerX publication check failed\n' || '' }}${{ steps.database_writes.outputs.summary_json && format('```{0}```\n', steps.database_writes.outputs.summary_json) || '' }}<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" } + + # Report the outcome to InferenceX, whose callback comments on the PR that + # published the changelog entry. Only merge ingests arrive by dispatch; + # staging and manual runs have no merged PR to report to. + - name: Report ingest to InferenceX + if: always() && github.event_name == 'repository_dispatch' && (github.event.action == 'ingest-results' || github.event.action == 'ingest-agentic-results') + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + MERGE_RUN_ID: ${{ github.event.client_payload.merge-run-id || github.event.client_payload.run-id }} + SOURCE_RUN_ID: ${{ steps.artifacts.outputs.source-run-id || + github.event.client_payload.source-run-id || github.event.client_payload.run-id }} + INGEST_SUCCEEDED: ${{ job.status == 'success' }} + INGEST_REPORT: ${{ github.workspace }}/ingest-report.json + BENCHMARK_REPO: ${{ vars.BENCHMARK_REPO }} + with: + github-token: ${{ secrets.PAT }} + script: | + const fs = require('fs'); + if (!process.env.BENCHMARK_REPO) { + throw new Error('BENCHMARK_REPO repository variable is not set'); + } + let report = null; + try { + report = JSON.parse(fs.readFileSync(process.env.INGEST_REPORT, 'utf8')); + } catch { + core.warning('Ingest report not found; reporting status only'); + } + const [owner, repo] = process.env.BENCHMARK_REPO.split('/'); + await github.rest.repos.createDispatchEvent({ + owner, + repo, + event_type: 'ingest-results-completed', + client_payload: { + 'source-repository': `${context.repo.owner}/${context.repo.repo}`, + 'merge-run-id': process.env.MERGE_RUN_ID, + 'source-run-id': process.env.SOURCE_RUN_ID, + 'app-run-id': String(context.runId), + 'app-run-attempt': String(context.runAttempt), + 'database-target': process.env.REQUESTED_DATABASE_TARGET, + 'ingest-succeeded': process.env.INGEST_SUCCEEDED, + report: report && { counts: report.counts, detail: report.detail, unmapped: report.unmapped }, + }, + }); diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index a2a11de96..9435484b1 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -654,6 +654,19 @@ and API values; ingest success alone does not establish complete coverage. To un repair, restore the snapshotted rows or original bytes under the same identities. Do not delete a shared series to repair one point. +### Per-run ingest report + +`admin:db:ingest:ci` writes `INGEST_REPORT_OUTPUT` (JSON, `packages/db/src/etl/ingest-run-report.ts`) +and appends the same counts to the job summary, for success and failure alike. It counts what this +run wrote, unlike `admin:db:verify`, which prints whole-database totals. Rows are `benchmark` +(benchmark_results rows), `eval` (eval_results rows), `trace` (agentic benchmark rows linked to a +trace_replay blob) and `power` (gpu_metrics series; `unmapped` counts power result rows the mapper +could not place). Columns are `new`, `duplicate`, `skipped` (failed-run rows, rows held by per-job +files, purged points, missing siblings, telemetry not collected), `unmapped` and `failed` (database +write or telemetry errors). For +dispatched merge ingests it also sends `ingest-results-completed` to `BENCHMARK_REPO`, whose +`ingest-results-callback` workflow comments the counts on the merged PR. + ### PowerX publication receipts The normal CI importer writes `POWER_PUBLICATION_MANIFEST` when configured. Each diff --git a/docs/deployment-configuration.md b/docs/deployment-configuration.md index d112cf6f1..900fab4d5 100644 --- a/docs/deployment-configuration.md +++ b/docs/deployment-configuration.md @@ -94,19 +94,19 @@ before doing any other work. `stage-results` also checks `PAT` up front. `backfill-server-log-files`, and `recompute-agentic-metrics` check `DATABASE_WRITE_URL` before writing to the production database. -| Secret | Required | Used for | Workflows | -| --------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `stage-results` (passed to `ingest-results`) | -| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | -| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | -| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | -| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`. | `ingest-results`, `ingest-operatorx`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | -| `PAT` | yes, by `stage-results` | Sends the `stage-results-started` and `stage-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results` | -| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | -| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | -| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | -| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | -| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | +| Secret | Required | Used for | Workflows | +| --------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `stage-results` (passed to `ingest-results`) | +| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | +| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | +| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | +| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`. | `ingest-results`, `ingest-operatorx`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | +| `PAT` | yes, by `stage-results`; optional for `ingest-results` | Sends the `stage-results-started`, `stage-results-completed` and `ingest-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results`, `ingest-results` (dispatched merge ingests only; a failed report never fails the ingest) | +| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | +| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | +| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | +| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | +| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | Values reach scripts only through step `env:`, never by inline expression in a `run:` block. diff --git a/packages/db/src/etl/ingest-run-report.test.ts b/packages/db/src/etl/ingest-run-report.test.ts new file mode 100644 index 000000000..14f206aa9 --- /dev/null +++ b/packages/db/src/etl/ingest-run-report.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; + +import { + createIngestRunReport, + INGEST_REPORT_CATEGORIES, + INGEST_REPORT_OUTCOMES, + renderIngestRunReportMarkdown, +} from './ingest-run-report'; + +describe('ingest run report', () => { + it('starts every category and outcome at zero', () => { + const report = createIngestRunReport(40141873883, 1); + for (const category of INGEST_REPORT_CATEGORIES) { + for (const outcome of INGEST_REPORT_OUTCOMES) { + expect(report.counts[category][outcome]).toBe(0); + } + } + expect(report.status).toBe('succeeded'); + }); + + it('renders one table row per category with counts in outcome order', () => { + const report = createIngestRunReport(40141873883, 2); + report.counts.benchmark.new = 3; + report.counts.benchmark.duplicate = 1; + report.counts.trace.skipped = 2; + report.prLinks = ['https://github.com/SemiAnalysisAI/InferenceX/pull/4456']; + const markdown = renderIngestRunReportMarkdown(report); + expect(markdown).toContain('run 40141873883 attempt 2'); + expect(markdown).toContain('| benchmark | 3 | 1 | 0 | 0 | 0 |'); + expect(markdown).toContain('| trace | 0 | 0 | 2 | 0 | 0 |'); + expect(markdown).toContain('PR: https://github.com/SemiAnalysisAI/InferenceX/pull/4456'); + }); + + it('keeps failure text and unmapped names inside one table-safe line', () => { + const report = createIngestRunReport(1, 1); + report.status = 'failed'; + report.error = 'boom | `x`\nnext'; + report.unmapped.models = ['a|b']; + const markdown = renderIngestRunReportMarkdown(report); + expect(markdown).toContain('Status: **failed**'); + expect(markdown).toContain("Error: `boom 'x' next`"); + expect(markdown).toContain('- unmapped models: a b'); + }); +}); diff --git a/packages/db/src/etl/ingest-run-report.ts b/packages/db/src/etl/ingest-run-report.ts new file mode 100644 index 000000000..cc8c6a1db --- /dev/null +++ b/packages/db/src/etl/ingest-run-report.ts @@ -0,0 +1,127 @@ +/** + * Per-run ingest report: what one `ingest-ci-run` invocation wrote, as counts + * per category. The workflow publishes it as the job summary, retains the JSON + * as an artifact, and forwards it to InferenceX so the merged PR gets a comment. + * + * Units per category: + * benchmark — benchmark_results rows + * eval — eval_results rows (aggregate and per-config) + * trace — agentic benchmark rows linked to a trace_replay blob + * power — gpu_metrics telemetry series (unmapped: power result rows the + * mapper could not place) + */ + +export const INGEST_REPORT_CATEGORIES = ['benchmark', 'eval', 'trace', 'power'] as const; +export type IngestReportCategory = (typeof INGEST_REPORT_CATEGORIES)[number]; + +export const INGEST_REPORT_OUTCOMES = [ + 'new', + 'duplicate', + 'skipped', + 'unmapped', + 'failed', +] as const; +export type IngestReportOutcome = (typeof INGEST_REPORT_OUTCOMES)[number]; + +export type IngestReportCounts = Record>; + +export interface IngestRunReport { + version: 1; + runId: number; + runAttempt: number; + status: 'succeeded' | 'failed' | 'purged'; + error: string | null; + evalsOnly: boolean; + prLinks: string[]; + counts: IngestReportCounts; + detail: { + benchmarkFiles: number; + unreadableBenchmarkFiles: number; + rawBenchmarkRows: number; + coveredByPerJobFiles: number; + purgedPoints: number; + failedRunRows: number; + runStatsNew: number; + runStatsDuplicate: number; + evalSamplesNew: number; + evalSampleFiles: number; + gpuMetricSamplesNew: number; + powerPublicationPoints: number; + changelogEntriesWritten: number; + }; + unmapped: { models: string[]; hardware: string[]; precisions: string[]; datasets: string[] }; +} + +export function createIngestRunReport(runId: number, runAttempt: number): IngestRunReport { + const counts = Object.fromEntries( + INGEST_REPORT_CATEGORIES.map((category) => [ + category, + Object.fromEntries(INGEST_REPORT_OUTCOMES.map((outcome) => [outcome, 0])), + ]), + ) as IngestReportCounts; + return { + version: 1, + runId, + runAttempt, + status: 'succeeded', + error: null, + evalsOnly: false, + prLinks: [], + counts, + detail: { + benchmarkFiles: 0, + unreadableBenchmarkFiles: 0, + rawBenchmarkRows: 0, + coveredByPerJobFiles: 0, + purgedPoints: 0, + failedRunRows: 0, + runStatsNew: 0, + runStatsDuplicate: 0, + evalSamplesNew: 0, + evalSampleFiles: 0, + gpuMetricSamplesNew: 0, + powerPublicationPoints: 0, + changelogEntriesWritten: 0, + }, + unmapped: { models: [], hardware: [], precisions: [], datasets: [] }, + }; +} + +function cell(text: string): string { + return text.replaceAll(/[|\r\n]/gu, ' '); +} + +/** Render the report as GitHub-flavored Markdown for $GITHUB_STEP_SUMMARY. */ +export function renderIngestRunReportMarkdown(report: IngestRunReport): string { + const status = + report.status === 'succeeded' + ? 'succeeded' + : report.status === 'purged' + ? 'skipped (run attempt purged via run-overrides)' + : 'failed'; + const lines = [ + `### Ingest report: run ${report.runId} attempt ${report.runAttempt}`, + '', + `Status: **${status}**${report.evalsOnly ? ' (evals-only run)' : ''}`, + ]; + if (report.error) lines.push('', `Error: \`${cell(report.error).replaceAll('`', "'")}\``); + if (report.prLinks.length > 0) lines.push('', `PR: ${report.prLinks.join(', ')}`); + lines.push( + '', + `| Category | ${INGEST_REPORT_OUTCOMES.join(' | ')} |`, + `| --- | ${INGEST_REPORT_OUTCOMES.map(() => '---:').join(' | ')} |`, + ...INGEST_REPORT_CATEGORIES.map( + (category) => + `| ${category} | ${INGEST_REPORT_OUTCOMES.map((o) => report.counts[category][o]).join(' | ')} |`, + ), + '', + '
Detail', + '', + ...Object.entries(report.detail).map(([key, value]) => `- ${key}: ${value}`), + ); + for (const [kind, names] of Object.entries(report.unmapped)) { + if (names.length > 0) lines.push(`- unmapped ${kind}: ${cell(names.join(', '))}`); + } + lines.push('', '
', ''); + return lines.join('\n'); +} diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index 50130cb57..1eca188b5 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -35,6 +35,7 @@ import { configWriteFailurePublicationError, } from './etl/ingest-receipts'; import { printIngestSummaryFooter } from './etl/ingest-summary'; +import { createIngestRunReport, renderIngestRunReportMarkdown } from './etl/ingest-run-report'; import { createConfigCache } from './etl/config-cache'; import { createWorkflowRunServices } from './etl/workflow-run'; import { @@ -244,6 +245,7 @@ if (reusedIngestMetadata) { } const runIdNum = parseInt(runIdStr, 10); +const report = createIngestRunReport(runIdNum, runAttemptNum); const GITHUB_TOKEN = process.env.GITHUB_TOKEN!; @@ -284,6 +286,7 @@ async function main(): Promise { console.log( ` Run ${runIdStr} attempt ${runAttemptNum} is purged via run-overrides — skipping.`, ); + report.status = 'purged'; return; } @@ -420,6 +423,10 @@ async function main(): Promise { } const appendOnly = hasAppendOnlyFlag(changelogs); const evalsOnly = hasEvalsOnlyFlag(changelogs); + report.evalsOnly = evalsOnly; + report.prLinks = [ + ...new Set(changelogs.flatMap((c) => c.entries.flatMap((e) => (e.prLink ? [e.prLink] : [])))), + ]; if (evalsOnly && requiredPower) throw new Error('Required power: benchmark scope cannot be published as an evals-only run'); @@ -446,6 +453,7 @@ async function main(): Promise { console.log( ` Run ${runId} attempt ${runAttemptNum} is purged via run-overrides — skipping ingest.`, ); + report.status = 'purged'; return; } console.log(` Workflow run DB id: ${workflowRunId}`); @@ -523,6 +531,7 @@ async function main(): Promise { ].sort(); const seenPointIdentities = new Map(); console.log(` Found ${files.length} benchmark JSON file(s)`); + report.detail.benchmarkFiles = files.length; const perJobPoints = perJobPointKeys(files); for (const [fileIndex, file] of files.entries()) { @@ -535,11 +544,14 @@ async function main(): Promise { console.warn(` [WARN] Failed to parse ${relativeFile}: ${file.unreadable}`); telemetryExpectationsUnknown = true; powerPublicationErrors.push(`Unreadable benchmark JSON: ${relativeFile}`); + report.detail.unreadableBenchmarkFiles++; console.log(` skipped unreadable JSON (${elapsed(fileStart)})`); console.log(` raw rows: ${file.rows.length + file.nonObjectRows}`); continue; } console.log(` raw rows: ${file.rows.length + file.nonObjectRows}`); + report.detail.rawBenchmarkRows += file.rows.length + file.nonObjectRows; + report.counts.benchmark.unmapped += file.nonObjectRows; for (const { raw } of file.rows) { const datasetSlug = datasetSlugFromBenchmarkRow(raw); @@ -556,7 +568,12 @@ async function main(): Promise { } if (mapped) return [mapped]; if (!failed) fileExpectationsUnknown = true; + if (failed) { + report.counts.benchmark.skipped++; + report.detail.failedRunRows++; + } else report.counts.benchmark.unmapped++; const issue = unmappedPowerResultIssue(raw, failed, relativeFile); + if (issue) report.counts.power.unmapped++; if (issue && 'error' in issue) powerPublicationErrors.push(issue.error); else if (issue) powerPublicationWarnings.push(issue.warning); return []; @@ -564,6 +581,8 @@ async function main(): Promise { if (fileExpectationsUnknown) telemetryExpectationsUnknown = true; console.log(` mapped rows: ${rows.length}`); + report.counts.benchmark.skipped += coveredRows; + report.detail.coveredByPerJobFiles += coveredRows; if (coveredRows > 0) { console.log(` skipped ${coveredRows} row(s) already held by per-job files`); } @@ -601,6 +620,7 @@ async function main(): Promise { } catch (error: any) { telemetryExpectationsUnknown = true; tracker.recordDbError('config', path.posix.basename(file.path), error); + report.counts.benchmark.failed++; const publicationError = configWriteFailurePublicationError(row, relativeFile); if (publicationError) powerPublicationErrors.push(publicationError); continue; @@ -616,6 +636,8 @@ async function main(): Promise { `isl ${row.isl}, osl ${row.osl}, conc ${row.conc}, offload ${row.offloadMode}, ` + `recipe ${row.recipeFingerprint ?? 'legacy'}`, ); + report.counts.benchmark.skipped++; + report.detail.purgedPoints++; continue; } if (plan.backfillId) { @@ -685,6 +707,8 @@ async function main(): Promise { ); totalNewBmk += newCount; totalDupBmk += dupCount; + report.counts.benchmark.new += newCount; + report.counts.benchmark.duplicate += dupCount; if (requiredPower) retainedPowerPoints.push(...toInsert); for (const r of toInsert) { @@ -730,6 +754,7 @@ async function main(): Promise { } if (gpuMetricsArtifact && !gpuMetricsArtifactHoldsTelemetry(gpuMetricsArtifact)) { telemetryNotCollectedArtifacts.add(gpuMetricsArtifact.artifactName); + report.counts.power.skipped++; console.log( ` gpu_metrics ${gpuMetricsArtifact.artifactName} held no telemetry; not collected`, ); @@ -757,6 +782,9 @@ async function main(): Promise { ); totalGpuMetricSeries += ingested.seriesIds.length; totalGpuMetricSamples += ingested.samplesInserted; + report.counts.power.new += ingested.seriesIds.length - ingested.seriesSkipped; + report.counts.power.duplicate += ingested.seriesSkipped; + report.detail.gpuMetricSamplesNew += ingested.samplesInserted; console.log( ` gpu_metrics ${ingested.seriesIds.length} series, ` + `+${ingested.samplesInserted} sample(s), ` + @@ -764,6 +792,7 @@ async function main(): Promise { ); } catch (error: any) { tracker.recordTelemetryError(`gpu_metrics for ${configKey}`, error); + report.counts.power.failed++; for (const row of toInsert) { const point = telemetryObservations.get( stablePowerPointIdentity(benchmarkPublicationIdentity(row)), @@ -803,6 +832,7 @@ async function main(): Promise { console.log( ` trace_replay ${suffix}: found ${unlinkedIds.length} unlinked row(s) (${elapsed(linkStart)})`, ); + report.counts.trace.duplicate += insertedIds.length - unlinkedIds.length; if (unlinkedIds.length === 0) { console.log( ` trace_replay ${suffix}: skipping blob insert; all benchmark rows already linked`, @@ -845,24 +875,29 @@ async function main(): Promise { ), ); totalTraceReplayLinked += linked; + report.counts.trace.new += linked; console.log(` trace_replay ${suffix}: done (${elapsed(traceStart)})`); }, ) .catch((error: any) => { tracker.recordDbError('trace_replay', traceSubject, error); + report.counts.trace.failed += unlinkedIds.length; }); traceTasks.push(task); } } catch (error: any) { tracker.recordDbError('trace_replay', traceSubject, error); + report.counts.trace.failed += insertedIds.length; } } else { console.log(` trace_replay ${suffix}: missing sibling artifact`); tracker.skips.traceReplayMissing++; + report.counts.trace.skipped += insertedIds.length; } } } catch (error: any) { tracker.recordDbError('benchmark', path.posix.basename(file.path), error); + report.counts.benchmark.failed += toInsert.length; } } console.log(` finished ${relativeFile} (${elapsed(fileStart)})`); @@ -947,6 +982,8 @@ async function main(): Promise { ); totalNewStats = newCount; totalDupStats = dupCount; + report.detail.runStatsNew = newCount; + report.detail.runStatsDuplicate = dupCount; } catch (error: any) { tracker.recordDbError('run_stats', `${statsRows.length} row(s)`, error); } @@ -965,14 +1002,19 @@ async function main(): Promise { for (const row of data) { if (typeof row !== 'object' || row === null) continue; const mapped = mapAggEvalRow(row as Record, tracker, runIdStr); - if (!mapped) continue; + if (!mapped) { + report.counts.eval.unmapped++; + continue; + } try { const configId = await getOrCreateConfig(mapped.config); const { outcome } = await ingestEvalRow(sql, configId, mapped, workflowRunId, date); if (outcome === 'new') totalEvals++; + report.counts.eval[outcome === 'new' ? 'new' : 'duplicate']++; } catch (error: any) { tracker.recordDbError('eval', `${path.basename(file)} ${mapped.task}`, error); + report.counts.eval.failed++; } } } @@ -1000,14 +1042,21 @@ async function main(): Promise { const resultsName = files.find((f) => f.startsWith('results_') && f.endsWith('.json')); if (!metaPath || !resultsName) { console.warn(` [WARN] ${path.basename(dir)}: missing meta_env.json or results_*.json`); + report.counts.eval.skipped++; continue; } const meta = readJson(metaPath) as Record | null; const results = readJson(path.join(dir, resultsName)) as Record | null; - if (!meta || !results) continue; + if (!meta || !results) { + report.counts.eval.skipped++; + continue; + } const evalParamsList = mapEvalRow(meta, results, tracker, runIdStr); - if (evalParamsList.length === 0) continue; + if (evalParamsList.length === 0) { + report.counts.eval.unmapped++; + continue; + } const samplesByTask = new Map(); for (const f of files) { @@ -1025,13 +1074,14 @@ async function main(): Promise { for (const params of evalParamsList) { try { const configId = await getOrCreateConfig(params.config); - const { id: evalResultId } = await ingestEvalRow( + const { id: evalResultId, outcome } = await ingestEvalRow( sql, configId, params, workflowRunId, date, ); + report.counts.eval[outcome === 'new' ? 'new' : 'duplicate']++; const samplesText = samplesByTask.get(params.task); if (!samplesText) continue; @@ -1044,6 +1094,7 @@ async function main(): Promise { totalSampleFiles++; } catch (error: any) { tracker.recordDbError('eval', `${path.basename(dir)} ${params.task}`, error); + report.counts.eval.failed++; } } } @@ -1068,6 +1119,9 @@ async function main(): Promise { } } console.log(` Changelog: ${totalChangelogs} written`); + report.detail.evalSamplesNew = totalSamples; + report.detail.evalSampleFiles = totalSampleFiles; + report.detail.changelogEntriesWritten = totalChangelogs; const [configCount] = await sql`select count(*)::int as n from configs`; const [resultCount] = await sql`select count(*)::int as n from benchmark_results`; @@ -1100,6 +1154,12 @@ async function main(): Promise { ); const { unmappedModels, unmappedHws, unmappedPrecisions } = tracker; + report.unmapped = { + models: [...unmappedModels].toSorted(), + hardware: [...unmappedHws].toSorted(), + precisions: [...unmappedPrecisions].toSorted(), + datasets: [...missingDatasets].toSorted(), + }; const unmappedOutPath = process.env.UNMAPPED_ENTITIES_OUTPUT; if ( @@ -1126,13 +1186,36 @@ async function main(): Promise { console.log(' Invalidate API cache: bun run admin:cache:invalidate'); } +/** Write INGEST_REPORT_OUTPUT (JSON) and append Markdown to GITHUB_STEP_SUMMARY when set. */ +function writeIngestRunReport(): void { + try { + const reportPath = process.env.INGEST_REPORT_OUTPUT; + if (reportPath) { + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(` Ingest report: ${reportPath}`); + } + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + if (summaryPath) fs.appendFileSync(summaryPath, renderIngestRunReportMarkdown(report)); + } catch (error) { + console.warn(` [WARN] failed to write ingest report: ${String(error)}`); + } +} + main() .catch((error) => { powerPublicationErrors.push(error instanceof Error ? error.message : String(error)); console.error('ingest-ci-run failed:', error); process.exitCode = 1; + report.status = 'failed'; + report.error = (error instanceof Error ? error.message : String(error)).slice(0, 500); }) .finally(async () => { + report.detail.powerPublicationPoints = powerPublicationPoints.size; + if (report.status === 'succeeded' && tracker.dbWriteErrors.length > 0) { + report.status = 'failed'; + report.error = `${tracker.dbWriteErrors.length} database write failure(s)`; + } + writeIngestRunReport(); const publicationPath = process.env.POWER_PUBLICATION_MANIFEST; const writeReceiptPath = process.env.DATABASE_WRITE_RECEIPT; const receiptInput = {