From d8f4fecfabcf19e1fd32ed7f1c636a0bfe9fbd7d Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:36:50 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat(collectivex):=20fill=20the=20Collectiv?= =?UTF-8?q?eX=20database=20from=20committed=20results=20/=20=E7=94=A8?= =?UTF-8?q?=E5=B7=B2=E6=8F=90=E4=BA=A4=E7=9A=84=E7=BB=93=E6=9E=9C=E5=A1=AB?= =?UTF-8?q?=E5=85=85=20CollectiveX=20=E6=95=B0=E6=8D=AE=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CollectiveX results are now committed to SemiAnalysisAI/InferenceX under collectivex/results/ instead of being uploaded as GitHub Actions artifacts. The dashboard keeps serving from its Neon database, but that database is now filled from those files rather than by the lazy artifact ingest. - committed-results.ts resolves a ref to the newest results commit at or before it (which must be on main), lists collectivex/results/ with the git trees API, reads each file from raw.githubusercontent.com, and validates it like the benchmark repo's own test: /.json matches the document, the case is runnable in cases.json, the result is from committed code, and its version matches cases.json. Failures name the file. - One run per results commit: run_id is the commit time in epoch seconds (so ORDER BY run_id DESC still means latest), run_attempt 1, source_sha the commit, matrix cases.json. upsertCollectiveXRun replaces a re-ingested commit's row and documents in one statement and refuses a run id held by another commit. - POST /api/v1/collectivex/ingest (Bearer COLLECTIVEX_ADMIN_SECRET, body {sha} or {ref}) ingests one commit and purges the CollectiveX cache; admin:db:ingest:collectivex takes --sha/--ref or --backfill. - The read routes read Neon only. The lazy ingest, artifact selection, the DELETE run handler and its admin UI, client polling, and the short CDN TTL are gone; responses keep their shapes and discovery_complete is always true. Migration 004 drops the unused deleted_at tombstone. - Reader: swap_blocks cases come from cases.json and must each be answered once; a version mismatch names the case; withheld EP tails stay null. Links point at collectivex/ at the results commit. - Docs (including an archive/wipe/migrate/backfill runbook), API reference, route catalog, skills reference and integrity, and tests. 中文：用已提交的结果填充 CollectiveX 数据库 CollectiveX 结果现在提交到 SemiAnalysisAI/InferenceX 的 collectivex/results/， 不再作为 GitHub Actions 产物上传。仪表板仍从 Neon 数据库提供数据，但该数据库 改为由这些文件填充，而不是按需导入产物。 - committed-results.ts 把 ref 解析为其之前（含自身）最新的结果提交（该提交必须 位于 main），用 git trees API 列出 collectivex/results/，从 raw.githubusercontent.com 读取每个文件，并按基准仓库自身测试的规则校验： /.json 与文档一致、用例在 cases.json 中可运行、结果来自已提交的 代码、版本与 cases.json 一致。报错会指明具体文件。 - 每个结果提交对应一次运行：run_id 为提交时间的 Unix 秒级时间戳（因此 ORDER BY run_id DESC 仍表示最新），run_attempt 为 1，source_sha 为该提交， matrix 为 cases.json。upsertCollectiveXRun 在一条语句中替换重复导入提交的行与 文档，并拒绝被其他提交占用的 run_id。 - POST /api/v1/collectivex/ingest（Bearer COLLECTIVEX_ADMIN_SECRET，请求体为 {sha} 或 {ref}）导入一个提交并清除 CollectiveX 缓存； admin:db:ingest:collectivex 支持 --sha/--ref 或 --backfill。 - 读取路由只读 Neon。移除按需导入、产物选择、DELETE 运行处理器及其管理界面、 客户端轮询和较短的 CDN TTL；响应结构不变，discovery_complete 始终为 true。 迁移 004 删除不再使用的 deleted_at 墓碑列。 - 读取器：swap_blocks 用例取自 cases.json，且每个用例只能被回答一次；版本不一致 时报错会指明用例；被保留的 EP 尾部延迟保持为 null。链接指向结果提交处的 collectivex/。 - 更新文档（含归档/清空/迁移/回填操作手册）、API 参考、路由目录、skills 参考及其 integrity 清单，以及测试。 --- .env.example | 11 +- docs/architecture.md | 2 +- docs/collectivex.md | 197 ++--- docs/deployment-configuration.md | 36 +- docs/index.md | 2 +- packages/app/cypress/e2e/collectivex.cy.ts | 206 +---- .../api/v1/collectivex/ingest/route.test.ts | 163 ++++ .../app/api/v1/collectivex/ingest/route.ts | 91 +++ .../api/v1/collectivex/latest/route.test.ts | 46 +- .../app/api/v1/collectivex/latest/route.ts | 33 +- .../runs/[runId]/route.fixtures.test.ts | 8 - .../v1/collectivex/runs/[runId]/route.test.ts | 150 +--- .../api/v1/collectivex/runs/[runId]/route.ts | 84 +-- .../app/api/v1/collectivex/runs/route.test.ts | 78 +- .../src/app/api/v1/collectivex/runs/route.ts | 37 +- .../collectivex/CollectiveXChart.tsx | 4 +- .../collectivex/CollectiveXDisplay.tsx | 171 +---- .../collectivex/CollectiveXRunsTable.tsx | 91 +-- .../src/components/collectivex/data.test.ts | 28 - .../app/src/components/collectivex/data.ts | 37 +- packages/app/src/hooks/api/use-collectivex.ts | 23 +- packages/app/src/lib/api/api-documentation.ts | 90 +-- packages/app/src/lib/api/api-route-catalog.ts | 40 +- packages/app/src/lib/api/api.test.ts | 22 - packages/app/src/lib/api/api.ts | 20 +- packages/app/src/lib/cache/api-cache.ts | 21 +- .../collectivex-lazy-ingest.test.ts | 703 ------------------ .../collectivex/collectivex-lazy-ingest.ts | 624 ---------------- .../004_committed_results.sql | 18 + packages/db/package.json | 2 +- .../collectivex/artifact-selection.test.ts | 63 -- .../db/src/collectivex/artifact-selection.ts | 54 -- .../src/collectivex/committed-results.test.ts | 313 ++++++++ .../db/src/collectivex/committed-results.ts | 443 +++++++++++ packages/db/src/collectivex/reader.test.ts | 27 +- packages/db/src/collectivex/reader.ts | 54 +- .../db/src/collectivex/swap-reader.test.ts | 121 ++- packages/db/src/collectivex/swap-reader.ts | 43 +- .../db/src/collectivex/swap-test-fixture.ts | 41 +- packages/db/src/collectivex/test-fixture.ts | 3 +- packages/db/src/collectivex/types.ts | 15 +- packages/db/src/ingest-collectivex.test.ts | 178 ++--- packages/db/src/ingest-collectivex.ts | 251 +++---- packages/db/src/queries/collectivex.test.ts | 109 +-- packages/db/src/queries/collectivex.ts | 168 +---- .../skills/inferencex-api/integrity.json | 2 +- .../inferencex-api/references/collectivex.md | 40 +- 47 files changed, 1806 insertions(+), 3157 deletions(-) create mode 100644 packages/app/src/app/api/v1/collectivex/ingest/route.test.ts create mode 100644 packages/app/src/app/api/v1/collectivex/ingest/route.ts delete mode 100644 packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts delete mode 100644 packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts create mode 100644 packages/db/migrations-collectivex/004_committed_results.sql delete mode 100644 packages/db/src/collectivex/artifact-selection.test.ts delete mode 100644 packages/db/src/collectivex/artifact-selection.ts create mode 100644 packages/db/src/collectivex/committed-results.test.ts create mode 100644 packages/db/src/collectivex/committed-results.ts diff --git a/.env.example b/.env.example index 9fcc227f4..77777ca69 100644 --- a/.env.example +++ b/.env.example @@ -17,8 +17,8 @@ # DATABASE_DRIVER=neon # DATABASE_SSL=true -# GitHub PAT — required for CollectiveX lazy ingest (sweep artifact reads); also used -# for framework releases, star counts, and unofficial-run artifact reads +# GitHub PAT — used for framework releases, star counts, unofficial-run artifact reads, +# and (optional, raises the rate limit) the CollectiveX committed-results ingest # Create at: https://github.com/settings/personal-access-tokens # GITHUB_TOKEN= # Extra benchmark source repos (owner/repo, comma-separated) searched after @@ -31,8 +31,10 @@ # Directory contains .json with { run, benchmarks, evaluations }. # INFERENCEX_LOCAL_ARTIFACT_DIR=/absolute/path/to/local-artifacts -# CollectiveX database (separate Neon instance). The app routes read and write through -# DATABASE_COLLECTIVEX_WRITE_URL; they do not read DATABASE_COLLECTIVEX_READONLY_URL. +# CollectiveX database (separate Neon instance), filled from InferenceX's committed +# collectivex/results/ by POST /api/v1/collectivex/ingest or admin:db:ingest:collectivex. +# The app routes read and write through DATABASE_COLLECTIVEX_WRITE_URL; they do not read +# DATABASE_COLLECTIVEX_READONLY_URL. # DATABASE_COLLECTIVEX_READONLY_URL= # DATABASE_COLLECTIVEX_WRITE_URL= @@ -84,6 +86,7 @@ # argument or run URL names the repository. # INGEST_REPO= # FEEDBACK_SECRET= +# Bearer token for POST /api/v1/collectivex/ingest (the InferenceX ingest workflow). # COLLECTIVEX_ADMIN_SECRET= # OperatorX durable raw-artifact cache (primary, write-capable connection). diff --git a/docs/architecture.md b/docs/architecture.md index d44145f9f..eb6dbf88e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,7 +16,7 @@ Raw-data API routes return raw DB rows with zero transformation, validation, or Some routes serve a purpose that raw DB rows cannot satisfy: - Dashboard projections under `/api/v1/views/*` use shared pure UI transformations server-side and publish their selectors and assumptions. Keep raw-data contracts separate from these projections. -- CollectiveX assembles stored sweep documents through its shared reader. +- CollectiveX assembles its stored committed-result documents through its shared reader. - OperatorX uses versioned ingest-time comparison projections for its page-owned API. The reader merges compact timing rows across runs while preserving source revisions, statuses, and timeline references; raw documents remain available for drilldowns. An optional revision-keyed Blob cache shares the merged comparison across instances; partial reads are excluded and OperatorX invalidation purges its deployment subtree. See [OperatorX](./operatorx.md#compact-comparison-reads). - `tco-feed` performs server-side frontier interpolation for spreadsheet consumers that cannot run the TypeScript transforms. Its assumptions (tier weights, workload mix, α) enter only as explicit query params with documented defaults, so a published sheet's URL fully records its methodology. - `pareto` returns frontier and hinterland coordinates over stored benchmark rows with explicit axis preferences. It reuses the chart dominance algorithm without UI filters. See the [Pareto frontier and hinterland reference](../packages/skills/skills/inferencex-api/references/pareto.md). diff --git a/docs/collectivex.md b/docs/collectivex.md index 5c65e2bc0..74ab2ec1b 100644 --- a/docs/collectivex.md +++ b/docs/collectivex.md @@ -1,93 +1,97 @@ # CollectiveX -Design rationale for the CollectiveX tab's data pipeline. Unlike every other tab -(Neon DB → ETL ingest → `/api/v1/*`), CollectiveX uses **lazy ingest-on-read**: its -database is a durable cache of GitHub Actions, populated by the API routes themselves. - -## Why lazy ingest instead of the main pipeline - -- **Sweep artifacts expire after 14 days.** The sweep workflow - (`collectivex-sweep.yml` in the harness repo) uploads a matrix artifact - (`cxsweep-matrix-{run_id}`) and per-cell result artifacts - (`cxshard-{cell}-{run_id}-{attempt}`) with 14-day retention. Persisting on first - view makes a run outlive its artifacts once anyone has looked at it. -- **The sweep JSON contract is expected to change.** The DB stores the RAW documents - verbatim; the shared reader (`packages/db/src/collectivex/reader.ts`) is the single - transform point and runs at API-read time, so a reader fix retroactively applies to - already-stored runs — no re-ingest. A contract change = reader change + a bump of the - numeric `version` in the harness's `experimental/CollectiveX/configs/sweep.json`. -- **No CI plumbing.** There is no ingest workflow, no cross-repo dispatch, and no GH - secrets. Runs launched via `gh api` on any harness branch appear on the dashboard - within the CDN TTL of someone viewing the page — only the workflow identity is - checked, never the branch. - -## How it works - -`packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts` exposes three `ensure*` functions the -routes call before reading the DB (`packages/db/src/queries/collectivex.ts`): - -- `ensureLatestCollectiveXRun` — walk GitHub's completed sweep runs newest-first; stop at - the first live requested-version run; persist it if absent. -- `ensureCollectiveXRunsList` — progressively backfill every requested-version run whose - 14-day artifacts may still be downloaded. Each request changes at most eight rows; an - incomplete response is uncached and the client refetches until the workflow history is - exhausted. Known rows do not consume the batch, so discovery advances past the newest runs. - GitHub discovery is limited to runs created within the last 44 days: the 30-day workflow - rerun window plus 14-day artifact retention, covering the oldest rerun whose artifacts can - still exist without rescanning permanent workflow history on every cold-origin request. -- `ensureCollectiveXRun` — fetch one run by id, or compare a stored run's `run_attempt` - against GitHub and refresh it when a rerun is newer. Only completed runs are persisted. - -Key invariants: - -- **Writes are atomic and race-safe**: one CTE statement with - `ON CONFLICT (run_id) DO NOTHING`; concurrent first-viewers can't double-ingest or - expose a partial run. A GitHub re-run (newer `run_attempt`) is replaced through a - `FOR UPDATE`-guarded refresh statement. -- **Reads use the primary**: discovery state checks, writes, and every route query use - `DATABASE_COLLECTIVEX_WRITE_URL`, so a newly inserted or refreshed run is visible in the - same request. -- **Deletion tombstones** (`cx_runs.deleted_at`, documents freed): discovery must never - resurrect a deleted run. Re-ingesting via the CLI - (`bun run admin:db:ingest:collectivex `) clears the tombstone — that CLI is - the operator tool for pre-warming runs before artifact expiry, backfills, and un-deletes. -- **"Latest" orders by `run_id`** (monotonic with run creation, matching the discovery - walk) — not by completion time, where a long-failing older run would shadow a newer - successful one. -- **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` - result shards (case-insensitive). Other vendor values are omitted from chart series, - coverage, and SKU summaries; runs with no supported-vendor cases stay hidden. -- **GitHub being down never takes the page down**: routes serve whatever the DB holds and - only surface an error when there is no stored fallback. -- **Run-list completeness is progressive**: `/api/v1/collectivex/runs` returns - `discovery_complete: false` while another bounded ingest pass is required. Those responses - use `private, no-store`; the client polls once per second until the field becomes `true`. - Stored runs remain visible indefinitely, while never-ingested runs disappear with their - upstream artifacts and can no longer be reconstructed. -- **Caching**: responses carry the `collectivex` CDN tag with a 60s - `s-maxage` (freshness bound for lazy discovery). Run deletion and - `POST /api/v1/invalidate?scope=collectivex` purge only that tag; the main dashboard's - blob cache is untouched by CollectiveX operations. -- **Env**: `DATABASE_COLLECTIVEX_WRITE_URL` is the primary used by lazy persistence, all - route reads, deletion, and migrations via `bun run admin:db:migrate:collectivex`. +Design rationale for the CollectiveX tab's data pipeline. CollectiveX keeps its own Neon +database, filled from the benchmark results that `SemiAnalysisAI/InferenceX` commits under +`collectivex/results/`: `cases.json` (the requested cases, with the contract `version`) plus +one `/.json` per measured case. The API routes only read that database. + +## One run per results commit + +Each InferenceX commit on `main` that changed `collectivex/results/` is one stored run: + +- `run_id` is the commit's committer time in epoch seconds. It grows with `main`'s history, + so "latest" still means `ORDER BY run_id DESC`. Two results commits landing in the same + second would collide; the ingest refuses the second one (HTTP 409 / CLI error) rather than + overwrite it. +- `run_attempt` is always 1, `source_sha` is the results commit, `source_branch` is `main`, + `conclusion` is `success`, `matrix` is `cases.json` and `summary` is the shared reader's + `buildRunSummary`. `cx_run_docs` holds the commit's result documents verbatim. +- Re-ingesting a commit is idempotent: one statement upserts the row and replaces its + documents, so a reader sees the old contents or the new ones, never a mix. +- A ref resolves to the newest results commit at or before it, and that commit must be in + `main`'s results history. Ingesting `main` after an unrelated merge therefore refreshes + the run already published for those files instead of creating a duplicate, and results + that never landed on `main` are never published. + +The DB stores raw documents; the shared reader (`packages/db/src/collectivex/reader.ts`) +assembles the dataset at read time, so a reader fix applies to stored runs without +re-ingesting. A contract change is a reader change plus a bump of the numeric `version` that +`cases.json` and every EP result carry. + +## Ingest + +`packages/db/src/collectivex/committed-results.ts` is shared by both entry points. It reads +exactly `collectivex/results/**` at the pinned commit: the git trees API lists the files and +each blob comes from `raw.githubusercontent.com`, which does not count against the REST rate +limit. It then checks every file the way the benchmark repo's own test does: the path is +`/.json` matching the document's identity (EP case attempts by +`identity.case_id` and `identity.case_factors.sku`; swap_blocks results by +`schema: collectivex-swap-blocks-v1`, `case_id` and `runtime.sku`), the case is runnable in +`cases.json`, the result was measured from committed code (`source_dirty: false`), and an EP +result carries the `cases.json` version. Each document is then assembled alone, so every +failure names its file. Files above 4 MiB, or a tree above 256 MiB, are rejected. + +- **HTTP**: `POST /api/v1/collectivex/ingest` with `Authorization: Bearer +` and a JSON body of exactly `{"sha": "<7-40 hex>"}` or + `{"ref": ""}`. It ingests through `DATABASE_COLLECTIVEX_WRITE_URL` and then + purges the CollectiveX cache scope. Responses: 200 `{ingested: "inserted" | "replaced", +run_id, source_sha, generated_at, version, requested_cases, measured_cases, documents}`; + 400 for any other body; 401 for a missing or wrong token; 404 when no results commit on + `main` matches; 409 for a run id held by another commit; 422 when the files break the + contract (the message names the file); 502 when GitHub cannot be read; 500 when the + secret is unset or the database write fails. An InferenceX workflow calls it on pushes to + `main` that touch `collectivex/results/`. One call reads a few GitHub API pages plus one + raw file per result and runs within the route's 300 s limit. +- **CLI**: `bun run admin:db:ingest:collectivex [--sha | --ref ] [--backfill] +[--yes] [--no-ssl]` ingests one commit (default: `main`), or with `--backfill` every + results commit on `main`, oldest first, reusing unchanged files between commits. It asks + for confirmation unless `--yes` is given and stops at the first failing commit. + `GITHUB_TOKEN` is optional but raises GitHub's API rate limit. + +## Operator runbook: re-seed from committed results + +Run these against the CollectiveX database only, in order: + +1. **Archive** the current contents: + `pg_dump --table=cx_runs --table=cx_run_docs "$DATABASE_COLLECTIVEX_WRITE_URL" > cx-archive.sql` +2. **Wipe**: `psql "$DATABASE_COLLECTIVEX_WRITE_URL" -c 'TRUNCATE cx_run_docs, cx_runs'`. + Until the backfill finishes, `/api/v1/collectivex/latest` returns 404 and the run list is empty. +3. **Migrate**: `bun run admin:db:migrate:collectivex` (applies `004_committed_results.sql`, + which drops the unused `deleted_at` tombstone column). +4. **Backfill**: `bun run admin:db:ingest:collectivex --backfill`, then purge the cache with + `POST /api/v1/invalidate?scope=collectivex` (the CLI does not reach the CDN). + +## Serving + +- **Reads use the primary**: every route reads through `DATABASE_COLLECTIVEX_WRITE_URL`, so a + response cached right after an ingest's purge never comes from a lagging replica. `DATABASE_COLLECTIVEX_READONLY_URL` is not read by the app routes. - `COLLECTIVEX_ADMIN_SECRET` is the delete route Bearer token (deliberately not - `INVALIDATE_SECRET`, since it is remembered in browser localStorage). When it is unset, - the delete route returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. `GITHUB_TOKEN` - authorizes artifact reads. -- **Source repositories**: sweeps are discovered in `SemiAnalysisAI/InferenceX` and in - every repository listed in `INFERENCEX_SOURCE_REPOS`, merged newest-first by run id. A - listed repository whose sweep-workflow listing returns 404 is skipped; GitHub also - returns 404 for a misspelled repository or one the token cannot see. One that fails to - list for another reason is logged, and the pass then counts as failed: the runs list - keeps `discovery_complete: false` and both passes use the failure cooldown. Discovery - throws when every repository is skipped or fails. Per-run lookups try each repository - in order. A run from any other repository records it as `source_repository` in its - summary, and source, methodology, and run links use it; runs without the field link to - `SemiAnalysisAI/InferenceX`. The ingest CLI takes the repository from its `[repo]` - argument, then from a full run URL, then `INGEST_REPO`, and exits when none is set. - `GITHUB_TOKEN` must be able to - read Actions runs and artifacts in every listed repository. +- **Caching**: responses carry the `collectivex` CDN tag with the default one-day `s-maxage`. + The ingest route and `POST /api/v1/invalidate?scope=collectivex` purge only that tag; the + main dashboard's blob cache is untouched by CollectiveX operations. +- **Run list**: `/api/v1/collectivex/runs` always reports `discovery_complete: true`; the + field stays for existing clients. +- **No deletion**: runs are removed by changing the committed results, not through the app. + `COLLECTIVEX_ADMIN_SECRET` authorizes only the ingest route; when it is unset that route + returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. +- **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` results + (case-insensitive). Other vendor values are omitted from chart series, coverage, and SKU + summaries; runs with no supported-vendor cases stay hidden. +- **Withheld tails**: an EP result may publish p90/p95/p99 as `null` when too many of a + series' samples were misaligned or host-stalled (p50 is always present). The reader keeps + the derived rates `null` there, the alpha-beta fit skips them, and the tooltip shows them + as unavailable. +- **Links**: a run links to `collectivex/results/` and the methodology at its `source_sha`. ## Known-support matrix @@ -111,12 +115,13 @@ same shared transform across the wire. ## vLLM swap_blocks -Standalone `backend=swap-blocks` sweep runs are discovered through the same matrix/shard -artifact path. Their execution-only matrix (`include` with one or more swap-blocks cells) maps to -app contract version 1. The shared reader validates `collectivex-swap-blocks-v1` documents, -including correctness, payload arithmetic, finite positive ordered latency percentiles, -sample counts and the run's source SHA. EP/KV documents keep their existing reader path. -No database migration is needed: raw JSON documents remain the source of truth. +swap_blocks cases are requested in `cases.json` with `suite: "swap-blocks"` and answered by +`collectivex-swap-blocks-v1` documents at `/.json`. Each result must answer +exactly one requested swap_blocks case of its own GPU pool; requested swap_blocks counts come +from `cases.json`, and those cases never enter the EP ladder parser. The shared reader +validates correctness, payload arithmetic, finite positive ordered latency percentiles and +sample counts. EP/KV documents keep their existing reader path. No database migration is +needed: raw JSON documents remain the source of truth. The additive `swap_blocks` dataset field holds runtime provenance and measured points; `swap_cases` labels their run summaries separately from EP/KV. One verified artifact counts @@ -145,6 +150,6 @@ References: [H200 specifications](https://www.nvidia.com/en-us/data-center/h200/ [GB200 bidirectional C2C](https://developer.nvidia.com/blog/nvidia-gb200-nvl72-delivers-trillion-parameter-llm-training-and-real-time-inference/), [HGX CPU uplink topology](https://docs.nvidia.com/enterprise-reference-architectures/whitepaper/hgx-servers-and-spectrum-x.pdf). -Multi-pool artifacts must carry `runtime.sku` matching a requested matrix cell. Older -single-pool artifacts may omit it. Measurement identity includes SKU so equivalent grid -points from different GPU pools remain independent. +Every swap_blocks result carries `runtime.sku`, which must match its requested case. +Measurement identity includes SKU so equivalent grid points from different GPU pools remain +independent. diff --git a/docs/deployment-configuration.md b/docs/deployment-configuration.md index 900fab4d5..d56034df4 100644 --- a/docs/deployment-configuration.md +++ b/docs/deployment-configuration.md @@ -15,10 +15,10 @@ scripts. | Variable | Required | Effect | | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `INFERENCEX_SOURCE_REPOS` | optional | Comma-separated `owner/repo` list searched after `SemiAnalysisAI/InferenceX` for unofficial-run overlays, live GPU metrics, live eval samples, CollectiveX sweeps, and ETL workflow-run lookups. Entries are trimmed and deduplicated; a malformed entry is an error. | +| `INFERENCEX_SOURCE_REPOS` | optional | Comma-separated `owner/repo` list searched after `SemiAnalysisAI/InferenceX` for unofficial-run overlays, live GPU metrics, live eval samples, and ETL workflow-run lookups. Entries are trimmed and deduplicated; a malformed entry is an error. | | `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Sent as `x-vercel-protection-bypass` by the cache invalidate and warmup scripts, the H3 media warmer, and the PowerX publication verifier. | | `INFERENCEX_APP_URL` | `warm-video-results.ts`, and `verify-power-publication.ts` without an origin argument | Origin for those two scripts. The verifier's origin argument wins when given. The cache scripts keep their localhost default. | -| `INGEST_REPO` | ingest, backfill, and artifact-preparation scripts when no argument or run URL names the repository | Repository for those scripts. `ingest-ci-run`, `ingest-collectivex`, `ingest-operatorx`, and the backfills take a `[repo]` argument or a full run URL first. CI-mode `ingest-ci-run` and `prepare-ci-artifacts` always require it. | +| `INGEST_REPO` | ingest, backfill, and artifact-preparation scripts when no argument or run URL names the repository | Repository for those scripts. `ingest-ci-run`, `ingest-operatorx`, and the backfills take a `[repo]` argument or a full run URL first. CI-mode `ingest-ci-run` and `prepare-ci-artifacts` always require it. | | `DATABASE_READONLY_URL` | yes, for the app | Read-only connection for the dashboard API routes. Read-only admin scripts and the MCP server use it too. | | `DATABASE_DRIVER` | optional | `neon` (serverless HTTP) or `postgres` (TCP) for the app's database clients. When unset, hosts ending in `.neon.tech` use `neon` and all others use `postgres`. Admin scripts always use `postgres`. | | `DATABASE_SSL` | optional | `false` disables TLS, and `true` or `require` requires it. When unset, TLS is required except for loopback hosts. Read by the app, the admin scripts, and the MCP server. | @@ -26,21 +26,15 @@ scripts. | `BLOB_CACHE_PREFIX`, `BLOB_READ_WRITE_TOKEN` | optional | Vercel Blob storage. The blob response cache runs only when both are set. A full invalidation deletes every blob under the prefix, so deployments that share a Blob store need distinct prefixes. H3 video storage needs only `BLOB_READ_WRITE_TOKEN`. Both features turn themselves off when their variables are unset. | | `NEXT_PUBLIC_POSTHOG_KEY`, `NEXT_PUBLIC_POSTHOG_HOST` | optional | PostHog project key and host for browser analytics and server-side events. Without the key, analytics is off. Without the host, `https://us.i.posthog.com` is used. Both are inlined at build time (see [Vercel project](#vercel-project)). | | `POSTHOG_PERSONAL_API_KEY`, `POSTHOG_PROJECT_ID` | optional, at build time, set together | Production builds upload source maps to PostHog only when both are set. | -| `GITHUB_TOKEN` | yes, for GitHub-backed features | Reads Actions runs and artifacts. Without it, `/api/unofficial-run`, `/api/gpu-metrics` live telemetry, and `/api/v1/eval-samples-live` return 500, CollectiveX discovery reports `unavailable`, and framework release and GitHub star requests go out unauthenticated. | +| `GITHUB_TOKEN` | yes, for GitHub-backed features | Reads Actions runs and artifacts. Without it, `/api/unofficial-run`, `/api/gpu-metrics` live telemetry, and `/api/v1/eval-samples-live` return 500, the CollectiveX ingest reads GitHub unauthenticated, and framework release and GitHub star requests go out unauthenticated. | | `OPERATORX_SOURCE` | optional | `database`, the behavior when unset, reads `DATABASE_OPERATORX_READONLY_URL`. `local` reads `OPERATORX_LOCAL_ARTIFACT_DIR` and works only in development. Any other value makes `/api/v1/operatorx/*` return 503. | `GITHUB_TOKEN` must be able to read Actions runs and artifacts in every listed source repository. Artifacts from every listed repository are served to any caller through -`/api/unofficial-run`, `/api/gpu-metrics`, `/api/v1/eval-samples-live`, and -`/api/v1/collectivex/*`. +`/api/unofficial-run`, `/api/gpu-metrics`, and `/api/v1/eval-samples-live`. -CollectiveX discovery skips a listed repository whose sweep-workflow listing returns 404. -GitHub also returns 404 for a misspelled repository or one the token cannot see, so -check the token's access when a repository's sweeps do not appear. A repository that -fails to list for another reason is logged, and the pass counts as failed: the runs list -stays incomplete and retries after a short cooldown. Runs from a repository other than -`SemiAnalysisAI/InferenceX` record it as `source_repository` in the stored summary, and -their links use it. +The CollectiveX ingest reads only `SemiAnalysisAI/InferenceX`'s committed +`collectivex/results/`, never `INFERENCEX_SOURCE_REPOS`. ### Feature variables @@ -48,15 +42,15 @@ The production block of `.env.example` lists the core connection, cache, and ana variables. The deployed app also reads the variables below. Each one backs a single feature, which fails as described when the variable is unset. -| Variable | Read by | When unset | -| ---------------------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `INVALIDATE_SECRET` | `POST /api/v1/invalidate` | Requests return 500 with `INVALIDATE_SECRET is not set`, except requests to the shared `staging` preview that carry `x-vercel-protection-bypass`. When it is set, a missing or wrong bearer token returns 401. Set it to the value of the `VERCEL_INVALIDATE_SECRET` GitHub secret. | -| `FEEDBACK_SECRET` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | -| `DATABASE_WRITE_URL` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | -| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read and lazily ingest through this primary connection, so they return errors. | -| `COLLECTIVEX_ADMIN_SECRET` | `DELETE /api/v1/collectivex/runs/[runId]` | Every request returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. When it is set, a missing or wrong bearer token returns 401. | -| `DATABASE_OPERATORX_READONLY_URL` | `/api/v1/operatorx/*` | The routes return 503 with `The OperatorX database is not configured`. | -| `DATABASE_PROXYTRACE_READONLY_URL` | `/api/v1/agentic-workload-explorer/*` | The routes return 500. | +| Variable | Read by | When unset | +| ---------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `INVALIDATE_SECRET` | `POST /api/v1/invalidate` | Requests return 500 with `INVALIDATE_SECRET is not set`, except requests to the shared `staging` preview that carry `x-vercel-protection-bypass`. When it is set, a missing or wrong bearer token returns 401. Set it to the value of the `VERCEL_INVALIDATE_SECRET` GitHub secret. | +| `FEEDBACK_SECRET` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | +| `DATABASE_WRITE_URL` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | +| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read, and the ingest route writes, through this primary connection, so they return errors. | +| `COLLECTIVEX_ADMIN_SECRET` | `POST /api/v1/collectivex/ingest` | Every request returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. When it is set, a missing or wrong bearer token returns 401. The InferenceX ingest workflow sends the same value. | +| `DATABASE_OPERATORX_READONLY_URL` | `/api/v1/operatorx/*` | The routes return 503 with `The OperatorX database is not configured`. | +| `DATABASE_PROXYTRACE_READONLY_URL` | `/api/v1/agentic-workload-explorer/*` | The routes return 500. | `DATABASE_COLLECTIVEX_READONLY_URL` is not read by the app routes. Every CollectiveX route reads through `DATABASE_COLLECTIVEX_WRITE_URL`. diff --git a/docs/index.md b/docs/index.md index bea8036af..5dd168434 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,7 +28,7 @@ Design rationale and non-obvious conventions. Repo-wide rules are in [AGENTS.md] - [Data Transforms](./data-transforms.md) covers power boundaries, prefix-cache tier shares, and hardware-key gotchas in the chart data pipeline. - [State Ownership](./state-ownership.md) explains where shared chart state lives and how URL state is restored across navigation. - [Blog](./blog.md) covers MDX content, frontmatter and scheduled publishing, captions, heading IDs, and math. -- [CollectiveX](./collectivex.md) covers the lazy ingest-on-read pipeline, its separate Neon database, tombstoned deletes, and the shared reader. +- [CollectiveX](./collectivex.md) covers the separate Neon database filled from committed results, run identity, the ingest route and CLI, the re-seed runbook, and the shared reader. - [Chinese Pages (/zh)](./i18n.md) explains why /zh pages are hand-authored and how hreflang and blog translation pairing work. - [Chinese Copy Editorial Guide](./chinese-copy.md) covers audience, register, terminology, the review workflow, and the PR checklist for Chinese copy. diff --git a/packages/app/cypress/e2e/collectivex.cy.ts b/packages/app/cypress/e2e/collectivex.cy.ts index eecbc8ed5..1e1a29e8f 100644 --- a/packages/app/cypress/e2e/collectivex.cy.ts +++ b/packages/app/cypress/e2e/collectivex.cy.ts @@ -30,7 +30,6 @@ const incompleteDataset = buildDataset({ meta: { run_id: '161', generated_at: '2026-07-09T12:20:00Z', - conclusion: 'failure', }, }); const kvDataset = buildDataset({ @@ -72,7 +71,6 @@ const kvOnlyDataset = buildDataset({ kv: [{}], meta: { run_id: '163', generated_at: '2026-08-08T12:20:00Z', source_sha: 'f'.repeat(40) }, }); -const ADMIN_TOKEN_KEY = 'collectivex-admin-token'; function installRuns(bodies: CollectiveXDataset[] = [dataset]) { cy.intercept('GET', '/api/v1/collectivex/runs?*', { @@ -103,9 +101,8 @@ describe('CollectiveX neutral run view', () => { }); it('shows the run header, coverage stats, and revision-pinned source links', () => { - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', `#${runId}`) - .and('contain.text', 'success'); + cy.get('[data-testid="collectivex-run-conclusion"]').should('not.exist'); + cy.get('[data-testid^="collectivex-delete"]').should('not.exist'); cy.get('[data-testid="collectivex-display"]') .should('contain.text', `${dataset.run.measured_cases}/${dataset.run.requested_cases}`) .and('contain.text', String(dataset.series.length)); @@ -114,19 +111,24 @@ describe('CollectiveX neutral run view', () => { cy.get('[data-testid="collectivex-source-link"]').should( 'have.attr', 'href', - `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/experimental/CollectiveX`, + `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/collectivex`, ); cy.get('[data-testid="collectivex-methodology-link"]') .should('contain.text', 'Methodology') .and( 'have.attr', 'href', - `https://github.com/SemiAnalysisAI/InferenceX/blob/${SOURCE_SHA}/experimental/CollectiveX/docs/methodology.md`, + `https://github.com/SemiAnalysisAI/InferenceX/blob/${SOURCE_SHA}/collectivex/docs/methodology.md`, + ); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`) + .should('contain.text', `#${runId}`) + .and( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/collectivex/results`, ); cy.get(`[data-testid="collectivex-run-row-${runId}"] a[href*="/actions/runs/"]`).should( - 'have.attr', - 'href', - `https://github.com/SemiAnalysisAI/InferenceX/actions/runs/${runId}`, + 'not.exist', ); installRuns(); @@ -137,32 +139,6 @@ describe('CollectiveX neutral run view', () => { cy.get(`[data-testid="collectivex-run-suite-kv-${runId}"]`).should('not.exist'); }); - it('keeps loading bounded discovery batches until every run is listed', () => { - let requests = 0; - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - requests += 1; - request.reply({ - body: { - version: 1, - runs: (requests === 1 ? [] : [dataset, comparisonDataset]).map(buildRunSummary), - discovery_complete: requests > 1, - }, - }); - }).as('progressiveRuns'); - - cy.reload(); - cy.wait('@progressiveRuns'); - cy.wait('@progressiveRuns'); - cy.wait('@run'); - - cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( - 'be.visible', - ); - cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); - cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); - cy.then(() => expect(requests).to.be.gte(2)); - }); - it('selects the EP16 series through the identity controls', () => { cy.get('[data-testid="collectivex-ep-select"]').should('be.visible'); cy.get('[data-testid="collectivex-phase-toggle"]').should('be.visible'); @@ -369,9 +345,10 @@ describe('CollectiveX neutral run view', () => { }); cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).uncheck(); - cy.get('[data-testid="collectivex-run-conclusion"]').should( - 'contain.text', - `#${comparisonDataset.run.run_id}`, + cy.get('[data-testid="collectivex-source-link"]').should( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/tree/${'d'.repeat(40)}/collectivex`, ); cy.get(`[data-testid="collectivex-run-line-style-${runId}"]`).should('not.exist'); cy.get( @@ -442,7 +419,10 @@ describe('CollectiveX neutral run view', () => { 'not.be.checked', ); cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); - cy.get('[data-testid="collectivex-run-conclusion"]').should('contain.text', `#${runId}`); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`).should('be.visible'); + cy.get(`[data-testid="collectivex-run-source-${incompleteDataset.run.run_id}"]`).should( + 'not.exist', + ); cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); }); @@ -546,9 +526,11 @@ describe('CollectiveX neutral run view', () => { cy.wait('@runs'); cy.wait('@run'); - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', `#${runId}`) - .and('contain.text', '成功'); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`).should( + 'have.attr', + 'aria-label', + `打开运行 #${runId} 对应的已提交结果`, + ); cy.get('[data-testid="collectivex-runs"]') .should('contain.text', '运行记录') .and('contain.text', '终态数据点'); @@ -566,21 +548,6 @@ describe('CollectiveX neutral run view', () => { .and('contain.text', '常规') .and('contain.text', '解码') .and('contain.text', '延迟 p50 / p90 / p95 / p99'); - - const cancelled = buildDataset({ - shards: [makeRawShard()], - meta: { run_id: '179', generated_at: '2026-08-29T12:20:00Z', conclusion: 'cancelled' }, - }); - installRuns([cancelled]); - installRun(cancelled, 'cancelledRun'); - cy.visit('/zh/collectivex'); - cy.wait('@runs'); - cy.wait('@cancelledRun'); - - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', '已取消') - .and('not.contain.text', '待处理') - .and('not.contain.text', 'cancelled'); }); it('keeps the Chinese explorer and runs table reachable at 375px', () => { @@ -597,121 +564,6 @@ describe('CollectiveX neutral run view', () => { }); }); -describe('CollectiveX run deletion', () => { - beforeEach(() => { - installRuns(); - installRun(); - openCollectiveX(); - }); - - it('deletes a table row after confirm + token prompt and remembers the token', () => { - let deleted = false; - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - request.reply({ - body: { - version: 1, - runs: deleted ? [] : [buildRunSummary(dataset)], - }, - }); - }).as('runsAfterDelete'); - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, (request) => { - expect(request.headers.authorization).to.eq('Bearer test-token'); - deleted = true; - request.reply({ deleted: true, runId }); - }).as('deleteRun'); - cy.window().then((win) => { - win.localStorage.removeItem(ADMIN_TOKEN_KEY); - cy.stub(win, 'confirm').returns(true); - cy.stub(win, 'prompt').returns('test-token'); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.wait('@deleteRun'); - cy.wait('@runsAfterDelete'); - cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('not.exist'); - cy.window().then((win) => { - expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq('test-token'); - }); - }); - - it('deletes every shown run with one confirmation and one token prompt', () => { - const deletedRunIds = new Set(); - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - request.reply({ - body: { - version: 1, - runs: [dataset, comparisonDataset] - .filter((item) => !deletedRunIds.has(item.run.run_id)) - .map(buildRunSummary), - discovery_complete: true, - }, - }); - }).as('runsAfterBulkDelete'); - installRun(comparisonDataset, 'comparisonRunForDelete'); - cy.intercept('DELETE', '/api/v1/collectivex/runs/*', (request) => { - expect(request.headers.authorization).to.eq('Bearer bulk-test-token'); - deletedRunIds.add(request.url.split('/').at(-1) ?? ''); - request.reply({ deleted: true }); - }).as('deleteShownRun'); - - cy.reload(); - cy.wait('@runsAfterBulkDelete'); - cy.wait('@run'); - cy.get(`[data-testid="collectivex-run-visible-${comparisonDataset.run.run_id}"]`).check(); - cy.wait('@comparisonRunForDelete'); - cy.window().then((win) => { - win.localStorage.removeItem(ADMIN_TOKEN_KEY); - cy.stub(win, 'confirm').as('bulkDeleteConfirm').returns(true); - cy.stub(win, 'prompt').as('bulkDeletePrompt').returns('bulk-test-token'); - }); - - cy.get('[data-testid="collectivex-delete-shown-runs"]').click(); - cy.wait('@deleteShownRun'); - cy.wait('@deleteShownRun'); - cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('not.exist'); - cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( - 'not.exist', - ); - cy.get('@bulkDeleteConfirm').should('have.been.calledOnce'); - cy.get('@bulkDeletePrompt').should('have.been.calledOnce'); - cy.then(() => { - expect(deletedRunIds).to.deep.eq(new Set([runId, comparisonDataset.run.run_id])); - }); - }); - - it('clears a stale stored token and reports unauthorized on 401', () => { - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, { statusCode: 401 }).as( - 'delete401', - ); - cy.window().then((win) => { - win.localStorage.setItem(ADMIN_TOKEN_KEY, 'stale-token'); - cy.stub(win, 'confirm').returns(true); - cy.stub(win, 'alert').as('unauthorizedAlert'); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.wait('@delete401'); - cy.get('@unauthorizedAlert').should('have.been.calledWith', 'Invalid admin token.'); - cy.window().then((win) => { - expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq(null); - }); - }); - - it('does nothing when the confirmation is declined', () => { - let deleteRequests = 0; - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, () => { - deleteRequests += 1; - }); - cy.window().then((win) => { - cy.stub(win, 'confirm').returns(false); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.get('[data-testid="collectivex-display"]').should('be.visible'); - cy.then(() => expect(deleteRequests).to.eq(0)); - }); -}); - describe('CollectiveX availability states', () => { it('reports a missing run list', () => { cy.intercept('GET', '/api/v1/collectivex/runs?*', { @@ -815,17 +667,17 @@ describe('CollectiveX source repository links', () => { cy.get('[data-testid="collectivex-source-link"]').should( 'have.attr', 'href', - `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/experimental/CollectiveX`, + `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/collectivex`, ); cy.get('[data-testid="collectivex-methodology-link"]').should( 'have.attr', 'href', - `https://github.com/ExampleOrg/ExampleBench/blob/${SOURCE_SHA}/experimental/CollectiveX/docs/methodology.md`, + `https://github.com/ExampleOrg/ExampleBench/blob/${SOURCE_SHA}/collectivex/docs/methodology.md`, ); - cy.get('[data-testid="collectivex-run-row-166"] a[href*="/actions/runs/"]').should( + cy.get('[data-testid="collectivex-run-source-166"]').should( 'have.attr', 'href', - 'https://github.com/ExampleOrg/ExampleBench/actions/runs/166', + `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/collectivex/results`, ); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts b/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts new file mode 100644 index 000000000..d2082aee5 --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts @@ -0,0 +1,163 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type * as CommittedResults from '@semianalysisai/inferencex-db/collectivex/committed-results'; + +const { mockResolve, mockLoad, mockUpsert, mockGetWriteDb, mockPurge } = vi.hoisted(() => ({ + mockResolve: vi.fn(), + mockLoad: vi.fn(), + mockUpsert: vi.fn(), + mockGetWriteDb: vi.fn(() => 'mock-write-sql'), + mockPurge: vi.fn(), +})); + +vi.mock('@semianalysisai/inferencex-db/collectivex/committed-results', async (importOriginal) => ({ + ...(await importOriginal()), + resolveResultsCommit: mockResolve, + loadCommittedRun: mockLoad, +})); + +vi.mock('@semianalysisai/inferencex-db/connection', () => ({ + getCollectiveXWriteDb: mockGetWriteDb, +})); + +vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ + upsertCollectiveXRun: mockUpsert, +})); + +vi.mock('@/lib/cache/api-cache', () => ({ + purgeCollectiveX: mockPurge, +})); + +import { CommittedResultsError } from '@semianalysisai/inferencex-db/collectivex/committed-results'; + +import { POST } from './route'; + +const SECRET = 'test-admin-secret'; +const SHA = 'c'.repeat(40); +const commit = { sha: SHA, committedAt: '2026-10-15T18:35:51Z' }; +const run = { + run_id: '1792089351', + run_attempt: 1, + version: 1, + generated_at: '2026-10-15T18:35:51Z', + source_sha: SHA, + source_branch: 'main', + conclusion: 'success', + matrix: {}, + summary: { requested_cases: 195, measured_cases: 2 }, +}; + +function post(body: unknown, authorization: string | null = `Bearer ${SECRET}`) { + return POST( + new Request('http://localhost/api/v1/collectivex/ingest', { + method: 'POST', + headers: authorization === null ? {} : { Authorization: authorization }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }), + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', SECRET); + vi.stubEnv('GITHUB_TOKEN', 'gh-token'); + mockResolve.mockResolvedValue(commit); + mockLoad.mockResolvedValue({ run, docs: [{}, {}] }); + mockUpsert.mockResolvedValue('inserted'); +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('POST /api/v1/collectivex/ingest', () => { + it('returns 500 when COLLECTIVEX_ADMIN_SECRET is not set', async () => { + vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', ''); + const res = await post({ sha: SHA }); + expect(res.status).toBe(500); + expect(await res.text()).toBe('COLLECTIVEX_ADMIN_SECRET is not set'); + expect(mockResolve).not.toHaveBeenCalled(); + }); + + it.each([ + ['no Authorization header', null], + ['a wrong secret', 'Bearer wrong-secret'], + ['the secret without the Bearer scheme', SECRET], + ])('returns 401 with %s and does nothing', async (_name, authorization) => { + const res = await post({ sha: SHA }, authorization); + expect(res.status).toBe(401); + expect(mockResolve).not.toHaveBeenCalled(); + expect(mockUpsert).not.toHaveBeenCalled(); + expect(mockPurge).not.toHaveBeenCalled(); + }); + + it.each([ + ['an empty object', {}], + ['both sha and ref', { sha: SHA, ref: 'main' }], + ['a malformed sha', { sha: 'not-hex' }], + ['a ref with ..', { ref: 'main/../x' }], + ['an extra field', { ref: 'main', force: true }], + ['an array', [SHA]], + ['invalid JSON', '{'], + ])('returns 400 for %s', async (_name, body) => { + const res = await post(body); + expect(res.status).toBe(400); + expect(mockResolve).not.toHaveBeenCalled(); + }); + + it('stores the resolved results commit and purges the CollectiveX cache', async () => { + const res = await post({ sha: SHA }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + ingested: 'inserted', + run_id: run.run_id, + source_sha: SHA, + generated_at: run.generated_at, + version: 1, + requested_cases: 195, + measured_cases: 2, + documents: 2, + }); + expect(mockResolve).toHaveBeenCalledWith(SHA, { token: 'gh-token' }); + expect(mockLoad).toHaveBeenCalledWith(commit, { token: 'gh-token' }); + expect(mockUpsert).toHaveBeenCalledWith('mock-write-sql', run, [{}, {}]); + expect(mockPurge).toHaveBeenCalledOnce(); + }); + + it('accepts a ref and reports a replaced run', async () => { + mockUpsert.mockResolvedValue('replaced'); + const res = await post({ ref: 'main' }); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.ingested).toBe('replaced'); + expect(mockResolve).toHaveBeenCalledWith('main', { token: 'gh-token' }); + }); + + it('returns 409 and keeps the cache when the run id belongs to another commit', async () => { + mockUpsert.mockResolvedValue('conflict'); + const res = await post({ sha: SHA }); + expect(res.status).toBe(409); + expect(mockPurge).not.toHaveBeenCalled(); + }); + + it.each([ + ['invalid', 422], + ['not-found', 404], + ['unavailable', 502], + ] as const)('maps a %s results error to %i without writing', async (kind, status) => { + mockLoad.mockRejectedValue(new CommittedResultsError(kind, `${kind} detail`)); + const res = await post({ sha: SHA }); + expect(res.status).toBe(status); + expect(await res.json()).toEqual({ error: `${kind} detail` }); + expect(mockUpsert).not.toHaveBeenCalled(); + expect(mockPurge).not.toHaveBeenCalled(); + }); + + it('returns 500 without leaking details on DB failure', async () => { + mockUpsert.mockRejectedValue(new Error('connection string leaked')); + const res = await post({ sha: SHA }); + expect(res.status).toBe(500); + expect(await res.json()).toEqual({ error: 'Internal server error' }); + }); +}); diff --git a/packages/app/src/app/api/v1/collectivex/ingest/route.ts b/packages/app/src/app/api/v1/collectivex/ingest/route.ts new file mode 100644 index 000000000..10355681e --- /dev/null +++ b/packages/app/src/app/api/v1/collectivex/ingest/route.ts @@ -0,0 +1,91 @@ +import { NextResponse } from 'next/server'; + +import { + CommittedResultsError, + loadCommittedRun, + resolveResultsCommit, +} from '@semianalysisai/inferencex-db/collectivex/committed-results'; +import { getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; +import { upsertCollectiveXRun } from '@semianalysisai/inferencex-db/queries/collectivex'; + +import { bearerMatches } from '@/lib/api/bearer-auth'; +import { purgeCollectiveX } from '@/lib/cache/api-cache'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; +// One commit is a few GitHub API calls plus one raw read per result file. +export const maxDuration = 300; + +const SHA = /^[0-9a-f]{7,40}$/u; +const REF = /^(?!.*\.\.)[\w./-]{1,255}$/u; + +const ERROR_STATUS = { invalid: 422, 'not-found': 404, unavailable: 502 } as const; + +/** `{sha}` or `{ref}`, exactly one; null when the body is anything else. */ +function targetOf(body: unknown): string | null { + if (body === null || typeof body !== 'object' || Array.isArray(body)) return null; + const { sha, ref, ...rest } = body as { sha?: unknown; ref?: unknown }; + if (Object.keys(rest).length > 0 || (sha === undefined) === (ref === undefined)) return null; + if (sha !== undefined) return typeof sha === 'string' && SHA.test(sha) ? sha : null; + return typeof ref === 'string' && REF.test(ref) ? ref : null; +} + +/** + * Store one InferenceX commit's committed CollectiveX results in the CollectiveX + * database and purge the CollectiveX cache scope. The InferenceX workflow that + * runs on pushes to main calls this. The ref resolves to the newest commit at or + * before it that changed collectivex/results/, which must be on main; storing the + * same commit again replaces its run. + */ +export async function POST(request: Request) { + const secret = process.env.COLLECTIVEX_ADMIN_SECRET; + if (!secret) { + return new NextResponse('COLLECTIVEX_ADMIN_SECRET is not set', { status: 500 }); + } + if (!bearerMatches(request.headers.get('Authorization') ?? '', secret)) { + return new NextResponse('Unauthorized', { status: 401 }); + } + + let target: string | null; + try { + target = targetOf(await request.json()); + } catch { + target = null; + } + if (target === null) { + return NextResponse.json( + { error: 'Body must be {"sha": ""} or {"ref": ""}' }, + { status: 400 }, + ); + } + + const token = process.env.GITHUB_TOKEN || undefined; + try { + const commit = await resolveResultsCommit(target, { token }); + const { run, docs } = await loadCommittedRun(commit, { token }); + const result = await upsertCollectiveXRun(getCollectiveXWriteDb(), run, docs); + if (result === 'conflict') { + return NextResponse.json( + { error: `Run ${run.run_id} already holds a different commit than ${commit.sha}` }, + { status: 409 }, + ); + } + purgeCollectiveX(); + return NextResponse.json({ + ingested: result, + run_id: run.run_id, + source_sha: run.source_sha, + generated_at: run.generated_at, + version: run.version, + requested_cases: run.summary.requested_cases, + measured_cases: run.summary.measured_cases, + documents: docs.length, + }); + } catch (error) { + if (error instanceof CommittedResultsError) { + return NextResponse.json({ error: error.message }, { status: ERROR_STATUS[error.kind] }); + } + console.error('Error ingesting CollectiveX results:', error); + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.test.ts b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts index fdd431c77..7381b79bf 100644 --- a/packages/app/src/app/api/v1/collectivex/latest/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts @@ -2,11 +2,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockGetLatest, mockFromRow, mockGetWriteDb, mockEnsureLatest } = vi.hoisted(() => ({ +const { mockGetLatest, mockFromRow, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ mockGetLatest: vi.fn(), mockFromRow: vi.fn(), mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockEnsureLatest: vi.fn(), + mockCachedJson: vi.fn((data: unknown) => Response.json(data)), })); vi.mock('@semianalysisai/inferencex-db/connection', () => ({ @@ -19,21 +19,8 @@ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ collectiveXDatasetFromRow: mockFromRow, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureLatestCollectiveXRun: mockEnsureLatest, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', - cachedJson: (data: unknown) => Response.json(data), + cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', })); @@ -45,16 +32,11 @@ function req(url: string): NextRequest { return new NextRequest(new URL(url, 'http://localhost')); } -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - const dataset = makeCollectiveXDataset(); beforeEach(() => { vi.clearAllMocks(); vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureLatest.mockResolvedValue(undefined); mockGetLatest.mockResolvedValue({ run_id: dataset.run.run_id }); mockFromRow.mockReturnValue(dataset); }); @@ -65,32 +47,18 @@ describe('GET /api/v1/collectivex/latest', () => { const res = await GET(req(url)); expect(res.status).toBe(400); } - expect(mockEnsureLatest).not.toHaveBeenCalled(); + expect(mockGetLatest).not.toHaveBeenCalled(); }); - it('lazily ingests then serves the latest run assembled as a dataset', async () => { + it('serves the newest stored run assembled as a dataset under the CollectiveX tag', async () => { const res = await GET(req('/api/v1/collectivex/latest?version=1')); expect(res.status).toBe(200); expect(await res.json()).toEqual(dataset); - expect(mockEnsureLatest).toHaveBeenCalledWith(1); expect(mockGetLatest).toHaveBeenCalledWith('mock-write-sql', 1); + expect(mockCachedJson).toHaveBeenCalledWith(dataset, { tag: 'collectivex' }); }); - it('serves the stored run when GitHub discovery fails', async () => { - mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); - const res = await GET(req('/api/v1/collectivex/latest?version=1')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual(dataset); - }); - - it('returns 503 when discovery fails and nothing is stored', async () => { - mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); - mockGetLatest.mockResolvedValue(null); - const res = await GET(req('/api/v1/collectivex/latest?version=1')); - expect(res.status).toBe(503); - }); - - it('returns 404 when neither GitHub nor the DB has a run', async () => { + it('returns 404 when nothing is stored for the version', async () => { mockGetLatest.mockResolvedValue(null); const res = await GET(req('/api/v1/collectivex/latest?version=1')); expect(res.status).toBe(404); diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.ts b/packages/app/src/app/api/v1/collectivex/latest/route.ts index 08c220976..3bdaecd0f 100644 --- a/packages/app/src/app/api/v1/collectivex/latest/route.ts +++ b/packages/app/src/app/api/v1/collectivex/latest/route.ts @@ -7,11 +7,7 @@ import { getLatestCollectiveXRun, } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; -import { - collectiveXSweepErrorStatus, - ensureLatestCollectiveXRun, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; @@ -24,32 +20,11 @@ export async function GET(request: NextRequest) { } if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-latest')); - // Discovery failures must not take the page down — serve whatever the DB - // already holds and only surface the error when there is no fallback. - let ensureError: unknown = null; - try { - await ensureLatestCollectiveXRun(version); - } catch (error) { - ensureError = error; - } - try { - // Discovery may have persisted a run, so read from the same primary. + // The primary, so a response cached right after an ingest's purge is never a stale replica read. const row = await getLatestCollectiveXRun(getCollectiveXWriteDb(), version); - if (row !== null) { - if (ensureError) - console.error('CollectiveX discovery failed; serving stored run:', ensureError); - return cachedJson(collectiveXDatasetFromRow(row), { - tag: collectiveXCacheTag(), - cacheControl: COLLECTIVEX_CACHE_CONTROL, - }); - } - if (ensureError) { - console.error('CollectiveX discovery failed with no stored fallback:', ensureError); - const status = collectiveXSweepErrorStatus(ensureError) ?? 502; - return NextResponse.json({ error: 'Unavailable' }, { status }); - } - return NextResponse.json({ error: 'Not found' }, { status: 404 }); + if (row === null) return NextResponse.json({ error: 'Not found' }, { status: 404 }); + return cachedJson(collectiveXDatasetFromRow(row), { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error fetching CollectiveX latest run:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts index 1df319a8b..7ae876995 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts @@ -17,20 +17,12 @@ vi.mock('@semianalysisai/inferencex-db/connection', () => ({ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ collectiveXDatasetFromRow: vi.fn(), - deleteCollectiveXRun: vi.fn(), getCollectiveXRun: vi.fn(), })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - collectiveXSweepErrorStatus: vi.fn(), - ensureCollectiveXRun: vi.fn(), -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', cachedJson: (data: unknown) => Response.json(data), collectiveXCacheTag: () => 'collectivex', - purgeCollectiveX: vi.fn(), })); vi.mock('@/lib/testing/test-fixtures', () => ({ diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts index 5ddf4be86..e58b1a452 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts @@ -1,16 +1,13 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockGetRun, mockDelete, mockFromRow, mockGetWriteDb, mockPurge, mockEnsureRun } = - vi.hoisted(() => ({ - mockGetRun: vi.fn(), - mockDelete: vi.fn(), - mockFromRow: vi.fn(), - mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockPurge: vi.fn(() => Promise.resolve(0)), - mockEnsureRun: vi.fn(), - })); +const { mockGetRun, mockFromRow, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ + mockGetRun: vi.fn(), + mockFromRow: vi.fn(), + mockGetWriteDb: vi.fn(() => 'mock-write-sql'), + mockCachedJson: vi.fn((data: unknown) => Response.json(data)), +})); vi.mock('@semianalysisai/inferencex-db/connection', () => ({ getCollectiveXWriteDb: mockGetWriteDb, @@ -19,151 +16,70 @@ vi.mock('@semianalysisai/inferencex-db/connection', () => ({ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ getCollectiveXRun: mockGetRun, - deleteCollectiveXRun: mockDelete, collectiveXDatasetFromRow: mockFromRow, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureCollectiveXRun: mockEnsureRun, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', - cachedJson: (data: unknown) => Response.json(data), + cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', - purgeCollectiveX: mockPurge, })); import { NextRequest } from 'next/server'; -import { DELETE, GET } from './route'; +import * as route from './route'; -const SECRET = 'test-admin-secret'; const dataset = makeCollectiveXDataset(); const runId = dataset.run.run_id; -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - function get(url: string, id: string) { - return GET(new NextRequest(new URL(url, 'http://localhost')), { + return route.GET(new NextRequest(new URL(url, 'http://localhost')), { params: Promise.resolve({ runId: id }), }); } -function del(id: string, token?: string) { - return DELETE( - new NextRequest(new URL(`/api/v1/collectivex/runs/${id}`, 'http://localhost'), { - method: 'DELETE', - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }), - { params: Promise.resolve({ runId: id }) }, - ); -} - beforeEach(() => { vi.clearAllMocks(); - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', SECRET); - mockEnsureRun.mockResolvedValue(undefined); + vi.spyOn(console, 'error').mockImplementation(() => {}); mockGetRun.mockResolvedValue({ run_id: runId }); mockFromRow.mockReturnValue(dataset); - mockDelete.mockResolvedValue(true); -}); - -afterEach(() => { - vi.unstubAllEnvs(); }); describe('GET /api/v1/collectivex/runs/[runId]', () => { - it('returns 400 for malformed version or run id', async () => { - const badRunId = await get(`/x?version=1`, 'abc'); - const badVersion = await get(`/x?version=99`, runId); - expect(badRunId.status).toBe(400); - expect(badVersion.status).toBe(400); - expect(mockEnsureRun).not.toHaveBeenCalled(); + it('returns 400 for an unknown version or a malformed run id', async () => { + for (const [url, id] of [ + ['/x', runId], + ['/x?version=99', runId], + ['/x?version=1', '0'], + ['/x?version=1', 'abc'], + ]) { + const res = await get(url, id); + expect(res.status).toBe(400); + } + expect(mockGetRun).not.toHaveBeenCalled(); }); - it('lazily ingests then serves the run assembled as a dataset', async () => { - const res = await get(`/x?version=1`, runId); + it('serves a stored run assembled as a dataset under the CollectiveX tag', async () => { + const res = await get('/x?version=1', runId); expect(res.status).toBe(200); expect(await res.json()).toEqual(dataset); - expect(mockEnsureRun).toHaveBeenCalledWith(1, runId); expect(mockGetRun).toHaveBeenCalledWith('mock-write-sql', 1, runId); + expect(mockCachedJson).toHaveBeenCalledWith(dataset, { tag: 'collectivex' }); }); - it.each([ - ['not-found', 404], - ['unavailable', 503], - ['invalid', 502], - ] as const)('maps %s ingest failures without exposing details', async (code, status) => { - vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureRun.mockRejectedValue(sweepError(code)); + it('returns 404 for a run that is not stored', async () => { mockGetRun.mockResolvedValue(null); - const res = await get(`/x?version=1`, runId); - expect(res.status).toBe(status); - }); - - it('serves a stored run when its GitHub refresh fails', async () => { - vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureRun.mockRejectedValue(sweepError('unavailable')); - - const res = await get(`/x?version=1`, runId); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual(dataset); - }); - - it('returns 404 when the run is absent after a clean ensure', async () => { - mockGetRun.mockResolvedValue(null); - const res = await get(`/x?version=1`, runId); + const res = await get('/x?version=1', '1792089351'); expect(res.status).toBe(404); }); -}); - -describe('DELETE /api/v1/collectivex/runs/[runId]', () => { - it('rejects missing or wrong bearer tokens', async () => { - const missing = await del(runId); - const wrong = await del(runId, 'wrong-token'); - expect(missing.status).toBe(401); - expect(wrong.status).toBe(401); - expect(mockDelete).not.toHaveBeenCalled(); - }); - it('returns 500 when the secret is not configured', async () => { - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', ''); - const res = await del(runId, 'anything'); + it('returns 500 without leaking details on DB failure', async () => { + mockGetRun.mockRejectedValue(new Error('boom')); + const res = await get('/x?version=1', runId); expect(res.status).toBe(500); - expect(await res.text()).toBe('COLLECTIVEX_ADMIN_SECRET is not set'); - expect(mockDelete).not.toHaveBeenCalled(); - }); - - it('tombstones the run and purges only the CollectiveX cache scope', async () => { - const res = await del(runId, SECRET); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ deleted: true, runId }); - expect(mockDelete).toHaveBeenCalledWith('mock-write-sql', runId); - expect(mockPurge).toHaveBeenCalledTimes(1); - }); - - it('returns 404 without purging when the run does not exist', async () => { - mockDelete.mockResolvedValue(false); - const res = await del(runId, SECRET); - expect(res.status).toBe(404); - expect(mockPurge).not.toHaveBeenCalled(); + expect(await res.json()).toEqual({ error: 'Internal server error' }); }); - it('returns 400 for malformed run ids', async () => { - const res = await del('not-a-run-id', SECRET); - expect(res.status).toBe(400); - expect(mockDelete).not.toHaveBeenCalled(); + it('exposes no mutation handler', () => { + expect(Object.keys(route)).not.toContain('DELETE'); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts index b6277ea20..489774e94 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts @@ -8,21 +8,10 @@ import { import { FIXTURES_MODE, getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; import { collectiveXDatasetFromRow, - deleteCollectiveXRun, getCollectiveXRun, } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { - COLLECTIVEX_CACHE_CONTROL, - collectiveXCacheTag, - cachedJson, - purgeCollectiveX, -} from '@/lib/cache/api-cache'; -import { bearerMatches } from '@/lib/api/bearer-auth'; -import { - collectiveXSweepErrorStatus, - ensureCollectiveXRun, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { collectiveXCacheTag, cachedJson } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; @@ -48,79 +37,12 @@ export async function GET(request: NextRequest, context: { params: Promise<{ run return cachedJson(loadFixture(`collectivex-run-${runId}`)); } - let ensureError: unknown = null; - try { - await ensureCollectiveXRun(version, runId); - } catch (error) { - ensureError = error; - } - try { - // Ensure may have inserted or refreshed this run, so read from the same primary. const row = await getCollectiveXRun(getCollectiveXWriteDb(), version, runId); - if (row === null) { - if (ensureError) { - const status = collectiveXSweepErrorStatus(ensureError); - if (status !== null) { - return NextResponse.json( - { error: status === 404 ? 'Not found' : 'Unavailable' }, - { status }, - ); - } - throw ensureError; - } - return NextResponse.json({ error: 'Not found' }, { status: 404 }); - } - if (ensureError) { - console.error('CollectiveX run refresh failed; serving stored run:', ensureError); - } - // Short window like the sibling routes: a GitHub re-run of failed shards - // refreshes this run's stored contents, and deletion must not linger. - return cachedJson(collectiveXDatasetFromRow(row), { - tag: collectiveXCacheTag(), - cacheControl: COLLECTIVEX_CACHE_CONTROL, - }); + if (row === null) return NextResponse.json({ error: 'Not found' }, { status: 404 }); + return cachedJson(collectiveXDatasetFromRow(row), { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error fetching CollectiveX run:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); } } - -/** - * Admin deletion of an ingested run. Authenticated with a dedicated Bearer - * secret — the token is remembered in browser localStorage, so it must not - * be the CI-held INVALIDATE_SECRET (scoped blast radius, independently - * rotatable). Deletion tombstones the run (lazy discovery must never - * re-ingest it) and purges the CollectiveX cache scope so the run table and - * latest views drop it immediately. - */ -export async function DELETE( - request: NextRequest, - context: { params: Promise<{ runId: string }> }, -) { - const secret = process.env.COLLECTIVEX_ADMIN_SECRET; - const authHeader = request.headers.get('Authorization') ?? ''; - if (!secret) { - return new NextResponse('COLLECTIVEX_ADMIN_SECRET is not set', { status: 500 }); - } - if (!bearerMatches(authHeader, secret)) { - return new NextResponse('Unauthorized', { status: 401 }); - } - - const { runId } = await context.params; - if (!RUN_ID.test(runId)) { - return NextResponse.json({ error: 'Unknown run id' }, { status: 400 }); - } - - try { - const deleted = await deleteCollectiveXRun(getCollectiveXWriteDb(), runId); - if (!deleted) { - return NextResponse.json({ error: 'Not found' }, { status: 404 }); - } - purgeCollectiveX(); - return NextResponse.json({ deleted: true, runId }); - } catch (error) { - console.error('Error deleting CollectiveX run:', error); - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); - } -} diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts index 47c482d8a..6ee7a8e56 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts @@ -3,10 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockList, mockGetWriteDb, mockEnsureList, mockCachedJson } = vi.hoisted(() => ({ +const { mockList, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ mockList: vi.fn(), mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockEnsureList: vi.fn(), mockCachedJson: vi.fn((data: unknown) => Response.json(data)), })); @@ -19,20 +18,7 @@ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ listCollectiveXRuns: mockList, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureCollectiveXRunsList: mockEnsureList, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', })); @@ -45,80 +31,33 @@ function req(url: string): NextRequest { return new NextRequest(new URL(url, 'http://localhost')); } -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - const summary = buildRunSummary(makeCollectiveXDataset()); beforeEach(() => { vi.clearAllMocks(); vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureList.mockResolvedValue(true); mockList.mockResolvedValue([summary]); }); describe('GET /api/v1/collectivex/runs', () => { it('returns 400 for a missing or unknown version', async () => { - for (const url of ['/api/v1/collectivex/runs', '/api/v1/collectivex/runs?version=abc']) { + for (const url of ['/api/v1/collectivex/runs', '/api/v1/collectivex/runs?version=99']) { const res = await GET(req(url)); expect(res.status).toBe(400); } - expect(mockEnsureList).not.toHaveBeenCalled(); + expect(mockList).not.toHaveBeenCalled(); }); - it('backfills recent runs then lists stored summaries newest first', async () => { + it('lists stored runs with a complete-discovery flag under the CollectiveX tag', async () => { const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: true, - }); - expect(mockEnsureList).toHaveBeenCalledWith(1); + const body = { version: 1, runs: [summary], discovery_complete: true }; + expect(await res.json()).toEqual(body); expect(mockList).toHaveBeenCalledWith('mock-write-sql', 1); + expect(mockCachedJson).toHaveBeenCalledWith(body, { tag: 'collectivex' }); }); - it('marks the list incomplete when another bounded discovery pass is needed', async () => { - mockEnsureList.mockResolvedValue(false); - - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: false, - }); - expect(mockCachedJson).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ cacheControl: 'private, no-store' }), - ); - }); - - it('serves the stored list as incomplete when the GitHub backfill fails', async () => { - mockEnsureList.mockRejectedValue(sweepError('unavailable')); - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: false, - }); - expect(mockCachedJson).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ cacheControl: 'private, no-store' }), - ); - }); - - it('returns 503 when the backfill fails and nothing is stored', async () => { - mockEnsureList.mockRejectedValue(sweepError('unavailable')); - mockList.mockResolvedValue([]); - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - expect(res.status).toBe(503); - }); - - it('returns an empty list when GitHub has no runs yet', async () => { + it('returns an empty list, not an error, when nothing is stored', async () => { mockList.mockResolvedValue([]); const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(200); @@ -129,5 +68,6 @@ describe('GET /api/v1/collectivex/runs', () => { mockList.mockRejectedValue(new Error('boom')); const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(500); + expect(await res.json()).toEqual({ error: 'Internal server error' }); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.ts b/packages/app/src/app/api/v1/collectivex/runs/route.ts index ff844a8db..ec69813b1 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/route.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/route.ts @@ -4,18 +4,12 @@ import { parseCollectiveXVersion } from '@semianalysisai/inferencex-db/collectiv import { FIXTURES_MODE, getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; import { listCollectiveXRuns } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; -import { - collectiveXSweepErrorStatus, - ensureCollectiveXRunsList, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; export const runtime = 'nodejs'; -const DISCOVERY_CACHE_CONTROL = 'private, no-store'; - export async function GET(request: NextRequest) { const version = parseCollectiveXVersion(request.nextUrl.searchParams.get('version') ?? ''); if (!version) { @@ -23,35 +17,10 @@ export async function GET(request: NextRequest) { } if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-runs')); - // Backfill failures must not take the run table down — serve the stored list - // and only surface the error when there is nothing at all to show. - let ensureError: unknown = null; - let discoveryComplete: boolean; - try { - discoveryComplete = await ensureCollectiveXRunsList(version); - } catch (error) { - ensureError = error; - discoveryComplete = false; - } - try { - // Backfill may have persisted runs, so read from the same primary. const runs = await listCollectiveXRuns(getCollectiveXWriteDb(), version); - if (runs.length === 0 && ensureError) { - console.error('CollectiveX run backfill failed with no stored fallback:', ensureError); - const status = collectiveXSweepErrorStatus(ensureError) ?? 502; - return NextResponse.json({ error: 'Unavailable' }, { status }); - } - if (ensureError) { - console.error('CollectiveX run backfill failed; serving stored list:', ensureError); - } - return cachedJson( - { version, runs, discovery_complete: discoveryComplete }, - { - tag: collectiveXCacheTag(), - cacheControl: discoveryComplete ? COLLECTIVEX_CACHE_CONTROL : DISCOVERY_CACHE_CONTROL, - }, - ); + // Every run is ingested ahead of time, so the stored list is always complete. + return cachedJson({ version, runs, discovery_complete: true }, { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error listing CollectiveX runs:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/components/collectivex/CollectiveXChart.tsx b/packages/app/src/components/collectivex/CollectiveXChart.tsx index b9fd045a4..2cea5d47e 100644 --- a/packages/app/src/components/collectivex/CollectiveXChart.tsx +++ b/packages/app/src/components/collectivex/CollectiveXChart.tsx @@ -121,7 +121,9 @@ function formatPercentiles( unavailable: string, ): string { if (value === null) return unavailable; - return `${value.latency_us.p50.toFixed(1)} / ${value.latency_us.p90.toFixed(1)} / ${value.latency_us.p95.toFixed(1)} / ${value.latency_us.p99.toFixed(1)} µs`; + const { p50, p90, p95, p99 } = value.latency_us; + const us = (v: number | null) => (v === null ? unavailable : v.toFixed(1)); + return `${us(p50)} / ${us(p90)} / ${us(p95)} / ${us(p99)} µs`; } function escapeHtml(value: string): string { diff --git a/packages/app/src/components/collectivex/CollectiveXDisplay.tsx b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx index 4f8902acf..4de0b6e60 100644 --- a/packages/app/src/components/collectivex/CollectiveXDisplay.tsx +++ b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx @@ -1,6 +1,6 @@ 'use client'; -import { BookOpen, ExternalLink, Loader2, RefreshCw, Trash2 } from 'lucide-react'; +import { BookOpen, ExternalLink, Loader2, RefreshCw } from 'lucide-react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; @@ -19,11 +19,7 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; -import { - useCollectiveXRunDatasets, - useCollectiveXRuns, - useDeleteCollectiveXRun, -} from '@/hooks/api/use-collectivex'; +import { useCollectiveXRunDatasets, useCollectiveXRuns } from '@/hooks/api/use-collectivex'; import { useThemeColors } from '@/hooks/useThemeColors'; import { track } from '@/lib/analytics/analytics'; import { useLocale } from '@/lib/i18n/use-locale'; @@ -34,7 +30,6 @@ import { CollectiveXKvSection } from './CollectiveXKvSection'; import { CollectiveXRunsTable } from './CollectiveXRunsTable'; import { CollectiveXSupportMatrices } from './CollectiveXSupportMatrices'; import { - collectiveXConclusionLabel, collectiveXColorKey, collectiveXLegendLabel, collectiveXRunDasharray, @@ -118,7 +113,6 @@ const STRINGS = { suiteAria: 'Filter CollectiveX runs by suite', allSuites: 'All', noSuiteRuns: 'No runs contain the selected suite.', - shownRunCount: (count: number) => `${count} runs shown`, selectRuns: 'Select one or more runs from the table to show their data.', selectedRunsFailed: 'One or more selected runs failed to load.', runControl: 'Run', @@ -146,18 +140,6 @@ const STRINGS = { 'Activation-data rate is derived at the selected latency percentile and is not physical link bandwidth.', payloadBandwidthNote: 'Payload bandwidth is the full logical payload (incl. FP8 scale bytes) ÷ latency, per chip — a derived rate over logical bytes, not physical link bandwidth. The tooltip β/α is a least-squares fit of latency vs bytes across the ladder (β = per-chip bandwidth term, α = fixed overhead).', - deleteRun: 'Delete run', - deleteShownRuns: 'Delete shown runs', - deletingShownRuns: 'Deleting shown runs…', - deleteConfirm: (id: string) => - `Delete run #${id} from the dashboard database? This cannot be undone.`, - deleteShownConfirm: (ids: readonly string[]) => - `Delete ${ids.length} shown ${ids.length === 1 ? 'run' : 'runs'} from the dashboard database? This cannot be undone.\n\n${ids.map((id) => `#${id}`).join('\n')}`, - deleteTokenPrompt: 'Admin token required to delete runs:', - deleteUnauthorized: 'Invalid admin token.', - deleteFailed: 'Deleting the run failed. Try again.', - deleteShownFailed: (deleted: number, total: number) => - `Deleted ${deleted} of ${total} shown runs before the operation failed. Try again.`, atLatency: (percentile: string) => `at ${percentile} latency`, chartControlsHeading: 'Chart filters', chartControlsDescription: @@ -203,7 +185,6 @@ const STRINGS = { runsHeading: '运行记录', runsDescription: '该基准测试版本下保存的全部运行记录。勾选一条或多条记录，即可在图表中对比。', runsShown: '已显示运行', - shownRunCount: (count: number) => `已显示 ${count} 个运行`, suiteControl: '测试套件', suiteAria: '按测试套件筛选 CollectiveX 运行', allSuites: '全部', @@ -234,29 +215,11 @@ const STRINGS = { payloadNote: '逻辑载荷速率按所选延迟分位点派生，不代表物理链路带宽。', payloadBandwidthNote: '载荷带宽为完整逻辑载荷（含 FP8 缩放字节）÷ 延迟（每芯片），是基于逻辑字节的派生速率，不代表物理链路带宽。工具提示中的 β/α 根据各 token 档位下的字节数与延迟做最小二乘拟合（β = 每芯片带宽项，α = 固定开销）。', - deleteRun: '删除运行', - deleteShownRuns: '删除已显示的运行', - deletingShownRuns: '正在删除已显示的运行……', - deleteConfirm: (id: string) => `从仪表板数据库中删除运行 #${id}？此操作无法撤销。`, - deleteShownConfirm: (ids: readonly string[]) => - `从仪表板数据库中删除当前显示的 ${ids.length} 个运行？此操作无法撤销。\n\n${ids.map((id) => `#${id}`).join('\n')}`, - deleteTokenPrompt: '删除运行需要管理员令牌：', - deleteUnauthorized: '管理员令牌无效。', - deleteFailed: '删除运行失败，请重试。', - deleteShownFailed: (deleted: number, total: number) => - `操作失败前已删除 ${total} 个已显示运行中的 ${deleted} 个，请重试。`, atLatency: (percentile: string) => `${percentile} 延迟分位点`, chartControlsHeading: '图表筛选', chartControlsDescription: '选择下方图表要展示的测量切片和指标。', }, } as const; -const CONCLUSION_CLASSES: Record = { - success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', -}; -const CONCLUSION_FALLBACK_CLASS = - 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; -const ADMIN_TOKEN_STORAGE_KEY = 'collectivex-admin-token'; function ControlGroup({ label, @@ -292,7 +255,6 @@ export default function CollectiveXDisplay() { const [version, setVersion] = useState(COLLECTIVEX_DEFAULT_VERSION); const [suiteFilter, setSuiteFilter] = useState('all'); const [visibleRunIds, setVisibleRunIds] = useState>(new Set()); - const [bulkDeletingRunIds, setBulkDeletingRunIds] = useState>(new Set()); const initializedVersionRef = useRef(null); const runsQuery = useCollectiveXRuns(version); const runList = runsQuery.data?.runs ?? []; @@ -373,7 +335,6 @@ export default function CollectiveXDisplay() { useEffect(() => { if (!runsQuery.data || initializedVersionRef.current === version) return; - if (runsQuery.data.runs.length === 0 && !runsQuery.data.discovery_complete) return; const initial = runsQuery.data.runs.find((run) => run.measured_cases > 0) ?? runsQuery.data.runs[0]; setVisibleRunIds(initial ? new Set([initial.run_id]) : new Set()); @@ -554,12 +515,6 @@ export default function CollectiveXDisplay() { void runsQuery.refetch(); for (const query of runQueries) void query.refetch(); }, [runQueries, runsQuery]); - const deleteRun = useDeleteCollectiveXRun(); - const deletingRunIds = useMemo(() => { - if (bulkDeletingRunIds.size > 0) return bulkDeletingRunIds; - const runId = deleteRun.isPending ? deleteRun.variables?.runId : undefined; - return new Set(runId ? [runId] : []); - }, [bulkDeletingRunIds, deleteRun.isPending, deleteRun.variables?.runId]); const handleVisibleRunChange = useCallback((runIds: string[], visible: boolean) => { setVisibleRunIds((previous) => { const next = new Set(previous); @@ -570,82 +525,6 @@ export default function CollectiveXDisplay() { return next; }); }, []); - const handleDeleteRun = useCallback( - async (runId: string) => { - track('collectivex_run_delete_prompted', { run: runId }); - if (!window.confirm(t.deleteConfirm(runId))) return; - const stored = localStorage.getItem(ADMIN_TOKEN_STORAGE_KEY) ?? ''; - const token = stored || (window.prompt(t.deleteTokenPrompt)?.trim() ?? ''); - if (!token) return; - try { - const deleted = await deleteRun.mutateAsync({ runId, token }); - if (!deleted) { - localStorage.removeItem(ADMIN_TOKEN_STORAGE_KEY); - track('collectivex_run_delete_failed', { run: runId, reason: 'unauthorized' }); - window.alert(t.deleteUnauthorized); - return; - } - localStorage.setItem(ADMIN_TOKEN_STORAGE_KEY, token); - setVisibleRunIds((previous) => { - const next = new Set(previous); - next.delete(runId); - return next; - }); - track('collectivex_run_delete_confirmed', { run: runId }); - } catch { - track('collectivex_run_delete_failed', { run: runId, reason: 'error' }); - window.alert(t.deleteFailed); - } - }, - [deleteRun, t], - ); - const handleDeleteShownRuns = useCallback(async () => { - const runIds = [...orderedVisibleRunIds]; - if (runIds.length === 0) return; - track('collectivex_shown_runs_delete_prompted', { count: runIds.length, runs: runIds }); - if (!window.confirm(t.deleteShownConfirm(runIds))) return; - const stored = localStorage.getItem(ADMIN_TOKEN_STORAGE_KEY) ?? ''; - const token = stored || (window.prompt(t.deleteTokenPrompt)?.trim() ?? ''); - if (!token) return; - - setBulkDeletingRunIds(new Set(runIds)); - let deletedCount = 0; - try { - for (const runId of runIds) { - const deleted = await deleteRun.mutateAsync({ runId, token }); - if (!deleted) { - localStorage.removeItem(ADMIN_TOKEN_STORAGE_KEY); - track('collectivex_shown_runs_delete_failed', { - count: runIds.length, - deleted: deletedCount, - reason: 'unauthorized', - }); - window.alert(t.deleteUnauthorized); - return; - } - deletedCount += 1; - setVisibleRunIds((previous) => { - const next = new Set(previous); - next.delete(runId); - return next; - }); - } - localStorage.setItem(ADMIN_TOKEN_STORAGE_KEY, token); - track('collectivex_shown_runs_delete_confirmed', { - count: runIds.length, - runs: runIds, - }); - } catch { - track('collectivex_shown_runs_delete_failed', { - count: runIds.length, - deleted: deletedCount, - reason: 'error', - }); - window.alert(t.deleteShownFailed(deletedCount, runIds.length)); - } finally { - setBulkDeletingRunIds(new Set()); - } - }, [deleteRun, orderedVisibleRunIds, t]); if (runsQuery.isLoading) { return ( @@ -701,30 +580,18 @@ export default function CollectiveXDisplay() { ), 0, ); - const singleConclusionClass = - (singleDataset?.run.conclusion && CONCLUSION_CLASSES[singleDataset.run.conclusion]) ?? - CONCLUSION_FALLBACK_CLASS; + const sourceShaByRunId = new Map( + datasets.map((dataset) => [dataset.run.run_id, dataset.run.source_sha]), + ); return (
-
- - CollectiveX - - - {singleDataset - ? `#${singleDataset.run.run_id} · ${collectiveXConclusionLabel(singleDataset.run.conclusion, locale)}` - : t.shownRunCount(datasets.length)} - -
+ + CollectiveX +

{t.description}

@@ -732,7 +599,7 @@ export default function CollectiveXDisplay() { <> @@ -746,7 +613,7 @@ export default function CollectiveXDisplay() { @@ -804,21 +671,6 @@ export default function CollectiveXDisplay() { testId="collectivex-suite-filter" /> -
void handleDeleteRun(runId)} emptyMessage={suiteFilter === 'all' ? undefined : t.noSuiteRuns} /> diff --git a/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx index dfbaa8152..2928ecd76 100644 --- a/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx +++ b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx @@ -1,16 +1,15 @@ 'use client'; -import { ExternalLink, Loader2, Trash2 } from 'lucide-react'; +import { ExternalLink, Loader2 } from 'lucide-react'; import { useRef } from 'react'; import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; -import { Button } from '@/components/ui/button'; import { track } from '@/lib/analytics/analytics'; import { useLocale } from '@/lib/i18n/use-locale'; import { cn } from '@/lib/shared/utils'; -import { collectiveXConclusionLabel, collectiveXRunDasharray, collectiveXSkuLabel } from './data'; +import { collectiveXRunDasharray, collectiveXSkuLabel } from './data'; import type { CollectiveXRunSummary } from './types'; interface CollectiveXRunsTableProps { @@ -18,9 +17,10 @@ interface CollectiveXRunsTableProps { selectedRunIndexById: ReadonlyMap; visibleRunIds: ReadonlySet; loadingRunIds: ReadonlySet; - deletingRunIds: ReadonlySet; + /** InferenceX commit of each loaded run; a run links to its committed results once known. */ + sourceShaByRunId: ReadonlyMap; + /** Applies one visibility state to every listed run (one id, or a shift-click range). */ onVisibleChange: (runIds: string[], visible: boolean) => void; - onDelete: (runId: string) => void; emptyMessage?: string; } @@ -29,18 +29,15 @@ const STRINGS = { shown: 'Shown', shownHint: 'Shift-click a checkbox to set every run between it and the last one clicked', run: 'Run', - result: 'Result', swapSuite: (m: number, n: number) => `swap_blocks: ${m}/${n} measured`, suites: 'Suites', cases: 'Measured cases', points: 'Terminal points', skus: 'SKUs', published: 'Published (UTC)', - actions: 'Actions', showRun: (id: string) => `Show run #${id}`, lineStyle: (id: string) => `Line style for run #${id}`, - openRun: (id: string) => `Open GitHub Actions run #${id}`, - deleteRun: (id: string) => `Delete run #${id}`, + openRun: (id: string) => `Open the results committed for run #${id}`, empty: 'No runs match this benchmark version.', epSuite: (measured: number, requested: number) => `EP: ${measured}/${requested} measured`, kvSuite: (measured: number, requested: number) => @@ -50,18 +47,15 @@ const STRINGS = { shown: '显示', shownHint: '按住 Shift 点击复选框，可一次性勾选或取消勾选它与上次点击行之间的全部运行', run: '运行', - result: '结果', swapSuite: (m: number, n: number) => `swap_blocks：已测量 ${m}/${n}`, suites: '测试套件', cases: '实测用例', points: '终态数据点', skus: 'SKUs', published: '发布时间（UTC）', - actions: '操作', showRun: (id: string) => `显示运行 #${id}`, lineStyle: (id: string) => `运行 #${id} 的线型`, - openRun: (id: string) => `打开 GitHub Actions 运行 #${id}`, - deleteRun: (id: string) => `删除运行 #${id}`, + openRun: (id: string) => `打开运行 #${id} 对应的已提交结果`, empty: '该基准测试版本下暂无运行记录。', epSuite: (measured: number, requested: number) => `EP：已测量 ${measured}/${requested}`, kvSuite: (measured: number, requested: number) => `KV 传输：已测量 ${measured}/${requested}`, @@ -73,13 +67,6 @@ const SUITE_BADGE_CLASSES = { kv: 'border-violet-600/40 bg-violet-500/10 text-violet-700 dark:text-violet-300', } as const; -const CONCLUSION_CLASSES: Record = { - success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', -}; -const CONCLUSION_FALLBACK_CLASS = - 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; - function formatDate(value: string, locale: 'en' | 'zh'): string { return new Intl.DateTimeFormat(locale === 'zh' ? 'zh-CN' : 'en', { dateStyle: 'medium', @@ -93,9 +80,8 @@ export function CollectiveXRunsTable({ selectedRunIndexById, visibleRunIds, loadingRunIds, - deletingRunIds, + sourceShaByRunId, onVisibleChange, - onDelete, emptyMessage, }: CollectiveXRunsTableProps) { const locale = useLocale(); @@ -120,21 +106,18 @@ export function CollectiveXRunsTable({ {t.shown} {t.run} - {t.result} {t.suites} {t.cases} {t.points} {t.skus} {t.published} - {t.actions} {runs.map((run, rowIndex) => { const visible = visibleRunIds.has(run.run_id); const loading = loadingRunIds.has(run.run_id); - const deleting = deletingRunIds.has(run.run_id); - const conclusion = run.conclusion; + const sourceSha = sourceShaByRunId.get(run.run_id); const selectedRunIndex = selectedRunIndexById.get(run.run_id); const kvRequested = run.kv_cases?.requested ?? 0; const kvMeasured = run.kv_cases?.measured ?? 0; @@ -158,7 +141,6 @@ export function CollectiveXRunsTable({ 0} aria-label={t.showRun(run.run_id)} data-testid={`collectivex-run-visible-${run.run_id}`} title={t.shownHint} @@ -219,28 +201,23 @@ export function CollectiveXRunsTable({ /> )} - track('collectivex_run_source_opened', { run: run.run_id })} - className="inline-flex items-center gap-1 hover:underline" - > - #{run.run_id} - - -
- - - #{run.run_id} + ) : ( + track('collectivex_run_source_opened', { run: run.run_id })} + className="inline-flex items-center gap-1 hover:underline" + > + #{run.run_id} + + )} - > - {collectiveXConclusionLabel(conclusion, locale)} - +
@@ -293,24 +270,6 @@ export function CollectiveXRunsTable({ {formatDate(run.generated_at, locale)} - - - ); })} diff --git a/packages/app/src/components/collectivex/data.test.ts b/packages/app/src/components/collectivex/data.test.ts index 9884c3a05..04b22e4f9 100644 --- a/packages/app/src/components/collectivex/data.test.ts +++ b/packages/app/src/components/collectivex/data.test.ts @@ -4,7 +4,6 @@ import { chartPoints, collectiveXCaseLabel, collectiveXColorKey, - collectiveXConclusionLabel, collectiveXLegendLabel, collectiveXRunDasharray, collectiveXSeriesForRun, @@ -31,33 +30,6 @@ import { makeCollectiveXDataset, makeCollectiveXSeries } from './test-fixture'; const dataset = makeCollectiveXDataset(); const [scaleUp, scaleOut] = dataset.series; -describe('collectiveXConclusionLabel', () => { - it.each([ - ['success', '成功'], - ['failure', '失败'], - ['cancelled', '已取消'], - ['skipped', '已跳过'], - ['timed_out', '超时'], - ['startup_failure', '启动失败'], - ['action_required', '需要处理'], - ['neutral', '中立'], - ['stale', '已过期'], - ])('localizes the %s workflow conclusion in Chinese', (conclusion, expected) => { - expect(collectiveXConclusionLabel(conclusion, 'zh')).toBe(expected); - }); - - it('derives pending only from a null workflow conclusion', () => { - expect(collectiveXConclusionLabel(null, 'zh')).toBe('待处理'); - expect(collectiveXConclusionLabel(null, 'en')).toBe('pending'); - }); - - it('preserves known English values and hides unknown values on Chinese pages', () => { - expect(collectiveXConclusionLabel('startup_failure', 'en')).toBe('startup_failure'); - expect(collectiveXConclusionLabel('future_status', 'en')).toBe('future_status'); - expect(collectiveXConclusionLabel('future_status', 'zh')).toBe('未知状态'); - }); -}); - describe('collectiveXTopologyLabel', () => { it('shows only the scale-up transport when there is no scale-out fabric', () => { expect(collectiveXTopologyLabel(scaleUp.system)).toBe( diff --git a/packages/app/src/components/collectivex/data.ts b/packages/app/src/components/collectivex/data.ts index 929958ac0..119d1f7ff 100644 --- a/packages/app/src/components/collectivex/data.ts +++ b/packages/app/src/components/collectivex/data.ts @@ -27,31 +27,6 @@ export interface CollectiveXSeriesSelection { const BASE_RUN_DASHARRAYS = ['none', '9 4', '3 3', '10 3 2 3', '2 3', '12 3 2 3'] as const; -const COLLECTIVEX_CONCLUSION_LABELS = { - en: { - action_required: 'action_required', - cancelled: 'cancelled', - failure: 'failure', - neutral: 'neutral', - skipped: 'skipped', - stale: 'stale', - startup_failure: 'startup_failure', - success: 'success', - timed_out: 'timed_out', - }, - zh: { - action_required: '需要处理', - cancelled: '已取消', - failure: '失败', - neutral: '中立', - skipped: '已跳过', - stale: '已过期', - startup_failure: '启动失败', - success: '成功', - timed_out: '超时', - }, -} as const; - const COLLECTIVEX_MODE_LABELS_ZH: Record = { normal: '常规', 'low-latency': '低延迟', @@ -62,16 +37,6 @@ const COLLECTIVEX_PHASE_LABELS_ZH: Record = { prefill: '预填充', }; -export function collectiveXConclusionLabel(conclusion: string | null, locale: Locale): string { - if (conclusion === null) return locale === 'zh' ? '待处理' : 'pending'; - const labels = COLLECTIVEX_CONCLUSION_LABELS[locale]; - return conclusion in labels - ? labels[conclusion as keyof typeof labels] - : locale === 'zh' - ? '未知状态' - : conclusion; -} - export function normalizeCollectiveXSku(sku: string): string { const normalized = sku.trim().toLowerCase(); const base = normalized.split(':').at(-1)?.split('-')[0] ?? normalized; @@ -211,7 +176,7 @@ export function fitAlphaBeta( const component = point.components[operation]; if (component === null || component.payload_bytes === null) continue; const latency = component.latency_us[percentile]; - if (latency <= 0) continue; + if (latency === null || latency <= 0) continue; bytesPerGpu.push(component.payload_bytes / ep); latencies.push(latency); } diff --git a/packages/app/src/hooks/api/use-collectivex.ts b/packages/app/src/hooks/api/use-collectivex.ts index bc51c4102..06c32eb6f 100644 --- a/packages/app/src/hooks/api/use-collectivex.ts +++ b/packages/app/src/hooks/api/use-collectivex.ts @@ -1,6 +1,6 @@ -import { useMutation, useQueries, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useQueries, useQuery } from '@tanstack/react-query'; -import { deleteCollectiveXRun, fetchCollectiveXRun, fetchCollectiveXRunList } from '@/lib/api/api'; +import { fetchCollectiveXRun, fetchCollectiveXRunList } from '@/lib/api/api'; import { COLLECTIVEX_DEFAULT_VERSION, type CollectiveXVersion, @@ -10,9 +10,6 @@ export function useCollectiveXRuns(version: CollectiveXVersion = COLLECTIVEX_DEF return useQuery({ queryKey: ['collectivex-runs', version], queryFn: ({ signal }) => fetchCollectiveXRunList(version, signal), - staleTime: 0, - refetchOnMount: 'always', - refetchInterval: (query) => (query.state.data?.discovery_complete === false ? 1_000 : false), }); } @@ -21,22 +18,6 @@ export function useCollectiveXRunDatasets(version: CollectiveXVersion, runIds: r queries: runIds.map((runId) => ({ queryKey: ['collectivex-run', version, runId], queryFn: ({ signal }: { signal: AbortSignal }) => fetchCollectiveXRun(version, runId, signal), - staleTime: 0, - refetchOnMount: 'always' as const, })), }); } - -export function useDeleteCollectiveXRun() { - const queryClient = useQueryClient(); - return useMutation({ - mutationFn: ({ runId, token }: { runId: string; token: string }) => - deleteCollectiveXRun(runId, token), - onSuccess: (deleted) => { - if (!deleted) return; - for (const key of ['collectivex-runs', 'collectivex-run']) { - void queryClient.invalidateQueries({ queryKey: [key] }); - } - }, - }); -} diff --git a/packages/app/src/lib/api/api-documentation.ts b/packages/app/src/lib/api/api-documentation.ts index b76e6848f..e96a30f62 100644 --- a/packages/app/src/lib/api/api-documentation.ts +++ b/packages/app/src/lib/api/api-documentation.ts @@ -781,8 +781,8 @@ export const apiDocumentationGroups: readonly ApiDocumentationGroup[] = [ id: 'collectivex', title: text('CollectiveX', 'CollectiveX'), description: text( - 'Versioned collective communication sweep results and run discovery.', - '带版本的集合通信扫描结果与运行发现数据。', + 'Versioned collective communication benchmark results, one run per results commit.', + '带版本的集合通信基准测试结果，每个结果提交对应一次运行。', ), }, { @@ -1949,8 +1949,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/latest', summary: text('Read the latest CollectiveX dataset', '读取最新 CollectiveX 数据集'), description: text( - 'Discovers and ingests the latest sweep when needed, then returns its versioned neutral dataset. run.source_repository names the GitHub repository that ran the sweep when it is not SemiAnalysisAI/InferenceX. A stored run is served if refresh fails. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', - '按需发现并导入最新扫描，然后返回带版本的中立数据集。若扫描并非由 SemiAnalysisAI/InferenceX 执行，run.source_repository 会标明实际执行的 GitHub 仓库。若刷新失败，会返回已存储的运行。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', + 'Returns the versioned neutral dataset of the newest stored run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed collectivex/results/: run_id is that commit time in epoch seconds, run_attempt is 1, and run.source_sha is the commit. Runs are ingested when results land on main, never on read. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', + '返回最新已存储运行的带版本中立数据集。每次运行对应 SemiAnalysisAI/InferenceX main 分支上一个修改了 collectivex/results/ 的提交：run_id 为该提交时间的 Unix 秒级时间戳，run_attempt 为 1，run.source_sha 为该提交。结果合入 main 时导入，读取时不会导入。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', ), audience: 'public', stability: 'beta', @@ -1974,11 +1974,11 @@ export const apiOperations: readonly ApiOperation[] = [ { version: 1, run: { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', - source_sha: '0123456789abcdef', + source_sha: 'e65c9045320c78f1fb86200802aceafa0c6310d8', requested_cases: 12, terminal_cases: 12, measured_cases: 10, @@ -2005,22 +2005,10 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '404', - 'No stored or discoverable run exists.', - '没有已存储或可发现的运行。', + 'No run is stored for the version.', + '该版本下没有已存储的运行。', 'Not found', ), - errorResponse( - '502', - 'Upstream sweep discovery is unavailable and no stored fallback exists.', - '上游扫描发现不可用，且没有已存储的回退数据。', - 'Unavailable', - ), - errorResponse( - '503', - 'Upstream sweep processing is temporarily unavailable.', - '上游扫描处理暂时不可用。', - 'Unavailable', - ), errorResponse( '500', 'The stored run query failed.', @@ -2038,8 +2026,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs', summary: text('List CollectiveX runs', '列出 CollectiveX 运行'), description: text( - 'Returns progressively discovered run summaries. discovery_complete=false means clients may poll while older runs are still being discovered.', - '返回逐步发现的运行汇总。discovery_complete=false 表示仍在发现更早的运行，客户端可以轮询。', + 'Returns every stored run summary, newest results commit first. Runs are ingested ahead of reads, so discovery_complete is always true; the field is kept for existing clients.', + '按结果提交从新到旧返回全部已存储的运行汇总。运行在读取前即已导入，因此 discovery_complete 始终为 true；保留该字段是为了兼容现有客户端。', ), audience: 'public', stability: 'beta', @@ -2057,8 +2045,8 @@ export const apiOperations: readonly ApiOperation[] = [ ], responses: [ success( - 'Version, run summaries, and discovery state.', - '版本、运行汇总和发现状态。', + 'Version, run summaries, and discovery_complete (always true).', + '版本、运行汇总以及 discovery_complete（始终为 true）。', objectSchema({ version: integerSchema, runs: arraySchema(collectiveRunSummarySchema), @@ -2068,9 +2056,9 @@ export const apiOperations: readonly ApiOperation[] = [ version: 1, runs: [ { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', covered_skus: ['h200_sxm'], requested_cases: 12, @@ -2090,18 +2078,6 @@ export const apiOperations: readonly ApiOperation[] = [ 'version 缺失或不受支持。', 'Unknown version', ), - errorResponse( - '502', - 'Discovery failed and no stored run list exists.', - '发现失败，且没有已存储的运行列表。', - 'Unavailable', - ), - errorResponse( - '503', - 'Discovery is temporarily unavailable.', - '发现暂时不可用。', - 'Unavailable', - ), errorResponse( '500', 'The stored run list query failed.', @@ -2119,8 +2095,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs/{runId}', summary: text('Read a CollectiveX run', '读取 CollectiveX 运行'), description: text( - 'Returns one positive numeric run ID as a versioned CollectiveX dataset, discovering and ingesting it on demand when possible.', - '按正整数运行 ID 返回带版本的 CollectiveX 数据集，并在可能时按需发现和导入。', + 'Returns one stored run as a versioned CollectiveX dataset. The run ID is the results commit time in epoch seconds, as listed by /api/v1/collectivex/runs.', + '以带版本的 CollectiveX 数据集返回一次已存储的运行。运行 ID 为结果提交时间的 Unix 秒级时间戳，与 /api/v1/collectivex/runs 列出的一致。', ), audience: 'public', stability: 'beta', @@ -2130,10 +2106,10 @@ export const apiOperations: readonly ApiOperation[] = [ 'path', true, 'integer', - 'Positive GitHub Actions run ID.', - 'GitHub Actions 正整数运行 ID。', + 'Positive run ID: the results commit time in epoch seconds.', + '正整数运行 ID：结果提交时间的 Unix 秒级时间戳。', positiveIdSchema, - 123456789, + 1792089351, ), parameter( 'version', @@ -2154,11 +2130,11 @@ export const apiOperations: readonly ApiOperation[] = [ { version: 1, run: { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', - source_sha: '0123456789abcdef', + source_sha: 'e65c9045320c78f1fb86200802aceafa0c6310d8', requested_cases: 12, terminal_cases: 12, measured_cases: 10, @@ -2183,23 +2159,11 @@ export const apiOperations: readonly ApiOperation[] = [ 'version 或 runId 无效。', 'Unknown version or run id', ), - errorResponse('404', 'The run does not exist.', '该运行不存在。', 'Not found'), - errorResponse( - '502', - 'The run cannot be fetched from the upstream source.', - '无法从上游来源获取该运行。', - 'Unavailable', - ), - errorResponse( - '503', - 'Upstream processing is temporarily unavailable.', - '上游处理暂时不可用。', - 'Unavailable', - ), + errorResponse('404', 'The run is not stored.', '该运行未存储。', 'Not found'), errorResponse('500', 'The run query failed.', '运行查询失败。', 'Internal server error'), ], responseShapeName: 'CollectiveXDataset', - curlUrl: `${API_BASE_URL}/api/v1/collectivex/runs/123456789?version=1`, + curlUrl: `${API_BASE_URL}/api/v1/collectivex/runs/1792089351?version=1`, }, { id: 'get-agentic-aggregates', @@ -3245,8 +3209,8 @@ const overview = { id: 'cache', title: text('Caching', '缓存'), description: text( - 'Read endpoints may be served from shared caches. CollectiveX uses short refresh windows, and framework releases use a one-hour shared cache.', - '只读端点可能由共享缓存提供。CollectiveX 使用较短的刷新窗口，框架版本使用一小时共享缓存。', + 'Read endpoints may be served from shared caches. CollectiveX responses stay cached until a results ingest purges them, and framework releases use a one-hour shared cache.', + '只读端点可能由共享缓存提供。CollectiveX 响应会一直缓存，直到结果导入时被清除；框架版本使用一小时共享缓存。', ), }, { diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index 874bc5b9b..f647e0013 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -679,13 +679,24 @@ export const apiRouteCatalog = [ operationId: 'list-benchmark-history', sourceSha256: '2b33a33b56ec5afb9c44ddc7db581686dec7488a397cd50c8efa810acff3a86c', }, + { + source: 'src/app/api/v1/collectivex/ingest/route.ts', + path: '/api/v1/collectivex/ingest', + method: 'POST', + classification: 'admin', + exclusionReason: { + en: 'Secret-protected CollectiveX ingest mutation called by the benchmark repository workflow; it is not part of the public read API.', + zh: '由基准测试仓库工作流调用、受密钥保护的 CollectiveX 导入写操作；它不属于公开只读 API。', + }, + sourceSha256: '017bebdc59684e37d4ac42862d0daa96ee7b4c1ffd28e92da967570ba3875247', + }, { source: 'src/app/api/v1/collectivex/latest/route.ts', path: '/api/v1/collectivex/latest', method: 'GET', classification: 'published-read', operationId: 'get-collectivex-latest', - sourceSha256: 'c60fff45af91fa38a071fee69b253983fe403738f7e35c56711708b5b3cb2d70', + sourceSha256: '1bd6b280006bc3ac40ac11382efb4e8e54bc0c7c22a97169791347525d2673c6', }, { source: 'src/app/api/v1/collectivex/runs/route.ts', @@ -693,7 +704,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-collectivex-runs', - sourceSha256: '32eaf00f56cb30db70b128f9aa8ec1132c56d9b02c2c6728072fd5bb4c465258', + sourceSha256: '17fbc05787bc21a9b7070914ba2457d197dc7954294d6014d494393b59f5c8db', }, { source: 'src/app/api/v1/collectivex/runs/[runId]/route.ts', @@ -701,18 +712,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'get-collectivex-run', - sourceSha256: 'e242fb048f89d0e5eff200e63f6baac7307f066a3fbe5984007390a2095f3699', - }, - { - source: 'src/app/api/v1/collectivex/runs/[runId]/route.ts', - path: '/api/v1/collectivex/runs/{runId}', - method: 'DELETE', - classification: 'admin', - exclusionReason: { - en: 'Authenticated CollectiveX administration mutation; deletion is not part of the public read API.', - zh: '需要身份验证的 CollectiveX 管理写操作；删除不属于公开只读 API。', - }, - sourceSha256: 'e242fb048f89d0e5eff200e63f6baac7307f066a3fbe5984007390a2095f3699', + sourceSha256: '9de0597df0db1923b11347072226e6b79f932ed07bf9edaa5807af6e3e1bb69b', }, { source: 'src/app/api/v1/datasets/route.ts', @@ -1580,7 +1580,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/cache/api-cache.ts', - sourceSha256: 'a96667af1379f6d9281da92923455d194e22da84ef8d31e1690436a23e7c2eb9', + sourceSha256: 'd168e625f68800e7bcf3c74f824491e4de7d201929e656809b8d4e225f6c53f5', reviewArea: { en: 'Public CDN tags, cache lifetimes, Blob key dimensions, and purge behavior.', zh: '公开 CDN 标签、缓存时长、Blob 键维度和清除行为。', @@ -1652,7 +1652,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/api/api.ts', - sourceSha256: 'bceb0916e3643f90f637530c7928bdf67860aab812362e48715432c88d93cdcc', + sourceSha256: '03c00a746ff4fc59cf9b3b01e5f69eee6c936d3d6d11887beeace7ce978a1624', reviewArea: { en: 'Public API client parameter serialization and TypeScript response contracts.', zh: '公开 API 客户端的参数序列化和 TypeScript 响应契约。', @@ -1704,7 +1704,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/collectivex/types.ts', - sourceSha256: '006f87eb54bca94142d6c8f1aa57679ce96330286b30654e0c4bd37600cd0d00', + sourceSha256: '3108c08a60ff154577fb93b52d6008f21f5597090ab5e8de289e6977dc73c517', reviewArea: { en: 'CollectiveX version negotiation and versioned dataset/run response types.', zh: 'CollectiveX 版本协商以及带版本的数据集与运行响应类型。', @@ -1747,10 +1747,10 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/collectivex.ts', - sourceSha256: '5b4cd412a1b543a8f26b5daf72ab252bc6c95edd7ca975262eefc1bbddbed3c7', + sourceSha256: '73ff29402f25c514bc5e2578703aacd4d6c81b7eb045266188d50b55391ed40c', reviewArea: { - en: 'CollectiveX dataset projection, run summaries, coverage, series, and discovery state.', - zh: 'CollectiveX 数据集投影、运行汇总、覆盖范围、序列和发现状态。', + en: 'CollectiveX dataset projection, run summaries, coverage, series, and latest-run ordering.', + zh: 'CollectiveX 数据集投影、运行汇总、覆盖范围、序列和最新运行排序。', }, }, { diff --git a/packages/app/src/lib/api/api.test.ts b/packages/app/src/lib/api/api.test.ts index a51e21087..e5d605a73 100644 --- a/packages/app/src/lib/api/api.test.ts +++ b/packages/app/src/lib/api/api.test.ts @@ -4,7 +4,6 @@ import { fetchBenchmarks, fetchWorkflowInfo, fetchAvailability, - deleteCollectiveXRun, fetchCollectiveXRun, fetchCollectiveXRunList, fetchReliability, @@ -219,25 +218,4 @@ describe('CollectiveX API', () => { expect(response.runs).toHaveLength(1); expect(response.runs[0].run_id).toBe(dataset.run.run_id); }); - - it('sends the bearer token on delete and reports success', async () => { - mockFetch.mockResolvedValueOnce({ ok: true, status: 200 }); - - await expect(deleteCollectiveXRun('160', 'secret-token')).resolves.toBe(true); - - expect(mockFetch).toHaveBeenCalledWith('/api/v1/collectivex/runs/160', { - method: 'DELETE', - headers: { Authorization: 'Bearer secret-token' }, - }); - }); - - it('resolves false on 401 so callers can clear a stale token', async () => { - mockFetch.mockResolvedValueOnce({ ok: false, status: 401 }); - await expect(deleteCollectiveXRun('160', 'stale')).resolves.toBe(false); - }); - - it('throws on non-auth delete failures', async () => { - mockFetch.mockResolvedValueOnce({ ok: false, status: 500 }); - await expect(deleteCollectiveXRun('160', 'secret-token')).rejects.toThrow(/500/); - }); }); diff --git a/packages/app/src/lib/api/api.ts b/packages/app/src/lib/api/api.ts index 46888a2c7..7538a5265 100644 --- a/packages/app/src/lib/api/api.ts +++ b/packages/app/src/lib/api/api.ts @@ -265,7 +265,7 @@ export function fetchSubmissions(signal?: AbortSignal) { return fetchJson('/api/v1/submissions', signal); } -/** Stored sweep runs plus whether lazy discovery has reached the end of history. */ +/** Stored CollectiveX run summaries, one per results commit; `discovery_complete` is always true. */ export function fetchCollectiveXRunList( version: CollectiveXVersion = COLLECTIVEX_DEFAULT_VERSION, signal?: AbortSignal, @@ -277,7 +277,7 @@ export function fetchCollectiveXRunList( }>(`/api/v1/collectivex/runs?version=${version}`, signal); } -/** Resolve a specific ingested sweep run's neutral view dataset by run_id. */ +/** Resolve a CollectiveX run's neutral view dataset by run_id. */ export function fetchCollectiveXRun( version: CollectiveXVersion, runId: string, @@ -289,22 +289,6 @@ export function fetchCollectiveXRun( ); } -/** - * Admin deletion of an ingested run. Requires the dedicated CollectiveX admin - * Bearer secret (COLLECTIVEX_ADMIN_SECRET — deliberately not the CI-held - * INVALIDATE_SECRET); resolves false on 401 so callers can clear a stale - * stored token, throws on other failures. - */ -export async function deleteCollectiveXRun(runId: string, token: string): Promise { - const response = await fetch(`/api/v1/collectivex/runs/${runId}`, { - method: 'DELETE', - headers: { Authorization: `Bearer ${token}` }, - }); - if (response.status === 401) return false; - if (!response.ok) throw new Error(`CollectiveX delete failed (${response.status}).`); - return true; -} - export interface FeedbackListRow { id: string; created_at: string; diff --git a/packages/app/src/lib/cache/api-cache.ts b/packages/app/src/lib/cache/api-cache.ts index 775f1d1ed..5c2c9c5f6 100644 --- a/packages/app/src/lib/cache/api-cache.ts +++ b/packages/app/src/lib/cache/api-cache.ts @@ -6,9 +6,11 @@ import { blobGet, blobPurge, blobSet } from './blob-cache'; /** * CollectiveX data lives in its own database and gets its own cache scope: * every CollectiveX response carries this CDN/unstable_cache tag (via - * `cachedJson(..., { tag })`), so run deletion can purge the CollectiveX + * `cachedJson(..., { tag })`), so a CollectiveX ingest can purge the CollectiveX * cache without dropping the main dashboard's. CollectiveX routes read their - * DB directly — no blob layer, so the tag is the entire scope. + * DB directly — no blob layer, so the tag is the entire scope. Responses stay + * cached until an ingest purges it (`POST /api/v1/collectivex/ingest`, or + * `/api/v1/invalidate?scope=collectivex`). */ export const COLLECTIVEX_CACHE_SCOPE = 'collectivex'; @@ -25,14 +27,6 @@ export function collectiveXCacheTag(): string { return namespace ? `${COLLECTIVEX_CACHE_SCOPE}:${namespace}` : COLLECTIVEX_CACHE_SCOPE; } -/** - * Short CDN window shared by the CollectiveX latest/runs routes: new sweep - * runs are discovered lazily at the origin, so freshness is bounded by this - * TTL rather than by an ingest-time purge. - */ -export const COLLECTIVEX_CACHE_CONTROL = - 'public, max-age=0, s-maxage=60, stale-while-revalidate=300'; - /** * OperatorX reads its own database and has its own scope, like CollectiveX: its * responses carry this tag; merged comparisons also use a revision-keyed Blob subtree. @@ -202,8 +196,7 @@ export async function purgeAll(): Promise { const deleted = await blobPurge('', [`${OPERATORX_CACHE_SCOPE}/`]); revalidateTag(cacheTag(), { expire: 0 }); // CollectiveX responses are cached only under their own tag (no blobs), so - // a full purge must drop that tag explicitly — this line is the sole - // mechanism that clears CollectiveX from the CDN. + // a full purge must drop that tag explicitly. purgeCollectiveX(); return deleted; } @@ -219,8 +212,8 @@ export function purgeCollectiveX(): void { /** * 1 day unless overridden. Purged on demand via revalidateTag with the matching - * tag. `tag` defaults to the environment-scoped database tag; CollectiveX routes - * pass their own scope (and a shorter cacheControl). + * tag. `tag` defaults to the environment-scoped database tag; CollectiveX and + * OperatorX routes pass their own scope. */ function cdnHeaders(tag: string = cacheTag(), cacheControl?: string): Record { return { diff --git a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts b/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts deleted file mode 100644 index 2ef6f9c29..000000000 --- a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts +++ /dev/null @@ -1,703 +0,0 @@ -import { - makeSwapDoc, - swapMatrix, -} from '@semianalysisai/inferencex-db/collectivex/swap-test-fixture'; -import AdmZip from 'adm-zip'; -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { makeRawMatrix, makeRawShard } from '@/components/collectivex/test-fixture'; - -const { mockGetStates, mockInsert, mockRefresh, mockGetWriteDb } = vi.hoisted(() => ({ - mockGetStates: vi.fn(), - mockInsert: vi.fn(), - mockRefresh: vi.fn(), - mockGetWriteDb: vi.fn(() => 'mock-write-sql'), -})); - -vi.mock('@semianalysisai/inferencex-db/connection', () => ({ - getCollectiveXWriteDb: mockGetWriteDb, -})); - -vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ - getCollectiveXRunStates: mockGetStates, - insertCollectiveXRun: mockInsert, - refreshCollectiveXRunAttempt: mockRefresh, -})); - -import { - collectiveXSweepErrorCode, - ensureCollectiveXRun, - ensureCollectiveXRunsList, - ensureLatestCollectiveXRun, - resetCollectiveXDiscoveryCooldown, -} from './collectivex-lazy-ingest'; - -const mockFetch = vi.fn(); -vi.stubGlobal('fetch', mockFetch); - -const shardA = makeRawShard({ backend: 'deepep-v2', ep: 8 }); -const shardB = makeRawShard({ - sku: 'mi355x', - backend: 'mori', - implName: 'mori', - vendor: 'amd', - ep: 16, - scaleUpTransport: 'xgmi', - scaleOutTransport: 'rdma', - topologyClass: 'mi355x-xgmi-rdma', - nodes: 2, - gpusPerNode: 8, - scaleUpDomain: 8, -}); - -function requestedOf(shard: Record) { - const identity = shard.identity as Record; - const factors = identity.case_factors as Record; - return { - caseId: identity.case_id as string, - sku: factors.sku as string, - disposition: 'runnable' as const, - case: factors.case as Record, - }; -} - -const matrix = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)]); -const matrixV2 = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)], 2); - -function zipDocs(...docs: unknown[]): ArrayBuffer { - const zip = new AdmZip(); - docs.forEach((doc, index) => zip.addFile(`doc-${index}.json`, Buffer.from(JSON.stringify(doc)))); - const bytes = zip.toBuffer(); - const archive = new ArrayBuffer(bytes.byteLength); - new Uint8Array(archive).set(bytes); - return archive; -} - -const matrixZip = zipDocs(matrix); -const matrixZipV2 = zipDocs(matrixV2); -const shardZip = zipDocs(shardA, shardB); -const shardZipV2 = zipDocs({ ...shardA, version: 2 }, { ...shardB, version: 2 }); - -function runObject(overrides: Record = {}) { - return { - id: 160, - name: 'CollectiveX Sweep', - path: '.github/workflows/collectivex-sweep.yml', - head_branch: 'collectivex-fp8-precision', - head_sha: 'a'.repeat(40), - status: 'completed', - conclusion: 'success', - run_attempt: 1, - updated_at: new Date().toISOString(), - ...overrides, - }; -} - -function runListing(...runs: ReturnType[]) { - return Response.json({ total_count: runs.length, workflow_runs: runs }); -} - -const twoRunListing = () => runListing(runObject({ id: 161 }), runObject({ id: 160 })); - -function artifactsBody(runId = 160, runAttempt = 1) { - return { - total_count: 2, - artifacts: [ - { - id: 1, - name: `cxsweep-matrix-${runId}`, - archive_download_url: 'https://example.test/matrix.zip', - expired: false, - }, - { - id: 2, - name: `cxshard-cases-${runId}-${runAttempt}`, - archive_download_url: 'https://example.test/shards.zip', - expired: false, - }, - ], - }; -} - -beforeEach(() => { - vi.restoreAllMocks(); - mockFetch.mockReset(); - mockGetStates.mockReset().mockResolvedValue({}); - mockInsert.mockReset().mockResolvedValue(true); - mockRefresh.mockReset().mockResolvedValue(true); - process.env.GITHUB_TOKEN = 'test-token'; - delete process.env.INFERENCEX_SOURCE_REPOS; - resetCollectiveXDiscoveryCooldown(); -}); - -afterAll(() => { - delete process.env.GITHUB_TOKEN; - delete process.env.INFERENCEX_SOURCE_REPOS; - vi.unstubAllGlobals(); -}); - -describe('ensureLatestCollectiveXRun', () => { - it('discovers a standalone swap artifact and persists its computed summary', async () => { - const doc = makeSwapDoc(); - doc.runtime.source_sha = 'a'.repeat(40); - mockFetch - .mockResolvedValueOnce(runListing(runObject())) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(zipDocs(swapMatrix))) - .mockResolvedValueOnce(new Response(zipDocs(doc))); - await ensureLatestCollectiveXRun(1); - expect(mockInsert.mock.calls[0][1].summary).toMatchObject({ - run_id: '160', - measured_cases: 1, - requested_points: 3, - swap_cases: { requested: 1, measured: 1 }, - kv_cases: { requested: 0, measured: 0 }, - }); - }); - - it('discovers the newest absent run and persists its raw documents', async () => { - mockFetch - .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - const [sql, run, docs] = mockInsert.mock.calls[0]; - expect(sql).toBe('mock-write-sql'); - expect(run).toMatchObject({ - run_id: '160', - run_attempt: 1, - version: 1, - source_branch: 'collectivex-fp8-precision', - conclusion: 'success', - matrix, - }); - expect(run.summary).toMatchObject({ run_id: '160', measured_cases: 2 }); - expect(run.summary.source_repository).toBeUndefined(); - expect(mockFetch.mock.calls[0][0]).toContain( - '/repos/SemiAnalysisAI/InferenceX/actions/workflows/collectivex-sweep.yml/runs?', - ); - expect(mockFetch.mock.calls[1][0]).toContain( - '/repos/SemiAnalysisAI/InferenceX/actions/runs/160/artifacts?', - ); - expect(docs).toEqual([shardA, shardB]); - }); - - it('stops without artifact downloads when the newest matching run is live', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject()] }), - ); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockFetch).toHaveBeenCalledTimes(1); - }); - - it('never re-ingests a tombstoned run — the next candidate wins', async () => { - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve( - ids[0] === '161' ? { '161': { state: 'deleted', version: 1, run_attempt: 1 } } : {}, - ), - ); - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('skips runs tagged for another version', async () => { - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody(161))) - .mockResolvedValueOnce(new Response(matrixZipV2)) - .mockResolvedValueOnce(new Response(shardZipV2)) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(2); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ version: 2, run_id: '161' }); - expect(mockInsert.mock.calls[1][1]).toMatchObject({ version: 1, run_id: '160' }); - }); - - it('refreshes a live run when GitHub reports a newer attempt', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch - .mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), - ) - .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockRefresh).toHaveBeenCalledTimes(1); - expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); - }); - - it('downloads only the highest usable attempt per shard', async () => { - const artifacts = { - total_count: 3, - artifacts: [ - { - id: 1, - name: 'cxsweep-matrix-160', - archive_download_url: 'https://example.test/matrix.zip', - expired: false, - }, - { - id: 2, - name: 'cxshard-cases-160-1', - archive_download_url: 'https://example.test/shard-attempt1.zip', - expired: false, - }, - { - id: 3, - name: 'cxshard-cases-160-2', - archive_download_url: 'https://example.test/shard-attempt2.zip', - expired: false, - }, - ], - }; - mockFetch - .mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), - ) - .mockResolvedValueOnce(Response.json(artifacts)) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockFetch).toHaveBeenCalledTimes(4); - expect(mockFetch.mock.calls[3][0]).toBe('https://example.test/shard-attempt2.zip'); - }); - - it('classifies a matrix artifact without a matrix document as invalid', async () => { - mockFetch - .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('invalid'); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('reports GitHub outages as unavailable', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 500 })); - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); - - it('reports a missing GITHUB_TOKEN as unavailable', async () => { - delete process.env.GITHUB_TOKEN; - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); -}); - -describe('discovery cooldown', () => { - it('serves a warm target from the cooldown without calling GitHub again', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); - - await ensureLatestCollectiveXRun(1); - const afterFirst = mockFetch.mock.calls.length; - expect(afterFirst).toBeGreaterThan(0); - - await ensureLatestCollectiveXRun(1); - await ensureLatestCollectiveXRun(1); - expect(mockFetch.mock.calls.length).toBe(afterFirst); - }); - - it('walks again once the cooldown window has elapsed', async () => { - vi.useFakeTimers(); - try { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); - - await ensureLatestCollectiveXRun(1); - const afterFirst = mockFetch.mock.calls.length; - - vi.advanceTimersByTime(61_000); - await ensureLatestCollectiveXRun(1); - expect(mockFetch.mock.calls.length).toBeGreaterThan(afterFirst); - } finally { - vi.useRealTimers(); - } - }); - - it('rethrows a remembered failure so an outage stays 502/503 rather than 404', async () => { - mockFetch.mockResolvedValue(new Response('nope', { status: 500 })); - - const first = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(first)).toBe('unavailable'); - const afterFirst = mockFetch.mock.calls.length; - - const second = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(second).toBe(first); - expect(mockFetch.mock.calls.length).toBe(afterFirst); - }); -}); - -describe('ensureCollectiveXRunsList', () => { - it('bounds GitHub discovery to runs that could still have rerun artifacts', async () => { - vi.useFakeTimers(); - vi.setSystemTime('2026-07-29T00:00:00.000Z'); - try { - mockFetch.mockResolvedValueOnce(runListing()); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - const discoveryUrl = new URL(mockFetch.mock.calls[0][0] as string); - expect(discoveryUrl.searchParams.get('created')).toBe('>=2026-06-15T00:00:00.000Z'); - } finally { - vi.useRealTimers(); - } - }); - - it('does not probe artifacts for unknown runs beyond the retention window', async () => { - mockFetch.mockResolvedValueOnce(runListing(runObject({ updated_at: '2020-01-01T00:00:00Z' }))); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockGetStates).not.toHaveBeenCalled(); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('continues past more than eight known rows to backfill an older run', async () => { - const runs = Array.from({ length: 10 }, (_unused, index) => runObject({ id: 169 - index })); - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve( - ids[0] === '160' ? {} : { [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }, - ), - ); - mockFetch - .mockResolvedValueOnce(runListing(...runs)) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - const complete = await ensureCollectiveXRunsList(1); - - expect(complete).toBe(true); - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('continues past one malformed run instead of hiding later valid runs', async () => { - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody(161))) - .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('reports an incomplete pass after changing eight runs', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - expect(mockInsert).toHaveBeenCalledTimes(8); - }); - - it('does not consume the mutation budget when concurrent inserts are no-ops', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); - mockInsert.mockResolvedValue(false); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - expect(mockInsert).toHaveBeenCalledTimes(8); - }); - - it('does not consume the mutation budget when guarded refreshes are no-ops', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => - runObject({ id: 167 - index, run_attempt: 2 }), - ); - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve({ [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }), - ); - mockRefresh.mockResolvedValue(false); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - expect(mockRefresh).toHaveBeenCalledTimes(8); - }); -}); - -describe('ensureCollectiveXRun', () => { - it('treats tombstoned runs as not found', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'deleted', version: 1, run_attempt: 1 } }); - const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).not.toHaveBeenCalled(); - }); - - it('checks the GitHub attempt for a live matching run without downloading artifacts', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockResolvedValueOnce(Response.json(runObject())); - - await ensureCollectiveXRun(1, '160'); - expect(mockGetStates).toHaveBeenCalledWith('mock-write-sql', ['160']); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockRefresh).not.toHaveBeenCalled(); - }); - - it('refreshes a live matching run when GitHub has a newer attempt', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch - .mockResolvedValueOnce(Response.json(runObject({ run_attempt: 2 }))) - .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureCollectiveXRun(1, '160'); - - expect(mockRefresh).toHaveBeenCalledTimes(1); - expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); - }); - - it('persists an absent run fetched by id', async () => { - mockFetch - .mockResolvedValueOnce(Response.json(runObject())) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureCollectiveXRun(1, '160'); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('rejects runs from other workflows as not found', async () => { - mockFetch.mockResolvedValueOnce( - Response.json(runObject({ path: '.github/workflows/run-sweep.yml' })), - ); - const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('rejects malformed run ids without touching GitHub', async () => { - const caught = await ensureCollectiveXRun(1, 'abc').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).not.toHaveBeenCalled(); - }); -}); - -describe('benchmark source repositories', () => { - const PRIMARY = 'SemiAnalysisAI/InferenceX'; - const EXTRA = 'ExampleOrg/ExampleBench'; - const PRIMARY_LISTING = `/repos/${PRIMARY}/actions/workflows/collectivex-sweep.yml/runs?`; - const EXTRA_LISTING = `/repos/${EXTRA}/actions/workflows/collectivex-sweep.yml/runs?`; - const downloads: [string, () => Response][] = [ - ['https://example.test/matrix.zip', () => new Response(matrixZip)], - ['https://example.test/shards.zip', () => new Response(shardZip)], - ]; - - // Answers each request from the first route whose fragment its URL contains, else 404. - function routeFetch(routes: [string, () => Response][]) { - mockFetch.mockImplementation((input: string) => { - const route = routes.find(([fragment]) => input.includes(fragment)); - return Promise.resolve(route ? route[1]() : new Response(null, { status: 404 })); - }); - } - - beforeEach(() => { - process.env.INFERENCEX_SOURCE_REPOS = EXTRA; - }); - - it('walks every repository newest-first by run id', async () => { - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve({ [ids[0]]: { state: 'live', version: 2, run_attempt: 1 } }), - ); - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 165 }), runObject({ id: 150 }))], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }), runObject({ id: 160 }))], - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockGetStates.mock.calls.map(([, ids]) => ids[0])).toEqual(['170', '165', '160', '150']); - }); - - it('reads artifacts from and records the repository that ran the newest sweep', async () => { - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 160 }))], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }))], - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - ...downloads, - ]); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ - run_id: '170', - summary: { source_repository: EXTRA }, - }); - expect( - mockFetch.mock.calls - .map(([url]) => String(url)) - .filter((url) => url.includes(`/repos/${PRIMARY}/actions/runs/`)), - ).toEqual([]); - }); - - it('keeps discovering the other repositories when one cannot be listed, but not as complete', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => new Response(null, { status: 500 })], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }))], - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - ...downloads, - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('170'); - }); - - it('skips another repository without a sweep workflow quietly', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([[PRIMARY_LISTING, () => runListing()]]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(warn).not.toHaveBeenCalled(); - - expect( - mockFetch.mock.calls.map(([url]) => String(url)).some((url) => url.includes(EXTRA_LISTING)), - ).toBe(true); - }); - - it('keeps a failed list pass incomplete without walking again during the cooldown', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => runListing()], - [EXTRA_LISTING, () => new Response(null, { status: 403 })], - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - const requests = mockFetch.mock.calls.length; - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - - expect(mockFetch.mock.calls.length).toBe(requests); - }); - - it('fails discovery when the only listable repository fails', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([[PRIMARY_LISTING, () => new Response(null, { status: 500 })]]); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); - - it('reports the latest pass as failed when another repository cannot be listed', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 160 }))], - [EXTRA_LISTING, () => new Response(null, { status: 500 })], - [`/repos/${PRIMARY}/actions/runs/160/artifacts?`, () => Response.json(artifactsBody(160))], - ...downloads, - ]); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('fails discovery only when every repository fails', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 500 })); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - expect( - mockFetch.mock.calls.map(([url]) => String(url)).some((url) => url.includes(EXTRA_LISTING)), - ).toBe(true); - }); - - it('looks a run up in each repository and records where it was found', async () => { - routeFetch([ - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - [`/repos/${EXTRA}/actions/runs/170`, () => Response.json(runObject({ id: 170 }))], - ...downloads, - ]); - - await ensureCollectiveXRun(1, '170'); - - expect(mockFetch.mock.calls.map(([url]) => String(url))[0]).toContain( - `/repos/${PRIMARY}/actions/runs/170`, - ); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ - run_id: '170', - summary: { source_repository: EXTRA }, - }); - }); - - it('reports a run absent from every repository as not found', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 404 })); - - const caught = await ensureCollectiveXRun(1, '170').catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).toHaveBeenCalledTimes(2); - }); - - it('keeps a failed lookup in one repository from reading as not found', async () => { - routeFetch([[`/repos/${PRIMARY}/actions/runs/170`, () => new Response(null, { status: 502 })]]); - - const caught = await ensureCollectiveXRun(1, '170').catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); -}); diff --git a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts b/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts deleted file mode 100644 index 0f717d9b3..000000000 --- a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts +++ /dev/null @@ -1,624 +0,0 @@ -import AdmZip from 'adm-zip'; - -import { - benchmarkSourceRepos, - GITHUB_API_BASE, - GITHUB_REPO_FULL, -} from '@semianalysisai/inferencex-constants'; - -import { - buildDatasetFromNeutral, - buildRunSummary, - isMatrixDoc, - matrixVersion, -} from '@semianalysisai/inferencex-db/collectivex/reader'; -import { - matrixArtifactName, - selectShardArtifacts, -} from '@semianalysisai/inferencex-db/collectivex/artifact-selection'; -import type { CollectiveXVersion } from '@semianalysisai/inferencex-db/collectivex/types'; -import { getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; -import { - getCollectiveXRunStates, - insertCollectiveXRun, - refreshCollectiveXRunAttempt, -} from '@semianalysisai/inferencex-db/queries/collectivex'; - -const WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; -const WORKFLOW_FILE = 'collectivex-sweep.yml'; -const WORKFLOW_NAME = 'CollectiveX Sweep'; -const RUNS_PER_PAGE = 100; -const ARTIFACTS_PER_PAGE = 100; - -const MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; -const MAX_RUN_BYTES = 256 * 1024 * 1024; -const REQUEST_TIMEOUT_MS = 30_000; -const MAX_REQUEST_ATTEMPTS = 3; -const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); -const ARTIFACT_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; -const WORKFLOW_RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; -const DISCOVERY_LOOKBACK_MS = WORKFLOW_RERUN_WINDOW_MS + ARTIFACT_RETENTION_MS; -const MAX_CHANGED_RUNS_PER_PASS = 8; - -type SweepErrorCode = 'invalid' | 'not-found' | 'unavailable'; - -interface GithubWorkflowRun { - id: number; - name: string; - path: string; - head_branch: string | null; - head_sha: string; - status: string | null; - conclusion: string | null; - run_attempt: number; - run_started_at?: string | null; - updated_at?: string | null; - created_at?: string | null; -} - -interface WorkflowRun extends GithubWorkflowRun { - /** `owner/repo` the run was read from. */ - source_repository: string; -} - -interface GithubArtifact { - id: number; - name: string; - archive_download_url: string; - expired?: boolean; - size_in_bytes?: number; -} - -class CollectiveXSweepError extends Error { - readonly code: SweepErrorCode; - - constructor(code: SweepErrorCode, message: string, options?: ErrorOptions) { - super(message, options); - this.name = 'CollectiveXSweepError'; - this.code = code; - } -} - -export function collectiveXSweepErrorCode(error: unknown): SweepErrorCode | null { - return error instanceof CollectiveXSweepError ? error.code : null; -} - -export function collectiveXSweepErrorStatus(error: unknown): 404 | 502 | 503 | null { - const code = collectiveXSweepErrorCode(error); - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; -} - -const inFlight = new Map>(); - -function dedupe(key: string, work: () => Promise): Promise { - const existing = inFlight.get(key) as Promise | undefined; - if (existing) return existing; - const promise = work().finally(() => inFlight.delete(key)); - inFlight.set(key, promise); - return promise; -} - -const DISCOVERY_COOLDOWN_MS = 60_000; -const DISCOVERY_FAILURE_COOLDOWN_MS = 10_000; - -interface DiscoveryOutcome { - until: number; - error: unknown; -} - -const discoveryCooldown = new Map(); - -function throttled(key: string, work: () => Promise): () => Promise { - return async () => { - const settled = discoveryCooldown.get(key); - if (settled && Date.now() < settled.until) { - if (settled.error !== null) throw settled.error; - return; - } - try { - await work(); - } catch (error) { - discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_FAILURE_COOLDOWN_MS, error }); - throw error; - } - discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_COOLDOWN_MS, error: null }); - }; -} - -export function resetCollectiveXDiscoveryCooldown(): void { - discoveryCooldown.clear(); -} - -function githubHeaders(token: string) { - return { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - }; -} - -async function waitBeforeRetry(attempt: number): Promise { - const delay = process.env.NODE_ENV === 'test' ? 0 : Math.min(250 * 2 ** (attempt - 1), 2000); - await new Promise((resolve) => { - setTimeout(resolve, delay); - }); -} - -async function githubFetch(url: string, token: string): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= MAX_REQUEST_ATTEMPTS; attempt++) { - try { - const response = await fetch(url, { - cache: 'no-store', - headers: githubHeaders(token), - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - if ( - response.ok || - !RETRYABLE_STATUSES.has(response.status) || - attempt === MAX_REQUEST_ATTEMPTS - ) { - return response; - } - lastError = new Error(`GitHub returned ${response.status}`); - } catch (error) { - lastError = error; - if (attempt === MAX_REQUEST_ATTEMPTS) break; - } - await waitBeforeRetry(attempt); - } - throw new CollectiveXSweepError('unavailable', 'GitHub request failed', { cause: lastError }); -} - -function requireToken(): string { - const token = process.env.GITHUB_TOKEN; - if (!token) throw new CollectiveXSweepError('unavailable', 'GITHUB_TOKEN is not configured'); - return token; -} - -function isSweepRun(run: GithubWorkflowRun): boolean { - return ( - run.name === WORKFLOW_NAME && - run.path === WORKFLOW_PATH && - Number.isSafeInteger(run.id) && - run.id > 0 && - Number.isSafeInteger(run.run_attempt) && - run.run_attempt > 0 - ); -} - -function runGeneratedAt(run: WorkflowRun): string { - return run.updated_at || run.run_started_at || run.created_at || ''; -} - -function artifactsMayBeAvailable(run: WorkflowRun): boolean { - const timestamp = Date.parse(runGeneratedAt(run)); - return !Number.isFinite(timestamp) || Date.now() - timestamp <= ARTIFACT_RETENTION_MS; -} - -async function* repositorySweepRuns( - token: string, - repository: string, -): AsyncGenerator { - let page = 1; - let visited = 0; - let total: number | null = null; - const createdSince = new Date(Date.now() - DISCOVERY_LOOKBACK_MS).toISOString(); - while (total === null || visited < total) { - const parameters = new URLSearchParams({ - status: 'completed', - created: `>=${createdSince}`, - per_page: String(RUNS_PER_PAGE), - page: String(page), - }); - const response = await githubFetch( - `${GITHUB_API_BASE}/repos/${repository}/actions/workflows/${WORKFLOW_FILE}/runs?${parameters}`, - token, - ); - // An extra source repository without the sweep workflow has no sweeps to list. - if (response.status === 404 && page === 1 && repository !== GITHUB_REPO_FULL) { - throw new CollectiveXSweepError('not-found', 'no sweep workflow'); - } - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub run discovery failed (${response.status})`, - ); - } - const payload = (await response.json()) as { - total_count?: number; - workflow_runs?: GithubWorkflowRun[]; - }; - const runs = payload.workflow_runs ?? []; - if ( - total === null && - Number.isSafeInteger(payload.total_count) && - (payload.total_count ?? -1) >= 0 - ) { - total = payload.total_count!; - } - if (runs.length === 0) break; - visited += runs.length; - for (const run of runs) { - if (isSweepRun(run)) yield { ...run, source_repository: repository }; - } - if (runs.length < RUNS_PER_PAGE || (total !== null && visited >= total)) break; - page += 1; - } -} - -interface SweepListing { - /** First error from a repository that has the sweep workflow but could not be fully listed. */ - failure: unknown; -} - -// Merges each source repository's newest-first runs by global run id, which is -// also how getLatestCollectiveXRun orders stored rows. An extra repository without -// the sweep workflow is skipped; one that fails is skipped and logged, and its error -// is recorded on the listing. Discovery throws when every repository is skipped or fails. -async function* sweepRuns(token: string, listing: SweepListing): AsyncGenerator { - const repositories = benchmarkSourceRepos(); - const sources = repositories.map((repository) => repositorySweepRuns(token, repository)); - const heads: (WorkflowRun | null)[] = sources.map(() => null); - const failures = new Map(); - let skipped = 0; - const advance = async (index: number): Promise => { - try { - const next = await sources[index].next(); - heads[index] = next.done ? null : next.value; - } catch (error) { - heads[index] = null; - if ( - repositories[index] !== GITHUB_REPO_FULL && - collectiveXSweepErrorCode(error) === 'not-found' - ) { - skipped += 1; - return; - } - failures.set(index, error); - listing.failure ??= error; - if (sources.length > 1) { - console.warn(`[collectivex] skipping ${repositories[index]} during discovery:`, error); - } - } - }; - await Promise.all(sources.map((_source, index) => advance(index))); - for (;;) { - let newest = -1; - for (const [index, run] of heads.entries()) { - if (run && (newest < 0 || run.id > heads[newest]!.id)) newest = index; - } - if (newest < 0) break; - yield heads[newest]!; - await advance(newest); - } - if (failures.size > 0 && failures.size + skipped === sources.length) throw failures.get(0); -} - -// Run ids are global, so at most one source repository holds a run. A failed -// lookup in any repository keeps a miss from reading as not found. -async function fetchRun(runId: number, token: string): Promise { - let failure: unknown = null; - for (const repository of benchmarkSourceRepos()) { - let response: Response; - try { - response = await githubFetch( - `${GITHUB_API_BASE}/repos/${repository}/actions/runs/${runId}`, - token, - ); - } catch (error) { - failure ??= error; - continue; - } - if (response.ok) { - const run = (await response.json()) as GithubWorkflowRun; - return { ...run, source_repository: repository }; - } - if (response.status !== 404) { - failure ??= new CollectiveXSweepError( - 'unavailable', - `GitHub run lookup failed (${response.status})`, - ); - } - } - throw failure ?? new CollectiveXSweepError('not-found', 'sweep run not found'); -} - -async function listArtifacts(run: WorkflowRun, token: string): Promise { - const artifacts: GithubArtifact[] = []; - let page = 1; - let total: number | null = null; - while (total === null || artifacts.length < total) { - const response = await githubFetch( - `${GITHUB_API_BASE}/repos/${run.source_repository}/actions/runs/${run.id}/artifacts?per_page=${ARTIFACTS_PER_PAGE}&page=${page}`, - token, - ); - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub artifact discovery failed (${response.status})`, - ); - } - const payload = (await response.json()) as { - total_count?: number; - artifacts?: GithubArtifact[]; - }; - const page_artifacts = payload.artifacts ?? []; - if ( - total === null && - Number.isSafeInteger(payload.total_count) && - (payload.total_count ?? -1) >= 0 - ) { - total = payload.total_count!; - } - if (page_artifacts.length === 0) break; - artifacts.push(...page_artifacts); - if (page_artifacts.length < ARTIFACTS_PER_PAGE) break; - page += 1; - } - return artifacts.filter((artifact) => !artifact.expired); -} - -function hasMatrixArtifact(artifacts: GithubArtifact[], run: WorkflowRun): boolean { - return artifacts.some((artifact) => artifact.name === matrixArtifactName(String(run.id))); -} - -async function collectDocs(artifact: GithubArtifact, token: string): Promise { - if (artifact.size_in_bytes !== undefined && artifact.size_in_bytes > MAX_ARTIFACT_BYTES) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is oversized`); - } - const response = await githubFetch(artifact.archive_download_url, token); - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub artifact download failed (${response.status})`, - ); - } - const archive = Buffer.from(await response.arrayBuffer()); - if (archive.byteLength > MAX_ARTIFACT_BYTES) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} archive is oversized`); - } - let zip: AdmZip; - try { - zip = new AdmZip(archive); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is not a ZIP`, { - cause: error, - }); - } - const docs: unknown[] = []; - for (const entry of zip.getEntries()) { - if (entry.isDirectory || !entry.entryName.endsWith('.json')) continue; - let text: string; - try { - text = new TextDecoder('utf-8', { fatal: true }).decode(entry.getData()); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has non-UTF-8 entry`, { - cause: error, - }); - } - try { - docs.push(JSON.parse(text)); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has invalid JSON`, { - cause: error, - }); - } - } - return docs; -} - -interface MatrixCandidate { - matrixDoc: unknown; - version: number; - matrixArtifacts: GithubArtifact[]; - resultArtifacts: GithubArtifact[]; -} - -function resultArtifactsForRun(artifacts: GithubArtifact[], run: WorkflowRun): GithubArtifact[] { - return selectShardArtifacts(artifacts, String(run.id), run.run_attempt).toSorted((left, right) => - left.name.localeCompare(right.name), - ); -} - -async function loadMatrixCandidate( - artifacts: GithubArtifact[], - token: string, - run: WorkflowRun, -): Promise { - const matrixArtifacts = artifacts - .filter((artifact) => artifact.name === matrixArtifactName(String(run.id))) - .toSorted((left, right) => right.id - left.id) - .slice(0, 1); - if (matrixArtifacts.length === 0) { - throw new CollectiveXSweepError('not-found', 'sweep run has no matrix artifact'); - } - const matrixDocs: unknown[] = []; - for (const artifact of matrixArtifacts) matrixDocs.push(...(await collectDocs(artifact, token))); - const matrixCandidates = matrixDocs.filter((doc) => isMatrixDoc(doc)); - if (matrixCandidates.length !== 1) { - throw new CollectiveXSweepError('invalid', 'sweep run must carry exactly one matrix document'); - } - const version = matrixVersion(matrixCandidates[0]); - if (version === null) { - throw new CollectiveXSweepError('invalid', 'matrix document has no valid version tag'); - } - const resultArtifacts = resultArtifactsForRun(artifacts, run); - return { matrixDoc: matrixCandidates[0], version, matrixArtifacts, resultArtifacts }; -} - -async function persistRun( - run: WorkflowRun, - candidate: MatrixCandidate, - token: string, - refresh = false, -): Promise { - const generatedAt = runGeneratedAt(run); - if (!generatedAt) { - throw new CollectiveXSweepError('invalid', 'sweep run is missing a timestamp'); - } - - let totalBytes = 0; - for (const artifact of [...candidate.matrixArtifacts, ...candidate.resultArtifacts]) { - totalBytes += artifact.size_in_bytes ?? 0; - if (totalBytes > MAX_RUN_BYTES) { - throw new CollectiveXSweepError('invalid', 'sweep run artifacts exceed the size budget'); - } - } - - const docs: unknown[] = []; - for (const artifact of candidate.resultArtifacts) { - docs.push(...(await collectDocs(artifact, token))); - } - - const meta = { - run_id: String(run.id), - run_attempt: run.run_attempt, - generated_at: generatedAt, - conclusion: run.conclusion, - source_sha: run.head_sha, - ...(run.source_repository === GITHUB_REPO_FULL - ? {} - : { source_repository: run.source_repository }), - }; - - let summary; - try { - summary = buildRunSummary(buildDatasetFromNeutral(candidate.matrixDoc, docs, meta)); - } catch (error) { - throw new CollectiveXSweepError('invalid', 'sweep run artifacts failed validation', { - cause: error, - }); - } - - const row = { - ...meta, - version: candidate.version, - source_branch: run.head_branch, - matrix: candidate.matrixDoc, - summary, - }; - return refresh - ? refreshCollectiveXRunAttempt(getCollectiveXWriteDb(), row, docs) - : insertCollectiveXRun(getCollectiveXWriteDb(), row, docs); -} - -async function matrixCandidateFor( - run: WorkflowRun, - version: CollectiveXVersion, - token: string, -): Promise { - const artifacts = await listArtifacts(run, token); - if (!hasMatrixArtifact(artifacts, run)) return null; - const candidate = await loadMatrixCandidate(artifacts, token, run); - return candidate.version === version ? candidate : null; -} - -type CandidateResult = 'changed-match' | 'changed-other' | 'match' | 'skip'; - -async function considerCandidate( - run: WorkflowRun, - version: CollectiveXVersion, - token: string, -): Promise { - const states = await getCollectiveXRunStates(getCollectiveXWriteDb(), [String(run.id)]); - const known = states[String(run.id)]; - if (known) { - if (known.version !== version || known.state !== 'live') return 'skip'; - if (run.run_attempt > known.run_attempt) { - const candidate = await matrixCandidateFor(run, version, token); - if (candidate) { - const changed = await persistRun(run, candidate, token, true); - return changed ? 'changed-match' : 'match'; - } - } - return 'match'; - } - const artifacts = await listArtifacts(run, token); - if (!hasMatrixArtifact(artifacts, run)) return 'skip'; - const candidate = await loadMatrixCandidate(artifacts, token, run); - const changed = await persistRun(run, candidate, token); - if (!changed) return candidate.version === version ? 'match' : 'skip'; - return candidate.version === version ? 'changed-match' : 'changed-other'; -} - -export function ensureLatestCollectiveXRun(version: CollectiveXVersion): Promise { - const key = `latest:${version}`; - return dedupe( - key, - throttled(key, async () => { - const token = requireToken(); - const listing: SweepListing = { failure: undefined }; - for await (const run of sweepRuns(token, listing)) { - const result = await considerCandidate(run, version, token); - if (result === 'match' || result === 'changed-match') break; - } - // A repository that could not be listed may hold a newer sweep. - if (listing.failure !== undefined) throw listing.failure; - }), - ); -} - -export function ensureCollectiveXRunsList(version: CollectiveXVersion): Promise { - const key = `list:${version}`; - return dedupe(key, async () => { - // A source repository that recently failed to list keeps the pass incomplete - // without walking every repository again on each poll. - const settled = discoveryCooldown.get(key); - if (settled && Date.now() < settled.until) return false; - const token = requireToken(); - let changed = 0; - const listing: SweepListing = { failure: undefined }; - for await (const run of sweepRuns(token, listing)) { - if (!artifactsMayBeAvailable(run)) continue; - let result: CandidateResult; - try { - result = await considerCandidate(run, version, token); - } catch (error) { - const code = collectiveXSweepErrorCode(error); - if (code === 'invalid' || code === 'not-found') continue; - throw error; - } - if (result === 'changed-match' || result === 'changed-other') changed += 1; - if (changed >= MAX_CHANGED_RUNS_PER_PASS) return false; - } - if (listing.failure === undefined) return true; - discoveryCooldown.set(key, { - until: Date.now() + DISCOVERY_FAILURE_COOLDOWN_MS, - error: listing.failure, - }); - return false; - }); -} - -export function ensureCollectiveXRun(version: CollectiveXVersion, runId: string): Promise { - return dedupe(`run:${version}:${runId}`, async () => { - const numericId = Number(runId); - if (!Number.isSafeInteger(numericId) || numericId <= 0) { - throw new CollectiveXSweepError('not-found', 'invalid run id'); - } - const states = await getCollectiveXRunStates(getCollectiveXWriteDb(), [runId]); - const known = states[runId]; - if (known && (known.state !== 'live' || known.version !== version)) { - throw new CollectiveXSweepError('not-found', 'run is not available'); - } - const token = requireToken(); - const run = await fetchRun(numericId, token); - if (!isSweepRun(run)) { - throw new CollectiveXSweepError('not-found', 'run is not a CollectiveX sweep'); - } - if (run.status !== 'completed') { - throw new CollectiveXSweepError('not-found', 'sweep run has not completed'); - } - if (known && run.run_attempt <= known.run_attempt) return; - const artifacts = await listArtifacts(run, token); - const candidate = await loadMatrixCandidate(artifacts, token, run); - if (candidate.version !== version) { - throw new CollectiveXSweepError('not-found', 'run does not match the requested version'); - } - await persistRun(run, candidate, token, known !== undefined); - }); -} diff --git a/packages/db/migrations-collectivex/004_committed_results.sql b/packages/db/migrations-collectivex/004_committed_results.sql new file mode 100644 index 000000000..35dc97f02 --- /dev/null +++ b/packages/db/migrations-collectivex/004_committed_results.sql @@ -0,0 +1,18 @@ +-- CollectiveX runs are now the benchmark repo's committed results, one row per +-- InferenceX commit that touched collectivex/results/ on main. +-- +-- run_id is that commit's committer time in epoch seconds (still monotonic with +-- history, so "latest" keeps ordering by run_id desc), run_attempt is always 1, +-- source_sha is the results commit, matrix is collectivex/results/cases.json, and +-- cx_run_docs holds the commit's result documents verbatim. Re-ingesting a commit +-- replaces its row and documents. +-- +-- Lazy GitHub Actions discovery is gone, so a deleted run no longer needs a +-- tombstone to stay deleted: removing results is a commit to the benchmark repo. + +alter table cx_runs drop column if exists deleted_at; + +comment on table cx_runs is + 'One row per InferenceX commit that changed collectivex/results/ on main; run_id = commit time in epoch seconds.'; +comment on table cx_run_docs is + 'The committed collectivex/results//.json documents of each cx_runs row, verbatim.'; diff --git a/packages/db/package.json b/packages/db/package.json index fd67436ef..a850e3a4b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -17,7 +17,7 @@ "scripts": { "db:prepare:ci": "bun src/prepare-ci-artifacts.ts", "db:ingest:ci": "bun src/ingest-ci-run.ts", - "db:ingest:collectivex": "bun --env-file=../../.env src/ingest-collectivex.ts --download", + "db:ingest:collectivex": "bun --env-file=../../.env src/ingest-collectivex.ts", "db:ingest:run": "bun --env-file=../../.env src/ingest-ci-run.ts --download", "db:ingest:gcs": "bun --env-file=../../.env src/ingest-gcs-backup.ts", "db:ingest:supplemental": "bun --env-file=../../.env src/ingest-supplemental.ts", diff --git a/packages/db/src/collectivex/artifact-selection.test.ts b/packages/db/src/collectivex/artifact-selection.test.ts deleted file mode 100644 index 33e7392f1..000000000 --- a/packages/db/src/collectivex/artifact-selection.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { - matrixArtifactName, - selectShardArtifactNames, - selectShardArtifacts, -} from './artifact-selection'; - -describe('shard artifact selection', () => { - it('selects one artifact per cell, sorted by name', () => { - expect( - selectShardArtifactNames( - ['cxshard-b-160-1', 'cxshard-a-160-1', 'cxsweep-matrix-160'], - '160', - 1, - ), - ).toEqual(['cxshard-a-160-1', 'cxshard-b-160-1']); - }); - - it('prefers the highest attempt not above the run attempt', () => { - const names = ['cxshard-a-160-1', 'cxshard-a-160-2', 'cxshard-a-160-3', 'cxshard-b-160-1']; - expect(selectShardArtifactNames(names, '160', 2)).toEqual([ - 'cxshard-a-160-2', - 'cxshard-b-160-1', - ]); - }); - - it('keeps cells with hyphenated names intact across run-id collisions', () => { - const names = ['cxshard-h200-ep8-160-1', 'cxshard-h200-ep8-161-1']; - expect(selectShardArtifactNames(names, '160', 1)).toEqual(['cxshard-h200-ep8-160-1']); - }); - - it('ignores foreign names and zero attempts', () => { - expect( - selectShardArtifactNames(['cxshard-a-160-0', 'other-160-1', 'cxshard-a-999-1'], '160', 1), - ).toEqual([]); - }); - it('breaks same-attempt ties on the later artifact id', () => { - const artifacts = [ - { name: 'cxshard-a-160-1', id: 10 }, - { name: 'cxshard-a-160-1', id: 42 }, - ]; - expect(selectShardArtifacts(artifacts, '160', 1)).toEqual([ - { name: 'cxshard-a-160-1', id: 42 }, - ]); - }); - - it('keeps the first match when callers supply no ids', () => { - expect( - selectShardArtifacts([{ name: 'cxshard-a-160-1' }, { name: 'cxshard-a-160-1' }], '160', 1), - ).toHaveLength(1); - }); - - it('treats a run id with regex metacharacters literally', () => { - expect(selectShardArtifactNames(['cxshard-a-1x0-1'], '1.0', 1)).toEqual([]); - }); -}); - -describe('matrixArtifactName', () => { - it('derives the per-run matrix artifact name', () => { - expect(matrixArtifactName('160')).toBe('cxsweep-matrix-160'); - }); -}); diff --git a/packages/db/src/collectivex/artifact-selection.ts b/packages/db/src/collectivex/artifact-selection.ts deleted file mode 100644 index eba4fd705..000000000 --- a/packages/db/src/collectivex/artifact-selection.ts +++ /dev/null @@ -1,54 +0,0 @@ -const MATRIX_PREFIX = 'cxsweep-matrix-'; -const SHARD_PREFIX = 'cxshard-'; - -export function matrixArtifactName(runId: string): string { - return `${MATRIX_PREFIX}${runId}`; -} - -function escapeRegExp(value: string): string { - return value.replaceAll(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`); -} - -interface ShardArtifactRef { - name: string; - id?: number; -} - -export function selectShardArtifacts( - artifacts: readonly T[], - runId: string, - runAttempt: number, -): T[] { - const pattern = new RegExp( - `^${SHARD_PREFIX}(?.+)-${escapeRegExp(runId)}-(?[1-9][0-9]*)$`, - 'u', - ); - const selected = new Map(); - for (const artifact of artifacts) { - const match = pattern.exec(artifact.name); - if (!match) continue; - const attempt = Number(match.groups!.attempt); - if (attempt > runAttempt) continue; - const previous = selected.get(match.groups!.cell); - const supersedes = - !previous || - attempt > previous.attempt || - (attempt === previous.attempt && (artifact.id ?? -1) > (previous.artifact.id ?? -1)); - if (supersedes) selected.set(match.groups!.cell, { artifact, attempt }); - } - return [...selected.values()].map((entry) => entry.artifact); -} - -export function selectShardArtifactNames( - names: readonly string[], - runId: string, - runAttempt: number, -): string[] { - return selectShardArtifacts( - names.map((name) => ({ name })), - runId, - runAttempt, - ) - .map((entry) => entry.name) - .toSorted(); -} diff --git a/packages/db/src/collectivex/committed-results.test.ts b/packages/db/src/collectivex/committed-results.test.ts new file mode 100644 index 000000000..83b9590c0 --- /dev/null +++ b/packages/db/src/collectivex/committed-results.test.ts @@ -0,0 +1,313 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + committedRun, + CommittedResultsError, + fetchResultsSource, + listResultsCommits, + RESULTS_DIR, + resolveResultsCommit, + validateCommittedResults, + type ResultsSource, +} from './committed-results'; +import { makeRawMatrix, makeRawShard } from './test-fixture'; +import { makeSwapDoc, makeSwapRequestedCase, swapCaseId } from './swap-test-fixture'; + +const SHA = 'c'.repeat(40); +const OTHER_SHA = 'd'.repeat(40); +const COMMITTED_AT = '2026-10-15T18:35:51Z'; +const COMMITTED_SECONDS = String(Date.parse(COMMITTED_AT) / 1000); + +type Json = Record; + +function epResult(options: Parameters[0] = {}): Json { + return { + ...makeRawShard(options), + provenance: { source_sha: 'e'.repeat(40), source_dirty: false }, + }; +} + +function requestedOf(shard: Json) { + return { + caseId: shard.identity.case_id as string, + sku: shard.identity.case_factors.sku as string, + case: shard.identity.case_factors.case as Json, + }; +} + +function casesFor(epDocs: Json[], swapSkus: string[] = [], version = 1): Json { + const matrix = makeRawMatrix(epDocs.map(requestedOf), version); + matrix.requested_cases = [ + ...(matrix.requested_cases as unknown[]), + ...swapSkus.map((sku) => makeSwapRequestedCase(sku)), + ]; + return matrix; +} + +function encode(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)); +} + +function sourceOf(cases: unknown, files: Record): ResultsSource { + return { + sha: SHA, + committedAt: COMMITTED_AT, + files: new Map([ + ['cases.json', encode(cases)], + ...Object.entries(files).map(([path, doc]): [string, Uint8Array] => [path, encode(doc)]), + ]), + }; +} + +function pathOf(doc: Json): string { + return `${doc.identity.case_factors.sku}/${doc.identity.case_id}.json`; +} + +describe('validateCommittedResults', () => { + it('accepts EP and swap_blocks results that answer runnable cases', () => { + const ep = epResult(); + const swap = makeSwapDoc('h200-dgxc'); + const results = validateCommittedResults( + sourceOf(casesFor([ep], ['h200-dgxc']), { + [pathOf(ep)]: ep, + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ); + expect(results.version).toBe(1); + expect(results.docs).toEqual([ep, swap]); + }); + + it('accepts a commit whose cases.json has no results yet', () => { + const ep = epResult(); + expect(validateCommittedResults(sourceOf(casesFor([ep]), {})).docs).toEqual([]); + }); + + it('rejects a file whose name does not match its case_id', () => { + const ep = epResult(); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { 'h200-dgxc/renamed.json': ep })), + ).toThrow( + `${RESULTS_DIR}/h200-dgxc/renamed.json holds case ${ep.identity.case_id} on h200-dgxc; the file must be h200-dgxc/${ep.identity.case_id}.json`, + ); + }); + + it('rejects a result filed under another SKU folder', () => { + const ep = epResult(); + expect(() => + validateCommittedResults( + sourceOf(casesFor([ep]), { [`b200-nscale/${ep.identity.case_id}.json`]: ep }), + ), + ).toThrow(/the file must be h200-dgxc\//u); + }); + + it('names the file whose version differs from cases.json', () => { + const ep = { ...epResult(), version: 2 }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + `${RESULTS_DIR}/${pathOf(ep)} has version 2, cases.json has 1`, + ); + }); + + it('recognizes swap_blocks documents by schema and checks their case and pool', () => { + const swap = makeSwapDoc('h200-dgxc'); + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc', 'random')}.json`]: swap, + }), + ), + ).toThrow(/the file must be h200-dgxc\/h200-dgxc-swap-blocks-smoke-contiguous\.json/u); + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['b200-nscale']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ), + ).toThrow(/is not a runnable case in cases\.json/u); + }); + + it('rejects documents that are not results, unparsable JSON, and stray paths', () => { + const ep = epResult(); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: { schema: 'other' } })), + ).toThrow(/is not a CollectiveX result document/u); + const broken = sourceOf(casesFor([ep]), {}); + broken.files.set(pathOf(ep), new TextEncoder().encode('{')); + expect(() => validateCommittedResults(broken)).toThrow( + `${RESULTS_DIR}/${pathOf(ep)} is not UTF-8 JSON`, + ); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { 'nested/deeper/x.json': ep })), + ).toThrow(/is not at \/\.json/u); + }); + + it('rejects results measured from uncommitted code', () => { + const ep = { ...epResult(), provenance: { source_dirty: true } }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + /was not measured from committed collectivex\/ code/u, + ); + }); + + it('requires cases.json with a version and requested cases', () => { + expect(() => + validateCommittedResults({ sha: SHA, committedAt: COMMITTED_AT, files: new Map() }), + ).toThrow(`${RESULTS_DIR}/cases.json is missing at ${SHA}`); + expect(() => validateCommittedResults(sourceOf({ requested_cases: [] }, {}))).toThrow( + /has no version and requested_cases/u, + ); + }); + + it('names the file the shared reader rejects', () => { + const swap = { ...makeSwapDoc('h200-dgxc'), iterations: 0 }; + const validate = () => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ); + expect(validate).toThrow(CommittedResultsError); + expect(validate).toThrow( + new RegExp(`^${RESULTS_DIR}/h200-dgxc/${swapCaseId('h200-dgxc')}\\.json: `, 'u'), + ); + expect(() => validate()).toThrowError(expect.objectContaining({ kind: 'invalid' })); + }); +}); + +describe('committedRun', () => { + it('identifies the run by the commit time in epoch seconds, one attempt per commit', () => { + const ep = epResult(); + const { run, docs } = committedRun( + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep })), + ); + expect(run).toMatchObject({ + run_id: COMMITTED_SECONDS, + run_attempt: 1, + generated_at: COMMITTED_AT, + source_sha: SHA, + source_branch: 'main', + conclusion: 'success', + version: 1, + }); + expect(run.matrix).toEqual(casesFor([ep])); + expect(run.summary).toMatchObject({ + run_id: COMMITTED_SECONDS, + measured_cases: 1, + requested_cases: 1, + }); + expect(docs).toEqual([ep]); + }); + + it('orders later commits after earlier ones', () => { + const ep = epResult(); + const later = { ...sourceOf(casesFor([ep]), {}), committedAt: '2026-10-16T00:00:00Z' }; + expect(Number(committedRun(validateCommittedResults(later)).run.run_id)).toBeGreaterThan( + Number(COMMITTED_SECONDS), + ); + }); +}); + +function jsonResponse(body: unknown): Response { + return Response.json(body); +} + +function commitBody(sha: string, date: string) { + return { sha, commit: { committer: { date } } }; +} + +describe('GitHub reads', () => { + it('resolves a ref to its newest results commit when that commit is on main', async () => { + const fetch = vi.fn((url: string) => { + const query = new URL(url).searchParams; + expect(query.get('path')).toBe(RESULTS_DIR); + return query.get('sha') === 'main' + ? jsonResponse([ + commitBody(SHA, COMMITTED_AT), + commitBody(OTHER_SHA, '2026-10-01T00:00:00Z'), + ]) + : jsonResponse([commitBody(SHA, COMMITTED_AT)]); + }); + await expect( + resolveResultsCommit('feature', { fetch: fetch as unknown as typeof globalThis.fetch }), + ).resolves.toEqual({ sha: SHA, committedAt: COMMITTED_AT }); + }); + + it('refuses a results commit that never landed on main', async () => { + const fetch = vi.fn((url: string) => + new URL(url).searchParams.get('sha') === 'main' + ? jsonResponse([commitBody(OTHER_SHA, COMMITTED_AT)]) + : jsonResponse([commitBody(SHA, COMMITTED_AT)]), + ); + await expect( + resolveResultsCommit(SHA, { fetch: fetch as unknown as typeof globalThis.fetch }), + ).rejects.toMatchObject({ kind: 'not-found' }); + }); + + it('lists main history oldest first across pages', async () => { + const page1 = Array.from({ length: 100 }, (_, index) => + commitBody(index.toString(16).padStart(40, '0'), '2026-10-15T00:00:00Z'), + ); + const fetch = vi.fn((url: string) => + jsonResponse( + new URL(url).searchParams.get('page') === '1' ? page1 : [commitBody(SHA, COMMITTED_AT)], + ), + ); + const commits = await listResultsCommits('main', { + fetch: fetch as unknown as typeof globalThis.fetch, + }); + expect(commits).toHaveLength(101); + expect(commits[0].sha).toBe(SHA); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it('reads exactly the results tree at the commit and reuses cached blobs', async () => { + const urls: string[] = []; + const fetch = vi.fn((url: string, init?: RequestInit) => { + urls.push(url); + if (url.includes('/git/trees/')) { + return jsonResponse({ + truncated: false, + tree: [ + { path: 'cases.json', type: 'blob', sha: 'b1', size: 10 }, + { path: 'h200-dgxc', type: 'tree', sha: 't1' }, + { path: 'h200-dgxc/a.json', type: 'blob', sha: 'b2', size: 10 }, + ], + }); + } + expect(init?.headers).toEqual({ Authorization: 'Bearer token' }); + return new Response('{}'); + }); + const blobCache = new Map([['b2', new TextEncoder().encode('cached')]]); + const source = await fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT }, + { fetch: fetch as unknown as typeof globalThis.fetch, token: 'token', blobCache }, + ); + expect(urls[0]).toContain(`/git/trees/${SHA}:${RESULTS_DIR}?recursive=1`); + expect(urls.slice(1)).toEqual([ + `https://raw.githubusercontent.com/SemiAnalysisAI/InferenceX/${SHA}/collectivex/results/cases.json`, + ]); + expect([...source.files.keys()].toSorted()).toEqual(['cases.json', 'h200-dgxc/a.json']); + expect(new TextDecoder().decode(source.files.get('h200-dgxc/a.json'))).toBe('cached'); + expect(blobCache.has('b1')).toBe(true); + }); + + it('reports a commit without a results directory as not found and oversize files as invalid', async () => { + const missing = vi.fn(() => new Response('', { status: 404 })); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT }, + { fetch: missing as unknown as typeof globalThis.fetch }, + ), + ).rejects.toThrow(`${RESULTS_DIR} does not exist at ${SHA}`); + + const oversize = vi.fn(() => + jsonResponse({ + tree: [{ path: 'a/b.json', type: 'blob', sha: 'x', size: 64 * 1024 * 1024 }], + }), + ); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT }, + { fetch: oversize as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ kind: 'invalid' }); + }); +}); diff --git a/packages/db/src/collectivex/committed-results.ts b/packages/db/src/collectivex/committed-results.ts new file mode 100644 index 000000000..d66b97e98 --- /dev/null +++ b/packages/db/src/collectivex/committed-results.ts @@ -0,0 +1,443 @@ +/** + * CollectiveX results committed to the benchmark repo: `collectivex/results/cases.json` + * (the requested cases) plus one `collectivex/results//.json` per measured + * case. Each InferenceX commit that carries results becomes one stored run, so this + * module pins a ref to a commit, reads exactly those files at it, validates them, and + * turns them into a cx_runs row. Shared by the ingest CLI and the app's ingest route. + */ + +import { + GITHUB_API_BASE, + GITHUB_OWNER, + GITHUB_REPO, + GITHUB_REPO_FULL, +} from '@semianalysisai/inferencex-constants'; + +import type { CollectiveXRunInsert } from '../queries/collectivex'; +import { buildDatasetFromNeutral, buildRunSummary, type CollectiveXNeutralRunMeta } from './reader'; + +export const RESULTS_DIR = 'collectivex/results'; +export const RESULTS_BRANCH = 'main'; +const CASES_FILE = 'cases.json'; +const SWAP_SCHEMA = 'collectivex-swap-blocks-v1'; +// The benchmark repo's own test caps a result at 1 MiB; leave headroom without +// letting one commit exhaust a function's memory. +const MAX_FILE_BYTES = 4 * 1024 * 1024; +const MAX_TOTAL_BYTES = 256 * 1024 * 1024; +const FETCH_ATTEMPTS = 3; +const FETCH_TIMEOUT_MS = 30_000; +const FETCH_CONCURRENCY = 8; +const COMMITS_PER_PAGE = 100; +const SHA = /^[0-9a-f]{40}$/u; +const RESULT_PATH = /^(?[^/]+)\/(?[^/]+)\.json$/u; + +export interface GithubReadOptions { + /** Authenticates API and raw reads; without it GitHub's unauthenticated rate limit applies. */ + token?: string; + fetch?: typeof globalThis.fetch; +} + +export interface ResultsCommit { + sha: string; + /** Committer time (ISO 8601): when the commit landed on its branch. */ + committedAt: string; +} + +/** One commit's files under RESULTS_DIR, keyed by path relative to it. */ +export interface ResultsSource extends ResultsCommit { + files: Map; +} + +export interface CommittedResults extends ResultsCommit { + version: number; + matrix: unknown; + docs: unknown[]; +} + +/** + * `invalid`: the committed files break the results contract. `not-found`: the ref, + * the results directory, or a main-history results commit does not exist. + * `unavailable`: GitHub could not be read. + */ +export type CommittedResultsErrorKind = 'invalid' | 'not-found' | 'unavailable'; + +export class CommittedResultsError extends Error { + readonly kind: CommittedResultsErrorKind; + + constructor(kind: CommittedResultsErrorKind, message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'CommittedResultsError'; + this.kind = kind; + } +} + +function apiHeaders(token?: string): Record { + return { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + +async function request( + url: string, + headers: Record, + fetchImpl: typeof globalThis.fetch, +): Promise { + let failure: unknown; + for (let attempt = 1; attempt <= FETCH_ATTEMPTS; attempt++) { + try { + const response = await fetchImpl(url, { + headers, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (response.ok) return response; + failure = new Error(`${url} returned HTTP ${response.status}`); + if (response.status === 404 || response.status === 422) { + throw new CommittedResultsError('not-found', `${url} returned HTTP ${response.status}`); + } + if (response.status < 500 && response.status !== 429) break; + } catch (error) { + if (error instanceof CommittedResultsError) throw error; + failure = error; + } + if (attempt < FETCH_ATTEMPTS) { + await new Promise((settle) => { + setTimeout(settle, 1000 * attempt); + }); + } + } + throw new CommittedResultsError('unavailable', `Reading ${url} failed`, { cause: failure }); +} + +async function getJson(url: string, options: GithubReadOptions): Promise { + const response = await request(url, apiHeaders(options.token), options.fetch ?? fetch); + return response.json(); +} + +function toResultsCommit(body: unknown): ResultsCommit { + const commit = body as { sha?: unknown; commit?: { committer?: { date?: unknown } } } | null; + const date = commit?.commit?.committer?.date; + if ( + typeof commit?.sha !== 'string' || + !SHA.test(commit.sha) || + typeof date !== 'string' || + !Number.isFinite(Date.parse(date)) + ) { + throw new CommittedResultsError('unavailable', 'GitHub returned a malformed commit'); + } + return { sha: commit.sha, committedAt: date }; +} + +async function resultsCommitsPage( + ref: string, + page: number, + perPage: number, + options: GithubReadOptions, +): Promise { + const query = new URLSearchParams({ + sha: ref, + path: RESULTS_DIR, + per_page: String(perPage), + page: String(page), + }); + const body = await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/commits?${query}`, + options, + ); + if (!Array.isArray(body)) { + throw new CommittedResultsError('unavailable', 'GitHub commit list is not an array'); + } + return body.map(toResultsCommit); +} + +/** Every commit on `branch` that touched RESULTS_DIR, oldest first. */ +export async function listResultsCommits( + branch: string = RESULTS_BRANCH, + options: GithubReadOptions = {}, +): Promise { + const commits: ResultsCommit[] = []; + for (let page = 1; ; page++) { + const batch = await resultsCommitsPage(branch, page, COMMITS_PER_PAGE, options); + commits.push(...batch); + if (batch.length < COMMITS_PER_PAGE) break; + } + return commits.toReversed(); +} + +/** + * Pin a branch, tag, or commit to the newest RESULTS_DIR commit at or before it, + * and require that commit to be on RESULTS_BRANCH. A ref whose own commit did not + * touch the results (for example main's head after an unrelated merge) therefore + * resolves to the run already published for those files instead of a duplicate, + * and results that never landed on main are never published. + */ +export async function resolveResultsCommit( + ref: string, + options: GithubReadOptions = {}, +): Promise { + const [commit] = await resultsCommitsPage(ref, 1, 1, options); + if (!commit) { + throw new CommittedResultsError( + 'not-found', + `No commit at or before ${ref} touched ${RESULTS_DIR}`, + ); + } + const history = await listResultsCommits(RESULTS_BRANCH, options); + if (!history.some((entry) => entry.sha === commit.sha)) { + throw new CommittedResultsError( + 'not-found', + `${commit.sha} (from ${ref}) is not a ${RESULTS_DIR} commit on ${RESULTS_BRANCH}`, + ); + } + return commit; +} + +interface TreeEntry { + path: string; + type: string; + sha: string; + size?: number; +} + +/** + * Read RESULTS_DIR at one commit: list it with the git trees API, then fetch each + * blob from raw.githubusercontent.com, which does not count against the REST rate + * limit. `blobCache` (keyed by blob sha) lets a backfill skip unchanged files. + */ +export async function fetchResultsSource( + commit: ResultsCommit, + options: GithubReadOptions & { blobCache?: Map } = {}, +): Promise { + const fetchImpl = options.fetch ?? fetch; + let tree: { tree?: TreeEntry[]; truncated?: boolean }; + try { + tree = (await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/git/trees/${commit.sha}:${RESULTS_DIR}?recursive=1`, + options, + )) as typeof tree; + } catch (error) { + if (error instanceof CommittedResultsError && error.kind === 'not-found') { + throw new CommittedResultsError( + 'not-found', + `${RESULTS_DIR} does not exist at ${commit.sha}`, + { + cause: error, + }, + ); + } + throw error; + } + if (tree.truncated || !Array.isArray(tree.tree)) { + throw new CommittedResultsError( + 'unavailable', + `GitHub truncated the ${RESULTS_DIR} tree at ${commit.sha}`, + ); + } + const blobs = tree.tree.filter((entry) => entry.type === 'blob' && entry.path.endsWith('.json')); + let total = 0; + for (const blob of blobs) { + const size = blob.size ?? 0; + if (size > MAX_FILE_BYTES) { + throw new CommittedResultsError( + 'invalid', + `${RESULTS_DIR}/${blob.path} exceeds ${MAX_FILE_BYTES} bytes`, + ); + } + total += size; + } + if (total > MAX_TOTAL_BYTES) { + throw new CommittedResultsError( + 'invalid', + `${RESULTS_DIR} at ${commit.sha} exceeds ${MAX_TOTAL_BYTES} bytes`, + ); + } + + const headers: Record = options.token + ? { Authorization: `Bearer ${options.token}` } + : {}; + const files = new Map(); + let next = 0; + const worker = async () => { + while (next < blobs.length) { + const blob = blobs[next++]; + let data = options.blobCache?.get(blob.sha); + if (!data) { + const path = `${RESULTS_DIR}/${blob.path}`.split('/').map(encodeURIComponent).join('/'); + const response = await request( + `https://raw.githubusercontent.com/${GITHUB_REPO_FULL}/${commit.sha}/${path}`, + headers, + fetchImpl, + ); + data = new Uint8Array(await response.arrayBuffer()); + options.blobCache?.set(blob.sha, data); + } + files.set(blob.path, data); + } + }; + await Promise.all(Array.from({ length: Math.min(FETCH_CONCURRENCY, blobs.length) }, worker)); + return { ...commit, files }; +} + +interface ResultDoc { + record_type?: unknown; + schema?: unknown; + case_id?: unknown; + version?: unknown; + identity?: { case_id?: unknown; case_factors?: { sku?: unknown } }; + provenance?: { source_dirty?: unknown }; + runtime?: { sku?: unknown; source_dirty?: unknown }; +} + +/** Identity fields of an EP case attempt or a swap_blocks result; null for anything else. */ +function resultIdentity(doc: ResultDoc | null) { + if (doc?.record_type === 'case-attempt') { + return { + caseId: doc.identity?.case_id, + sku: doc.identity?.case_factors?.sku, + sourceDirty: doc.provenance?.source_dirty, + }; + } + if (doc?.schema === SWAP_SCHEMA) { + return { caseId: doc.case_id, sku: doc.runtime?.sku, sourceDirty: doc.runtime?.source_dirty }; + } + return null; +} + +function parseJson(path: string, data: Uint8Array): unknown { + try { + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(data)); + } catch (error) { + throw new CommittedResultsError('invalid', `${path} is not UTF-8 JSON`, { cause: error }); + } +} + +function runMeta(source: ResultsCommit): CollectiveXNeutralRunMeta { + const seconds = Math.floor(Date.parse(source.committedAt) / 1000); + return { + run_id: String(seconds), + run_attempt: 1, + generated_at: new Date(seconds * 1000).toISOString().replace(/\.\d{3}Z$/u, 'Z'), + conclusion: 'success', + source_sha: source.sha, + }; +} + +/** + * Check one commit's files the way the benchmark repo's own test does: every + * result sits at `/.json` matching its document, answers a + * runnable case in cases.json, was measured from committed code, and carries + * the cases.json version. Every failure names the offending file. + */ +export function validateCommittedResults(source: ResultsSource): CommittedResults { + const casesPath = `${RESULTS_DIR}/${CASES_FILE}`; + const casesBytes = source.files.get(CASES_FILE); + if (!casesBytes) + throw new CommittedResultsError('invalid', `${casesPath} is missing at ${source.sha}`); + const matrix = parseJson(casesPath, casesBytes) as { + version?: unknown; + requested_cases?: { sku?: unknown; disposition?: unknown; case?: { case_id?: unknown } }[]; + }; + if ( + typeof matrix?.version !== 'number' || + !Number.isSafeInteger(matrix.version) || + matrix.version <= 0 || + !Array.isArray(matrix.requested_cases) + ) { + throw new CommittedResultsError('invalid', `${casesPath} has no version and requested_cases`); + } + const { version } = matrix; + const runnable = new Set( + matrix.requested_cases + .filter((entry) => entry?.disposition === 'runnable') + .map((entry) => JSON.stringify([entry.sku, entry.case?.case_id])), + ); + + const results: { path: string; doc: unknown }[] = []; + for (const [file, data] of [...source.files].toSorted(([a], [b]) => a.localeCompare(b))) { + if (file === CASES_FILE) continue; + const path = `${RESULTS_DIR}/${file}`; + const location = RESULT_PATH.exec(file)?.groups; + if (!location) + throw new CommittedResultsError('invalid', `${path} is not at /.json`); + const doc = parseJson(path, data) as ResultDoc | null; + const identity = resultIdentity(doc); + if (!identity) + throw new CommittedResultsError('invalid', `${path} is not a CollectiveX result document`); + if (identity.caseId !== location.caseId || identity.sku !== location.sku) { + throw new CommittedResultsError( + 'invalid', + `${path} holds case ${String(identity.caseId)} on ${String(identity.sku)}; the file must be ${String(identity.sku)}/${String(identity.caseId)}.json`, + ); + } + if (!runnable.has(JSON.stringify([location.sku, location.caseId]))) { + throw new CommittedResultsError('invalid', `${path} is not a runnable case in ${CASES_FILE}`); + } + if (identity.sourceDirty !== false) { + throw new CommittedResultsError( + 'invalid', + `${path} was not measured from committed collectivex/ code`, + ); + } + if (doc?.record_type === 'case-attempt' && Number(doc.version) !== version) { + throw new CommittedResultsError( + 'invalid', + `${path} has version ${String(doc.version)}, ${CASES_FILE} has ${version}`, + ); + } + results.push({ path, doc }); + } + + // Assemble each document alone first so a reader failure names its file. + const meta = runMeta(source); + for (const { path, doc } of results) { + try { + buildDatasetFromNeutral(matrix, [doc], meta); + } catch (error) { + throw new CommittedResultsError( + 'invalid', + `${path}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + } + return { + sha: source.sha, + committedAt: source.committedAt, + version, + matrix, + docs: results.map(({ doc }) => doc), + }; +} + +/** + * The cx_runs row for one results commit. run_id is the commit time in epoch + * seconds, so it grows with main's history and `ORDER BY run_id DESC` still means + * "latest"; each commit is one attempt whose documents are replaced wholesale when + * it is ingested again. + */ +export function committedRun(results: CommittedResults): { + run: CollectiveXRunInsert; + docs: unknown[]; +} { + const meta = runMeta(results); + const summary = buildRunSummary(buildDatasetFromNeutral(results.matrix, results.docs, meta)); + return { + run: { + ...meta, + version: results.version, + source_branch: RESULTS_BRANCH, + matrix: results.matrix, + summary, + }, + docs: results.docs, + }; +} + +/** Fetch, validate, and assemble one main-history results commit into its cx_runs row. */ +export async function loadCommittedRun( + commit: ResultsCommit, + options: GithubReadOptions & { blobCache?: Map } = {}, +): Promise<{ run: CollectiveXRunInsert; docs: unknown[] }> { + return committedRun(validateCommittedResults(await fetchResultsSource(commit, options))); +} diff --git a/packages/db/src/collectivex/reader.test.ts b/packages/db/src/collectivex/reader.test.ts index 07b3e7eaf..1f5321623 100644 --- a/packages/db/src/collectivex/reader.test.ts +++ b/packages/db/src/collectivex/reader.test.ts @@ -238,13 +238,36 @@ describe('CollectiveX artifact assembly', () => { ).toBeNull(); }); - it('rejects malformed and cross-version artifacts', () => { + it('keeps a withheld tail null through the derived rates', () => { + const shard = makeRawShard({ rows: [{}] }); + const row = (shard.measurement as { rows: Record[] }).rows[0]; + row.components.dispatch.percentiles_us.p99 = null; + row.token_rate_at_latency_percentile.p99 = null; + const point = buildDatasetFromNeutral( + makeRawMatrix([requestedOf(shard)]), + [shard], + makeRunMeta(), + ).series[0].points[0]; + const dispatch = point.components.dispatch; + expect(dispatch?.latency_us.p99).toBeNull(); + expect(dispatch?.activation_data_rate_gbps_at_latency_percentile?.p99).toBeNull(); + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p99).toBeNull(); + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p50).toBeCloseTo( + (400000000 / 8 / 417) * 1e-3, + 3, + ); + expect(point.roundtrip_token_rate_at_latency_percentile.p99).toBeNull(); + }); + + it('rejects malformed and cross-version artifacts, naming the mismatched case', () => { expect(() => buildDatasetFromNeutral({}, [], makeRunMeta())).toThrow(/matrix/); const shard = makeRawShard(); shard.version = 2; expect(() => buildDatasetFromNeutral(makeRawMatrix([requestedOf(shard)]), [shard], makeRunMeta()), - ).toThrow(/version/); + ).toThrow( + 'CollectiveX version mismatch: h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16 has version 2, cases.json has 1', + ); }); }); diff --git a/packages/db/src/collectivex/reader.ts b/packages/db/src/collectivex/reader.ts index fd62085ad..89c84ba97 100644 --- a/packages/db/src/collectivex/reader.ts +++ b/packages/db/src/collectivex/reader.ts @@ -1,4 +1,4 @@ -import { readSwapResults, isSwapMatrix } from './swap-reader'; +import { readSwapResults } from './swap-reader'; import type { CollectiveXComponent, @@ -10,11 +10,11 @@ import type { CollectiveXKvRow, CollectiveXMode, CollectiveXOutcome, - CollectiveXPercentiles, CollectiveXPoint, CollectiveXPrecision, CollectiveXRunSummary, CollectiveXSeries, + CollectiveXTailPercentiles, CollectiveXTerminalStatus, } from './types'; @@ -38,13 +38,13 @@ interface RawCase { interface RawComponent { availability: string; - percentiles_us: CollectiveXPercentiles | null; + percentiles_us: CollectiveXTailPercentiles | null; } interface RawRow { tokens_per_rank: number; global_tokens: number; - token_rate_at_latency_percentile: CollectiveXPercentiles; + token_rate_at_latency_percentile: CollectiveXTailPercentiles; components: Record; byte_provenance: Record; wire_byte_provenance?: Record< @@ -115,7 +115,6 @@ function toSupportedVendor(raw: string): CollectiveXVendor | null { } function matrixOf(value: unknown): RawMatrix { - if (isSwapMatrix(value)) return { version: 1, requested_cases: [] }; const matrix = value as RawMatrix; if (!Number.isSafeInteger(matrix?.version) || !Array.isArray(matrix?.requested_cases)) { throw new TypeError('invalid CollectiveX matrix'); @@ -152,10 +151,11 @@ function toMode(raw: string | undefined): CollectiveXMode { function ratesFrom( bytes: number, - latency: CollectiveXPercentiles, + latency: CollectiveXTailPercentiles, divisor = 1, -): CollectiveXPercentiles { - const rate = (us: number) => (bytes / divisor / us) * 1e-3; +): CollectiveXTailPercentiles { + const rate = (us: number | null) => + us !== null && us > 0 ? (bytes / divisor / us) * 1e-3 : null; return { p50: rate(latency.p50), p90: rate(latency.p90), @@ -285,6 +285,11 @@ function isKvCase(kase: RawCase): boolean { return kase.suite === 'kv-transfer'; } +// swap_blocks cases have no EP ladder; their measurements come from swap-reader. +function isSwapCase(kase: RawCase): boolean { + return kase.suite === 'swap-blocks'; +} + function mapKvRow(row: RawKvRow): CollectiveXKvRow { return { kind: row.kind === 'bulk' ? 'bulk' : 'paged', @@ -365,12 +370,17 @@ export function buildDatasetFromNeutral( run: CollectiveXNeutralRunMeta, ): CollectiveXDataset { const matrix = matrixOf(matrixRaw); - const swap = readSwapResults(matrixRaw, docs, run.source_sha); + const swapRequested = matrix.requested_cases.filter((requested) => isSwapCase(requested.case)); + const swap = readSwapResults(matrix.requested_cases, docs); const swapPoints = swap.reduce((count, result) => count + result.points.length, 0); const shards = docs.flatMap((doc) => { const shard = shardOf(doc); if (!shard) return []; - if (Number(shard.version) !== matrix.version) throw new Error('CollectiveX version mismatch'); + if (Number(shard.version) !== matrix.version) { + throw new Error( + `CollectiveX version mismatch: ${shard.identity.case_id} has version ${String(shard.version)}, cases.json has ${matrix.version}`, + ); + } return [shard]; }); const supportedShards = shards.flatMap((shard): SupportedShard[] => { @@ -397,7 +407,7 @@ export function buildDatasetFromNeutral( const coverage: CollectiveXCoverage[] = matrix.requested_cases.flatMap((requested) => { const kase = requested.case; const caseId = kase.case_id; - if (isKvCase(kase) || !caseId || hiddenCaseIds.has(caseId)) return []; + if (isKvCase(kase) || isSwapCase(kase) || !caseId || hiddenCaseIds.has(caseId)) return []; const measured = successful.get(caseId)?.shard; const failed = terminal.get(caseId)?.shard; let outcome: CollectiveXOutcome; @@ -446,7 +456,7 @@ export function buildDatasetFromNeutral( version: matrix.version, run: { ...run, - requested_cases: coverage.length + kv.length + swap.length, + requested_cases: coverage.length + kv.length + swapRequested.length, terminal_cases: coverage.filter((item) => item.points.every((point) => point.terminal_status !== 'pending')) .length + @@ -472,10 +482,10 @@ export function buildDatasetFromNeutral( ...new Set([ ...coverage.map((item) => item.sku), ...kv.map((item) => item.sku), - ...swap.map((item) => item.sku), + ...swapRequested.map((item) => item.sku), ]), ].toSorted(), - swap_requested_cases: swap.length, + swap_requested_cases: swapRequested.length, swap_measured_cases: swap.length, kv_requested_cases: kv.length, kv_measured_cases: kv.filter((item) => item.outcome === 'success').length, @@ -517,19 +527,3 @@ export function buildRunSummary(dataset: CollectiveXDataset): CollectiveXRunSumm }, }; } - -export function isMatrixDoc(doc: unknown): boolean { - if (isSwapMatrix(doc)) return true; - const candidate = doc as { requested_cases?: unknown; include?: unknown } | null; - return ( - Array.isArray(candidate?.requested_cases) && - Array.isArray(candidate?.include) && - matrixVersion(doc) !== null - ); -} - -export function matrixVersion(doc: unknown): number | null { - if (isSwapMatrix(doc)) return 1; - const value = (doc as { version?: unknown } | null)?.version; - return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null; -} diff --git a/packages/db/src/collectivex/swap-reader.test.ts b/packages/db/src/collectivex/swap-reader.test.ts index bc3edb027..8a49f9ba1 100644 --- a/packages/db/src/collectivex/swap-reader.test.ts +++ b/packages/db/src/collectivex/swap-reader.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { buildDatasetFromNeutral, buildRunSummary, isMatrixDoc, matrixVersion } from './reader'; -import { makeSwapDoc, swapMatrix, swapMeta } from './swap-test-fixture'; +import { buildDatasetFromNeutral, buildRunSummary } from './reader'; +import { + makeSwapDoc, + makeSwapRequestedCase, + swapCaseId, + swapMatrix, + swapMeta, +} from './swap-test-fixture'; +import { makeRawMatrix, makeRawShard } from './test-fixture'; -describe('standalone swap_blocks artifacts', () => { +describe('swap_blocks results', () => { it('assembles verified copies through GiB and exposes the run without EP cases', () => { - expect(isMatrixDoc(swapMatrix)).toBe(true); - expect(matrixVersion(swapMatrix)).toBe(1); const dataset = buildDatasetFromNeutral( swapMatrix, [makeSwapDoc(), { log: 'unrelated' }], @@ -32,6 +37,7 @@ describe('standalone swap_blocks artifacts', () => { covered_skus: ['h200-dgxc'], }); }); + it.each(['correctness', 'payload', 'latency', 'samples', 'provenance', 'duplicate'])( 'rejects %s corruption instead of plotting success', (corruption) => { @@ -40,45 +46,104 @@ describe('standalone swap_blocks artifacts', () => { if (corruption === 'payload') doc.cases[0].payload_bytes = 2048; if (corruption === 'latency') doc.cases[0].latency.percentiles_us.p50 = NaN; if (corruption === 'samples') doc.cases[0].latency.sample_count = 0; - if (corruption === 'provenance') doc.runtime.source_sha = 'other'; + if (corruption === 'provenance') (doc.runtime as Record).source_sha = null; if (corruption === 'duplicate') doc.cases.push(doc.cases[0]); expect(() => buildDatasetFromNeutral(swapMatrix, [doc], swapMeta)).toThrow(/invalid/); }, ); - it('does not misclassify arbitrary execution matrices', () => { - const matrix = { include: [{ backend: 'other', sku: 'h200-dgxc' }] }; - expect(isMatrixDoc(matrix)).toBe(false); - expect(matrixVersion(matrix)).toBeNull(); - expect(() => buildDatasetFromNeutral(matrix, [], swapMeta)).toThrow( + + it('keeps the measuring revision instead of requiring the snapshot revision', () => { + const doc = makeSwapDoc(); + doc.runtime.source_sha = 'f'.repeat(40); + const dataset = buildDatasetFromNeutral(swapMatrix, [doc], swapMeta); + expect(dataset.run.source_sha).toBe('abc123'); + expect(dataset.swap_blocks?.[0].runtime.source_sha).toBe('f'.repeat(40)); + }); + + it('rejects the retired execution-only swap matrix', () => { + const matrix = { include: [{ backend: 'swap-blocks', sku: 'h200-dgxc' }] }; + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc()], swapMeta)).toThrow( 'invalid CollectiveX matrix', ); }); }); +describe('swap_blocks cases in a unified cases.json', () => { + it('skips ladder-free swap cases in EP coverage and keeps unanswered ones pending', () => { + const shard = makeRawShard(); + const identity = shard.identity as { + case_id: string; + case_factors: { sku: string; case: Record }; + }; + const ep = makeRawMatrix([ + { + caseId: identity.case_id, + sku: identity.case_factors.sku, + case: identity.case_factors.case, + }, + ]); + const matrix = { + ...ep, + requested_cases: [ + ...(ep.requested_cases as unknown[]), + makeSwapRequestedCase('h200-dgxc', 'contiguous'), + makeSwapRequestedCase('h200-dgxc', 'random'), + makeSwapRequestedCase('mi300x', 'contiguous'), + ], + }; + + const dataset = buildDatasetFromNeutral(matrix, [shard, makeSwapDoc()], swapMeta); + + expect(dataset.coverage.map((row) => row.case_id)).toEqual([identity.case_id]); + expect(dataset.series).toHaveLength(1); + expect(dataset.swap_blocks?.map((result) => result.sku)).toEqual(['h200-dgxc']); + expect(dataset.run).toMatchObject({ + requested_cases: 4, + measured_cases: 2, + terminal_cases: 2, + swap_requested_cases: 3, + swap_measured_cases: 1, + covered_skus: ['h200-dgxc', 'mi300x'], + }); + }); + + it('assembles a snapshot with requested cases and no results', () => { + const dataset = buildDatasetFromNeutral( + { version: 1, requested_cases: [makeSwapRequestedCase()] }, + [], + swapMeta, + ); + expect(dataset.swap_blocks).toEqual([]); + expect(buildRunSummary(dataset)).toMatchObject({ + requested_cases: 1, + measured_cases: 0, + swap_cases: { requested: 1, measured: 0 }, + }); + }); +}); + it('keeps identical measurements from separate GPU pools and rejects ambiguous provenance', () => { const matrix = { - include: [ - { backend: 'swap-blocks', sku: 'h200-dgxc' }, - { backend: 'swap-blocks', sku: 'mi355x' }, - ], + version: 1, + requested_cases: [makeSwapRequestedCase('h200-dgxc'), makeSwapRequestedCase('mi355x')], }; - const first = makeSwapDoc(); - const docs = ['h200-dgxc', 'mi355x'].map((sku) => ({ - ...first, - runtime: { ...first.runtime, sku }, - })); + const docs = ['h200-dgxc', 'mi355x'].map((sku) => makeSwapDoc(sku)); const dataset = buildDatasetFromNeutral(matrix, docs, swapMeta); expect(dataset.swap_blocks?.map((r) => [r.sku, r.points[0].latency_us.p50])).toEqual([ ['h200-dgxc', 2], ['mi355x', 2], ]); expect(new Set(dataset.swap_blocks?.map((r) => r.result_id)).size).toBe(2); - expect(() => buildDatasetFromNeutral(matrix, [first], swapMeta)).toThrow('GPU pool'); - expect(() => - buildDatasetFromNeutral( - matrix, - [{ ...first, runtime: { ...first.runtime, sku: 'unknown' } }], - swapMeta, - ), - ).toThrow('GPU pool'); + + const unlabeled = makeSwapDoc(); + delete (unlabeled.runtime as { sku?: string }).sku; + expect(() => buildDatasetFromNeutral(matrix, [unlabeled], swapMeta)).toThrow('GPU pool'); + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc('unknown')], swapMeta)).toThrow( + 'GPU pool', + ); + const unrequested = { ...makeSwapDoc(), case_id: swapCaseId('h200-dgxc', 'random') }; + expect(() => buildDatasetFromNeutral(matrix, [unrequested], swapMeta)).toThrow('requested case'); + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc(), makeSwapDoc()], swapMeta)).toThrow( + 'requested case', + ); }); diff --git a/packages/db/src/collectivex/swap-reader.ts b/packages/db/src/collectivex/swap-reader.ts index 2f6c17590..2067420a4 100644 --- a/packages/db/src/collectivex/swap-reader.ts +++ b/packages/db/src/collectivex/swap-reader.ts @@ -1,16 +1,9 @@ import type { CollectiveXPercentiles, CollectiveXSwapPoint, CollectiveXSwapResult } from './types'; -export function isSwapMatrix(value: unknown): boolean { - const matrix = value as { - version?: unknown; - include?: { backend?: unknown; sku?: unknown }[]; - } | null; - return ( - matrix?.version === undefined && - Array.isArray(matrix?.include) && - matrix.include.length > 0 && - matrix.include.every((cell) => cell?.backend === 'swap-blocks' && typeof cell?.sku === 'string') - ); +/** The cases.json entries this reader needs: swap_blocks cases carry `suite: 'swap-blocks'`. */ +export interface SwapRequestedCase { + sku: string; + case: { suite?: string; case_id?: string }; } function positive(value: unknown): value is number { @@ -32,16 +25,23 @@ function percentiles(value: unknown): value is CollectiveXPercentiles { } export function readSwapResults( - matrix: unknown, + requestedCases: readonly SwapRequestedCase[], docs: unknown[], - sourceSha: string, ): CollectiveXSwapResult[] { - if (!isSwapMatrix(matrix)) return []; - const skus = (matrix as { include: { sku: string }[] }).include.map((cell) => cell.sku); + // Each result must answer one requested case of its own GPU pool, exactly once. + const requested = new Map>(); + for (const entry of requestedCases) { + if (entry.case.suite !== 'swap-blocks' || !entry.case.case_id) continue; + const caseIds = requested.get(entry.sku) ?? new Set(); + caseIds.add(entry.case.case_id); + requested.set(entry.sku, caseIds); + } + const answered = new Set(); const seen = new Set(); return docs.flatMap((doc) => { const raw = doc as { schema?: string; + case_id?: string; operation?: string; timing?: string; warmup: number; @@ -66,15 +66,20 @@ export function readSwapResults( !Number.isSafeInteger(raw.warmup) || raw.warmup < 0 || !raw.runtime || - raw.runtime.source_sha !== sourceSha || - !['device', 'torch', 'vllm', 'image'].every( + !['device', 'torch', 'vllm', 'image', 'source_sha'].every( (key) => typeof raw.runtime[key as keyof typeof raw.runtime] === 'string', ) ) { throw new TypeError('invalid swap_blocks artifact or provenance'); } - const sku = raw.runtime.sku ?? (skus.length === 1 ? skus[0] : undefined); - if (!sku || !skus.includes(sku)) throw new TypeError('invalid swap_blocks GPU pool'); + const { sku } = raw.runtime; + const caseIds = sku ? requested.get(sku) : undefined; + if (!sku || !caseIds) throw new TypeError('invalid swap_blocks GPU pool'); + const caseKey = JSON.stringify([sku, raw.case_id]); + if (!raw.case_id || !caseIds.has(raw.case_id) || answered.has(caseKey)) { + throw new TypeError('swap_blocks result does not answer one requested case'); + } + answered.add(caseKey); const points = raw.cases.map((row): CollectiveXSwapPoint => { const key = JSON.stringify([ sku, diff --git a/packages/db/src/collectivex/swap-test-fixture.ts b/packages/db/src/collectivex/swap-test-fixture.ts index 146e1a11a..3d04eccdf 100644 --- a/packages/db/src/collectivex/swap-test-fixture.ts +++ b/packages/db/src/collectivex/swap-test-fixture.ts @@ -1,4 +1,36 @@ -export const swapMatrix = { include: [{ backend: 'swap-blocks', sku: 'h200-dgxc' }] }; +export function swapCaseId(sku: string, layout = 'contiguous'): string { + return `${sku}-swap-blocks-smoke-${layout}`; +} + +/** One cases.json entry: the swap_blocks suite requests one case per GPU pool and layout. */ +export function makeSwapRequestedCase(sku = 'h200-dgxc', layout = 'contiguous') { + return { + sku, + disposition: 'runnable' as const, + reason: null, + detail: null, + case: { + suite: 'swap-blocks', + backend: 'swap-blocks', + case_id: swapCaseId(sku, layout), + profile: 'smoke', + layout, + directions: 'h2d d2h d2d', + block_bytes: '1024 1073741824', + num_blocks: '1 4', + max_payload_bytes: 1073741824, + warmup: 4, + iterations: 20, + seed: 0, + device: 0, + nodes: 1, + gpus_per_node: 1, + scale_up_domain: 1, + }, + }; +} + +export const swapMatrix = { version: 1, requested_cases: [makeSwapRequestedCase()] }; export const swapMeta = { run_id: '170', run_attempt: 1, @@ -6,14 +38,18 @@ export const swapMeta = { conclusion: 'success', source_sha: 'abc123', }; -export function makeSwapDoc() { +export function makeSwapDoc(sku = 'h200-dgxc') { return { schema: 'collectivex-swap-blocks-v1', + case_id: swapCaseId(sku), + profile: 'smoke', + generated_at: '2026-09-16T19:58:41.123456+08:00', operation: 'vllm._custom_ops.swap_blocks', timing: 'drained-wall-clock-including-submission-and-synchronization', warmup: 4, iterations: 20, runtime: { + sku, device: 'NVIDIA H200', torch: '2.11', vllm: '0.25.1', @@ -21,6 +57,7 @@ export function makeSwapDoc() { hip: null, image: 'vllm/vllm-openai:v0.25.1', source_sha: 'abc123', + source_dirty: false, }, selection: { max_payload_bytes: 1073741824, diff --git a/packages/db/src/collectivex/test-fixture.ts b/packages/db/src/collectivex/test-fixture.ts index b8fd56c6f..f26e9e6de 100644 --- a/packages/db/src/collectivex/test-fixture.ts +++ b/packages/db/src/collectivex/test-fixture.ts @@ -100,6 +100,7 @@ function makeRawCase(options: ShardOverrides, caseId: string): Json { const phase = options.phase === 'prefill' ? 'prefill' : 'decode'; return { case_id: caseId, + suite: 'ep-core', backend: options.backend ?? 'deepep-v2', ep: options.ep ?? 8, gpus_per_node: options.gpusPerNode ?? 8, @@ -171,10 +172,10 @@ function requestedFromShard(shard: Json): RequestedCaseSpec { }; } +/** The shape of the benchmark repo's committed `collectivex/results/cases.json`. */ export function makeRawMatrix(requested: RequestedCaseSpec[], version = 1): Json { return { version, - include: [], requested_cases: requested.map((entry) => ({ case: entry.case, sku: entry.sku, diff --git a/packages/db/src/collectivex/types.ts b/packages/db/src/collectivex/types.ts index 2d10a90fa..36b163ed8 100644 --- a/packages/db/src/collectivex/types.ts +++ b/packages/db/src/collectivex/types.ts @@ -1,6 +1,6 @@ /** - * CollectiveX neutral contract — the version-tagged shape shared by the sweep - * artifacts' reader, the ingest script, and the dashboard frontend. + * CollectiveX neutral contract — the version-tagged shape shared by the + * committed-results reader, the Neon ingest, and the dashboard frontend. * * This module is pure TypeScript (no Node/DB imports) so it is safe to import * from client components. UI-only types (chart points, axis modes, display @@ -34,10 +34,13 @@ export type CollectiveXOutcome = export type CollectiveXTerminalStatus = Exclude | 'measured'; export type CollectiveXPercentiles = Record; +// EP rows: null where the harness withheld a tail because too many of its samples were misaligned +// or host-stalled. p50 is always published. +export type CollectiveXTailPercentiles = Record; export interface CollectiveXComponent { - latency_us: CollectiveXPercentiles; - activation_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + latency_us: CollectiveXTailPercentiles; + activation_data_rate_gbps_at_latency_percentile: CollectiveXTailPercentiles | null; /** * Per-GPU bandwidth over the FULL logical payload (activation bytes plus any * FP8 scale bytes), i.e. `total_logical_bytes / ep_size / latency`. Distinct @@ -45,7 +48,7 @@ export interface CollectiveXComponent { * and excludes scale bytes. Null when the component carries no byte * provenance (e.g. an unavailable component). */ - payload_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + payload_data_rate_gbps_at_latency_percentile: CollectiveXTailPercentiles | null; /** * Aggregate total logical payload bytes for this component at this point * (the numerator behind `payload_data_rate_*`). Carried raw so a consumer can @@ -59,7 +62,7 @@ export interface CollectiveXPoint { tokens_per_rank: number; global_tokens: number; components: Record; - roundtrip_token_rate_at_latency_percentile: CollectiveXPercentiles; + roundtrip_token_rate_at_latency_percentile: CollectiveXTailPercentiles; } export interface CollectiveXTopology { diff --git a/packages/db/src/ingest-collectivex.test.ts b/packages/db/src/ingest-collectivex.test.ts index f93aaa2ba..d4a2a19d5 100644 --- a/packages/db/src/ingest-collectivex.test.ts +++ b/packages/db/src/ingest-collectivex.test.ts @@ -1,124 +1,98 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; - import { afterEach, describe, expect, it, vi } from 'vitest'; +import type * as CommittedResults from './collectivex/committed-results'; + const mocks = vi.hoisted(() => ({ - fetchRunMeta: vi.fn(), - buildDataset: vi.fn(), - begin: vi.fn(), -})); -vi.mock('./lib/github-artifacts', () => ({ - fetchRunMeta: mocks.fetchRunMeta, - listRunArtifacts: () => [], - downloadArtifact: () => {}, + loadCommittedRun: vi.fn(), + upsert: vi.fn(), })); -vi.mock('./collectivex/reader', () => ({ - isMatrixDoc: () => true, - matrixVersion: () => 1, - buildDatasetFromNeutral: mocks.buildDataset, - buildRunSummary: () => ({ requested_cases: 0, measured_cases: 0, requested_points: 0 }), -})); -vi.mock('./etl/db-utils', () => ({ - createAdminSql: () => ({ begin: mocks.begin, end: () => Promise.resolve() }), +vi.mock('./collectivex/committed-results', async (importOriginal) => ({ + ...(await importOriginal()), + loadCommittedRun: mocks.loadCommittedRun, })); +vi.mock('./queries/collectivex', () => ({ upsertCollectiveXRun: mocks.upsert })); + +import { ingestCommits, parseIngestArgs } from './ingest-collectivex'; +import type { DbClient } from './connection.js'; -const RUN_ID = '170'; -const originalArgv = process.argv; -const originalExitCode = process.exitCode; +const sql = (() => Promise.resolve([])) as unknown as DbClient; afterEach(() => { - process.argv = originalArgv; - process.exitCode = originalExitCode; - vi.unstubAllEnvs(); vi.restoreAllMocks(); vi.clearAllMocks(); - vi.resetModules(); }); -// The script runs main() on import, so it must load after argv and the environment are set. -async function runCli(argv: string[]): Promise { - vi.spyOn(console, 'log').mockImplementation(() => {}); - vi.spyOn(console, 'error').mockImplementation(() => {}); - process.argv = ['bun', 'ingest-collectivex.ts', ...argv]; - await import('./ingest-collectivex'); -} - -describe('download-mode repository selection', () => { - // A non-sweep workflow path stops the run right after the repository lookup. - const otherWorkflow = { path: '.github/workflows/other.yml' }; - - it.each([ - { - name: 'the [repo] argument', - argv: [ - '--download', - `https://github.com/example-org/url-repo/actions/runs/${RUN_ID}`, - 'example-org/arg-repo', - ], - expected: 'example-org/arg-repo', - }, - { - name: 'a full run URL', - argv: ['--download', `https://github.com/example-org/url-repo/actions/runs/${RUN_ID}`], - expected: 'example-org/url-repo', - }, - { name: 'INGEST_REPO', argv: ['--download', RUN_ID], expected: 'example-org/env-repo' }, - ])('prefers $name', async ({ argv, expected }) => { - vi.stubEnv('INGEST_REPO', 'example-org/env-repo'); - mocks.fetchRunMeta.mockReturnValue(otherWorkflow); - - await runCli(argv); - - await vi.waitFor(() => expect(mocks.fetchRunMeta).toHaveBeenCalledWith(expected, RUN_ID)); +describe('parseIngestArgs', () => { + it('defaults to main', () => { + expect(parseIngestArgs([])).toEqual({ ref: 'main', backfill: false }); }); - it('exits when no argument, run URL, or INGEST_REPO names the repository', async () => { - vi.stubEnv('INGEST_REPO', ''); - const exit = vi.spyOn(process, 'exit').mockImplementation(() => { - throw new Error('process.exit'); + it('takes one commit from --sha or --ref', () => { + expect(parseIngestArgs(['--sha', 'a'.repeat(40)])).toEqual({ + ref: 'a'.repeat(40), + backfill: false, }); + expect(parseIngestArgs(['--ref', 'v1.2', '--yes'])).toEqual({ ref: 'v1.2', backfill: false }); + }); - await expect(runCli(['--download', RUN_ID])).rejects.toThrow('process.exit'); + it('walks main for --backfill', () => { + expect(parseIngestArgs(['--backfill', '-y', '--no-ssl'])).toEqual({ + ref: 'main', + backfill: true, + }); + }); - expect(exit).toHaveBeenCalledWith(1); - expect(console.error).toHaveBeenCalledWith('INGEST_REPO is not set'); - expect(mocks.fetchRunMeta).not.toHaveBeenCalled(); + it('rejects conflicting, malformed, and unknown arguments', () => { + expect(() => parseIngestArgs(['--sha', 'a'.repeat(40), '--ref', 'main'])).toThrow( + /mutually exclusive/u, + ); + expect(() => parseIngestArgs(['--backfill', '--ref', 'main'])).toThrow(/--backfill/u); + expect(() => parseIngestArgs(['--sha', 'not-a-sha'])).toThrow(/Invalid commit sha/u); + expect(() => parseIngestArgs(['12345'])).toThrow(); }); }); -describe('recorded source repository', () => { - it.each([ - { repo: 'example-org/bench', recorded: 'example-org/bench' }, - { repo: 'semianalysisai/inferencex', recorded: undefined }, - ])('records $repo as $recorded', async ({ repo, recorded }) => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cx-ingest-test-')); - try { - fs.mkdirSync(path.join(dir, `cxsweep-matrix-${RUN_ID}`)); - fs.writeFileSync(path.join(dir, `cxsweep-matrix-${RUN_ID}`, 'matrix.json'), '{}'); - vi.stubEnv('INGEST_RUN_ID', RUN_ID); - vi.stubEnv('INGEST_ARTIFACTS_PATH', dir); - vi.stubEnv('INGEST_REPO', repo); - mocks.fetchRunMeta.mockReturnValue({ - path: '.github/workflows/collectivex-sweep.yml', - name: 'CollectiveX sweep', - run_attempt: 1, - head_sha: 'abc', - head_branch: 'main', - conclusion: 'success', - created_at: '2026-08-01T00:00:00Z', - }); - mocks.buildDataset.mockReturnValue({}); - mocks.begin.mockResolvedValue(undefined); - - await runCli([]); - - await vi.waitFor(() => expect(mocks.begin).toHaveBeenCalled()); - const meta = mocks.buildDataset.mock.calls[0][2] as Record; - expect(meta.source_repository).toBe(recorded); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } +function runFor(sha: string) { + return { + run: { run_id: sha.slice(0, 3), summary: { measured_cases: 1, requested_cases: 2 } }, + docs: [{}], + }; +} + +describe('ingestCommits', () => { + const commits = [ + { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z' }, + { sha: 'b'.repeat(40), committedAt: '2026-10-16T00:00:00Z' }, + ]; + + it('stores commits in order and shares one blob cache across them', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValueOnce('inserted').mockResolvedValueOnce('replaced'); + + await ingestCommits(sql, commits, 'token'); + + expect(mocks.loadCommittedRun.mock.calls.map(([commit]) => commit.sha)).toEqual( + commits.map((commit) => commit.sha), + ); + const [first, second] = mocks.loadCommittedRun.mock.calls.map(([, options]) => options); + expect(first.token).toBe('token'); + expect(first.blobCache).toBe(second.blobCache); + expect(mocks.upsert).toHaveBeenCalledTimes(2); + }); + + it('stops at a run id held by a different commit', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('conflict'); + + await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow( + /already holds a different commit/u, + ); + expect(mocks.loadCommittedRun).toHaveBeenCalledTimes(1); }); }); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts index 69cf78028..724c08f43 100644 --- a/packages/db/src/ingest-collectivex.ts +++ b/packages/db/src/ingest-collectivex.ts @@ -1,165 +1,98 @@ -import fs from 'fs'; -import os from 'os'; -import path from 'path'; +import { parseArgs } from 'node:util'; -import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; - -import { hasNoSslFlag, requireIngestRepo } from './cli-utils'; +import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils'; import { createAdminSql } from './etl/db-utils'; import { - downloadArtifact, - fetchRunMeta, - listRunArtifacts, - type RunMeta, -} from './lib/github-artifacts'; -import { repositoryFromRunUrl } from './lib/runtime-metadata-artifacts'; -import { matrixArtifactName, selectShardArtifactNames } from './collectivex/artifact-selection'; -import { - buildDatasetFromNeutral, - buildRunSummary, - isMatrixDoc, - matrixVersion, - type CollectiveXNeutralRunMeta, -} from './collectivex/reader'; - -const SWEEP_WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; -const DOCS_INSERT_CHUNK = 200; - -const isDownloadMode = process.argv[2] === '--download'; - -let artifactsDir: string; -let runIdStr: string; -let REPO: string; -let tempDir: string | null = null; + listResultsCommits, + loadCommittedRun, + RESULTS_BRANCH, + RESULTS_DIR, + resolveResultsCommit, + type ResultsCommit, +} from './collectivex/committed-results'; +import { upsertCollectiveXRun } from './queries/collectivex'; +import type { DbClient } from './connection.js'; + +const USAGE = + 'Usage: bun run admin:db:ingest:collectivex [--sha | --ref ] [--backfill] [--yes] [--no-ssl]'; + +export interface IngestCollectiveXArgs { + ref: string; + backfill: boolean; +} -if (isDownloadMode) { - const args = process.argv.slice(3).filter((a) => !a.startsWith('--')); - const input = args[0]; - if (!input) { - console.error('Usage: bun run admin:db:ingest:collectivex [repo]'); - process.exit(1); +/** --sha and --ref name one commit; --backfill walks every results commit on main instead. */ +export function parseIngestArgs(argv: string[]): IngestCollectiveXArgs { + const { values } = parseArgs({ + args: argv, + options: { + sha: { type: 'string' }, + ref: { type: 'string' }, + backfill: { type: 'boolean', default: false }, + yes: { type: 'boolean', short: 'y', default: false }, + 'no-ssl': { type: 'boolean', default: false }, + }, + strict: true, + allowPositionals: false, + }); + if (values.sha !== undefined && values.ref !== undefined) { + throw new Error(`--sha and --ref are mutually exclusive\n${USAGE}`); } - const match = input.match(/\/runs\/(?\d+)/u); - const parsedId = match ? match.groups!.runId : /^\d+$/u.test(input) ? input : null; - if (!parsedId) { - console.error(`Could not parse run ID from: ${input}`); - process.exit(1); + if (values.backfill && (values.sha !== undefined || values.ref !== undefined)) { + throw new Error(`--backfill reads every commit on ${RESULTS_BRANCH}\n${USAGE}`); } - runIdStr = parsedId; - REPO = args[1] ?? repositoryFromRunUrl(input) ?? requireIngestRepo(); - tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cx-ingest-')); - artifactsDir = tempDir; -} else { - const runId = process.env.INGEST_RUN_ID; - const artifactsPath = process.env.INGEST_ARTIFACTS_PATH; - if (!runId || !artifactsPath) { - console.error('INGEST_RUN_ID and INGEST_ARTIFACTS_PATH are required without --download'); - process.exit(1); + if (values.sha !== undefined && !/^[0-9a-f]{7,40}$/u.test(values.sha)) { + throw new Error(`Invalid commit sha: ${values.sha}`); } - runIdStr = runId; - REPO = requireIngestRepo(); - artifactsDir = artifactsPath; + return { ref: values.sha ?? values.ref ?? RESULTS_BRANCH, backfill: values.backfill }; } -if (!/^[\w.-]+\/[\w.-]+$/u.test(REPO)) { - console.error(`Invalid repo slug: ${REPO}`); - process.exit(1); -} -if (!/^\d+$/u.test(runIdStr)) { - console.error(`Invalid run id: ${runIdStr}`); - process.exit(1); -} - -function readDocsDir(dir: string): unknown[] { - const docs: unknown[] = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - docs.push(...readDocsDir(full)); - continue; +/** Ingest each commit in order; one failed commit stops the walk before later ones. */ +export async function ingestCommits( + sql: DbClient, + commits: readonly ResultsCommit[], + token: string | undefined, +): Promise { + const blobCache = new Map(); + for (const commit of commits) { + const { run, docs } = await loadCommittedRun(commit, { token, blobCache }); + const result = await upsertCollectiveXRun(sql, run, docs); + if (result === 'conflict') { + throw new Error(`run ${run.run_id} already holds a different commit than ${commit.sha}`); } - if (!entry.name.endsWith('.json')) continue; - docs.push(JSON.parse(fs.readFileSync(full, 'utf8'))); + console.log( + ` ${result} run ${run.run_id} @ ${commit.sha.slice(0, 12)} (${commit.committedAt}): ` + + `${run.summary.measured_cases}/${run.summary.requested_cases} cases measured, ${docs.length} docs`, + ); } - return docs; -} - -function runGeneratedAt(run: RunMeta): string { - return run.updated_at || run.run_started_at || run.created_at || ''; } async function main(): Promise { - console.log(`=== db:ingest:collectivex — run ${runIdStr} (${REPO}) ===`); - - const run = fetchRunMeta(REPO, runIdStr); - if (run.path !== SWEEP_WORKFLOW_PATH) { - throw new Error(`run ${runIdStr} is not a CollectiveX sweep (workflow: ${run.path})`); + const args = parseIngestArgs(process.argv.slice(2)); + const token = process.env.GITHUB_TOKEN || undefined; + console.log('=== db:ingest:collectivex ==='); + + const commits = args.backfill + ? await listResultsCommits(RESULTS_BRANCH, { token }) + : [await resolveResultsCommit(args.ref, { token })]; + if (commits.length === 0) { + console.log(`No commit on ${RESULTS_BRANCH} touched ${RESULTS_DIR}.`); + return; } - const generatedAt = runGeneratedAt(run); - if (!generatedAt) throw new Error(`run ${runIdStr} is missing a timestamp`); console.log( - ` workflow: ${run.name} branch: ${run.head_branch ?? '?'} attempt: ${run.run_attempt} conclusion: ${run.conclusion ?? 'in-progress'}`, + args.backfill + ? `This will store ${commits.length} ${RESULTS_DIR} commits from ${RESULTS_BRANCH}, oldest first,` + : `This will store ${RESULTS_DIR} at ${commits[0].sha} (resolved from ${args.ref}),`, ); - - const matrixDirName = matrixArtifactName(runIdStr); - - if (isDownloadMode) { - const artifacts = listRunArtifacts(REPO, runIdStr); - const byName = new Map(); - for (const artifact of artifacts) { - const existing = byName.get(artifact.name); - if (!existing || artifact.created_at > existing.created_at) { - byName.set(artifact.name, artifact); - } - } - const wanted = [ - matrixDirName, - ...selectShardArtifactNames([...byName.keys()], runIdStr, run.run_attempt), - ]; - for (const name of wanted) { - const artifact = byName.get(name); - if (!artifact) continue; - console.log(` downloading ${name}`); - downloadArtifact(artifact, artifactsDir); + console.log('replacing any run already stored for the same commit.\n'); + if (!hasYesFlag()) { + const ok = await confirm('Continue? (y/N) '); + if (!ok) { + console.log('Aborted.'); + return; } } - const availableDirs = fs - .readdirSync(artifactsDir, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name); - - if (!availableDirs.includes(matrixDirName)) { - throw new Error(`run ${runIdStr} has no ${matrixDirName} artifact`); - } - const matrixDocs = readDocsDir(path.join(artifactsDir, matrixDirName)).filter((doc) => - isMatrixDoc(doc), - ); - if (matrixDocs.length !== 1) { - throw new Error(`expected exactly one matrix document, found ${matrixDocs.length}`); - } - const matrix = matrixDocs[0]; - const version = matrixVersion(matrix); - if (version === null) throw new Error('matrix document has no valid version tag'); - - const shardDirs = selectShardArtifactNames(availableDirs, runIdStr, run.run_attempt); - console.log(` matrix version: ${version} shard artifacts: ${shardDirs.length}`); - const docs = shardDirs.flatMap((dir) => readDocsDir(path.join(artifactsDir, dir))); - - const meta: CollectiveXNeutralRunMeta = { - run_id: runIdStr, - run_attempt: run.run_attempt, - generated_at: generatedAt, - conclusion: run.conclusion, - source_sha: run.head_sha, - ...(REPO.toLowerCase() === GITHUB_REPO_FULL.toLowerCase() ? {} : { source_repository: REPO }), - }; - const dataset = buildDatasetFromNeutral(matrix, docs, meta); - const summary = buildRunSummary(dataset); - console.log( - ` assembled: ${summary.requested_cases} cases (${summary.measured_cases} measured), ${summary.requested_points} points`, - ); - const sql = createAdminSql({ envVar: 'DATABASE_COLLECTIVEX_WRITE_URL', noSsl: hasNoSslFlag(), @@ -167,37 +100,21 @@ async function main(): Promise { onnotice: () => {}, }); try { - await sql.begin(async (tx) => { - await tx`DELETE FROM cx_runs WHERE run_id = ${runIdStr}`; - await tx` - INSERT INTO cx_runs - (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) - VALUES - (${runIdStr}, ${run.run_attempt}, ${version}, ${generatedAt}, ${run.head_sha}, - ${run.head_branch}, ${run.conclusion}, ${matrix as never}::jsonb, - ${summary as never}::jsonb) - `; - for (let i = 0; i < docs.length; i += DOCS_INSERT_CHUNK) { - const chunk = docs.slice(i, i + DOCS_INSERT_CHUNK).map((doc) => JSON.stringify(doc)); - await tx` - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT ${runIdStr}, ${run.run_attempt}, unnest(${tx.array(chunk)}::jsonb[]) - `; - } - }); - console.log(` stored run ${runIdStr} (version ${version}, ${docs.length} docs)`); + await ingestCommits( + ((strings: TemplateStringsArray, ...values: unknown[]) => + sql(strings, ...(values as never[]))) as DbClient, + commits, + token, + ); } finally { await sql.end(); } - console.log('=== db:ingest:collectivex complete ==='); } -main() - .catch((error) => { +if (import.meta.main) { + main().catch((error: unknown) => { console.error('db:ingest:collectivex failed:', error); process.exitCode = 1; - }) - .finally(() => { - if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); }); +} diff --git a/packages/db/src/queries/collectivex.test.ts b/packages/db/src/queries/collectivex.test.ts index ddeb8eb56..fbf07d09a 100644 --- a/packages/db/src/queries/collectivex.test.ts +++ b/packages/db/src/queries/collectivex.test.ts @@ -4,13 +4,10 @@ import { makeRawMatrix, makeRawShard, makeRunMeta } from '../collectivex/test-fi import type { DbClient } from '../connection.js'; import { collectiveXDatasetFromRow, - deleteCollectiveXRun, getCollectiveXRun, - getCollectiveXRunStates, getLatestCollectiveXRun, - insertCollectiveXRun, listCollectiveXRuns, - refreshCollectiveXRunAttempt, + upsertCollectiveXRun, type CollectiveXRunInsert, type CollectiveXRunRow, } from './collectivex'; @@ -30,17 +27,17 @@ function fakeSql(rowsQueue: Record[][]) { } const runInsert: CollectiveXRunInsert = { - run_id: '160', - run_attempt: 2, + run_id: '1783513200', + run_attempt: 1, version: 1, generated_at: '2026-07-08T12:20:00Z', source_sha: 'a'.repeat(40), - source_branch: 'collectivex', + source_branch: 'main', conclusion: 'success', - matrix: { version: 1, requested_cases: [], include: [] }, + matrix: { version: 1, requested_cases: [] }, summary: { - run_id: '160', - run_attempt: 2, + run_id: '1783513200', + run_attempt: 1, generated_at: '2026-07-08T12:20:00Z', conclusion: 'success', covered_skus: [], @@ -52,36 +49,14 @@ const runInsert: CollectiveXRunInsert = { }, }; -describe('getCollectiveXRunStates', () => { - it('maps deleted flags to tombstone states with version and attempt', async () => { - const { sql } = fakeSql([ - [ - { run_id: '160', version: 1, run_attempt: 2, deleted: false }, - { run_id: '161', version: 2, run_attempt: 1, deleted: true }, - ], - ]); - const states = await getCollectiveXRunStates(sql, ['160', '161']); - expect(states).toEqual({ - '160': { state: 'live', version: 1, run_attempt: 2 }, - '161': { state: 'deleted', version: 2, run_attempt: 1 }, - }); - }); - - it('skips the query entirely for an empty id list', async () => { - const { sql, calls } = fakeSql([]); - expect(await getCollectiveXRunStates(sql, [])).toEqual({}); - expect(calls).toHaveLength(0); - }); -}); - describe('read queries', () => { - it('excludes tombstoned and unsupported-vendor-only rows', async () => { + it('excludes unsupported-vendor-only rows and orders newest commit first', async () => { const { sql, calls } = fakeSql([[], [], []]); await getLatestCollectiveXRun(sql, 1); await getCollectiveXRun(sql, 1, '160'); await listCollectiveXRuns(sql, 1); for (const call of calls) { - expect(call.text).toContain('deleted_at IS NULL'); + expect(call.text).not.toContain('deleted_at'); expect(call.text).toContain("summary->>'requested_cases'"); } expect(calls[0].text).toContain('ORDER BY r.run_id DESC'); @@ -99,64 +74,40 @@ describe('read queries', () => { }); }); -describe('insertCollectiveXRun', () => { - it('binds raw objects (never pre-stringified JSON) and reports insertion', async () => { - const { sql, calls } = fakeSql([[{ runs_inserted: 1 }]]); +describe('upsertCollectiveXRun', () => { + it('binds raw objects (never pre-stringified JSON) and reports a first insert', async () => { + const { sql, calls } = fakeSql([[{ inserted: true }]]); const docs = [{ record_type: 'case-attempt' }]; - await expect(insertCollectiveXRun(sql, runInsert, docs)).resolves.toBe(true); + await expect(upsertCollectiveXRun(sql, runInsert, docs)).resolves.toBe('inserted'); - expect(calls[0].text).toContain('ON CONFLICT (run_id) DO NOTHING'); + expect(calls).toHaveLength(1); expect(calls[0].values).toContainEqual(runInsert.matrix); + expect(calls[0].values).toContainEqual(runInsert.summary); expect(calls[0].values).toContainEqual({ docs }); expect( calls[0].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), ).toBe(true); }); - it('reports a conflict no-op as not inserted', async () => { - const { sql } = fakeSql([[{ runs_inserted: 0 }]]); - await expect(insertCollectiveXRun(sql, runInsert, [])).resolves.toBe(false); + it('replaces the row and its documents when the same commit is ingested again', async () => { + const { sql, calls } = fakeSql([[{ inserted: false }]]); + await expect(upsertCollectiveXRun(sql, runInsert, [])).resolves.toBe('replaced'); + const { text } = calls[0]; + expect(text).toContain('ON CONFLICT (run_id) DO UPDATE'); + expect(text).toContain('WHERE cx_runs.source_sha = EXCLUDED.source_sha'); + expect(text).toContain('DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted)'); + expect(text).toContain('INSERT INTO cx_run_docs'); }); -}); -describe('refreshCollectiveXRunAttempt', () => { - it('guards on a strictly newer attempt and reports replacement', async () => { - const { sql, calls } = fakeSql([[{ runs_updated: 1 }]]); - await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(true); - expect(calls[0].text).toContain('run_attempt < '); - expect(calls[0].text).toContain('deleted_at IS NULL'); - expect(calls[0].text).toContain('FOR UPDATE'); + it('writes nothing when the run id already belongs to another commit', async () => { + const { sql } = fakeSql([[]]); + await expect(upsertCollectiveXRun(sql, runInsert, [{}])).resolves.toBe('conflict'); }); - it('reports a guarded no-op (older or equal attempt, or tombstoned) as false', async () => { - const { sql } = fakeSql([[{ runs_updated: 0 }]]); - await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(false); - }); -}); - -describe('deleteCollectiveXRun', () => { - it('tombstones the run and frees its documents in one atomic statement', async () => { - const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); - await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(true); - expect(calls).toHaveLength(1); - expect(calls[0].text).toContain('SET deleted_at = now()'); - expect(calls[0].text).toContain('deleted_at IS NULL'); - expect(calls[0].text).toContain('DELETE FROM cx_run_docs'); - }); - - it('reports absent or already-tombstoned runs as false', async () => { - const { sql } = fakeSql([[{ runs_deleted: 0 }]]); - await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(false); - }); - - it('cannot reach anything but the one run it is given', async () => { - const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); - await deleteCollectiveXRun(sql, '160'); - expect(calls[0].values).toEqual(['160']); - expect(calls[0].text).toContain( - 'DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned)', - ); + it('touches only the CollectiveX tables and never drops data wholesale', async () => { + const { sql, calls } = fakeSql([[{ inserted: true }]]); + await upsertCollectiveXRun(sql, runInsert, []); expect(new Set([...calls[0].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( new Set(['cx_runs', 'cx_run_docs']), ); @@ -178,7 +129,7 @@ describe('collectiveXDatasetFromRow', () => { version: 1, generated_at: meta.generated_at, source_sha: meta.source_sha, - source_branch: 'collectivex', + source_branch: 'main', source_repository: null, conclusion: meta.conclusion, matrix: makeRawMatrix([ diff --git a/packages/db/src/queries/collectivex.ts b/packages/db/src/queries/collectivex.ts index 756fb77dd..81ff58dc3 100644 --- a/packages/db/src/queries/collectivex.ts +++ b/packages/db/src/queries/collectivex.ts @@ -17,7 +17,7 @@ export interface CollectiveXRunRow { docs: unknown[]; } -/** Row metadata + raw docs as written by the lazy ingest. */ +/** One results commit's cx_runs row (see collectivex/committed-results.ts). */ export interface CollectiveXRunInsert { run_id: string; run_attempt: number; @@ -30,34 +30,18 @@ export interface CollectiveXRunInsert { summary: CollectiveXRunSummary; } -/** - * Known-run state. Runs are discovered lazily from GitHub on read, so a - * deleted run keeps a tombstoned cx_runs row — otherwise the next discovery - * pass would re-ingest it. - */ -type CollectiveXRunState = 'live' | 'deleted'; - -interface CollectiveXRunStateRow { - state: CollectiveXRunState; - version: number; - run_attempt: number; -} - function toRunRow(row: Record): CollectiveXRunRow { const { docs, ...rest } = row as unknown as Omit & { docs: unknown }; return { ...rest, docs: Array.isArray(docs) ? docs : [] }; } /** - * The most recent visible live run for a version. Zero-case summaries contain - * no AMD/NVIDIA data and stay hidden. Ordered by run_id — GitHub run ids - * increase monotonically with creation, matching lazy discovery's newest-first - * walk (completion time would let a long-failing older run shadow a newer - * successful one). + * The most recent visible run for a version. Zero-case summaries contain no + * AMD/NVIDIA data and stay hidden. Ordered by run_id, the results commit's + * time in epoch seconds, so the newest commit on main is the latest run. * * Row and documents come back in ONE query, with docs filtered to the row's - * CURRENT run_attempt: a reader can never observe one attempt's metadata with - * another attempt's documents, even while a refresh commits concurrently. + * run_attempt, so a reader never mixes one ingest's row with another's docs. */ export async function getLatestCollectiveXRun( sql: DbClient, @@ -75,7 +59,7 @@ export async function getLatestCollectiveXRun( FROM cx_run_docs WHERE run_id = r.run_id AND run_attempt = r.run_attempt ) d ON true - WHERE r.version = ${version} AND r.deleted_at IS NULL + WHERE r.version = ${version} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 ORDER BY r.run_id DESC LIMIT 1 @@ -84,8 +68,8 @@ export async function getLatestCollectiveXRun( } /** - * One specific visible live run by id, or null when absent, tombstoned, on - * another version, or carrying no supported-vendor cases. + * One specific visible run by id, or null when absent, on another version, or + * carrying no supported-vendor cases. */ export async function getCollectiveXRun( sql: DbClient, @@ -104,14 +88,14 @@ export async function getCollectiveXRun( FROM cx_run_docs WHERE run_id = r.run_id AND run_attempt = r.run_attempt ) d ON true - WHERE r.version = ${version} AND r.run_id = ${runId} AND r.deleted_at IS NULL + WHERE r.version = ${version} AND r.run_id = ${runId} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 `; return rows.length === 0 ? null : toRunRow(rows[0]); } /** - * Every visible live run summary for a version, newest-first, straight from + * Every visible run summary for a version, newest-first, straight from * the precomputed `summary` column — no document loading. Runs whose reader * summary contains no AMD/NVIDIA cases stay hidden. */ @@ -122,143 +106,65 @@ export async function listCollectiveXRuns( const rows = await sql` SELECT summary FROM cx_runs - WHERE version = ${version} AND deleted_at IS NULL + WHERE version = ${version} AND COALESCE((summary->>'requested_cases')::int, 0) > 0 ORDER BY run_id DESC `; return rows.map((row) => row.summary as CollectiveXRunSummary); } -/** State of each known run id (live or tombstoned); absent ids are omitted. */ -export async function getCollectiveXRunStates( - sql: DbClient, - runIds: readonly string[], -): Promise> { - if (runIds.length === 0) return {}; - const rows = await sql` - SELECT run_id::text, version, run_attempt, (deleted_at IS NOT NULL) AS deleted - FROM cx_runs - WHERE run_id = ANY(${runIds as string[]}::bigint[]) - `; - return Object.fromEntries( - rows.map((row) => [ - row.run_id as string, - { - state: row.deleted ? 'deleted' : 'live', - version: row.version as number, - run_attempt: row.run_attempt as number, - }, - ]), - ); -} +export type CollectiveXUpsertResult = 'inserted' | 'replaced' | 'conflict'; /** - * Atomically persist one run and its raw documents in a single statement - * (data-modifying CTEs), so concurrent lazy ingests can race safely: - * `ON CONFLICT DO NOTHING` turns the loser into a clean no-op, and a partial - * run can never become visible. Returns true when this call inserted the run. + * Store one results commit and its raw documents in a single statement + * (data-modifying CTEs), so a reader sees either the previous contents or the + * new ones, never a partial run. Re-ingesting the same commit replaces its row + * and documents. `conflict` means run_id already belongs to a different commit + * (two results commits landed in the same second); nothing is written. * * The `::jsonb`-cast parameters must be raw objects: both drivers * JSON-serialize objects exactly once, while pre-stringified values get * double-encoded by postgres.js. `docs` is wrapped in an object because the * neon driver would serialize a bare JS array as a Postgres array literal. */ -export async function insertCollectiveXRun( +export async function upsertCollectiveXRun( sql: DbClient, run: CollectiveXRunInsert, docs: unknown[], -): Promise { +): Promise { const rows = await sql` - WITH new_run AS ( + WITH upserted AS ( INSERT INTO cx_runs (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) VALUES (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) - ON CONFLICT (run_id) DO NOTHING - RETURNING run_id - ), - new_docs AS ( - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT new_run.run_id, ${run.run_attempt}, entries.value - FROM new_run, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) - RETURNING id - ) - SELECT (SELECT count(*)::int FROM new_run) AS runs_inserted - `; - return (rows[0]?.runs_inserted as number) > 0; -} - -/** - * Replace a live run's contents when GitHub reports a NEWER attempt (a re-run - * of failed shards after the run was already ingested). Single statement with - * `FOR UPDATE` + an attempt guard: concurrent refreshers serialize on the row - * lock and the loser re-evaluates the guard to a no-op. Readers filter docs - * by the row's current run_attempt, so any superseded docs this statement's - * snapshot could not see (and therefore could not DELETE) stay invisible until - * the next refresh garbage-collects them. Tombstoned runs are never refreshed. - * Returns true when replaced. - */ -export async function refreshCollectiveXRunAttempt( - sql: DbClient, - run: CollectiveXRunInsert, - docs: unknown[], -): Promise { - const rows = await sql` - WITH target AS ( - SELECT run_id FROM cx_runs - WHERE run_id = ${run.run_id} AND deleted_at IS NULL AND run_attempt < ${run.run_attempt} - FOR UPDATE + ON CONFLICT (run_id) DO UPDATE SET + run_attempt = EXCLUDED.run_attempt, + version = EXCLUDED.version, + generated_at = EXCLUDED.generated_at, + source_branch = EXCLUDED.source_branch, + conclusion = EXCLUDED.conclusion, + matrix = EXCLUDED.matrix, + summary = EXCLUDED.summary, + ingested_at = now() + WHERE cx_runs.source_sha = EXCLUDED.source_sha + RETURNING run_id, (xmax = 0) AS inserted ), removed AS ( - DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM target) - ), - updated AS ( - UPDATE cx_runs SET - run_attempt = ${run.run_attempt}, - generated_at = ${run.generated_at}, - source_sha = ${run.source_sha}, - source_branch = ${run.source_branch}, - conclusion = ${run.conclusion}, - matrix = ${run.matrix as never}::jsonb, - summary = ${run.summary as never}::jsonb, - ingested_at = now() - WHERE run_id IN (SELECT run_id FROM target) - RETURNING run_id + DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted) ), new_docs AS ( INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT updated.run_id, ${run.run_attempt}, entries.value - FROM updated, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) + SELECT upserted.run_id, ${run.run_attempt}, entries.value + FROM upserted, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) RETURNING id ) - SELECT (SELECT count(*)::int FROM updated) AS runs_updated - `; - return (rows[0]?.runs_updated as number) > 0; -} - -/** - * Tombstone a run: mark it deleted (so lazy discovery never re-ingests it) - * and drop its documents to free space — one atomic statement, so a partial - * failure can never tombstone the run while leaving its documents orphaned - * behind an unretryable 404. Returns false when the run is absent or already - * tombstoned. Re-ingesting via the CLI intentionally clears the tombstone - * (operator override; its hard DELETE also cascades any leftover docs). - */ -export async function deleteCollectiveXRun(sql: DbClient, runId: string): Promise { - const rows = await sql` - WITH tombstoned AS ( - UPDATE cx_runs SET deleted_at = now() - WHERE run_id = ${runId} AND deleted_at IS NULL - RETURNING run_id - ), - removed AS ( - DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned) - ) - SELECT (SELECT count(*)::int FROM tombstoned) AS runs_deleted + SELECT inserted FROM upserted `; - return (rows[0]?.runs_deleted as number) > 0; + if (rows.length === 0) return 'conflict'; + return rows[0].inserted ? 'inserted' : 'replaced'; } /** Assemble a stored run's raw documents into the dashboard dataset. */ diff --git a/packages/skills/skills/inferencex-api/integrity.json b/packages/skills/skills/inferencex-api/integrity.json index 83afdf01a..8e2dd0379 100644 --- a/packages/skills/skills/inferencex-api/integrity.json +++ b/packages/skills/skills/inferencex-api/integrity.json @@ -8,7 +8,7 @@ "references/chart-templates.md": "aed6854627142019a4a5df4a2d07523b1a64e5814dd2bbdcc74ee18beb083912", "references/cli-contract.md": "4f070877eeefffad2860c58f79f68432fbe27aa21fc578a1c696f0309a3d7d7b", "references/cli.md": "11e1183f7ed70d669019e8435ac16a6a9f873de027507b8850a9943f6659bfd9", - "references/collectivex.md": "f79cf8e2779a9289ab69f79f4a4b338e0b9a5f76a3750a3e969f33a744e726a1", + "references/collectivex.md": "949b3104b4dbb65db53ac9397eba28aac4981521f64ae336e7a7fdaf96b26ec0", "references/dashboard-views.md": "2095bef6178b7264db9e3c1e2bfcde1516fb190ddd5851dd5f8d104c8a0ab58b", "references/offline-exports.md": "95aa565dcd4c9e592159baa560a9a58217bc61e395de1c6daf0576795c7f4ec9", "references/pareto.md": "1b4d2d163f982e3f2d97addce310789eae9c1e5badd82dd7501708c9e0385027", diff --git a/packages/skills/skills/inferencex-api/references/collectivex.md b/packages/skills/skills/inferencex-api/references/collectivex.md index d6a6874e6..07ad18878 100644 --- a/packages/skills/skills/inferencex-api/references/collectivex.md +++ b/packages/skills/skills/inferencex-api/references/collectivex.md @@ -2,9 +2,9 @@ Use this cookbook to discover public communication sweeps, compare two returned runs, and export the evidence. These are existing observations; no new benchmark -is run. Public GETs can populate the service's documented lazy cache from existing -GitHub artifacts. No database credentials, admin operations, or launch tools are -needed. +is run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed +`collectivex/results/`; the service ingests it when it lands, never on read. No +database credentials, admin operations, or launch tools are needed. For a replayable contract 1 comparison: @@ -44,8 +44,9 @@ node .claude/skills/inferencex-api/scripts/inferencex.mjs collectivex compare \ Requires Node 24 or 26. The command checks the current OpenAPI operations, reads the run list **once**, and selects its two newest runs with `measured_cases > 0`, ordered by numeric run ID. The older selection is `left`; the newer is `right`. This is a -bounded example selection, not a representative sample. A cancelled or failed -workflow can still contain measured cases; its conclusion remains in the export. +bounded example selection, not a representative sample. The run ID is the results +commit time in epoch seconds and the dataset's `run.source_sha` is that commit; the +conclusion remains in the export. Run-list counts identify candidate pairs; exact EP and KV identities in the returned details establish whether they have matched groups. For a requested pair, supply @@ -67,31 +68,29 @@ The public operations require `version=1`: | Operation | Meaning | | ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest available run by run ID; it need not contain measurements. | +| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest stored run by run ID; it need not contain measurements. | | [`/collectivex/runs`](https://inferencex.semianalysis.com/api/v1/collectivex/runs?version=1) | Retained summaries and `discovery_complete`; the helper uses this for selection. | -| `/collectivex/runs/{runId}` | One run dataset, possibly refreshed to a newer attempt. | +| `/collectivex/runs/{runId}` | One run dataset; a re-ingest of the same commit can replace its contents. | All paths above are under `/api/v1`. Consult the [public API reference](https://inferencex.semianalysis.com/api) and [OpenAPI document](https://inferencex.semianalysis.com/api/openapi.json) for the -current contract. A missing/unsupported version is HTTP 400. HTTP 404, an upstream -502/503, malformed data, a redirect, or an interrupted response is a failed read, +current contract. A missing/unsupported version is HTTP 400. HTTP 404, a 5xx, +malformed data, a redirect, or an interrupted response is a failed read, not an empty comparison. The command exits nonzero without a completed manifest on those failures; it never substitutes a different run. -`discovery_complete=false` means further bounded discovery passes may reveal more -runs. Report that flag even when two selections were available. The route has no +The service now always reports `discovery_complete=true`; a `false` value would mean +further passes may reveal more runs. Report the flag as returned. The route has no documented `limit`, `offset`, or cursor: do not invent pagination parameters. If fewer than two measured summaries were returned, the helper exports `fewer_than_two_measured_runs` and makes no detail requests. Further discovery, if needed, should have an explicit small request budget and retain each response. -Even `discovery_complete=true` does **not** mean complete workflow history. Recent -discovery has a bounded upstream window, artifacts expire, and retained runs can -outlive their artifacts. The export therefore always sets `history_complete=false`. -Stored fallback data can also be served when upstream refresh fails. Compare each -detail's returned `run_attempt` with its discovery summary; a later attempt is a -different snapshot, even under the same run ID. Equal attempt numbers do not prove -identical snapshots; data can refresh within an attempt. +Even `discovery_complete=true` does **not** prove complete history: the stored runs +are whatever results commits the service has ingested. The export therefore always +sets `history_complete=false`. Compare each detail's returned `run_attempt` with its +summary; equal attempt numbers do not prove identical snapshots, because re-ingesting +a commit replaces its contents under the same run ID and attempt. ## 2. Check comparability before interpreting differences @@ -183,6 +182,11 @@ effect. Equal SHAs also do not establish identical runtime conditions. | Swap `swap_blocks[].points[].latency_us` | Copy latency in microseconds at the named percentile. | | Swap `swap_blocks[].points[].payload_gbps_at_latency_percentile` | Payload GB/s at the named latency percentile, with bytes counted once. | +EP p90, p95 and p99 are null where the harness withheld that tail: too many of the +series' samples were misaligned across ranks or host-stalled. The rates at a +withheld tail are null too; p50 is always present. Report a null tail as withheld, +not as zero or as a missing measurement. + The saved OpenAPI operation description defines the optional swap metrics even when their property schemas omit descriptions. Swap points remain in the raw datasets, outside the formal EP/KV comparison groups. For supplemental summaries, From 228b31172a15c42e1b5f70440129e46444cb7924 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:40:39 +0800 Subject: [PATCH 2/5] =?UTF-8?q?feat(collectivex):=20ingest=20committed=20r?= =?UTF-8?q?esults=20from=20a=20dispatch=20workflow=20/=20=E6=94=B9?= =?UTF-8?q?=E7=94=B1=20dispatch=20=E5=B7=A5=E4=BD=9C=E6=B5=81=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E5=B7=B2=E6=8F=90=E4=BA=A4=E7=9A=84=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the secret-protected POST /api/v1/collectivex/ingest route with .github/workflows/ingest-collectivex.yml, mirroring ingest-operatorx: repository_dispatch type ingest-collectivex with client_payload {sha}, plus workflow_dispatch with exactly one of sha (validated as 40 hex), ref, or backfill. It runs the CollectiveX migrations, the ingest CLI with --sha/--ref/--backfill and --yes against DATABASE_COLLECTIVEX_WRITE_URL (INFX_MAIN_PAT for GitHub reads), purges the collectivex cache scope with VERCEL_INVALIDATE_SECRET, and alerts Slack on failure. The app now has no CollectiveX mutation, so COLLECTIVEX_ADMIN_SECRET is gone from the docs and .env.example, and the route leaves the catalog. Add a PGlite test over the real CollectiveX migrations: a first commit with only mi300x and mi325x results serves those series while the other requested pools stay pending; a later commit adding pools is a newer superset run that latest and the run list pick first; re-ingesting a commit replaces it and a run id held by another commit is refused. The docs describe partial-result commits. 中文：改由 dispatch 工作流导入已提交的结果 用 .github/workflows/ingest-collectivex.yml 取代受密钥保护的 POST /api/v1/collectivex/ingest 路由，结构与 ingest-operatorx 一致： repository_dispatch 类型为 ingest-collectivex，client_payload 为 {sha}； workflow_dispatch 需且只能提供 sha（校验为 40 位十六进制）、ref 或 backfill 之一。工作流先运行 CollectiveX 迁移，再以 --sha/--ref/--backfill 和 --yes 调用导入 CLI（写入 DATABASE_COLLECTIVEX_WRITE_URL，GitHub 读取使用 INFX_MAIN_PAT），随后用 VERCEL_INVALIDATE_SECRET 清除 collectivex 缓存范围， 失败时发送 Slack 告警。应用不再包含任何 CollectiveX 写操作，因此从文档和 .env.example 中移除 COLLECTIVEX_ADMIN_SECRET，并从路由目录中删除该路由。 新增基于真实 CollectiveX 迁移的 PGlite 测试：只包含 mi300x 和 mi325x 结果的 首个提交会展示这两个 GPU 池的序列，其余已请求的 GPU 池保持 pending；之后新增 GPU 池的提交会成为更新的超集运行，latest 与运行列表都优先返回它；重新导入同一 提交会就地替换，被其他提交占用的 run_id 会被拒绝。文档补充了仅含部分结果的 提交的说明。 --- .env.example | 8 +- .github/workflows/ingest-collectivex.yml | 131 ++++++++++++++ docs/collectivex.md | 35 ++-- docs/deployment-configuration.md | 37 ++-- .../api/v1/collectivex/ingest/route.test.ts | 163 ------------------ .../app/api/v1/collectivex/ingest/route.ts | 91 ---------- packages/app/src/lib/api/api-route-catalog.ts | 13 +- packages/app/src/lib/cache/api-cache.ts | 4 +- .../db/src/queries/collectivex-db.test.ts | 149 ++++++++++++++++ 9 files changed, 324 insertions(+), 307 deletions(-) create mode 100644 .github/workflows/ingest-collectivex.yml delete mode 100644 packages/app/src/app/api/v1/collectivex/ingest/route.test.ts delete mode 100644 packages/app/src/app/api/v1/collectivex/ingest/route.ts create mode 100644 packages/db/src/queries/collectivex-db.test.ts diff --git a/.env.example b/.env.example index 77777ca69..b2987181b 100644 --- a/.env.example +++ b/.env.example @@ -32,7 +32,7 @@ # INFERENCEX_LOCAL_ARTIFACT_DIR=/absolute/path/to/local-artifacts # CollectiveX database (separate Neon instance), filled from InferenceX's committed -# collectivex/results/ by POST /api/v1/collectivex/ingest or admin:db:ingest:collectivex. +# collectivex/results/ by the ingest-collectivex workflow or admin:db:ingest:collectivex. # The app routes read and write through DATABASE_COLLECTIVEX_WRITE_URL; they do not read # DATABASE_COLLECTIVEX_READONLY_URL. # DATABASE_COLLECTIVEX_READONLY_URL= @@ -71,8 +71,8 @@ # ║ Secrets and database administration (DO NOT SHARE) ║ # ║ ║ # ║ DB scripts (migrate, ingest, reset, etc) read these. The app also reads ║ -# ║ INVALIDATE_SECRET, FEEDBACK_SECRET, and COLLECTIVEX_ADMIN_SECRET (and ║ -# ║ DATABASE_WRITE_URL for feedback) at runtime; set them in Vercel too. ║ +# ║ INVALIDATE_SECRET and FEEDBACK_SECRET (and DATABASE_WRITE_URL for ║ +# ║ feedback) at runtime; set them in Vercel too. ║ # ╚══════════════════════════════════════════════════════════════════════════╝ # DATABASE_WRITE_URL= @@ -86,8 +86,6 @@ # argument or run URL names the repository. # INGEST_REPO= # FEEDBACK_SECRET= -# Bearer token for POST /api/v1/collectivex/ingest (the InferenceX ingest workflow). -# COLLECTIVEX_ADMIN_SECRET= # OperatorX durable raw-artifact cache (primary, write-capable connection). # Apply admin:db:migrate:operatorx before enabling the hidden OperatorX tab. diff --git a/.github/workflows/ingest-collectivex.yml b/.github/workflows/ingest-collectivex.yml new file mode 100644 index 000000000..fd67b28d1 --- /dev/null +++ b/.github/workflows/ingest-collectivex.yml @@ -0,0 +1,131 @@ +name: Ingest CollectiveX Results + +on: + repository_dispatch: + types: [ingest-collectivex] + workflow_dispatch: + inputs: + sha: + description: InferenceX commit (40 hex) whose collectivex/results/ to ingest + required: false + ref: + description: InferenceX branch or tag, instead of sha + required: false + backfill: + description: Ingest every collectivex/results/ commit on main instead + type: boolean + default: false + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.client_payload.sha || inputs.sha || inputs.ref || 'backfill' }} + cancel-in-progress: false + +jobs: + ingest: + name: Ingest CollectiveX results + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check inputs and secrets + env: + APP_URL: ${{ vars.APP_URL }} + VERCEL_INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SHA: ${{ github.event.client_payload.sha || inputs.sha }} + REF: ${{ inputs.ref }} + BACKFILL: ${{ inputs.backfill }} + run: | + : "${SLACK_WEBHOOK_URL:?SLACK_WEBHOOK_URL secret is not set}" + : "${APP_URL:?APP_URL repository variable is not set}" + : "${VERCEL_INVALIDATE_SECRET:?VERCEL_INVALIDATE_SECRET secret is not set}" + : "${DATABASE_COLLECTIVEX_WRITE_URL:?DATABASE_COLLECTIVEX_WRITE_URL secret is not set}" + : "${GH_TOKEN:?INFX_MAIN_PAT secret is not set}" + selected=0 + [ -n "$SHA" ] && selected=$((selected + 1)) + [ -n "$REF" ] && selected=$((selected + 1)) + [ "$BACKFILL" = true ] && selected=$((selected + 1)) + if [ "$selected" -ne 1 ]; then + echo "give exactly one of sha, ref, or backfill" >&2 + exit 1 + fi + if [ -n "$SHA" ] && ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "sha must be 40 lowercase hex characters" >&2 + exit 1 + fi + if [ -n "$REF" ] && { ! [[ "$REF" =~ ^[A-Za-z0-9_./-]{1,255}$ ]] || [[ "$REF" == *..* ]]; }; then + echo "ref is not a valid branch or tag name" >&2 + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + id: setup-bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Cache Bun packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-${{ runner.arch }}-${{ + steps.setup-bun.outputs.bun-version }}-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun-${{ runner.arch }}-${{ steps.setup-bun.outputs.bun-version }}- + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + + - name: Run migrations + env: + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + run: bun run --cwd packages/db src/migrate-collectivex.ts --yes + + - name: Ingest results + env: + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + SHA: ${{ github.event.client_payload.sha || inputs.sha }} + REF: ${{ inputs.ref }} + BACKFILL: ${{ inputs.backfill }} + run: | + if [ "$BACKFILL" = true ]; then + target=(--backfill) + elif [ -n "$SHA" ]; then + target=(--sha "$SHA") + else + target=(--ref "$REF") + fi + bun run --cwd packages/db src/ingest-collectivex.ts "${target[@]}" --yes + + - name: Invalidate CollectiveX cache + env: + APP_URL: ${{ vars.APP_URL }} + VERCEL_INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + VERCEL_PROTECTION_BYPASS_SECRET: ${{ secrets.VERCEL_PROTECTION_BYPASS_SECRET }} + run: | + : "${APP_URL:?APP_URL repository variable is not set}" + headers=(-H "Authorization: Bearer $VERCEL_INVALIDATE_SECRET") + if [ -n "$VERCEL_PROTECTION_BYPASS_SECRET" ]; then + headers+=(-H "x-vercel-protection-bypass: $VERCEL_PROTECTION_BYPASS_SECRET") + fi + curl -sSf -X POST "${APP_URL%/}/api/v1/invalidate?scope=collectivex" "${headers[@]}" + + - name: Notify Slack on failure + if: failure() + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook + payload: | + { + "text": ":rotating_light: *Ingest CollectiveX workflow failed*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } diff --git a/docs/collectivex.md b/docs/collectivex.md index 74ab2ec1b..504084f48 100644 --- a/docs/collectivex.md +++ b/docs/collectivex.md @@ -41,17 +41,14 @@ limit. It then checks every file the way the benchmark repo's own test does: the result carries the `cases.json` version. Each document is then assembled alone, so every failure names its file. Files above 4 MiB, or a tree above 256 MiB, are rejected. -- **HTTP**: `POST /api/v1/collectivex/ingest` with `Authorization: Bearer -` and a JSON body of exactly `{"sha": "<7-40 hex>"}` or - `{"ref": ""}`. It ingests through `DATABASE_COLLECTIVEX_WRITE_URL` and then - purges the CollectiveX cache scope. Responses: 200 `{ingested: "inserted" | "replaced", -run_id, source_sha, generated_at, version, requested_cases, measured_cases, documents}`; - 400 for any other body; 401 for a missing or wrong token; 404 when no results commit on - `main` matches; 409 for a run id held by another commit; 422 when the files break the - contract (the message names the file); 502 when GitHub cannot be read; 500 when the - secret is unset or the database write fails. An InferenceX workflow calls it on pushes to - `main` that touch `collectivex/results/`. One call reads a few GitHub API pages plus one - raw file per result and runs within the route's 300 s limit. +- **Workflow**: `.github/workflows/ingest-collectivex.yml` runs on `repository_dispatch` + type `ingest-collectivex` with `client_payload: {"sha": "<40 hex>"}`, which the InferenceX + workflow sends on pushes to `main` that touch `collectivex/results/`, and on + `workflow_dispatch` with exactly one of `sha` (40 hex), `ref`, or `backfill`. It runs + `migrate-collectivex.ts --yes`, then the ingest CLI with `--sha`, `--ref` or `--backfill` + and `--yes` against the `DATABASE_COLLECTIVEX_WRITE_URL` secret (GitHub reads use + `INFX_MAIN_PAT`), then purges the cache with `POST /api/v1/invalidate?scope=collectivex` + (`VERCEL_INVALIDATE_SECRET`). It alerts Slack on failure, like `ingest-operatorx`. - **CLI**: `bun run admin:db:ingest:collectivex [--sha | --ref ] [--backfill] [--yes] [--no-ssl]` ingests one commit (default: `main`), or with `--backfill` every results commit on `main`, oldest first, reusing unchanged files between commits. It asks @@ -68,7 +65,8 @@ Run these against the CollectiveX database only, in order: Until the backfill finishes, `/api/v1/collectivex/latest` returns 404 and the run list is empty. 3. **Migrate**: `bun run admin:db:migrate:collectivex` (applies `004_committed_results.sql`, which drops the unused `deleted_at` tombstone column). -4. **Backfill**: `bun run admin:db:ingest:collectivex --backfill`, then purge the cache with +4. **Backfill**: run the `Ingest CollectiveX Results` workflow with `backfill` checked, which + also purges the cache. Locally, `bun run admin:db:ingest:collectivex --backfill`, then `POST /api/v1/invalidate?scope=collectivex` (the CLI does not reach the CDN). ## Serving @@ -77,13 +75,18 @@ Run these against the CollectiveX database only, in order: response cached right after an ingest's purge never comes from a lagging replica. `DATABASE_COLLECTIVEX_READONLY_URL` is not read by the app routes. - **Caching**: responses carry the `collectivex` CDN tag with the default one-day `s-maxage`. - The ingest route and `POST /api/v1/invalidate?scope=collectivex` purge only that tag; the + The ingest workflow purges only that tag through `POST /api/v1/invalidate?scope=collectivex`; the main dashboard's blob cache is untouched by CollectiveX operations. +- **Partial results**: a commit may hold results for only some pools (the first committed + results cover only `mi300x` and `mi325x`). Every runnable case in `cases.json` without a + result file stays in `coverage` with outcome `pending` (requested, not run), counts toward + `requested_cases` but not `measured_cases`, and adds no series, so the SKU controls list + only measured pools. A later commit that adds pools is a new run with a later `run_id`, + so it becomes the latest run while the earlier one stays listed. - **Run list**: `/api/v1/collectivex/runs` always reports `discovery_complete: true`; the field stays for existing clients. -- **No deletion**: runs are removed by changing the committed results, not through the app. - `COLLECTIVEX_ADMIN_SECRET` authorizes only the ingest route; when it is unset that route - returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. +- **No deletion or ingest in the app**: the app exposes no CollectiveX mutation. Runs are + added by the ingest workflow or CLI and removed by changing the committed results. - **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` results (case-insensitive). Other vendor values are omitted from chart series, coverage, and SKU summaries; runs with no supported-vendor cases stay hidden. diff --git a/docs/deployment-configuration.md b/docs/deployment-configuration.md index d56034df4..2b573c9f3 100644 --- a/docs/deployment-configuration.md +++ b/docs/deployment-configuration.md @@ -47,8 +47,7 @@ feature, which fails as described when the variable is unset. | `INVALIDATE_SECRET` | `POST /api/v1/invalidate` | Requests return 500 with `INVALIDATE_SECRET is not set`, except requests to the shared `staging` preview that carry `x-vercel-protection-bypass`. When it is set, a missing or wrong bearer token returns 401. Set it to the value of the `VERCEL_INVALIDATE_SECRET` GitHub secret. | | `FEEDBACK_SECRET` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | | `DATABASE_WRITE_URL` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | -| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read, and the ingest route writes, through this primary connection, so they return errors. | -| `COLLECTIVEX_ADMIN_SECRET` | `POST /api/v1/collectivex/ingest` | Every request returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. When it is set, a missing or wrong bearer token returns 401. The InferenceX ingest workflow sends the same value. | +| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read through this primary connection, so they return errors. | | `DATABASE_OPERATORX_READONLY_URL` | `/api/v1/operatorx/*` | The routes return 503 with `The OperatorX database is not configured`. | | `DATABASE_PROXYTRACE_READONLY_URL` | `/api/v1/agentic-workload-explorer/*` | The routes return 500. | @@ -73,34 +72,36 @@ project, redeploy for the new value to take effect. | Name | Required | Effect | | ----------------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `APP_URL` | yes, by every workflow that uses it | Site targeted by cache invalidation and warmup, OperatorX invalidation, post-deploy OperatorX warmup, H3 media warmup, and PowerX verification on the `production` ingest target. | +| `APP_URL` | yes, by every workflow that uses it | Site targeted by cache invalidation and warmup, OperatorX and CollectiveX invalidation, post-deploy OperatorX warmup, H3 media warmup, and PowerX verification on the `production` ingest target. | | `BENCHMARK_REPO` | yes, by every workflow that uses it | Repository whose benchmark artifacts the ingest workflows download and that dispatches `stage-results`. When it is not `SemiAnalysisAI/InferenceX`, also list it in `INFERENCEX_SOURCE_REPOS` for the app's GitHub lookups. | | `STAGING_APP_URL` | `staging` target and `stage-results` | Site invalidated by the `staging` ingest target and linked from staged chart comments. | ### Secrets `GITHUB_TOKEN` is provided by GitHub and is not listed. `SLACK_WEBHOOK_URL` is the -failure alarm, so `ingest-results`, `ingest-operatorx`, and `stage-results` check it +failure alarm, so `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, and `stage-results` check it before doing any other work. `stage-results` also checks `PAT` up front. `ingest-operatorx` checks `APP_URL`, `VERCEL_INVALIDATE_SECRET`, -`DATABASE_OPERATORX_WRITE_URL`, and `INFX_MAIN_PAT` in its first step. +`DATABASE_OPERATORX_WRITE_URL`, and `INFX_MAIN_PAT` in its first step, and +`ingest-collectivex` checks the same with `DATABASE_COLLECTIVEX_WRITE_URL`. `backfill-server-log-files` checks `INFX_MAIN_PAT` before the backfill. `apply-run-overrides`, `backfill-server-log-files`, and `recompute-agentic-metrics` check `DATABASE_WRITE_URL` before writing to the production database. -| Secret | Required | Used for | Workflows | -| --------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `stage-results` (passed to `ingest-results`) | -| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | -| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | -| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | -| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`. | `ingest-results`, `ingest-operatorx`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | -| `PAT` | yes, by `stage-results`; optional for `ingest-results` | Sends the `stage-results-started`, `stage-results-completed` and `ingest-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results`, `ingest-results` (dispatched merge ingests only; a failed report never fails the ingest) | -| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | -| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | -| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | -| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | -| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | +| Secret | Required | Used for | Workflows | +| --------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, `stage-results` (passed to `ingest-results`) | +| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | +| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | +| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | +| `DATABASE_COLLECTIVEX_WRITE_URL` | yes, by `ingest-collectivex` | CollectiveX database write connection for migrations and ingest. | `ingest-collectivex` | +| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`, and the committed CollectiveX results. | `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | +| `PAT` | yes, by `stage-results`; optional for `ingest-results` | Sends the `stage-results-started`, `stage-results-completed` and `ingest-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results`, `ingest-results` (dispatched merge ingests only; a failed report never fails the ingest) | +| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | +| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | +| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-collectivex`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | +| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-collectivex`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | +| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | Values reach scripts only through step `env:`, never by inline expression in a `run:` block. diff --git a/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts b/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts deleted file mode 100644 index d2082aee5..000000000 --- a/packages/app/src/app/api/v1/collectivex/ingest/route.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -import type * as CommittedResults from '@semianalysisai/inferencex-db/collectivex/committed-results'; - -const { mockResolve, mockLoad, mockUpsert, mockGetWriteDb, mockPurge } = vi.hoisted(() => ({ - mockResolve: vi.fn(), - mockLoad: vi.fn(), - mockUpsert: vi.fn(), - mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockPurge: vi.fn(), -})); - -vi.mock('@semianalysisai/inferencex-db/collectivex/committed-results', async (importOriginal) => ({ - ...(await importOriginal()), - resolveResultsCommit: mockResolve, - loadCommittedRun: mockLoad, -})); - -vi.mock('@semianalysisai/inferencex-db/connection', () => ({ - getCollectiveXWriteDb: mockGetWriteDb, -})); - -vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ - upsertCollectiveXRun: mockUpsert, -})); - -vi.mock('@/lib/cache/api-cache', () => ({ - purgeCollectiveX: mockPurge, -})); - -import { CommittedResultsError } from '@semianalysisai/inferencex-db/collectivex/committed-results'; - -import { POST } from './route'; - -const SECRET = 'test-admin-secret'; -const SHA = 'c'.repeat(40); -const commit = { sha: SHA, committedAt: '2026-10-15T18:35:51Z' }; -const run = { - run_id: '1792089351', - run_attempt: 1, - version: 1, - generated_at: '2026-10-15T18:35:51Z', - source_sha: SHA, - source_branch: 'main', - conclusion: 'success', - matrix: {}, - summary: { requested_cases: 195, measured_cases: 2 }, -}; - -function post(body: unknown, authorization: string | null = `Bearer ${SECRET}`) { - return POST( - new Request('http://localhost/api/v1/collectivex/ingest', { - method: 'POST', - headers: authorization === null ? {} : { Authorization: authorization }, - body: typeof body === 'string' ? body : JSON.stringify(body), - }), - ); -} - -beforeEach(() => { - vi.clearAllMocks(); - vi.spyOn(console, 'error').mockImplementation(() => {}); - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', SECRET); - vi.stubEnv('GITHUB_TOKEN', 'gh-token'); - mockResolve.mockResolvedValue(commit); - mockLoad.mockResolvedValue({ run, docs: [{}, {}] }); - mockUpsert.mockResolvedValue('inserted'); -}); - -afterEach(() => { - vi.unstubAllEnvs(); -}); - -describe('POST /api/v1/collectivex/ingest', () => { - it('returns 500 when COLLECTIVEX_ADMIN_SECRET is not set', async () => { - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', ''); - const res = await post({ sha: SHA }); - expect(res.status).toBe(500); - expect(await res.text()).toBe('COLLECTIVEX_ADMIN_SECRET is not set'); - expect(mockResolve).not.toHaveBeenCalled(); - }); - - it.each([ - ['no Authorization header', null], - ['a wrong secret', 'Bearer wrong-secret'], - ['the secret without the Bearer scheme', SECRET], - ])('returns 401 with %s and does nothing', async (_name, authorization) => { - const res = await post({ sha: SHA }, authorization); - expect(res.status).toBe(401); - expect(mockResolve).not.toHaveBeenCalled(); - expect(mockUpsert).not.toHaveBeenCalled(); - expect(mockPurge).not.toHaveBeenCalled(); - }); - - it.each([ - ['an empty object', {}], - ['both sha and ref', { sha: SHA, ref: 'main' }], - ['a malformed sha', { sha: 'not-hex' }], - ['a ref with ..', { ref: 'main/../x' }], - ['an extra field', { ref: 'main', force: true }], - ['an array', [SHA]], - ['invalid JSON', '{'], - ])('returns 400 for %s', async (_name, body) => { - const res = await post(body); - expect(res.status).toBe(400); - expect(mockResolve).not.toHaveBeenCalled(); - }); - - it('stores the resolved results commit and purges the CollectiveX cache', async () => { - const res = await post({ sha: SHA }); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - ingested: 'inserted', - run_id: run.run_id, - source_sha: SHA, - generated_at: run.generated_at, - version: 1, - requested_cases: 195, - measured_cases: 2, - documents: 2, - }); - expect(mockResolve).toHaveBeenCalledWith(SHA, { token: 'gh-token' }); - expect(mockLoad).toHaveBeenCalledWith(commit, { token: 'gh-token' }); - expect(mockUpsert).toHaveBeenCalledWith('mock-write-sql', run, [{}, {}]); - expect(mockPurge).toHaveBeenCalledOnce(); - }); - - it('accepts a ref and reports a replaced run', async () => { - mockUpsert.mockResolvedValue('replaced'); - const res = await post({ ref: 'main' }); - expect(res.status).toBe(200); - const body = await res.json(); - expect(body.ingested).toBe('replaced'); - expect(mockResolve).toHaveBeenCalledWith('main', { token: 'gh-token' }); - }); - - it('returns 409 and keeps the cache when the run id belongs to another commit', async () => { - mockUpsert.mockResolvedValue('conflict'); - const res = await post({ sha: SHA }); - expect(res.status).toBe(409); - expect(mockPurge).not.toHaveBeenCalled(); - }); - - it.each([ - ['invalid', 422], - ['not-found', 404], - ['unavailable', 502], - ] as const)('maps a %s results error to %i without writing', async (kind, status) => { - mockLoad.mockRejectedValue(new CommittedResultsError(kind, `${kind} detail`)); - const res = await post({ sha: SHA }); - expect(res.status).toBe(status); - expect(await res.json()).toEqual({ error: `${kind} detail` }); - expect(mockUpsert).not.toHaveBeenCalled(); - expect(mockPurge).not.toHaveBeenCalled(); - }); - - it('returns 500 without leaking details on DB failure', async () => { - mockUpsert.mockRejectedValue(new Error('connection string leaked')); - const res = await post({ sha: SHA }); - expect(res.status).toBe(500); - expect(await res.json()).toEqual({ error: 'Internal server error' }); - }); -}); diff --git a/packages/app/src/app/api/v1/collectivex/ingest/route.ts b/packages/app/src/app/api/v1/collectivex/ingest/route.ts deleted file mode 100644 index 10355681e..000000000 --- a/packages/app/src/app/api/v1/collectivex/ingest/route.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { NextResponse } from 'next/server'; - -import { - CommittedResultsError, - loadCommittedRun, - resolveResultsCommit, -} from '@semianalysisai/inferencex-db/collectivex/committed-results'; -import { getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; -import { upsertCollectiveXRun } from '@semianalysisai/inferencex-db/queries/collectivex'; - -import { bearerMatches } from '@/lib/api/bearer-auth'; -import { purgeCollectiveX } from '@/lib/cache/api-cache'; - -export const dynamic = 'force-dynamic'; -export const runtime = 'nodejs'; -// One commit is a few GitHub API calls plus one raw read per result file. -export const maxDuration = 300; - -const SHA = /^[0-9a-f]{7,40}$/u; -const REF = /^(?!.*\.\.)[\w./-]{1,255}$/u; - -const ERROR_STATUS = { invalid: 422, 'not-found': 404, unavailable: 502 } as const; - -/** `{sha}` or `{ref}`, exactly one; null when the body is anything else. */ -function targetOf(body: unknown): string | null { - if (body === null || typeof body !== 'object' || Array.isArray(body)) return null; - const { sha, ref, ...rest } = body as { sha?: unknown; ref?: unknown }; - if (Object.keys(rest).length > 0 || (sha === undefined) === (ref === undefined)) return null; - if (sha !== undefined) return typeof sha === 'string' && SHA.test(sha) ? sha : null; - return typeof ref === 'string' && REF.test(ref) ? ref : null; -} - -/** - * Store one InferenceX commit's committed CollectiveX results in the CollectiveX - * database and purge the CollectiveX cache scope. The InferenceX workflow that - * runs on pushes to main calls this. The ref resolves to the newest commit at or - * before it that changed collectivex/results/, which must be on main; storing the - * same commit again replaces its run. - */ -export async function POST(request: Request) { - const secret = process.env.COLLECTIVEX_ADMIN_SECRET; - if (!secret) { - return new NextResponse('COLLECTIVEX_ADMIN_SECRET is not set', { status: 500 }); - } - if (!bearerMatches(request.headers.get('Authorization') ?? '', secret)) { - return new NextResponse('Unauthorized', { status: 401 }); - } - - let target: string | null; - try { - target = targetOf(await request.json()); - } catch { - target = null; - } - if (target === null) { - return NextResponse.json( - { error: 'Body must be {"sha": ""} or {"ref": ""}' }, - { status: 400 }, - ); - } - - const token = process.env.GITHUB_TOKEN || undefined; - try { - const commit = await resolveResultsCommit(target, { token }); - const { run, docs } = await loadCommittedRun(commit, { token }); - const result = await upsertCollectiveXRun(getCollectiveXWriteDb(), run, docs); - if (result === 'conflict') { - return NextResponse.json( - { error: `Run ${run.run_id} already holds a different commit than ${commit.sha}` }, - { status: 409 }, - ); - } - purgeCollectiveX(); - return NextResponse.json({ - ingested: result, - run_id: run.run_id, - source_sha: run.source_sha, - generated_at: run.generated_at, - version: run.version, - requested_cases: run.summary.requested_cases, - measured_cases: run.summary.measured_cases, - documents: docs.length, - }); - } catch (error) { - if (error instanceof CommittedResultsError) { - return NextResponse.json({ error: error.message }, { status: ERROR_STATUS[error.kind] }); - } - console.error('Error ingesting CollectiveX results:', error); - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); - } -} diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index f647e0013..f8f61fef5 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -679,17 +679,6 @@ export const apiRouteCatalog = [ operationId: 'list-benchmark-history', sourceSha256: '2b33a33b56ec5afb9c44ddc7db581686dec7488a397cd50c8efa810acff3a86c', }, - { - source: 'src/app/api/v1/collectivex/ingest/route.ts', - path: '/api/v1/collectivex/ingest', - method: 'POST', - classification: 'admin', - exclusionReason: { - en: 'Secret-protected CollectiveX ingest mutation called by the benchmark repository workflow; it is not part of the public read API.', - zh: '由基准测试仓库工作流调用、受密钥保护的 CollectiveX 导入写操作；它不属于公开只读 API。', - }, - sourceSha256: '017bebdc59684e37d4ac42862d0daa96ee7b4c1ffd28e92da967570ba3875247', - }, { source: 'src/app/api/v1/collectivex/latest/route.ts', path: '/api/v1/collectivex/latest', @@ -1580,7 +1569,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/cache/api-cache.ts', - sourceSha256: 'd168e625f68800e7bcf3c74f824491e4de7d201929e656809b8d4e225f6c53f5', + sourceSha256: '29c35918b58926ade3e28c81459e5938bdc761a28250b3d923fcac9141741a67', reviewArea: { en: 'Public CDN tags, cache lifetimes, Blob key dimensions, and purge behavior.', zh: '公开 CDN 标签、缓存时长、Blob 键维度和清除行为。', diff --git a/packages/app/src/lib/cache/api-cache.ts b/packages/app/src/lib/cache/api-cache.ts index 5c2c9c5f6..e2840d248 100644 --- a/packages/app/src/lib/cache/api-cache.ts +++ b/packages/app/src/lib/cache/api-cache.ts @@ -9,8 +9,8 @@ import { blobGet, blobPurge, blobSet } from './blob-cache'; * `cachedJson(..., { tag })`), so a CollectiveX ingest can purge the CollectiveX * cache without dropping the main dashboard's. CollectiveX routes read their * DB directly — no blob layer, so the tag is the entire scope. Responses stay - * cached until an ingest purges it (`POST /api/v1/collectivex/ingest`, or - * `/api/v1/invalidate?scope=collectivex`). + * cached until the CollectiveX ingest workflow purges it + * (`/api/v1/invalidate?scope=collectivex`). */ export const COLLECTIVEX_CACHE_SCOPE = 'collectivex'; diff --git a/packages/db/src/queries/collectivex-db.test.ts b/packages/db/src/queries/collectivex-db.test.ts new file mode 100644 index 000000000..7f61d7a1e --- /dev/null +++ b/packages/db/src/queries/collectivex-db.test.ts @@ -0,0 +1,149 @@ +import fs from 'node:fs'; + +import { PGlite } from '@electric-sql/pglite'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + committedRun, + validateCommittedResults, + type ResultsSource, +} from '../collectivex/committed-results'; +import { makeRawMatrix, makeRawShard } from '../collectivex/test-fixture'; +import { pgliteSql, type PgliteSql } from '../lib/test-pglite'; +import { + collectiveXDatasetFromRow, + getCollectiveXRun, + getLatestCollectiveXRun, + listCollectiveXRuns, + upsertCollectiveXRun, +} from './collectivex'; + +type Json = Record; + +const FIRST_SHA = 'a'.repeat(40); +const SECOND_SHA = 'b'.repeat(40); +const FIRST_AT = '2026-10-28T00:00:00Z'; +const SECOND_AT = '2026-10-29T00:00:00Z'; + +async function collectiveXDb(): Promise { + const db = await PGlite.create(); + const dir = new URL('../../migrations-collectivex/', import.meta.url); + for (const name of fs.readdirSync(dir).toSorted()) { + if (name.endsWith('.sql')) await db.exec(fs.readFileSync(new URL(name, dir), 'utf8')); + } + return pgliteSql(db); +} + +function result(sku: string, vendor: 'amd' | 'nvidia'): Json { + return { + ...makeRawShard({ sku, vendor }), + provenance: { source_sha: 'e'.repeat(40), source_dirty: false }, + }; +} + +// The first committed results cover two AMD pools; cases.json requests every pool. +const mi300x = result('mi300x', 'amd'); +const mi325x = result('mi325x', 'amd'); +const h200 = result('h200-dgxc', 'nvidia'); +const b200 = result('b200-nscale', 'nvidia'); +const cases = makeRawMatrix( + [mi300x, mi325x, h200, b200].map((doc) => ({ + caseId: doc.identity.case_id, + sku: doc.identity.case_factors.sku, + case: doc.identity.case_factors.case, + })), +); + +function encode(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)); +} + +function source(sha: string, committedAt: string, docs: Json[]): ResultsSource { + return { + sha, + committedAt, + files: new Map([ + ['cases.json', encode(cases)], + ...docs.map((doc): [string, Uint8Array] => [ + `${doc.identity.case_factors.sku}/${doc.identity.case_id}.json`, + encode(doc), + ]), + ]), + }; +} + +let sql: PgliteSql; + +beforeEach(async () => { + sql = await collectiveXDb(); +}); + +describe('CollectiveX runs from committed results in PostgreSQL', () => { + it('serves a first commit holding only the mi300x and mi325x folders', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + await expect(upsertCollectiveXRun(sql, first.run, first.docs)).resolves.toBe('inserted'); + + const row = await getLatestCollectiveXRun(sql, 1); + expect(row).not.toBeNull(); + const dataset = collectiveXDatasetFromRow(row!); + expect(dataset.run).toMatchObject({ + run_id: first.run.run_id, + source_sha: FIRST_SHA, + requested_cases: 4, + measured_cases: 2, + }); + expect(dataset.series.map((series) => series.system.sku).toSorted()).toEqual([ + 'mi300x', + 'mi325x', + ]); + // Pools without a result file stay requested but not run, never an error. + expect(Object.fromEntries(dataset.coverage.map((item) => [item.sku, item.outcome]))).toEqual({ + mi300x: 'success', + mi325x: 'success', + 'h200-dgxc': 'pending', + 'b200-nscale': 'pending', + }); + }); + + it('stores a later commit that adds pools as a newer superset run and serves it as latest', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + const second = committedRun( + validateCommittedResults(source(SECOND_SHA, SECOND_AT, [mi300x, mi325x, h200, b200])), + ); + await upsertCollectiveXRun(sql, first.run, first.docs); + await expect(upsertCollectiveXRun(sql, second.run, second.docs)).resolves.toBe('inserted'); + + expect(Number(second.run.run_id)).toBeGreaterThan(Number(first.run.run_id)); + const latest = await getLatestCollectiveXRun(sql, 1); + expect(latest?.source_sha).toBe(SECOND_SHA); + expect(collectiveXDatasetFromRow(latest!).run.measured_cases).toBe(4); + + const runs = await listCollectiveXRuns(sql, 1); + expect(runs.map((run) => run.run_id)).toEqual([second.run.run_id, first.run.run_id]); + expect(runs.map((run) => run.measured_cases)).toEqual([4, 2]); + + const earlier = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(earlier?.docs).toHaveLength(2); + }); + + it('replaces a re-ingested commit in place and refuses a run id held by another commit', async () => { + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + await upsertCollectiveXRun(sql, first.run, first.docs); + const again = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + await expect(upsertCollectiveXRun(sql, again.run, again.docs)).resolves.toBe('replaced'); + const row = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(row?.docs).toHaveLength(2); + + const clash = committedRun(validateCommittedResults(source(SECOND_SHA, FIRST_AT, [h200]))); + await expect(upsertCollectiveXRun(sql, clash.run, clash.docs)).resolves.toBe('conflict'); + const kept = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(kept?.source_sha).toBe(FIRST_SHA); + expect(await listCollectiveXRuns(sql, 1)).toHaveLength(1); + }); +}); From e77a78d688ec821780baf6e29195f04868d3468b Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:50:19 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix(collectivex):=20serialize=20ingests,=20?= =?UTF-8?q?skip=20unmeasured=20commits,=20retire=20lazy-ingest=20rows,=20a?= =?UTF-8?q?dd=20prune=20/=20=E4=B8=B2=E8=A1=8C=E5=8C=96=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E3=80=81=E8=B7=B3=E8=BF=87=E6=9C=AA=E6=B5=8B=E9=87=8F=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E3=80=81=E6=B8=85=E9=99=A4=E6=87=92=E5=8A=A0=E8=BD=BD?= =?UTF-8?q?=E6=97=A7=E8=A1=8C=E5=B9=B6=E6=94=AF=E6=8C=81=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every CollectiveX write now runs in a transaction that first takes a fixed pg_advisory_xact_lock, and the ingest workflow uses one queued concurrency group, so two ingests of the same commit can no longer store its documents twice. Commits with nothing measured (such as cases.json-only commits) are skipped, and the read queries also hide any run whose summary measures nothing. Migration 005 deletes the retired lazy-ingest rows (GitHub run ids, which outrank every committed run) and adds a CHECK keeping run_id below 10^10. run_id is now derived from main's results history: the committer second, raised past the previous results commit on ties, so it is deterministic and unique. The CLI and workflow gain --backfill --prune and --delete-run; a backfill skips commits that 404. Validation now matches collectivex/tests/test_results.py (exact version, 40-hex source_sha, no IPv4 or host-naming keys). covered_skus lists only pools with a measured result. Docs: runbook (backfill+prune instead of truncate + ref=main), no HTTP 409, nullable EP tail percentiles in the API docs, stale module comment. 所有 CollectiveX 写操作现在都在事务中先获取固定的 pg_advisory_xact_lock， 导入工作流也改为单一排队的并发组，同一提交的两次导入不会再把文档存两遍。 没有任何测量结果的提交（例如只修改 cases.json 的提交）会被跳过，读取查询 也会隐藏汇总中没有测量结果的运行。迁移 005 删除已停用的懒加载旧行（其 GitHub 运行 ID 会压过所有已提交运行），并添加 CHECK 约束使 run_id 小于 10^10。 run_id 现在由 main 的结果提交历史推导：取提交者时间（秒），同一秒时递增到 上一个结果提交之后，因此确定且唯一。CLI 与工作流新增 --backfill --prune 与 --delete-run；回填会跳过返回 404 的提交。校验与 collectivex/tests/test_results.py 一致（版本严格相等、source_sha 为 40 位十六进制、不含 IPv4 或主机字段）。 covered_skus 只列出有已测量结果的资源池。文档：运维手册改为回填加清理（不再 手动清空表并以 ref=main 运行）、删除 HTTP 409 说明、在 API 文档中说明 EP 尾部 百分位可为 null，并修正过时的模块注释。 --- .github/workflows/ingest-collectivex.yml | 36 +++- docs/collectivex.md | 104 +++++++---- packages/app/src/lib/api/api-documentation.ts | 16 +- packages/app/src/lib/api/api-route-catalog.ts | 2 +- .../005_remove_lazy_ingest_runs.sql | 22 +++ .../src/collectivex/committed-results.test.ts | 102 +++++++++-- .../db/src/collectivex/committed-results.ts | 120 ++++++++++--- packages/db/src/collectivex/reader.ts | 7 +- .../db/src/collectivex/swap-reader.test.ts | 3 +- .../db/src/collectivex/swap-test-fixture.ts | 2 +- packages/db/src/ingest-collectivex.test.ts | 90 +++++++--- packages/db/src/ingest-collectivex.ts | 161 ++++++++++++++---- .../db/src/queries/collectivex-db.test.ts | 95 ++++++++++- packages/db/src/queries/collectivex.test.ts | 37 ++-- packages/db/src/queries/collectivex.ts | 142 ++++++++++----- .../skills/inferencex-api/integrity.json | 2 +- .../inferencex-api/references/collectivex.md | 9 +- 17 files changed, 755 insertions(+), 195 deletions(-) create mode 100644 packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql diff --git a/.github/workflows/ingest-collectivex.yml b/.github/workflows/ingest-collectivex.yml index fd67b28d1..3815fc942 100644 --- a/.github/workflows/ingest-collectivex.yml +++ b/.github/workflows/ingest-collectivex.yml @@ -15,11 +15,22 @@ on: description: Ingest every collectivex/results/ commit on main instead type: boolean default: false + prune: + description: With backfill, then delete every stored run the backfill did not store + type: boolean + default: false + delete_run: + description: Delete this one stored run_id instead (no other target) + required: false permissions: {} +# One group for every ingest, prune and delete: runs queue behind each other instead of +# writing at once (the database also serializes writers with an advisory lock). GitHub +# keeps a single pending run per group, so a burst of dispatches can supersede a pending +# one; a backfill then stores anything it skipped. concurrency: - group: ${{ github.workflow }}-${{ github.event.client_payload.sha || inputs.sha || inputs.ref || 'backfill' }} + group: ${{ github.workflow }} cancel-in-progress: false jobs: @@ -40,6 +51,8 @@ jobs: SHA: ${{ github.event.client_payload.sha || inputs.sha }} REF: ${{ inputs.ref }} BACKFILL: ${{ inputs.backfill }} + PRUNE: ${{ inputs.prune }} + DELETE_RUN: ${{ inputs.delete_run }} run: | : "${SLACK_WEBHOOK_URL:?SLACK_WEBHOOK_URL secret is not set}" : "${APP_URL:?APP_URL repository variable is not set}" @@ -50,15 +63,24 @@ jobs: [ -n "$SHA" ] && selected=$((selected + 1)) [ -n "$REF" ] && selected=$((selected + 1)) [ "$BACKFILL" = true ] && selected=$((selected + 1)) + [ -n "$DELETE_RUN" ] && selected=$((selected + 1)) if [ "$selected" -ne 1 ]; then - echo "give exactly one of sha, ref, or backfill" >&2 + echo "give exactly one of sha, ref, backfill, or delete_run" >&2 + exit 1 + fi + if [ "$PRUNE" = true ] && [ "$BACKFILL" != true ]; then + echo "prune needs backfill" >&2 + exit 1 + fi + if [ -n "$DELETE_RUN" ] && ! [[ "$DELETE_RUN" =~ ^[1-9][0-9]{0,9}$ ]]; then + echo "delete_run must be a stored run_id" >&2 exit 1 fi if [ -n "$SHA" ] && ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "sha must be 40 lowercase hex characters" >&2 exit 1 fi - if [ -n "$REF" ] && { ! [[ "$REF" =~ ^[A-Za-z0-9_./-]{1,255}$ ]] || [[ "$REF" == *..* ]]; }; then + if [ -n "$REF" ] && { ! [[ "$REF" =~ ^[A-Za-z0-9_][A-Za-z0-9_./-]{0,254}$ ]] || [[ "$REF" == *..* ]]; }; then echo "ref is not a valid branch or tag name" >&2 exit 1 fi @@ -96,8 +118,14 @@ jobs: SHA: ${{ github.event.client_payload.sha || inputs.sha }} REF: ${{ inputs.ref }} BACKFILL: ${{ inputs.backfill }} + PRUNE: ${{ inputs.prune }} + DELETE_RUN: ${{ inputs.delete_run }} run: | - if [ "$BACKFILL" = true ]; then + if [ -n "$DELETE_RUN" ]; then + target=(--delete-run "$DELETE_RUN") + elif [ "$BACKFILL" = true ] && [ "$PRUNE" = true ]; then + target=(--backfill --prune) + elif [ "$BACKFILL" = true ]; then target=(--backfill) elif [ -n "$SHA" ]; then target=(--sha "$SHA") diff --git a/docs/collectivex.md b/docs/collectivex.md index 504084f48..3b0795fd0 100644 --- a/docs/collectivex.md +++ b/docs/collectivex.md @@ -7,17 +7,29 @@ one `/.json` per measured case. The API routes only read that data ## One run per results commit -Each InferenceX commit on `main` that changed `collectivex/results/` is one stored run: - -- `run_id` is the commit's committer time in epoch seconds. It grows with `main`'s history, - so "latest" still means `ORDER BY run_id DESC`. Two results commits landing in the same - second would collide; the ingest refuses the second one (HTTP 409 / CLI error) rather than - overwrite it. +Each InferenceX commit on `main` that changed `collectivex/results/` and holds at least one +measured result is one stored run. A commit with nothing measured (for example one that only +changed `cases.json` before any result landed) is skipped, never stored as an empty run, and +the read queries hide any row whose summary measures nothing. + +- `run_id` is the commit's committer time in epoch seconds, raised to one past the previous + results commit's id when it would not exceed it (two commits in the same second, or a + committer clock behind history). It is computed from `main`'s results history alone, so + every ingest gives a commit the same id, ids are unique, and "latest" still means + `ORDER BY run_id DESC`. A stored id that belongs to another commit only happens if `main`'s + history was rewritten; the CLI then fails rather than overwrite it (there is no HTTP ingest + endpoint, so no HTTP status is involved). Ids are below 10^10, which migration + `005_remove_lazy_ingest_runs.sql` enforces with a CHECK after deleting the retired lazy + ingest's rows (keyed by GitHub Actions run id, so they would otherwise outrank every + committed run). - `run_attempt` is always 1, `source_sha` is the results commit, `source_branch` is `main`, `conclusion` is `success`, `matrix` is `cases.json` and `summary` is the shared reader's `buildRunSummary`. `cx_run_docs` holds the commit's result documents verbatim. -- Re-ingesting a commit is idempotent: one statement upserts the row and replaces its - documents, so a reader sees the old contents or the new ones, never a mix. +- Re-ingesting a commit is idempotent: one transaction takes a fixed advisory lock + (`pg_advisory_xact_lock`, shared by every CollectiveX write) and then one statement upserts + the row and replaces its documents, so a reader sees the old contents or the new ones, + never a mix, and two concurrent ingests of the same commit cannot both insert its + documents. - A ref resolves to the newest results commit at or before it, and that commit must be in `main`'s results history. Ingesting `main` after an unrelated merge therefore refreshes the run already published for those files instead of creating a duplicate, and results @@ -37,37 +49,61 @@ limit. It then checks every file the way the benchmark repo's own test does: the `/.json` matching the document's identity (EP case attempts by `identity.case_id` and `identity.case_factors.sku`; swap_blocks results by `schema: collectivex-swap-blocks-v1`, `case_id` and `runtime.sku`), the case is runnable in -`cases.json`, the result was measured from committed code (`source_dirty: false`), and an EP -result carries the `cases.json` version. Each document is then assembled alone, so every -failure names its file. Files above 4 MiB, or a tree above 256 MiB, are rejected. +`cases.json`, the result was measured from committed code (`source_dirty: false` and a 40-hex +`source_sha`), an EP result carries exactly the `cases.json` version, and no string holds an +IPv4 address and no key names a machine (`host`, `hosts`, `hostname`, `nodelist`, +`slurm_nodelist`, `master_addr`, case-insensitive). Each document is then assembled alone, so +every failure names its file. Files above 4 MiB, or a tree above 256 MiB, are rejected. - **Workflow**: `.github/workflows/ingest-collectivex.yml` runs on `repository_dispatch` type `ingest-collectivex` with `client_payload: {"sha": "<40 hex>"}`, which the InferenceX workflow sends on pushes to `main` that touch `collectivex/results/`, and on - `workflow_dispatch` with exactly one of `sha` (40 hex), `ref`, or `backfill`. It runs - `migrate-collectivex.ts --yes`, then the ingest CLI with `--sha`, `--ref` or `--backfill` - and `--yes` against the `DATABASE_COLLECTIVEX_WRITE_URL` secret (GitHub reads use + `workflow_dispatch` with exactly one of `sha` (40 hex), `ref`, `backfill` (optionally with + `prune`), or `delete_run`. Every run shares one concurrency group that queues rather than + cancels; GitHub keeps only one pending run per group, so a burst of dispatches can + supersede a pending ingest, which the next backfill makes up. It runs + `migrate-collectivex.ts --yes`, then the ingest CLI with `--sha`, `--ref`, `--backfill +[--prune]` or `--delete-run` and `--yes` against the `DATABASE_COLLECTIVEX_WRITE_URL` secret (GitHub reads use `INFX_MAIN_PAT`), then purges the cache with `POST /api/v1/invalidate?scope=collectivex` (`VERCEL_INVALIDATE_SECRET`). It alerts Slack on failure, like `ingest-operatorx`. -- **CLI**: `bun run admin:db:ingest:collectivex [--sha | --ref ] [--backfill] -[--yes] [--no-ssl]` ingests one commit (default: `main`), or with `--backfill` every - results commit on `main`, oldest first, reusing unchanged files between commits. It asks - for confirmation unless `--yes` is given and stops at the first failing commit. - `GITHUB_TOKEN` is optional but raises GitHub's API rate limit. +- **CLI**: `bun run admin:db:ingest:collectivex [--sha | --ref | --backfill +[--prune] | --delete-run ] [--yes] [--no-ssl]` ingests one commit (default: `main`), + or with `--backfill` every results commit on `main`, oldest first, reusing unchanged files + between commits. A backfill skips commits GitHub answers 404 for (for example one that + deleted `collectivex/results/`) and commits with nothing measured; any other failure, such + as a file that breaks the results contract, stops it. `--prune` then deletes every stored + run the backfill did not store: runs whose commit left `main`, no longer validates, or + measures nothing. `--delete-run ` deletes one stored run at once. It asks for + confirmation unless `--yes` is given. `GITHUB_TOKEN` is optional but raises GitHub's API + rate limit. -## Operator runbook: re-seed from committed results +## Operator runbook -Run these against the CollectiveX database only, in order: +**Cut over from the lazy ingest** (once, after this lands): 1. **Archive** the current contents: `pg_dump --table=cx_runs --table=cx_run_docs "$DATABASE_COLLECTIVEX_WRITE_URL" > cx-archive.sql` -2. **Wipe**: `psql "$DATABASE_COLLECTIVEX_WRITE_URL" -c 'TRUNCATE cx_run_docs, cx_runs'`. - Until the backfill finishes, `/api/v1/collectivex/latest` returns 404 and the run list is empty. -3. **Migrate**: `bun run admin:db:migrate:collectivex` (applies `004_committed_results.sql`, - which drops the unused `deleted_at` tombstone column). -4. **Backfill**: run the `Ingest CollectiveX Results` workflow with `backfill` checked, which - also purges the cache. Locally, `bun run admin:db:ingest:collectivex --backfill`, then - `POST /api/v1/invalidate?scope=collectivex` (the CLI does not reach the CDN). +2. **Backfill**: run the `Ingest CollectiveX Results` workflow with `backfill` and `prune` + checked. Its migration step applies `004_committed_results.sql` (drops the `deleted_at` + tombstone) and `005_remove_lazy_ingest_runs.sql` (deletes every lazy-ingest run and adds + the run_id CHECK), so no manual `TRUNCATE` is needed; the backfill then stores every + measured results commit on `main` and the cache is purged. Until a measured results + commit lands on `main`, `/api/v1/collectivex/latest` returns 404 and the run list is empty. + Do not use `ref=main` for this: it stores only the newest results commit, not the history. + +**Day to day** nothing is manual: each push to `main` that touches `collectivex/results/` +dispatches an ingest of that commit. Use `sha` or `ref` to re-ingest one commit (for +example after its dispatched ingest failed) and `backfill` to make up any missed one. + +**Retract a bad run**: the durable fix is a commit to the benchmark repo that removes or +corrects the result files, followed by a `backfill` with `prune` (the run's own commit is +still on `main`, so a backfill without `prune` would keep it). To take a run down at once, +run the workflow with `delete_run` set to its `run_id`; a later backfill re-adds it while its +commit still holds measured results, so follow up with the benchmark-repo fix. + +Locally, the same steps are `bun run admin:db:migrate:collectivex` and +`bun run admin:db:ingest:collectivex --backfill --prune` (or `--delete-run `), then +`POST /api/v1/invalidate?scope=collectivex` (the CLI does not reach the CDN). ## Serving @@ -86,14 +122,22 @@ Run these against the CollectiveX database only, in order: - **Run list**: `/api/v1/collectivex/runs` always reports `discovery_complete: true`; the field stays for existing clients. - **No deletion or ingest in the app**: the app exposes no CollectiveX mutation. Runs are - added by the ingest workflow or CLI and removed by changing the committed results. + added by the ingest workflow or CLI and removed by a `backfill` with `prune` after the + committed results change, or at once with `delete_run` (see the runbook). +- **Covered pools**: a run summary's `covered_skus` lists only pools with at least one + measured result (EP, KV or swap_blocks); pools that are requested but have no result yet + are not listed. - **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` results (case-insensitive). Other vendor values are omitted from chart series, coverage, and SKU summaries; runs with no supported-vendor cases stay hidden. - **Withheld tails**: an EP result may publish p90/p95/p99 as `null` when too many of a series' samples were misaligned or host-stalled (p50 is always present). The reader keeps the derived rates `null` there, the alpha-beta fit skips them, and the tooltip shows them - as unavailable. + as unavailable. This is an API response-shape change: in `series[].points[]`, + `components.*.latency_us`, `components.*.activation_data_rate_gbps_at_latency_percentile`, + `components.*.payload_data_rate_gbps_at_latency_percentile` and + `roundtrip_token_rate_at_latency_percentile` may now carry `null` for `p90`, `p95` and + `p99`; the API documentation says so. - **Links**: a run links to `collectivex/results/` and the methodology at its `source_sha`. ## Known-support matrix diff --git a/packages/app/src/lib/api/api-documentation.ts b/packages/app/src/lib/api/api-documentation.ts index e96a30f62..8dd8da0a3 100644 --- a/packages/app/src/lib/api/api-documentation.ts +++ b/packages/app/src/lib/api/api-documentation.ts @@ -1949,8 +1949,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/latest', summary: text('Read the latest CollectiveX dataset', '读取最新 CollectiveX 数据集'), description: text( - 'Returns the versioned neutral dataset of the newest stored run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed collectivex/results/: run_id is that commit time in epoch seconds, run_attempt is 1, and run.source_sha is the commit. Runs are ingested when results land on main, never on read. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', - '返回最新已存储运行的带版本中立数据集。每次运行对应 SemiAnalysisAI/InferenceX main 分支上一个修改了 collectivex/results/ 的提交：run_id 为该提交时间的 Unix 秒级时间戳，run_attempt 为 1，run.source_sha 为该提交。结果合入 main 时导入，读取时不会导入。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', + 'Returns the versioned neutral dataset of the newest stored run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed collectivex/results/ and holds at least one measured result: run_id is that commit time in epoch seconds (raised past the previous results commit when two land in the same second), run_attempt is 1, and run.source_sha is the commit. run.covered_skus lists only pools with a measured result. Runs are ingested when results land on main, never on read. In series[].points[], components.*.latency_us, components.*.activation_data_rate_gbps_at_latency_percentile, components.*.payload_data_rate_gbps_at_latency_percentile and roundtrip_token_rate_at_latency_percentile carry p50, p90, p95 and p99; p90, p95 and p99 are null where the harness withheld a tail because too many samples were misaligned or host-stalled, while p50 is always a number. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', + '返回最新已存储运行的带版本中立数据集。每次运行对应 SemiAnalysisAI/InferenceX main 分支上一个修改了 collectivex/results/ 且至少包含一个已测量结果的提交：run_id 为该提交时间的 Unix 秒级时间戳（若两个结果提交落在同一秒，则递增至上一个结果提交之后），run_attempt 为 1，run.source_sha 为该提交。run.covered_skus 只列出有已测量结果的资源池。结果合入 main 时导入，读取时不会导入。 在 series[].points[] 中，components.*.latency_us、components.*.activation_data_rate_gbps_at_latency_percentile、components.*.payload_data_rate_gbps_at_latency_percentile 与 roundtrip_token_rate_at_latency_percentile 包含 p50、p90、p95 和 p99；若测试工具因过多样本未对齐或被主机阻塞而未发布尾部延迟，p90、p95 和 p99 为 null，p50 始终为数值。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', ), audience: 'public', stability: 'beta', @@ -2026,8 +2026,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs', summary: text('List CollectiveX runs', '列出 CollectiveX 运行'), description: text( - 'Returns every stored run summary, newest results commit first. Runs are ingested ahead of reads, so discovery_complete is always true; the field is kept for existing clients.', - '按结果提交从新到旧返回全部已存储的运行汇总。运行在读取前即已导入，因此 discovery_complete 始终为 true；保留该字段是为了兼容现有客户端。', + 'Returns every stored run summary, newest results commit first. Only commits with at least one measured result are runs, and covered_skus lists only pools with a measured result. Runs are ingested ahead of reads, so discovery_complete is always true; the field is kept for existing clients.', + '按结果提交从新到旧返回全部已存储的运行汇总。只有至少包含一个已测量结果的提交才构成运行，covered_skus 只列出有已测量结果的资源池。运行在读取前即已导入，因此 discovery_complete 始终为 true；保留该字段是为了兼容现有客户端。', ), audience: 'public', stability: 'beta', @@ -2095,8 +2095,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs/{runId}', summary: text('Read a CollectiveX run', '读取 CollectiveX 运行'), description: text( - 'Returns one stored run as a versioned CollectiveX dataset. The run ID is the results commit time in epoch seconds, as listed by /api/v1/collectivex/runs.', - '以带版本的 CollectiveX 数据集返回一次已存储的运行。运行 ID 为结果提交时间的 Unix 秒级时间戳，与 /api/v1/collectivex/runs 列出的一致。', + 'Returns one stored run as a versioned CollectiveX dataset. The run ID is the results commit time in epoch seconds (raised past the previous results commit on a same-second tie), as listed by /api/v1/collectivex/runs. In series[].points[], components.*.latency_us, components.*.activation_data_rate_gbps_at_latency_percentile, components.*.payload_data_rate_gbps_at_latency_percentile and roundtrip_token_rate_at_latency_percentile carry p50, p90, p95 and p99; p90, p95 and p99 are null where the harness withheld a tail because too many samples were misaligned or host-stalled, while p50 is always a number.', + '以带版本的 CollectiveX 数据集返回一次已存储的运行。运行 ID 为结果提交时间的 Unix 秒级时间戳（同一秒出现多个结果提交时递增至上一个之后），与 /api/v1/collectivex/runs 列出的一致。 在 series[].points[] 中，components.*.latency_us、components.*.activation_data_rate_gbps_at_latency_percentile、components.*.payload_data_rate_gbps_at_latency_percentile 与 roundtrip_token_rate_at_latency_percentile 包含 p50、p90、p95 和 p99；若测试工具因过多样本未对齐或被主机阻塞而未发布尾部延迟，p90、p95 和 p99 为 null，p50 始终为数值。', ), audience: 'public', stability: 'beta', @@ -2106,8 +2106,8 @@ export const apiOperations: readonly ApiOperation[] = [ 'path', true, 'integer', - 'Positive run ID: the results commit time in epoch seconds.', - '正整数运行 ID：结果提交时间的 Unix 秒级时间戳。', + 'Positive run ID: the results commit time in epoch seconds, below 10^10.', + '正整数运行 ID：结果提交时间的 Unix 秒级时间戳，小于 10^10。', positiveIdSchema, 1792089351, ), diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index f8f61fef5..1d3e87f6c 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -1736,7 +1736,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/collectivex.ts', - sourceSha256: '73ff29402f25c514bc5e2578703aacd4d6c81b7eb045266188d50b55391ed40c', + sourceSha256: 'd330c47acd94331d815515287990a5e25ab76a0f0d49c57d47dea0852604c3cd', reviewArea: { en: 'CollectiveX dataset projection, run summaries, coverage, series, and latest-run ordering.', zh: 'CollectiveX 数据集投影、运行汇总、覆盖范围、序列和最新运行排序。', diff --git a/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql b/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql new file mode 100644 index 000000000..b9858c97d --- /dev/null +++ b/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql @@ -0,0 +1,22 @@ +-- Remove the runs the retired lazy GitHub Actions ingest stored, and keep them out. +-- +-- Those rows are keyed by GitHub Actions run id (>= 10^10), committed-results runs by +-- epoch seconds (< 10^10 until the year 2286). Left in place, every legacy row would +-- outrank every committed-results run under `ORDER BY run_id DESC`, so "latest" would +-- stay a stale CI sweep forever. Deleting cascades to their cx_run_docs. The archive of +-- the old contents is a pg_dump taken before this runs (see docs/collectivex.md). +-- +-- The CHECK makes the id range part of the schema, so a legacy-shaped id can never be +-- stored again. It matches MAX_COMMITTED_RUN_ID in +-- packages/db/src/collectivex/committed-results.ts. + +delete from cx_runs where run_id >= 10000000000 or run_id <= 0; + +-- Documents whose run is gone (none are expected: the foreign key cascades). +delete from cx_run_docs d where not exists (select 1 from cx_runs r where r.run_id = d.run_id); + +alter table cx_runs + add constraint cx_runs_committed_run_id check (run_id > 0 and run_id < 10000000000); + +comment on table cx_runs is + 'One row per InferenceX commit on main whose collectivex/results/ holds measured results; run_id = committer epoch second, raised past the previous results commit''s id on ties.'; diff --git a/packages/db/src/collectivex/committed-results.test.ts b/packages/db/src/collectivex/committed-results.test.ts index 83b9590c0..fc07283cb 100644 --- a/packages/db/src/collectivex/committed-results.test.ts +++ b/packages/db/src/collectivex/committed-results.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; import { + assignRunIds, committedRun, CommittedResultsError, fetchResultsSource, @@ -52,6 +53,7 @@ function sourceOf(cases: unknown, files: Record): ResultsSource return { sha: SHA, committedAt: COMMITTED_AT, + runId: COMMITTED_SECONDS, files: new Map([ ['cases.json', encode(cases)], ...Object.entries(files).map(([path, doc]): [string, Uint8Array] => [path, encode(doc)]), @@ -141,15 +143,65 @@ describe('validateCommittedResults', () => { }); it('rejects results measured from uncommitted code', () => { - const ep = { ...epResult(), provenance: { source_dirty: true } }; + const ep = { ...epResult(), provenance: { source_sha: 'e'.repeat(40), source_dirty: true } }; expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( /was not measured from committed collectivex\/ code/u, ); + const unknown = { ...epResult(), provenance: { source_sha: 'e'.repeat(40) } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([unknown]), { [pathOf(unknown)]: unknown })), + ).toThrow(/was not measured from committed collectivex\/ code/u); + }); + + it('requires a full 40-hex source_sha on EP and swap_blocks results', () => { + for (const sourceSha of [undefined, 'abc123', 'E'.repeat(40), 'e'.repeat(41)]) { + const ep = { ...epResult(), provenance: { source_sha: sourceSha, source_dirty: false } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep })), + ).toThrow(`${RESULTS_DIR}/${pathOf(ep)} has no 40-hex source_sha`); + } + const swap = makeSwapDoc('h200-dgxc'); + swap.runtime.source_sha = 'abc123'; + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ), + ).toThrow(/has no 40-hex source_sha/u); + }); + + it('requires the exact cases.json version, not a coercible one', () => { + const ep = { ...epResult(), version: '1' }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + /has version 1, cases\.json has 1/u, + ); + }); + + it('rejects results that name machines by IPv4 address or host field', () => { + const withIp = { ...epResult(), notes: ['rendezvous at 10.0.12.7:29500'] }; + expect(() => + validateCommittedResults(sourceOf(casesFor([withIp]), { [pathOf(withIp)]: withIp })), + ).toThrow(`${RESULTS_DIR}/${pathOf(withIp)} contains an IP address`); + const withHost = { ...epResult(), launch: { SLURM_NODELIST: 'node[1-2]' } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([withHost]), { [pathOf(withHost)]: withHost })), + ).toThrow(/contains the host-identifying field "SLURM_NODELIST"/u); + // Dotted versions that are not four octets, and host_* keys, are fine. + const fine = { ...epResult(), notes: ['vllm 0.25.1', 'v1.2.3.4.5'], host_memory: 'pinned' }; + expect( + validateCommittedResults(sourceOf(casesFor([fine]), { [pathOf(fine)]: fine })).docs, + ).toHaveLength(1); }); it('requires cases.json with a version and requested cases', () => { expect(() => - validateCommittedResults({ sha: SHA, committedAt: COMMITTED_AT, files: new Map() }), + validateCommittedResults({ + sha: SHA, + committedAt: COMMITTED_AT, + runId: COMMITTED_SECONDS, + files: new Map(), + }), ).toThrow(`${RESULTS_DIR}/cases.json is missing at ${SHA}`); expect(() => validateCommittedResults(sourceOf({ requested_cases: [] }, {}))).toThrow( /has no version and requested_cases/u, @@ -196,12 +248,36 @@ describe('committedRun', () => { expect(docs).toEqual([ep]); }); - it('orders later commits after earlier ones', () => { + it('stores the run under the commit run id and keeps generated_at at the commit time', () => { const ep = epResult(); - const later = { ...sourceOf(casesFor([ep]), {}), committedAt: '2026-10-16T00:00:00Z' }; - expect(Number(committedRun(validateCommittedResults(later)).run.run_id)).toBeGreaterThan( - Number(COMMITTED_SECONDS), - ); + const bumped = { + ...sourceOf(casesFor([ep]), {}), + runId: String(Number(COMMITTED_SECONDS) + 1), + }; + const { run } = committedRun(validateCommittedResults(bumped)); + expect(run.run_id).toBe(bumped.runId); + expect(run.generated_at).toBe(COMMITTED_AT); + }); +}); + +describe('assignRunIds', () => { + it('is the committer second, bumped past ties and clocks behind history', () => { + const ids = assignRunIds([ + { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z' }, + { sha: 'b'.repeat(40), committedAt: '2026-10-15T00:00:00.900Z' }, + { sha: 'c'.repeat(40), committedAt: '2026-10-14T00:00:00Z' }, + { sha: 'd'.repeat(40), committedAt: '2026-10-16T00:00:00Z' }, + ]).map((commit) => Number(commit.runId)); + const base = Date.parse('2026-10-15T00:00:00Z') / 1000; + expect(ids).toEqual([base, base + 1, base + 2, Date.parse('2026-10-16T00:00:00Z') / 1000]); + }); + + it('gives a commit the same id whatever ingest computes it', () => { + const history = [ + { sha: 'a'.repeat(40), committedAt: COMMITTED_AT }, + { sha: 'b'.repeat(40), committedAt: COMMITTED_AT }, + ]; + expect(assignRunIds(history)).toEqual(assignRunIds(history)); }); }); @@ -227,7 +303,7 @@ describe('GitHub reads', () => { }); await expect( resolveResultsCommit('feature', { fetch: fetch as unknown as typeof globalThis.fetch }), - ).resolves.toEqual({ sha: SHA, committedAt: COMMITTED_AT }); + ).resolves.toEqual({ sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }); }); it('refuses a results commit that never landed on main', async () => { @@ -255,6 +331,10 @@ describe('GitHub reads', () => { }); expect(commits).toHaveLength(101); expect(commits[0].sha).toBe(SHA); + // The 100 same-second commits after SHA get distinct, increasing ids. + const ids = commits.map((commit) => Number(commit.runId)); + expect(new Set(ids).size).toBe(101); + expect(ids).toEqual(ids.toSorted((a, b) => a - b)); expect(fetch).toHaveBeenCalledTimes(2); }); @@ -277,7 +357,7 @@ describe('GitHub reads', () => { }); const blobCache = new Map([['b2', new TextEncoder().encode('cached')]]); const source = await fetchResultsSource( - { sha: SHA, committedAt: COMMITTED_AT }, + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, { fetch: fetch as unknown as typeof globalThis.fetch, token: 'token', blobCache }, ); expect(urls[0]).toContain(`/git/trees/${SHA}:${RESULTS_DIR}?recursive=1`); @@ -293,7 +373,7 @@ describe('GitHub reads', () => { const missing = vi.fn(() => new Response('', { status: 404 })); await expect( fetchResultsSource( - { sha: SHA, committedAt: COMMITTED_AT }, + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, { fetch: missing as unknown as typeof globalThis.fetch }, ), ).rejects.toThrow(`${RESULTS_DIR} does not exist at ${SHA}`); @@ -305,7 +385,7 @@ describe('GitHub reads', () => { ); await expect( fetchResultsSource( - { sha: SHA, committedAt: COMMITTED_AT }, + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, { fetch: oversize as unknown as typeof globalThis.fetch }, ), ).rejects.toMatchObject({ kind: 'invalid' }); diff --git a/packages/db/src/collectivex/committed-results.ts b/packages/db/src/collectivex/committed-results.ts index d66b97e98..47545e0f2 100644 --- a/packages/db/src/collectivex/committed-results.ts +++ b/packages/db/src/collectivex/committed-results.ts @@ -1,9 +1,10 @@ /** * CollectiveX results committed to the benchmark repo: `collectivex/results/cases.json` * (the requested cases) plus one `collectivex/results//.json` per measured - * case. Each InferenceX commit that carries results becomes one stored run, so this - * module pins a ref to a commit, reads exactly those files at it, validates them, and - * turns them into a cx_runs row. Shared by the ingest CLI and the app's ingest route. + * case. Each InferenceX commit on main that carries measured results becomes one stored + * run, so this module pins a ref to a commit, reads exactly those files at it, validates + * them, and turns them into a cx_runs row. Used by the ingest CLI (ingest-collectivex.ts), + * which the ingest-collectivex workflow runs; the app itself never ingests. */ import { @@ -30,6 +31,23 @@ const FETCH_CONCURRENCY = 8; const COMMITS_PER_PAGE = 100; const SHA = /^[0-9a-f]{40}$/u; const RESULT_PATH = /^(?[^/]+)\/(?[^/]+)\.json$/u; +// Mirrors collectivex/tests/test_results.py in the benchmark repo: results must not name +// machines, by IPv4 address in any string or by a host-identifying key anywhere. +const IPV4 = /(?; @@ -115,7 +145,7 @@ async function getJson(url: string, options: GithubReadOptions): Promise { +): Promise { const query = new URLSearchParams({ sha: ref, path: RESULTS_DIR, @@ -148,21 +178,37 @@ async function resultsCommitsPage( if (!Array.isArray(body)) { throw new CommittedResultsError('unavailable', 'GitHub commit list is not an array'); } - return body.map(toResultsCommit); + return body.map(toGithubCommit); +} + +/** + * Give each results commit of a history (oldest first) its run_id: its committer + * second, or one more than the previous id when that is not larger. + */ +export function assignRunIds(history: readonly GithubCommit[]): ResultsCommit[] { + let previous = 0; + return history.map((commit) => { + const id = Math.max(Math.floor(Date.parse(commit.committedAt) / 1000), previous + 1); + if (id >= MAX_COMMITTED_RUN_ID) { + throw new CommittedResultsError('unavailable', `${commit.sha} has an out-of-range time`); + } + previous = id; + return { ...commit, runId: String(id) }; + }); } -/** Every commit on `branch` that touched RESULTS_DIR, oldest first. */ +/** Every commit on `branch` that touched RESULTS_DIR, oldest first, with its run_id. */ export async function listResultsCommits( branch: string = RESULTS_BRANCH, options: GithubReadOptions = {}, ): Promise { - const commits: ResultsCommit[] = []; + const commits: GithubCommit[] = []; for (let page = 1; ; page++) { const batch = await resultsCommitsPage(branch, page, COMMITS_PER_PAGE, options); commits.push(...batch); if (batch.length < COMMITS_PER_PAGE) break; } - return commits.toReversed(); + return assignRunIds(commits.toReversed()); } /** @@ -184,13 +230,14 @@ export async function resolveResultsCommit( ); } const history = await listResultsCommits(RESULTS_BRANCH, options); - if (!history.some((entry) => entry.sha === commit.sha)) { + const entry = history.find((candidate) => candidate.sha === commit.sha); + if (!entry) { throw new CommittedResultsError( 'not-found', `${commit.sha} (from ${ref}) is not a ${RESULTS_DIR} commit on ${RESULTS_BRANCH}`, ); } - return commit; + return entry; } interface TreeEntry { @@ -285,8 +332,8 @@ interface ResultDoc { case_id?: unknown; version?: unknown; identity?: { case_id?: unknown; case_factors?: { sku?: unknown } }; - provenance?: { source_dirty?: unknown }; - runtime?: { sku?: unknown; source_dirty?: unknown }; + provenance?: { source_sha?: unknown; source_dirty?: unknown }; + runtime?: { sku?: unknown; source_sha?: unknown; source_dirty?: unknown }; } /** Identity fields of an EP case attempt or a swap_blocks result; null for anything else. */ @@ -295,11 +342,37 @@ function resultIdentity(doc: ResultDoc | null) { return { caseId: doc.identity?.case_id, sku: doc.identity?.case_factors?.sku, + sourceSha: doc.provenance?.source_sha, sourceDirty: doc.provenance?.source_dirty, }; } if (doc?.schema === SWAP_SCHEMA) { - return { caseId: doc.case_id, sku: doc.runtime?.sku, sourceDirty: doc.runtime?.source_dirty }; + return { + caseId: doc.case_id, + sku: doc.runtime?.sku, + sourceSha: doc.runtime?.source_sha, + sourceDirty: doc.runtime?.source_dirty, + }; + } + return null; +} + +/** The first machine identifier in a document: an IPv4 address or a host-naming key. */ +function machineIdentifier(value: unknown): string | null { + if (typeof value === 'string') return IPV4.test(value) ? 'an IP address' : null; + if (Array.isArray(value)) { + for (const item of value) { + const found = machineIdentifier(item); + if (found) return found; + } + return null; + } + if (value && typeof value === 'object') { + for (const [key, item] of Object.entries(value)) { + if (HOST_KEYS.has(key.toLowerCase())) return `the host-identifying field "${key}"`; + const found = machineIdentifier(item); + if (found) return found; + } } return null; } @@ -315,7 +388,7 @@ function parseJson(path: string, data: Uint8Array): unknown { function runMeta(source: ResultsCommit): CollectiveXNeutralRunMeta { const seconds = Math.floor(Date.parse(source.committedAt) / 1000); return { - run_id: String(seconds), + run_id: source.runId, run_attempt: 1, generated_at: new Date(seconds * 1000).toISOString().replace(/\.\d{3}Z$/u, 'Z'), conclusion: 'success', @@ -373,18 +446,23 @@ export function validateCommittedResults(source: ResultsSource): CommittedResult if (!runnable.has(JSON.stringify([location.sku, location.caseId]))) { throw new CommittedResultsError('invalid', `${path} is not a runnable case in ${CASES_FILE}`); } + if (typeof identity.sourceSha !== 'string' || !SHA.test(identity.sourceSha)) { + throw new CommittedResultsError('invalid', `${path} has no 40-hex source_sha`); + } if (identity.sourceDirty !== false) { throw new CommittedResultsError( 'invalid', `${path} was not measured from committed collectivex/ code`, ); } - if (doc?.record_type === 'case-attempt' && Number(doc.version) !== version) { + if (doc?.record_type === 'case-attempt' && doc.version !== version) { throw new CommittedResultsError( 'invalid', `${path} has version ${String(doc.version)}, ${CASES_FILE} has ${version}`, ); } + const leaked = machineIdentifier(doc); + if (leaked) throw new CommittedResultsError('invalid', `${path} contains ${leaked}`); results.push({ path, doc }); } @@ -404,6 +482,7 @@ export function validateCommittedResults(source: ResultsSource): CommittedResult return { sha: source.sha, committedAt: source.committedAt, + runId: source.runId, version, matrix, docs: results.map(({ doc }) => doc), @@ -411,10 +490,9 @@ export function validateCommittedResults(source: ResultsSource): CommittedResult } /** - * The cx_runs row for one results commit. run_id is the commit time in epoch - * seconds, so it grows with main's history and `ORDER BY run_id DESC` still means - * "latest"; each commit is one attempt whose documents are replaced wholesale when - * it is ingested again. + * The cx_runs row for one results commit. run_id comes from assignRunIds, so it grows + * with main's history and `ORDER BY run_id DESC` still means "latest"; each commit is + * one attempt whose documents are replaced wholesale when it is ingested again. */ export function committedRun(results: CommittedResults): { run: CollectiveXRunInsert; diff --git a/packages/db/src/collectivex/reader.ts b/packages/db/src/collectivex/reader.ts index 89c84ba97..e7ae4148c 100644 --- a/packages/db/src/collectivex/reader.ts +++ b/packages/db/src/collectivex/reader.ts @@ -478,11 +478,12 @@ export function buildDatasetFromNeutral( points.filter((point) => point.terminal_status !== 'pending').length + swapPoints, measured_points: points.filter((point) => point.terminal_status === 'measured').length + swapPoints, + // Pools with at least one measured result; requested-only pools are not covered. covered_skus: [ ...new Set([ - ...coverage.map((item) => item.sku), - ...kv.map((item) => item.sku), - ...swapRequested.map((item) => item.sku), + ...coverage.filter((item) => item.outcome === 'success').map((item) => item.sku), + ...kv.filter((item) => item.outcome === 'success').map((item) => item.sku), + ...swap.map((item) => item.sku), ]), ].toSorted(), swap_requested_cases: swapRequested.length, diff --git a/packages/db/src/collectivex/swap-reader.test.ts b/packages/db/src/collectivex/swap-reader.test.ts index 8a49f9ba1..1d8ad5c74 100644 --- a/packages/db/src/collectivex/swap-reader.test.ts +++ b/packages/db/src/collectivex/swap-reader.test.ts @@ -103,7 +103,8 @@ describe('swap_blocks cases in a unified cases.json', () => { terminal_cases: 2, swap_requested_cases: 3, swap_measured_cases: 1, - covered_skus: ['h200-dgxc', 'mi300x'], + // mi300x is requested but has no result, so it is not a covered pool. + covered_skus: ['h200-dgxc'], }); }); diff --git a/packages/db/src/collectivex/swap-test-fixture.ts b/packages/db/src/collectivex/swap-test-fixture.ts index 3d04eccdf..7d0c3d598 100644 --- a/packages/db/src/collectivex/swap-test-fixture.ts +++ b/packages/db/src/collectivex/swap-test-fixture.ts @@ -56,7 +56,7 @@ export function makeSwapDoc(sku = 'h200-dgxc') { cuda: '13.0', hip: null, image: 'vllm/vllm-openai:v0.25.1', - source_sha: 'abc123', + source_sha: 'f'.repeat(40), source_dirty: false, }, selection: { diff --git a/packages/db/src/ingest-collectivex.test.ts b/packages/db/src/ingest-collectivex.test.ts index d4a2a19d5..454548987 100644 --- a/packages/db/src/ingest-collectivex.test.ts +++ b/packages/db/src/ingest-collectivex.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type * as CommittedResults from './collectivex/committed-results'; +import { CommittedResultsError } from './collectivex/committed-results'; const mocks = vi.hoisted(() => ({ loadCommittedRun: vi.fn(), @@ -10,12 +11,16 @@ vi.mock('./collectivex/committed-results', async (importOriginal) => ({ ...(await importOriginal()), loadCommittedRun: mocks.loadCommittedRun, })); -vi.mock('./queries/collectivex', () => ({ upsertCollectiveXRun: mocks.upsert })); +vi.mock('./queries/collectivex', () => ({ + upsertCollectiveXRun: mocks.upsert, + pruneCollectiveXRuns: vi.fn(), + deleteCollectiveXRun: vi.fn(), +})); import { ingestCommits, parseIngestArgs } from './ingest-collectivex'; -import type { DbClient } from './connection.js'; +import type { Sql } from './etl/db-utils'; -const sql = (() => Promise.resolve([])) as unknown as DbClient; +const sql = (() => Promise.resolve([])) as unknown as Sql; afterEach(() => { vi.restoreAllMocks(); @@ -24,21 +29,29 @@ afterEach(() => { describe('parseIngestArgs', () => { it('defaults to main', () => { - expect(parseIngestArgs([])).toEqual({ ref: 'main', backfill: false }); + expect(parseIngestArgs([])).toEqual({ mode: 'commit', ref: 'main' }); }); it('takes one commit from --sha or --ref', () => { expect(parseIngestArgs(['--sha', 'a'.repeat(40)])).toEqual({ + mode: 'commit', ref: 'a'.repeat(40), - backfill: false, }); - expect(parseIngestArgs(['--ref', 'v1.2', '--yes'])).toEqual({ ref: 'v1.2', backfill: false }); + expect(parseIngestArgs(['--ref', 'v1.2', '--yes'])).toEqual({ mode: 'commit', ref: 'v1.2' }); }); - it('walks main for --backfill', () => { + it('walks main for --backfill, optionally pruning', () => { expect(parseIngestArgs(['--backfill', '-y', '--no-ssl'])).toEqual({ - ref: 'main', - backfill: true, + mode: 'backfill', + prune: false, + }); + expect(parseIngestArgs(['--backfill', '--prune'])).toEqual({ mode: 'backfill', prune: true }); + }); + + it('deletes one named run', () => { + expect(parseIngestArgs(['--delete-run', '1792039236'])).toEqual({ + mode: 'delete', + runId: '1792039236', }); }); @@ -47,22 +60,27 @@ describe('parseIngestArgs', () => { /mutually exclusive/u, ); expect(() => parseIngestArgs(['--backfill', '--ref', 'main'])).toThrow(/--backfill/u); + expect(() => parseIngestArgs(['--prune'])).toThrow(/--prune needs --backfill/u); + expect(() => parseIngestArgs(['--delete-run', '5', '--backfill'])).toThrow(/--delete-run/u); + expect(() => parseIngestArgs(['--delete-run', '37459990486'])).toThrow(/Invalid run id/u); + expect(() => parseIngestArgs(['--delete-run', '0'])).toThrow(/Invalid run id/u); expect(() => parseIngestArgs(['--sha', 'not-a-sha'])).toThrow(/Invalid commit sha/u); expect(() => parseIngestArgs(['12345'])).toThrow(); }); }); -function runFor(sha: string) { +function runFor(sha: string, measured = 1) { return { - run: { run_id: sha.slice(0, 3), summary: { measured_cases: 1, requested_cases: 2 } }, - docs: [{}], + run: { run_id: sha.slice(0, 3), summary: { measured_cases: measured, requested_cases: 2 } }, + docs: measured > 0 ? [{}] : [], }; } describe('ingestCommits', () => { const commits = [ - { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z' }, - { sha: 'b'.repeat(40), committedAt: '2026-10-16T00:00:00Z' }, + { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z', runId: '1760486400' }, + { sha: 'b'.repeat(40), committedAt: '2026-10-16T00:00:00Z', runId: '1760572800' }, + { sha: 'c'.repeat(40), committedAt: '2026-10-17T00:00:00Z', runId: '1760659200' }, ]; it('stores commits in order and shares one blob cache across them', async () => { @@ -70,9 +88,9 @@ describe('ingestCommits', () => { mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => Promise.resolve(runFor(commit.sha)), ); - mocks.upsert.mockResolvedValueOnce('inserted').mockResolvedValueOnce('replaced'); + mocks.upsert.mockResolvedValue('inserted'); - await ingestCommits(sql, commits, 'token'); + const outcome = await ingestCommits(sql, commits, 'token'); expect(mocks.loadCommittedRun.mock.calls.map(([commit]) => commit.sha)).toEqual( commits.map((commit) => commit.sha), @@ -80,19 +98,53 @@ describe('ingestCommits', () => { const [first, second] = mocks.loadCommittedRun.mock.calls.map(([, options]) => options); expect(first.token).toBe('token'); expect(first.blobCache).toBe(second.blobCache); + expect(mocks.upsert).toHaveBeenCalledTimes(3); + expect(outcome).toEqual({ stored: commits, skipped: [] }); + }); + + it('skips a commit with nothing measured instead of storing an empty run', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha, commit.sha === commits[1].sha ? 0 : 1)), + ); + mocks.upsert.mockResolvedValue('inserted'); + + const outcome = await ingestCommits(sql, commits, undefined); + expect(mocks.upsert).toHaveBeenCalledTimes(2); + expect(outcome.skipped).toEqual([commits[1]]); + expect(outcome.stored).toEqual([commits[0], commits[2]]); + }); + + it('skips commits GitHub answers 404 for only when asked to', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + commit.sha === commits[0].sha + ? Promise.reject(new CommittedResultsError('not-found', 'gone')) + : Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('inserted'); + + await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow('gone'); + const outcome = await ingestCommits(sql, commits, undefined, { skipMissing: true }); + expect(outcome.skipped).toEqual([commits[0]]); + expect(outcome.stored).toEqual(commits.slice(1)); }); - it('stops at a run id held by a different commit', async () => { + it('stops at invalid results and at a run id held by a different commit', async () => { vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockRejectedValueOnce(new CommittedResultsError('invalid', 'bad')); + await expect(ingestCommits(sql, commits, undefined, { skipMissing: true })).rejects.toThrow( + 'bad', + ); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => Promise.resolve(runFor(commit.sha)), ); mocks.upsert.mockResolvedValue('conflict'); - await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow( /already holds a different commit/u, ); - expect(mocks.loadCommittedRun).toHaveBeenCalledTimes(1); + expect(mocks.loadCommittedRun).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts index 724c08f43..489f5fc3c 100644 --- a/packages/db/src/ingest-collectivex.ts +++ b/packages/db/src/ingest-collectivex.ts @@ -1,27 +1,37 @@ import { parseArgs } from 'node:util'; import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils'; -import { createAdminSql } from './etl/db-utils'; +import { createAdminSql, type Sql } from './etl/db-utils'; import { + CommittedResultsError, listResultsCommits, loadCommittedRun, + MAX_COMMITTED_RUN_ID, RESULTS_BRANCH, RESULTS_DIR, resolveResultsCommit, type ResultsCommit, } from './collectivex/committed-results'; -import { upsertCollectiveXRun } from './queries/collectivex'; -import type { DbClient } from './connection.js'; +import { + deleteCollectiveXRun, + pruneCollectiveXRuns, + upsertCollectiveXRun, +} from './queries/collectivex'; const USAGE = - 'Usage: bun run admin:db:ingest:collectivex [--sha | --ref ] [--backfill] [--yes] [--no-ssl]'; + 'Usage: bun run admin:db:ingest:collectivex ' + + '[--sha | --ref | --backfill [--prune] | --delete-run ] [--yes] [--no-ssl]'; -export interface IngestCollectiveXArgs { - ref: string; - backfill: boolean; -} +export type IngestCollectiveXArgs = + | { mode: 'commit'; ref: string } + | { mode: 'backfill'; prune: boolean } + | { mode: 'delete'; runId: string }; -/** --sha and --ref name one commit; --backfill walks every results commit on main instead. */ +/** + * --sha and --ref name one commit (default: main); --backfill walks every results + * commit on main, and with --prune then deletes stored runs it did not store; + * --delete-run removes one stored run. + */ export function parseIngestArgs(argv: string[]): IngestCollectiveXArgs { const { values } = parseArgs({ args: argv, @@ -29,42 +39,112 @@ export function parseIngestArgs(argv: string[]): IngestCollectiveXArgs { sha: { type: 'string' }, ref: { type: 'string' }, backfill: { type: 'boolean', default: false }, + prune: { type: 'boolean', default: false }, + 'delete-run': { type: 'string' }, yes: { type: 'boolean', short: 'y', default: false }, 'no-ssl': { type: 'boolean', default: false }, }, strict: true, allowPositionals: false, }); + const deleteRun = values['delete-run']; if (values.sha !== undefined && values.ref !== undefined) { throw new Error(`--sha and --ref are mutually exclusive\n${USAGE}`); } if (values.backfill && (values.sha !== undefined || values.ref !== undefined)) { throw new Error(`--backfill reads every commit on ${RESULTS_BRANCH}\n${USAGE}`); } + if (values.prune && !values.backfill) { + throw new Error( + `--prune needs --backfill: it keeps exactly the runs a backfill stores\n${USAGE}`, + ); + } + if ( + deleteRun !== undefined && + (values.backfill || values.sha !== undefined || values.ref !== undefined) + ) { + throw new Error(`--delete-run takes no other target\n${USAGE}`); + } if (values.sha !== undefined && !/^[0-9a-f]{7,40}$/u.test(values.sha)) { throw new Error(`Invalid commit sha: ${values.sha}`); } - return { ref: values.sha ?? values.ref ?? RESULTS_BRANCH, backfill: values.backfill }; + if (deleteRun !== undefined) { + if (!/^[1-9]\d{0,9}$/u.test(deleteRun) || Number(deleteRun) >= MAX_COMMITTED_RUN_ID) { + throw new Error(`Invalid run id: ${deleteRun}`); + } + return { mode: 'delete', runId: deleteRun }; + } + if (values.backfill) return { mode: 'backfill', prune: values.prune }; + return { mode: 'commit', ref: values.sha ?? values.ref ?? RESULTS_BRANCH }; } -/** Ingest each commit in order; one failed commit stops the walk before later ones. */ +export interface IngestOutcome { + /** Commits whose run is stored (inserted or replaced). */ + stored: ResultsCommit[]; + /** Commits not stored: no results tree or file at them, or nothing measured. */ + skipped: ResultsCommit[]; +} + +/** + * Ingest each commit in order. A commit with no measured result (for example one + * that only changed cases.json) is skipped, never stored as an empty run. With + * `skipMissing` (the backfill), a commit GitHub answers 404 for is skipped too; + * any other failure stops the walk before later commits. + */ export async function ingestCommits( - sql: DbClient, + sql: Sql, commits: readonly ResultsCommit[], token: string | undefined, -): Promise { + { skipMissing = false }: { skipMissing?: boolean } = {}, +): Promise { const blobCache = new Map(); + const outcome: IngestOutcome = { stored: [], skipped: [] }; for (const commit of commits) { - const { run, docs } = await loadCommittedRun(commit, { token, blobCache }); + const label = `${commit.sha.slice(0, 12)} (${commit.committedAt})`; + let loaded: Awaited>; + try { + loaded = await loadCommittedRun(commit, { token, blobCache }); + } catch (error) { + if (skipMissing && error instanceof CommittedResultsError && error.kind === 'not-found') { + console.log(` skipped ${label}: ${error.message}`); + outcome.skipped.push(commit); + continue; + } + throw error; + } + const { run, docs } = loaded; + if (run.summary.measured_cases === 0) { + console.log( + ` skipped ${label}: nothing measured (${run.summary.requested_cases} cases requested)`, + ); + outcome.skipped.push(commit); + continue; + } const result = await upsertCollectiveXRun(sql, run, docs); if (result === 'conflict') { throw new Error(`run ${run.run_id} already holds a different commit than ${commit.sha}`); } console.log( - ` ${result} run ${run.run_id} @ ${commit.sha.slice(0, 12)} (${commit.committedAt}): ` + + ` ${result} run ${run.run_id} @ ${label}: ` + `${run.summary.measured_cases}/${run.summary.requested_cases} cases measured, ${docs.length} docs`, ); + outcome.stored.push(commit); + } + return outcome; +} + +function describePlan(args: IngestCollectiveXArgs, commits: readonly ResultsCommit[]): string { + if (args.mode === 'delete') { + return `This will delete stored run ${args.runId} and its documents.\n`; + } + if (args.mode === 'commit') { + return ( + `This will store ${RESULTS_DIR} at ${commits[0].sha} (resolved from ${args.ref}) as run ` + + `${commits[0].runId}, replacing any run already stored for that commit.\n` + ); } + const then = args.prune ? ', then DELETE every stored run it did not store' : ''; + return `This will store ${commits.length} ${RESULTS_DIR} commits from ${RESULTS_BRANCH}, oldest first, replacing any run already stored for the same commit${then}.\n`; } async function main(): Promise { @@ -72,19 +152,17 @@ async function main(): Promise { const token = process.env.GITHUB_TOKEN || undefined; console.log('=== db:ingest:collectivex ==='); - const commits = args.backfill - ? await listResultsCommits(RESULTS_BRANCH, { token }) - : [await resolveResultsCommit(args.ref, { token })]; - if (commits.length === 0) { - console.log(`No commit on ${RESULTS_BRANCH} touched ${RESULTS_DIR}.`); - return; + let commits: ResultsCommit[] = []; + if (args.mode === 'backfill') { + commits = await listResultsCommits(RESULTS_BRANCH, { token }); + if (commits.length === 0 && !args.prune) { + console.log(`No commit on ${RESULTS_BRANCH} touched ${RESULTS_DIR}.`); + return; + } + } else if (args.mode === 'commit') { + commits = [await resolveResultsCommit(args.ref, { token })]; } - console.log( - args.backfill - ? `This will store ${commits.length} ${RESULTS_DIR} commits from ${RESULTS_BRANCH}, oldest first,` - : `This will store ${RESULTS_DIR} at ${commits[0].sha} (resolved from ${args.ref}),`, - ); - console.log('replacing any run already stored for the same commit.\n'); + console.log(describePlan(args, commits)); if (!hasYesFlag()) { const ok = await confirm('Continue? (y/N) '); if (!ok) { @@ -100,12 +178,29 @@ async function main(): Promise { onnotice: () => {}, }); try { - await ingestCommits( - ((strings: TemplateStringsArray, ...values: unknown[]) => - sql(strings, ...(values as never[]))) as DbClient, - commits, - token, - ); + if (args.mode === 'delete') { + const removed = await deleteCollectiveXRun(sql, args.runId); + console.log( + removed + ? ` deleted run ${removed.run_id} @ ${removed.source_sha.slice(0, 12)}` + : ` no stored run ${args.runId}`, + ); + } else { + const { stored, skipped } = await ingestCommits(sql, commits, token, { + skipMissing: args.mode === 'backfill', + }); + console.log(` ${stored.length} stored, ${skipped.length} skipped`); + if (args.mode === 'backfill' && args.prune) { + const removed = await pruneCollectiveXRuns( + sql, + stored.map((commit) => commit.sha), + ); + for (const run of removed) { + console.log(` pruned run ${run.run_id} @ ${run.source_sha.slice(0, 12)}`); + } + console.log(` ${removed.length} pruned`); + } + } } finally { await sql.end(); } diff --git a/packages/db/src/queries/collectivex-db.test.ts b/packages/db/src/queries/collectivex-db.test.ts index 7f61d7a1e..fb983f349 100644 --- a/packages/db/src/queries/collectivex-db.test.ts +++ b/packages/db/src/queries/collectivex-db.test.ts @@ -4,6 +4,7 @@ import { PGlite } from '@electric-sql/pglite'; import { beforeEach, describe, expect, it } from 'vitest'; import { + assignRunIds, committedRun, validateCommittedResults, type ResultsSource, @@ -12,9 +13,11 @@ import { makeRawMatrix, makeRawShard } from '../collectivex/test-fixture'; import { pgliteSql, type PgliteSql } from '../lib/test-pglite'; import { collectiveXDatasetFromRow, + deleteCollectiveXRun, getCollectiveXRun, getLatestCollectiveXRun, listCollectiveXRuns, + pruneCollectiveXRuns, upsertCollectiveXRun, } from './collectivex'; @@ -25,12 +28,19 @@ const SECOND_SHA = 'b'.repeat(40); const FIRST_AT = '2026-10-28T00:00:00Z'; const SECOND_AT = '2026-10-29T00:00:00Z'; +const MIGRATIONS = new URL('../../migrations-collectivex/', import.meta.url); + +async function applyMigrations(db: PGlite, filter: (name: string) => boolean): Promise { + for (const name of fs.readdirSync(MIGRATIONS).toSorted()) { + if (name.endsWith('.sql') && filter(name)) { + await db.exec(fs.readFileSync(new URL(name, MIGRATIONS), 'utf8')); + } + } +} + async function collectiveXDb(): Promise { const db = await PGlite.create(); - const dir = new URL('../../migrations-collectivex/', import.meta.url); - for (const name of fs.readdirSync(dir).toSorted()) { - if (name.endsWith('.sql')) await db.exec(fs.readFileSync(new URL(name, dir), 'utf8')); - } + await applyMigrations(db, () => true); return pgliteSql(db); } @@ -59,9 +69,11 @@ function encode(value: unknown): Uint8Array { } function source(sha: string, committedAt: string, docs: Json[]): ResultsSource { + const [{ runId }] = assignRunIds([{ sha, committedAt }]); return { sha, committedAt, + runId, files: new Map([ ['cases.json', encode(cases)], ...docs.map((doc): [string, Uint8Array] => [ @@ -146,4 +158,79 @@ describe('CollectiveX runs from committed results in PostgreSQL', () => { expect(kept?.source_sha).toBe(FIRST_SHA); expect(await listCollectiveXRuns(sql, 1)).toHaveLength(1); }); + + it('never serves a run with nothing measured', async () => { + const empty = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, []))); + expect(empty.run.summary.measured_cases).toBe(0); + await upsertCollectiveXRun(sql, empty.run, empty.docs); + expect(await getLatestCollectiveXRun(sql, 1)).toBeNull(); + expect(await getCollectiveXRun(sql, 1, empty.run.run_id)).toBeNull(); + expect(await listCollectiveXRuns(sql, 1)).toEqual([]); + }); + + it('stores each document once when two ingests of one commit overlap', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + const results = await Promise.all([ + upsertCollectiveXRun(sql, first.run, first.docs), + upsertCollectiveXRun(sql, first.run, first.docs), + upsertCollectiveXRun(sql, first.run, first.docs), + ]); + expect(results.toSorted()).toEqual(['inserted', 'replaced', 'replaced']); + const [{ count }] = await sql`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(2); + }); + + it('prunes runs outside the kept commits and deletes one named run', async () => { + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + const second = committedRun(validateCommittedResults(source(SECOND_SHA, SECOND_AT, [h200]))); + await upsertCollectiveXRun(sql, first.run, first.docs); + await upsertCollectiveXRun(sql, second.run, second.docs); + + await expect(pruneCollectiveXRuns(sql, [SECOND_SHA])).resolves.toEqual([ + { run_id: first.run.run_id, source_sha: FIRST_SHA }, + ]); + const kept = await listCollectiveXRuns(sql, 1); + expect(kept.map((run) => run.run_id)).toEqual([second.run.run_id]); + const [{ count }] = await sql`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(1); + + await expect(deleteCollectiveXRun(sql, second.run.run_id)).resolves.toEqual({ + run_id: second.run.run_id, + source_sha: SECOND_SHA, + }); + await expect(deleteCollectiveXRun(sql, second.run.run_id)).resolves.toBeNull(); + expect(await getLatestCollectiveXRun(sql, 1)).toBeNull(); + }); +}); + +describe('migration 005', () => { + it('drops lazy-ingest runs that would outrank committed runs, and forbids new ones', async () => { + const db = await PGlite.create(); + await applyMigrations(db, (name) => name < '005'); + const legacy = pgliteSql(db); + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + await upsertCollectiveXRun(legacy, first.run, first.docs); + await upsertCollectiveXRun( + legacy, + { ...first.run, run_id: '37459990486', source_sha: SECOND_SHA }, + first.docs, + ); + const before = await getLatestCollectiveXRun(legacy, 1); + expect(before?.run_id).toBe('37459990486'); + + await applyMigrations(db, (name) => name >= '005'); + const after = await getLatestCollectiveXRun(legacy, 1); + expect(after?.run_id).toBe(first.run.run_id); + const [{ count }] = await legacy`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(1); + await expect( + upsertCollectiveXRun( + legacy, + { ...first.run, run_id: '37459990487', source_sha: SECOND_SHA }, + first.docs, + ), + ).rejects.toThrow(/cx_runs_committed_run_id/u); + }); }); diff --git a/packages/db/src/queries/collectivex.test.ts b/packages/db/src/queries/collectivex.test.ts index fbf07d09a..e7372d036 100644 --- a/packages/db/src/queries/collectivex.test.ts +++ b/packages/db/src/queries/collectivex.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { makeRawMatrix, makeRawShard, makeRunMeta } from '../collectivex/test-fixture'; import type { DbClient } from '../connection.js'; +import type { Sql } from '../etl/db-utils'; import { collectiveXDatasetFromRow, getCollectiveXRun, @@ -19,10 +20,13 @@ interface Captured { function fakeSql(rowsQueue: Record[][]) { const calls: Captured[] = []; - const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { + const tag = ((strings: TemplateStringsArray, ...values: unknown[]) => { calls.push({ text: strings.join('?'), values }); return Promise.resolve(rowsQueue.shift() ?? []); }) as DbClient; + const sql = Object.assign(tag, { + begin: (fn: (tx: DbClient) => Promise) => fn(tag), + }) as unknown as Sql & DbClient; return { sql, calls }; } @@ -58,6 +62,7 @@ describe('read queries', () => { for (const call of calls) { expect(call.text).not.toContain('deleted_at'); expect(call.text).toContain("summary->>'requested_cases'"); + expect(call.text).toContain("summary->>'measured_cases')::int, 0) > 0"); } expect(calls[0].text).toContain('ORDER BY r.run_id DESC'); expect(calls[2].text).toContain('ORDER BY run_id DESC'); @@ -76,24 +81,30 @@ describe('read queries', () => { describe('upsertCollectiveXRun', () => { it('binds raw objects (never pre-stringified JSON) and reports a first insert', async () => { - const { sql, calls } = fakeSql([[{ inserted: true }]]); + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); const docs = [{ record_type: 'case-attempt' }]; await expect(upsertCollectiveXRun(sql, runInsert, docs)).resolves.toBe('inserted'); - expect(calls).toHaveLength(1); - expect(calls[0].values).toContainEqual(runInsert.matrix); - expect(calls[0].values).toContainEqual(runInsert.summary); - expect(calls[0].values).toContainEqual({ docs }); + expect(calls).toHaveLength(2); + expect(calls[1].values).toContainEqual(runInsert.matrix); + expect(calls[1].values).toContainEqual(runInsert.summary); + expect(calls[1].values).toContainEqual({ docs }); expect( - calls[0].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), + calls[1].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), ).toBe(true); }); + it('takes the CollectiveX write lock in the transaction before writing', async () => { + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); + await upsertCollectiveXRun(sql, runInsert, []); + expect(calls[0].text).toBe('SELECT pg_advisory_xact_lock(7384110219)'); + }); + it('replaces the row and its documents when the same commit is ingested again', async () => { - const { sql, calls } = fakeSql([[{ inserted: false }]]); + const { sql, calls } = fakeSql([[], [{ inserted: false }]]); await expect(upsertCollectiveXRun(sql, runInsert, [])).resolves.toBe('replaced'); - const { text } = calls[0]; + const { text } = calls[1]; expect(text).toContain('ON CONFLICT (run_id) DO UPDATE'); expect(text).toContain('WHERE cx_runs.source_sha = EXCLUDED.source_sha'); expect(text).toContain('DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted)'); @@ -101,17 +112,17 @@ describe('upsertCollectiveXRun', () => { }); it('writes nothing when the run id already belongs to another commit', async () => { - const { sql } = fakeSql([[]]); + const { sql } = fakeSql([[], []]); await expect(upsertCollectiveXRun(sql, runInsert, [{}])).resolves.toBe('conflict'); }); it('touches only the CollectiveX tables and never drops data wholesale', async () => { - const { sql, calls } = fakeSql([[{ inserted: true }]]); + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); await upsertCollectiveXRun(sql, runInsert, []); - expect(new Set([...calls[0].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( + expect(new Set([...calls[1].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( new Set(['cx_runs', 'cx_run_docs']), ); - expect(calls[0].text).not.toMatch(/\b(?:DROP|TRUNCATE|ALTER)\b/iu); + expect(calls[1].text).not.toMatch(/\b(?:DROP|TRUNCATE|ALTER)\b/iu); }); }); diff --git a/packages/db/src/queries/collectivex.ts b/packages/db/src/queries/collectivex.ts index 81ff58dc3..cb97b233a 100644 --- a/packages/db/src/queries/collectivex.ts +++ b/packages/db/src/queries/collectivex.ts @@ -1,4 +1,5 @@ import type { DbClient } from '../connection.js'; +import type { Sql } from '../etl/db-utils'; import { buildDatasetFromNeutral } from '../collectivex/reader'; import type { CollectiveXDataset, CollectiveXRunSummary } from '../collectivex/types'; @@ -36,9 +37,11 @@ function toRunRow(row: Record): CollectiveXRunRow { } /** - * The most recent visible run for a version. Zero-case summaries contain no - * AMD/NVIDIA data and stay hidden. Ordered by run_id, the results commit's - * time in epoch seconds, so the newest commit on main is the latest run. + * The most recent visible run for a version. A run is visible when its summary + * requests AMD/NVIDIA cases AND measured at least one of them; the ingest never + * stores an unmeasured commit, and this keeps any such row hidden regardless. + * Ordered by run_id, which grows with main's results history, so the newest + * results commit on main is the latest run. * * Row and documents come back in ONE query, with docs filtered to the row's * run_attempt, so a reader never mixes one ingest's row with another's docs. @@ -61,6 +64,7 @@ export async function getLatestCollectiveXRun( ) d ON true WHERE r.version = ${version} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((r.summary->>'measured_cases')::int, 0) > 0 ORDER BY r.run_id DESC LIMIT 1 `; @@ -69,7 +73,7 @@ export async function getLatestCollectiveXRun( /** * One specific visible run by id, or null when absent, on another version, or - * carrying no supported-vendor cases. + * carrying no measured supported-vendor cases. */ export async function getCollectiveXRun( sql: DbClient, @@ -90,6 +94,7 @@ export async function getCollectiveXRun( ) d ON true WHERE r.version = ${version} AND r.run_id = ${runId} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((r.summary->>'measured_cases')::int, 0) > 0 `; return rows.length === 0 ? null : toRunRow(rows[0]); } @@ -97,7 +102,7 @@ export async function getCollectiveXRun( /** * Every visible run summary for a version, newest-first, straight from * the precomputed `summary` column — no document loading. Runs whose reader - * summary contains no AMD/NVIDIA cases stay hidden. + * summary requests or measures no AMD/NVIDIA cases stay hidden. */ export async function listCollectiveXRuns( sql: DbClient, @@ -108,6 +113,7 @@ export async function listCollectiveXRuns( FROM cx_runs WHERE version = ${version} AND COALESCE((summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((summary->>'measured_cases')::int, 0) > 0 ORDER BY run_id DESC `; return rows.map((row) => row.summary as CollectiveXRunSummary); @@ -115,12 +121,21 @@ export async function listCollectiveXRuns( export type CollectiveXUpsertResult = 'inserted' | 'replaced' | 'conflict'; +// Every CollectiveX write below first takes the transaction-scoped advisory lock +// `pg_advisory_xact_lock(7384110219)` (an arbitrary fixed key), so ingests, prunes and +// deletions run one at a time. Without it two concurrent ingests of one commit each +// delete only the documents visible to their snapshot and then both insert, storing +// every document twice. + /** - * Store one results commit and its raw documents in a single statement - * (data-modifying CTEs), so a reader sees either the previous contents or the - * new ones, never a partial run. Re-ingesting the same commit replaces its row - * and documents. `conflict` means run_id already belongs to a different commit - * (two results commits landed in the same second); nothing is written. + * Store one results commit and its raw documents in one transaction: take the + * write lock, then one statement (data-modifying CTEs) upserts the row and + * replaces its documents. Readers see the previous contents or the new ones, + * never a partial run, and concurrent ingests of the same commit serialize on + * the lock, so the second one's DELETE sees and removes the first one's + * documents. Re-ingesting a commit replaces its row and documents. + * `conflict` means run_id already belongs to a different commit (main's + * results history was rewritten); nothing is written. * * The `::jsonb`-cast parameters must be raw objects: both drivers * JSON-serialize objects exactly once, while pre-stringified values get @@ -128,45 +143,88 @@ export type CollectiveXUpsertResult = 'inserted' | 'replaced' | 'conflict'; * neon driver would serialize a bare JS array as a Postgres array literal. */ export async function upsertCollectiveXRun( - sql: DbClient, + sql: Sql, run: CollectiveXRunInsert, docs: unknown[], ): Promise { - const rows = await sql` - WITH upserted AS ( - INSERT INTO cx_runs - (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) - VALUES - (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, - ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, - ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) - ON CONFLICT (run_id) DO UPDATE SET - run_attempt = EXCLUDED.run_attempt, - version = EXCLUDED.version, - generated_at = EXCLUDED.generated_at, - source_branch = EXCLUDED.source_branch, - conclusion = EXCLUDED.conclusion, - matrix = EXCLUDED.matrix, - summary = EXCLUDED.summary, - ingested_at = now() - WHERE cx_runs.source_sha = EXCLUDED.source_sha - RETURNING run_id, (xmax = 0) AS inserted - ), - removed AS ( - DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted) - ), - new_docs AS ( - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT upserted.run_id, ${run.run_attempt}, entries.value - FROM upserted, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) - RETURNING id - ) - SELECT inserted FROM upserted - `; + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx` + WITH upserted AS ( + INSERT INTO cx_runs + (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) + VALUES + (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, + ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, + ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) + ON CONFLICT (run_id) DO UPDATE SET + run_attempt = EXCLUDED.run_attempt, + version = EXCLUDED.version, + generated_at = EXCLUDED.generated_at, + source_branch = EXCLUDED.source_branch, + conclusion = EXCLUDED.conclusion, + matrix = EXCLUDED.matrix, + summary = EXCLUDED.summary, + ingested_at = now() + WHERE cx_runs.source_sha = EXCLUDED.source_sha + RETURNING run_id, (xmax = 0) AS inserted + ), + removed AS ( + DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted) + ), + new_docs AS ( + INSERT INTO cx_run_docs (run_id, run_attempt, doc) + SELECT upserted.run_id, ${run.run_attempt}, entries.value + FROM upserted, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) + RETURNING id + ) + SELECT inserted FROM upserted + `; + }); if (rows.length === 0) return 'conflict'; return rows[0].inserted ? 'inserted' : 'replaced'; } +/** A stored run removed by a prune or deletion. */ +export interface CollectiveXRemovedRun { + run_id: string; + source_sha: string; +} + +/** + * Delete every stored run whose results commit is not in `keepShas` (with its + * documents, by cascade). The backfill passes the commits it just stored, so a + * run whose commit left main, stopped validating or measures nothing is retracted. + */ +export async function pruneCollectiveXRuns( + sql: Sql, + keepShas: readonly string[], +): Promise { + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx` + DELETE FROM cx_runs + WHERE source_sha NOT IN ( + SELECT jsonb_array_elements_text((${{ shas: keepShas } as never}::jsonb)->'shas') + ) + RETURNING run_id::text, source_sha + `; + }); + return rows as unknown as CollectiveXRemovedRun[]; +} + +/** Delete one stored run and its documents; null when no such run is stored. */ +export async function deleteCollectiveXRun( + sql: Sql, + runId: string, +): Promise { + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx`DELETE FROM cx_runs WHERE run_id = ${runId} RETURNING run_id::text, source_sha`; + }); + return rows.length === 0 ? null : (rows[0] as unknown as CollectiveXRemovedRun); +} + /** Assemble a stored run's raw documents into the dashboard dataset. */ export function collectiveXDatasetFromRow(row: CollectiveXRunRow): CollectiveXDataset { return buildDatasetFromNeutral(row.matrix, row.docs, { diff --git a/packages/skills/skills/inferencex-api/integrity.json b/packages/skills/skills/inferencex-api/integrity.json index 8e2dd0379..73335f5ef 100644 --- a/packages/skills/skills/inferencex-api/integrity.json +++ b/packages/skills/skills/inferencex-api/integrity.json @@ -8,7 +8,7 @@ "references/chart-templates.md": "aed6854627142019a4a5df4a2d07523b1a64e5814dd2bbdcc74ee18beb083912", "references/cli-contract.md": "4f070877eeefffad2860c58f79f68432fbe27aa21fc578a1c696f0309a3d7d7b", "references/cli.md": "11e1183f7ed70d669019e8435ac16a6a9f873de027507b8850a9943f6659bfd9", - "references/collectivex.md": "949b3104b4dbb65db53ac9397eba28aac4981521f64ae336e7a7fdaf96b26ec0", + "references/collectivex.md": "509b8cb7682de751a557c2d9f1db865d419998c62025c1bf140a3aec3fd612dd", "references/dashboard-views.md": "2095bef6178b7264db9e3c1e2bfcde1516fb190ddd5851dd5f8d104c8a0ab58b", "references/offline-exports.md": "95aa565dcd4c9e592159baa560a9a58217bc61e395de1c6daf0576795c7f4ec9", "references/pareto.md": "1b4d2d163f982e3f2d97addce310789eae9c1e5badd82dd7501708c9e0385027", diff --git a/packages/skills/skills/inferencex-api/references/collectivex.md b/packages/skills/skills/inferencex-api/references/collectivex.md index 07ad18878..41d5e489a 100644 --- a/packages/skills/skills/inferencex-api/references/collectivex.md +++ b/packages/skills/skills/inferencex-api/references/collectivex.md @@ -45,8 +45,11 @@ Requires Node 24 or 26. The command checks the current OpenAPI operations, reads list **once**, and selects its two newest runs with `measured_cases > 0`, ordered by numeric run ID. The older selection is `left`; the newer is `right`. This is a bounded example selection, not a representative sample. The run ID is the results -commit time in epoch seconds and the dataset's `run.source_sha` is that commit; the -conclusion remains in the export. +commit time in epoch seconds (raised past the previous results commit on a same-second +tie) and the dataset's `run.source_sha` is that commit; the conclusion remains in the +export. EP tail percentiles (`p90`, `p95`, `p99`) of latencies and derived rates can be +`null` where the harness withheld them; `p50` is always a number. Report a `null` tail as +unavailable, never as zero. Run-list counts identify candidate pairs; exact EP and KV identities in the returned details establish whether they have matched groups. For a requested pair, supply @@ -68,7 +71,7 @@ The public operations require `version=1`: | Operation | Meaning | | ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest stored run by run ID; it need not contain measurements. | +| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest stored run by run ID; only runs with a measured result are stored. | | [`/collectivex/runs`](https://inferencex.semianalysis.com/api/v1/collectivex/runs?version=1) | Retained summaries and `discovery_complete`; the helper uses this for selection. | | `/collectivex/runs/{runId}` | One run dataset; a re-ingest of the same commit can replace its contents. | From 543731866a1799295bece7a681fbc4872466876c Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:09:49 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix(collectivex):=20retry=20raw-file=20404s?= =?UTF-8?q?=20and=20refuse=20to=20prune=20after=20an=20unreadable=20commit?= =?UTF-8?q?=20/=20=E9=87=8D=E8=AF=95=E5=8E=9F=E5=A7=8B=E6=96=87=E4=BB=B6?= =?UTF-8?q?=20404=EF=BC=8C=E5=B9=B6=E5=9C=A8=E6=8F=90=E4=BA=A4=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E8=AF=BB=E5=8F=96=E6=97=B6=E6=8B=92=E7=BB=9D=E6=B8=85?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A raw file the trees API already listed exists, so its 404 is a CDN miss and is retried, then reported unavailable. A backfill that skipped any commit for a 404 now fails instead of pruning, so a transient GitHub miss can no longer delete a stored run. trees API 已列出的原始文件必然存在，其 404 属于 CDN 未命中，因此会重试，失败后报告为不可用。 若回填因 404 跳过了任何提交，现在会直接失败而不执行清理，GitHub 的短暂故障不再会删除已存储的运行。 --- .../src/collectivex/committed-results.test.ts | 20 +++++++++++++++++++ .../db/src/collectivex/committed-results.ts | 13 +++++++++++- packages/db/src/ingest-collectivex.test.ts | 3 ++- packages/db/src/ingest-collectivex.ts | 14 +++++++++++-- 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/packages/db/src/collectivex/committed-results.test.ts b/packages/db/src/collectivex/committed-results.test.ts index fc07283cb..f8d27d818 100644 --- a/packages/db/src/collectivex/committed-results.test.ts +++ b/packages/db/src/collectivex/committed-results.test.ts @@ -369,6 +369,26 @@ describe('GitHub reads', () => { expect(blobCache.has('b1')).toBe(true); }); + it('retries a listed raw file that 404s and reports it unavailable, not missing', async () => { + vi.useFakeTimers(); + const fetch = vi.fn((url: string) => + new URL(url).hostname === 'raw.githubusercontent.com' + ? new Response('', { status: 404 }) + : jsonResponse({ tree: [{ path: 'cases.json', type: 'blob', sha: 'c1', size: 2 }] }), + ); + const read = fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: fetch as unknown as typeof globalThis.fetch }, + ); + const settled = expect(read).rejects.toMatchObject({ kind: 'unavailable' }); + await vi.runAllTimersAsync(); + await settled; + vi.useRealTimers(); + expect( + fetch.mock.calls.filter(([url]) => url.startsWith('https://raw.')).length, + ).toBeGreaterThan(1); + }); + it('reports a commit without a results directory as not found and oversize files as invalid', async () => { const missing = vi.fn(() => new Response('', { status: 404 })); await expect( diff --git a/packages/db/src/collectivex/committed-results.ts b/packages/db/src/collectivex/committed-results.ts index 47545e0f2..7619507ed 100644 --- a/packages/db/src/collectivex/committed-results.ts +++ b/packages/db/src/collectivex/committed-results.ts @@ -123,9 +123,20 @@ async function request( }); if (response.ok) return response; failure = new Error(`${url} returned HTTP ${response.status}`); - if (response.status === 404 || response.status === 422) { + // A raw file the trees API already listed exists, so its 404 is a CDN miss worth retrying; + // only the API's own 404/422 means the commit or tree is really absent. + const listedFile = new URL(url).hostname === 'raw.githubusercontent.com'; + if (!listedFile && (response.status === 404 || response.status === 422)) { throw new CommittedResultsError('not-found', `${url} returned HTTP ${response.status}`); } + if (listedFile && response.status === 404) { + if (attempt < FETCH_ATTEMPTS) { + await new Promise((settle) => { + setTimeout(settle, 1000 * attempt); + }); + } + continue; + } if (response.status < 500 && response.status !== 429) break; } catch (error) { if (error instanceof CommittedResultsError) throw error; diff --git a/packages/db/src/ingest-collectivex.test.ts b/packages/db/src/ingest-collectivex.test.ts index 454548987..187e55112 100644 --- a/packages/db/src/ingest-collectivex.test.ts +++ b/packages/db/src/ingest-collectivex.test.ts @@ -99,7 +99,7 @@ describe('ingestCommits', () => { expect(first.token).toBe('token'); expect(first.blobCache).toBe(second.blobCache); expect(mocks.upsert).toHaveBeenCalledTimes(3); - expect(outcome).toEqual({ stored: commits, skipped: [] }); + expect(outcome).toEqual({ stored: commits, skipped: [], missing: [] }); }); it('skips a commit with nothing measured instead of storing an empty run', async () => { @@ -128,6 +128,7 @@ describe('ingestCommits', () => { await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow('gone'); const outcome = await ingestCommits(sql, commits, undefined, { skipMissing: true }); expect(outcome.skipped).toEqual([commits[0]]); + expect(outcome.missing).toEqual([commits[0]]); expect(outcome.stored).toEqual(commits.slice(1)); }); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts index 489f5fc3c..c3507b9e4 100644 --- a/packages/db/src/ingest-collectivex.ts +++ b/packages/db/src/ingest-collectivex.ts @@ -83,6 +83,8 @@ export interface IngestOutcome { stored: ResultsCommit[]; /** Commits not stored: no results tree or file at them, or nothing measured. */ skipped: ResultsCommit[]; + /** The skipped commits GitHub answered 404 for: a prune must not trust this walk. */ + missing: ResultsCommit[]; } /** @@ -98,7 +100,7 @@ export async function ingestCommits( { skipMissing = false }: { skipMissing?: boolean } = {}, ): Promise { const blobCache = new Map(); - const outcome: IngestOutcome = { stored: [], skipped: [] }; + const outcome: IngestOutcome = { stored: [], skipped: [], missing: [] }; for (const commit of commits) { const label = `${commit.sha.slice(0, 12)} (${commit.committedAt})`; let loaded: Awaited>; @@ -108,6 +110,7 @@ export async function ingestCommits( if (skipMissing && error instanceof CommittedResultsError && error.kind === 'not-found') { console.log(` skipped ${label}: ${error.message}`); outcome.skipped.push(commit); + outcome.missing.push(commit); continue; } throw error; @@ -186,10 +189,17 @@ async function main(): Promise { : ` no stored run ${args.runId}`, ); } else { - const { stored, skipped } = await ingestCommits(sql, commits, token, { + const { stored, skipped, missing } = await ingestCommits(sql, commits, token, { skipMissing: args.mode === 'backfill', }); console.log(` ${stored.length} stored, ${skipped.length} skipped`); + if (args.mode === 'backfill' && args.prune && missing.length > 0) { + // A 404 can be a transient GitHub miss; pruning after it would delete a good run. + throw new Error( + `not pruning: ${missing.length} commit(s) could not be read ` + + `(${missing.map((commit) => commit.sha.slice(0, 12)).join(', ')}); rerun the backfill`, + ); + } if (args.mode === 'backfill' && args.prune) { const removed = await pruneCollectiveXRuns( sql, From 21f777aa52a37a4563b61915a5f278b730fbcea9 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:15:51 +0800 Subject: [PATCH 5/5] =?UTF-8?q?fix(collectivex):=20treat=20a=20commit=20th?= =?UTF-8?q?at=20deleted=20its=20results=20as=20absent,=20not=20unreadable?= =?UTF-8?q?=20/=20=E5=B0=86=E5=88=A0=E9=99=A4=E7=BB=93=E6=9E=9C=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=9A=84=E6=8F=90=E4=BA=A4=E8=A7=86=E4=B8=BA=E2=80=9C?= =?UTF-8?q?=E4=B8=8D=E5=AD=98=E5=9C=A8=E2=80=9D=E8=80=8C=E9=9D=9E=E2=80=9C?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E8=AF=BB=E5=8F=96=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trees 404 for collectivex/results is confirmed against the parent tree: when that tree is readable and has no results entry the commit removed them, which is skipped without blocking a prune. Any other 404 still counts as unreadable. collectivex/results 的 trees 404 会通过父目录树确认：父目录可读且没有 results 条目时，说明该提交删除了结果， 此时跳过且不阻止清理。其他 404 仍视为无法读取。 --- .../src/collectivex/committed-results.test.ts | 14 ++++++++++++++ .../db/src/collectivex/committed-results.ts | 17 ++++++++++++++--- packages/db/src/ingest-collectivex.test.ts | 14 ++++++++++++++ packages/db/src/ingest-collectivex.ts | 5 +++++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/packages/db/src/collectivex/committed-results.test.ts b/packages/db/src/collectivex/committed-results.test.ts index f8d27d818..b1893833a 100644 --- a/packages/db/src/collectivex/committed-results.test.ts +++ b/packages/db/src/collectivex/committed-results.test.ts @@ -369,6 +369,20 @@ describe('GitHub reads', () => { expect(blobCache.has('b1')).toBe(true); }); + it('reports a commit that deleted the results directory as absent', async () => { + const fetch = vi.fn((url: string) => + url.includes(':collectivex/results') + ? new Response('', { status: 404 }) + : jsonResponse({ tree: [{ path: 'bench', type: 'tree', sha: 't1' }] }), + ); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: fetch as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ kind: 'absent' }); + }); + it('retries a listed raw file that 404s and reports it unavailable, not missing', async () => { vi.useFakeTimers(); const fetch = vi.fn((url: string) => diff --git a/packages/db/src/collectivex/committed-results.ts b/packages/db/src/collectivex/committed-results.ts index 7619507ed..e92fe1bc5 100644 --- a/packages/db/src/collectivex/committed-results.ts +++ b/packages/db/src/collectivex/committed-results.ts @@ -86,10 +86,11 @@ export interface CommittedResults extends ResultsCommit { /** * `invalid`: the committed files break the results contract. `not-found`: the ref, - * the results directory, or a main-history results commit does not exist. + * the results directory, or a main-history results commit does not exist. `absent`: the + * commit is readable and has no results directory (it deleted them), so there is no run. * `unavailable`: GitHub could not be read. */ -export type CommittedResultsErrorKind = 'invalid' | 'not-found' | 'unavailable'; +export type CommittedResultsErrorKind = 'invalid' | 'not-found' | 'absent' | 'unavailable'; export class CommittedResultsError extends Error { readonly kind: CommittedResultsErrorKind; @@ -276,8 +277,18 @@ export async function fetchResultsSource( )) as typeof tree; } catch (error) { if (error instanceof CommittedResultsError && error.kind === 'not-found') { + // A 404 alone can be a GitHub miss; only a readable parent tree without results/ proves + // the commit removed them. + const parent = RESULTS_DIR.slice(0, RESULTS_DIR.lastIndexOf('/')); + const leaf = RESULTS_DIR.slice(RESULTS_DIR.lastIndexOf('/') + 1); + const listing = (await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/git/trees/${commit.sha}:${parent}`, + options, + ).catch(() => null)) as { tree?: TreeEntry[] } | null; + const absent = + Array.isArray(listing?.tree) && !listing.tree.some((entry) => entry.path === leaf); throw new CommittedResultsError( - 'not-found', + absent ? 'absent' : 'not-found', `${RESULTS_DIR} does not exist at ${commit.sha}`, { cause: error, diff --git a/packages/db/src/ingest-collectivex.test.ts b/packages/db/src/ingest-collectivex.test.ts index 187e55112..296f7013f 100644 --- a/packages/db/src/ingest-collectivex.test.ts +++ b/packages/db/src/ingest-collectivex.test.ts @@ -132,6 +132,20 @@ describe('ingestCommits', () => { expect(outcome.stored).toEqual(commits.slice(1)); }); + it('skips a commit that deleted its results without counting it as missing', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + commit.sha === commits[0].sha + ? Promise.reject(new CommittedResultsError('absent', 'deleted')) + : Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('inserted'); + const outcome = await ingestCommits(sql, commits, undefined); + expect(outcome.skipped).toEqual([commits[0]]); + expect(outcome.missing).toEqual([]); + expect(outcome.stored).toEqual(commits.slice(1)); + }); + it('stops at invalid results and at a run id held by a different commit', async () => { vi.spyOn(console, 'log').mockImplementation(() => {}); mocks.loadCommittedRun.mockRejectedValueOnce(new CommittedResultsError('invalid', 'bad')); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts index c3507b9e4..bbda5a4f3 100644 --- a/packages/db/src/ingest-collectivex.ts +++ b/packages/db/src/ingest-collectivex.ts @@ -107,6 +107,11 @@ export async function ingestCommits( try { loaded = await loadCommittedRun(commit, { token, blobCache }); } catch (error) { + if (error instanceof CommittedResultsError && error.kind === 'absent') { + console.log(` skipped ${label}: ${error.message}`); + outcome.skipped.push(commit); + continue; + } if (skipMissing && error instanceof CommittedResultsError && error.kind === 'not-found') { console.log(` skipped ${label}: ${error.message}`); outcome.skipped.push(commit);