diff --git a/.env.example b/.env.example index 9fcc227f4..b2987181b 100644 --- a/.env.example +++ b/.env.example @@ -17,8 +17,8 @@ # DATABASE_DRIVER=neon # DATABASE_SSL=true -# GitHub PAT — required for CollectiveX lazy ingest (sweep artifact reads); also used -# for framework releases, star counts, and unofficial-run artifact reads +# GitHub PAT — used for framework releases, star counts, unofficial-run artifact reads, +# and (optional, raises the rate limit) the CollectiveX committed-results ingest # Create at: https://github.com/settings/personal-access-tokens # GITHUB_TOKEN= # Extra benchmark source repos (owner/repo, comma-separated) searched after @@ -31,8 +31,10 @@ # Directory contains .json with { run, benchmarks, evaluations }. # INFERENCEX_LOCAL_ARTIFACT_DIR=/absolute/path/to/local-artifacts -# CollectiveX database (separate Neon instance). The app routes read and write through -# DATABASE_COLLECTIVEX_WRITE_URL; they do not read DATABASE_COLLECTIVEX_READONLY_URL. +# CollectiveX database (separate Neon instance), filled from InferenceX's committed +# collectivex/results/ by the ingest-collectivex workflow or admin:db:ingest:collectivex. +# The app routes read and write through DATABASE_COLLECTIVEX_WRITE_URL; they do not read +# DATABASE_COLLECTIVEX_READONLY_URL. # DATABASE_COLLECTIVEX_READONLY_URL= # DATABASE_COLLECTIVEX_WRITE_URL= @@ -69,8 +71,8 @@ # ║ Secrets and database administration (DO NOT SHARE) ║ # ║ ║ # ║ DB scripts (migrate, ingest, reset, etc) read these. The app also reads ║ -# ║ INVALIDATE_SECRET, FEEDBACK_SECRET, and COLLECTIVEX_ADMIN_SECRET (and ║ -# ║ DATABASE_WRITE_URL for feedback) at runtime; set them in Vercel too. ║ +# ║ INVALIDATE_SECRET and FEEDBACK_SECRET (and DATABASE_WRITE_URL for ║ +# ║ feedback) at runtime; set them in Vercel too. ║ # ╚══════════════════════════════════════════════════════════════════════════╝ # DATABASE_WRITE_URL= @@ -84,7 +86,6 @@ # argument or run URL names the repository. # INGEST_REPO= # FEEDBACK_SECRET= -# COLLECTIVEX_ADMIN_SECRET= # OperatorX durable raw-artifact cache (primary, write-capable connection). # Apply admin:db:migrate:operatorx before enabling the hidden OperatorX tab. diff --git a/.github/workflows/ingest-collectivex.yml b/.github/workflows/ingest-collectivex.yml new file mode 100644 index 000000000..3815fc942 --- /dev/null +++ b/.github/workflows/ingest-collectivex.yml @@ -0,0 +1,159 @@ +name: Ingest CollectiveX Results + +on: + repository_dispatch: + types: [ingest-collectivex] + workflow_dispatch: + inputs: + sha: + description: InferenceX commit (40 hex) whose collectivex/results/ to ingest + required: false + ref: + description: InferenceX branch or tag, instead of sha + required: false + backfill: + description: Ingest every collectivex/results/ commit on main instead + type: boolean + default: false + prune: + description: With backfill, then delete every stored run the backfill did not store + type: boolean + default: false + delete_run: + description: Delete this one stored run_id instead (no other target) + required: false + +permissions: {} + +# One group for every ingest, prune and delete: runs queue behind each other instead of +# writing at once (the database also serializes writers with an advisory lock). GitHub +# keeps a single pending run per group, so a burst of dispatches can supersede a pending +# one; a backfill then stores anything it skipped. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + ingest: + name: Ingest CollectiveX results + timeout-minutes: 60 + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check inputs and secrets + env: + APP_URL: ${{ vars.APP_URL }} + VERCEL_INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + SHA: ${{ github.event.client_payload.sha || inputs.sha }} + REF: ${{ inputs.ref }} + BACKFILL: ${{ inputs.backfill }} + PRUNE: ${{ inputs.prune }} + DELETE_RUN: ${{ inputs.delete_run }} + run: | + : "${SLACK_WEBHOOK_URL:?SLACK_WEBHOOK_URL secret is not set}" + : "${APP_URL:?APP_URL repository variable is not set}" + : "${VERCEL_INVALIDATE_SECRET:?VERCEL_INVALIDATE_SECRET secret is not set}" + : "${DATABASE_COLLECTIVEX_WRITE_URL:?DATABASE_COLLECTIVEX_WRITE_URL secret is not set}" + : "${GH_TOKEN:?INFX_MAIN_PAT secret is not set}" + selected=0 + [ -n "$SHA" ] && selected=$((selected + 1)) + [ -n "$REF" ] && selected=$((selected + 1)) + [ "$BACKFILL" = true ] && selected=$((selected + 1)) + [ -n "$DELETE_RUN" ] && selected=$((selected + 1)) + if [ "$selected" -ne 1 ]; then + echo "give exactly one of sha, ref, backfill, or delete_run" >&2 + exit 1 + fi + if [ "$PRUNE" = true ] && [ "$BACKFILL" != true ]; then + echo "prune needs backfill" >&2 + exit 1 + fi + if [ -n "$DELETE_RUN" ] && ! [[ "$DELETE_RUN" =~ ^[1-9][0-9]{0,9}$ ]]; then + echo "delete_run must be a stored run_id" >&2 + exit 1 + fi + if [ -n "$SHA" ] && ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "sha must be 40 lowercase hex characters" >&2 + exit 1 + fi + if [ -n "$REF" ] && { ! [[ "$REF" =~ ^[A-Za-z0-9_][A-Za-z0-9_./-]{0,254}$ ]] || [[ "$REF" == *..* ]]; }; then + echo "ref is not a valid branch or tag name" >&2 + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Bun + id: setup-bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Cache Bun packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.bun/install/cache + key: ${{ runner.os }}-bun-${{ runner.arch }}-${{ + steps.setup-bun.outputs.bun-version }}-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun-${{ runner.arch }}-${{ steps.setup-bun.outputs.bun-version }}- + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + + - name: Run migrations + env: + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + run: bun run --cwd packages/db src/migrate-collectivex.ts --yes + + - name: Ingest results + env: + DATABASE_COLLECTIVEX_WRITE_URL: ${{ secrets.DATABASE_COLLECTIVEX_WRITE_URL }} + GITHUB_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + SHA: ${{ github.event.client_payload.sha || inputs.sha }} + REF: ${{ inputs.ref }} + BACKFILL: ${{ inputs.backfill }} + PRUNE: ${{ inputs.prune }} + DELETE_RUN: ${{ inputs.delete_run }} + run: | + if [ -n "$DELETE_RUN" ]; then + target=(--delete-run "$DELETE_RUN") + elif [ "$BACKFILL" = true ] && [ "$PRUNE" = true ]; then + target=(--backfill --prune) + elif [ "$BACKFILL" = true ]; then + target=(--backfill) + elif [ -n "$SHA" ]; then + target=(--sha "$SHA") + else + target=(--ref "$REF") + fi + bun run --cwd packages/db src/ingest-collectivex.ts "${target[@]}" --yes + + - name: Invalidate CollectiveX cache + env: + APP_URL: ${{ vars.APP_URL }} + VERCEL_INVALIDATE_SECRET: ${{ secrets.VERCEL_INVALIDATE_SECRET }} + VERCEL_PROTECTION_BYPASS_SECRET: ${{ secrets.VERCEL_PROTECTION_BYPASS_SECRET }} + run: | + : "${APP_URL:?APP_URL repository variable is not set}" + headers=(-H "Authorization: Bearer $VERCEL_INVALIDATE_SECRET") + if [ -n "$VERCEL_PROTECTION_BYPASS_SECRET" ]; then + headers+=(-H "x-vercel-protection-bypass: $VERCEL_PROTECTION_BYPASS_SECRET") + fi + curl -sSf -X POST "${APP_URL%/}/api/v1/invalidate?scope=collectivex" "${headers[@]}" + + - name: Notify Slack on failure + if: failure() + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook + payload: | + { + "text": ":rotating_light: *Ingest CollectiveX workflow failed*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" + } diff --git a/docs/architecture.md b/docs/architecture.md index d44145f9f..eb6dbf88e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,7 +16,7 @@ Raw-data API routes return raw DB rows with zero transformation, validation, or Some routes serve a purpose that raw DB rows cannot satisfy: - Dashboard projections under `/api/v1/views/*` use shared pure UI transformations server-side and publish their selectors and assumptions. Keep raw-data contracts separate from these projections. -- CollectiveX assembles stored sweep documents through its shared reader. +- CollectiveX assembles its stored committed-result documents through its shared reader. - OperatorX uses versioned ingest-time comparison projections for its page-owned API. The reader merges compact timing rows across runs while preserving source revisions, statuses, and timeline references; raw documents remain available for drilldowns. An optional revision-keyed Blob cache shares the merged comparison across instances; partial reads are excluded and OperatorX invalidation purges its deployment subtree. See [OperatorX](./operatorx.md#compact-comparison-reads). - `tco-feed` performs server-side frontier interpolation for spreadsheet consumers that cannot run the TypeScript transforms. Its assumptions (tier weights, workload mix, α) enter only as explicit query params with documented defaults, so a published sheet's URL fully records its methodology. - `pareto` returns frontier and hinterland coordinates over stored benchmark rows with explicit axis preferences. It reuses the chart dominance algorithm without UI filters. See the [Pareto frontier and hinterland reference](../packages/skills/skills/inferencex-api/references/pareto.md). diff --git a/docs/collectivex.md b/docs/collectivex.md index 5c65e2bc0..3b0795fd0 100644 --- a/docs/collectivex.md +++ b/docs/collectivex.md @@ -1,93 +1,144 @@ # CollectiveX -Design rationale for the CollectiveX tab's data pipeline. Unlike every other tab -(Neon DB → ETL ingest → `/api/v1/*`), CollectiveX uses **lazy ingest-on-read**: its -database is a durable cache of GitHub Actions, populated by the API routes themselves. - -## Why lazy ingest instead of the main pipeline - -- **Sweep artifacts expire after 14 days.** The sweep workflow - (`collectivex-sweep.yml` in the harness repo) uploads a matrix artifact - (`cxsweep-matrix-{run_id}`) and per-cell result artifacts - (`cxshard-{cell}-{run_id}-{attempt}`) with 14-day retention. Persisting on first - view makes a run outlive its artifacts once anyone has looked at it. -- **The sweep JSON contract is expected to change.** The DB stores the RAW documents - verbatim; the shared reader (`packages/db/src/collectivex/reader.ts`) is the single - transform point and runs at API-read time, so a reader fix retroactively applies to - already-stored runs — no re-ingest. A contract change = reader change + a bump of the - numeric `version` in the harness's `experimental/CollectiveX/configs/sweep.json`. -- **No CI plumbing.** There is no ingest workflow, no cross-repo dispatch, and no GH - secrets. Runs launched via `gh api` on any harness branch appear on the dashboard - within the CDN TTL of someone viewing the page — only the workflow identity is - checked, never the branch. - -## How it works - -`packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts` exposes three `ensure*` functions the -routes call before reading the DB (`packages/db/src/queries/collectivex.ts`): - -- `ensureLatestCollectiveXRun` — walk GitHub's completed sweep runs newest-first; stop at - the first live requested-version run; persist it if absent. -- `ensureCollectiveXRunsList` — progressively backfill every requested-version run whose - 14-day artifacts may still be downloaded. Each request changes at most eight rows; an - incomplete response is uncached and the client refetches until the workflow history is - exhausted. Known rows do not consume the batch, so discovery advances past the newest runs. - GitHub discovery is limited to runs created within the last 44 days: the 30-day workflow - rerun window plus 14-day artifact retention, covering the oldest rerun whose artifacts can - still exist without rescanning permanent workflow history on every cold-origin request. -- `ensureCollectiveXRun` — fetch one run by id, or compare a stored run's `run_attempt` - against GitHub and refresh it when a rerun is newer. Only completed runs are persisted. - -Key invariants: - -- **Writes are atomic and race-safe**: one CTE statement with - `ON CONFLICT (run_id) DO NOTHING`; concurrent first-viewers can't double-ingest or - expose a partial run. A GitHub re-run (newer `run_attempt`) is replaced through a - `FOR UPDATE`-guarded refresh statement. -- **Reads use the primary**: discovery state checks, writes, and every route query use - `DATABASE_COLLECTIVEX_WRITE_URL`, so a newly inserted or refreshed run is visible in the - same request. -- **Deletion tombstones** (`cx_runs.deleted_at`, documents freed): discovery must never - resurrect a deleted run. Re-ingesting via the CLI - (`bun run admin:db:ingest:collectivex `) clears the tombstone — that CLI is - the operator tool for pre-warming runs before artifact expiry, backfills, and un-deletes. -- **"Latest" orders by `run_id`** (monotonic with run creation, matching the discovery - walk) — not by completion time, where a long-failing older run would shadow a newer - successful one. -- **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` - result shards (case-insensitive). Other vendor values are omitted from chart series, - coverage, and SKU summaries; runs with no supported-vendor cases stay hidden. -- **GitHub being down never takes the page down**: routes serve whatever the DB holds and - only surface an error when there is no stored fallback. -- **Run-list completeness is progressive**: `/api/v1/collectivex/runs` returns - `discovery_complete: false` while another bounded ingest pass is required. Those responses - use `private, no-store`; the client polls once per second until the field becomes `true`. - Stored runs remain visible indefinitely, while never-ingested runs disappear with their - upstream artifacts and can no longer be reconstructed. -- **Caching**: responses carry the `collectivex` CDN tag with a 60s - `s-maxage` (freshness bound for lazy discovery). Run deletion and - `POST /api/v1/invalidate?scope=collectivex` purge only that tag; the main dashboard's - blob cache is untouched by CollectiveX operations. -- **Env**: `DATABASE_COLLECTIVEX_WRITE_URL` is the primary used by lazy persistence, all - route reads, deletion, and migrations via `bun run admin:db:migrate:collectivex`. +Design rationale for the CollectiveX tab's data pipeline. CollectiveX keeps its own Neon +database, filled from the benchmark results that `SemiAnalysisAI/InferenceX` commits under +`collectivex/results/`: `cases.json` (the requested cases, with the contract `version`) plus +one `/.json` per measured case. The API routes only read that database. + +## One run per results commit + +Each InferenceX commit on `main` that changed `collectivex/results/` and holds at least one +measured result is one stored run. A commit with nothing measured (for example one that only +changed `cases.json` before any result landed) is skipped, never stored as an empty run, and +the read queries hide any row whose summary measures nothing. + +- `run_id` is the commit's committer time in epoch seconds, raised to one past the previous + results commit's id when it would not exceed it (two commits in the same second, or a + committer clock behind history). It is computed from `main`'s results history alone, so + every ingest gives a commit the same id, ids are unique, and "latest" still means + `ORDER BY run_id DESC`. A stored id that belongs to another commit only happens if `main`'s + history was rewritten; the CLI then fails rather than overwrite it (there is no HTTP ingest + endpoint, so no HTTP status is involved). Ids are below 10^10, which migration + `005_remove_lazy_ingest_runs.sql` enforces with a CHECK after deleting the retired lazy + ingest's rows (keyed by GitHub Actions run id, so they would otherwise outrank every + committed run). +- `run_attempt` is always 1, `source_sha` is the results commit, `source_branch` is `main`, + `conclusion` is `success`, `matrix` is `cases.json` and `summary` is the shared reader's + `buildRunSummary`. `cx_run_docs` holds the commit's result documents verbatim. +- Re-ingesting a commit is idempotent: one transaction takes a fixed advisory lock + (`pg_advisory_xact_lock`, shared by every CollectiveX write) and then one statement upserts + the row and replaces its documents, so a reader sees the old contents or the new ones, + never a mix, and two concurrent ingests of the same commit cannot both insert its + documents. +- A ref resolves to the newest results commit at or before it, and that commit must be in + `main`'s results history. Ingesting `main` after an unrelated merge therefore refreshes + the run already published for those files instead of creating a duplicate, and results + that never landed on `main` are never published. + +The DB stores raw documents; the shared reader (`packages/db/src/collectivex/reader.ts`) +assembles the dataset at read time, so a reader fix applies to stored runs without +re-ingesting. A contract change is a reader change plus a bump of the numeric `version` that +`cases.json` and every EP result carry. + +## Ingest + +`packages/db/src/collectivex/committed-results.ts` is shared by both entry points. It reads +exactly `collectivex/results/**` at the pinned commit: the git trees API lists the files and +each blob comes from `raw.githubusercontent.com`, which does not count against the REST rate +limit. It then checks every file the way the benchmark repo's own test does: the path is +`/.json` matching the document's identity (EP case attempts by +`identity.case_id` and `identity.case_factors.sku`; swap_blocks results by +`schema: collectivex-swap-blocks-v1`, `case_id` and `runtime.sku`), the case is runnable in +`cases.json`, the result was measured from committed code (`source_dirty: false` and a 40-hex +`source_sha`), an EP result carries exactly the `cases.json` version, and no string holds an +IPv4 address and no key names a machine (`host`, `hosts`, `hostname`, `nodelist`, +`slurm_nodelist`, `master_addr`, case-insensitive). Each document is then assembled alone, so +every failure names its file. Files above 4 MiB, or a tree above 256 MiB, are rejected. + +- **Workflow**: `.github/workflows/ingest-collectivex.yml` runs on `repository_dispatch` + type `ingest-collectivex` with `client_payload: {"sha": "<40 hex>"}`, which the InferenceX + workflow sends on pushes to `main` that touch `collectivex/results/`, and on + `workflow_dispatch` with exactly one of `sha` (40 hex), `ref`, `backfill` (optionally with + `prune`), or `delete_run`. Every run shares one concurrency group that queues rather than + cancels; GitHub keeps only one pending run per group, so a burst of dispatches can + supersede a pending ingest, which the next backfill makes up. It runs + `migrate-collectivex.ts --yes`, then the ingest CLI with `--sha`, `--ref`, `--backfill +[--prune]` or `--delete-run` and `--yes` against the `DATABASE_COLLECTIVEX_WRITE_URL` secret (GitHub reads use + `INFX_MAIN_PAT`), then purges the cache with `POST /api/v1/invalidate?scope=collectivex` + (`VERCEL_INVALIDATE_SECRET`). It alerts Slack on failure, like `ingest-operatorx`. +- **CLI**: `bun run admin:db:ingest:collectivex [--sha | --ref | --backfill +[--prune] | --delete-run ] [--yes] [--no-ssl]` ingests one commit (default: `main`), + or with `--backfill` every results commit on `main`, oldest first, reusing unchanged files + between commits. A backfill skips commits GitHub answers 404 for (for example one that + deleted `collectivex/results/`) and commits with nothing measured; any other failure, such + as a file that breaks the results contract, stops it. `--prune` then deletes every stored + run the backfill did not store: runs whose commit left `main`, no longer validates, or + measures nothing. `--delete-run ` deletes one stored run at once. It asks for + confirmation unless `--yes` is given. `GITHUB_TOKEN` is optional but raises GitHub's API + rate limit. + +## Operator runbook + +**Cut over from the lazy ingest** (once, after this lands): + +1. **Archive** the current contents: + `pg_dump --table=cx_runs --table=cx_run_docs "$DATABASE_COLLECTIVEX_WRITE_URL" > cx-archive.sql` +2. **Backfill**: run the `Ingest CollectiveX Results` workflow with `backfill` and `prune` + checked. Its migration step applies `004_committed_results.sql` (drops the `deleted_at` + tombstone) and `005_remove_lazy_ingest_runs.sql` (deletes every lazy-ingest run and adds + the run_id CHECK), so no manual `TRUNCATE` is needed; the backfill then stores every + measured results commit on `main` and the cache is purged. Until a measured results + commit lands on `main`, `/api/v1/collectivex/latest` returns 404 and the run list is empty. + Do not use `ref=main` for this: it stores only the newest results commit, not the history. + +**Day to day** nothing is manual: each push to `main` that touches `collectivex/results/` +dispatches an ingest of that commit. Use `sha` or `ref` to re-ingest one commit (for +example after its dispatched ingest failed) and `backfill` to make up any missed one. + +**Retract a bad run**: the durable fix is a commit to the benchmark repo that removes or +corrects the result files, followed by a `backfill` with `prune` (the run's own commit is +still on `main`, so a backfill without `prune` would keep it). To take a run down at once, +run the workflow with `delete_run` set to its `run_id`; a later backfill re-adds it while its +commit still holds measured results, so follow up with the benchmark-repo fix. + +Locally, the same steps are `bun run admin:db:migrate:collectivex` and +`bun run admin:db:ingest:collectivex --backfill --prune` (or `--delete-run `), then +`POST /api/v1/invalidate?scope=collectivex` (the CLI does not reach the CDN). + +## Serving + +- **Reads use the primary**: every route reads through `DATABASE_COLLECTIVEX_WRITE_URL`, so a + response cached right after an ingest's purge never comes from a lagging replica. `DATABASE_COLLECTIVEX_READONLY_URL` is not read by the app routes. - `COLLECTIVEX_ADMIN_SECRET` is the delete route Bearer token (deliberately not - `INVALIDATE_SECRET`, since it is remembered in browser localStorage). When it is unset, - the delete route returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. `GITHUB_TOKEN` - authorizes artifact reads. -- **Source repositories**: sweeps are discovered in `SemiAnalysisAI/InferenceX` and in - every repository listed in `INFERENCEX_SOURCE_REPOS`, merged newest-first by run id. A - listed repository whose sweep-workflow listing returns 404 is skipped; GitHub also - returns 404 for a misspelled repository or one the token cannot see. One that fails to - list for another reason is logged, and the pass then counts as failed: the runs list - keeps `discovery_complete: false` and both passes use the failure cooldown. Discovery - throws when every repository is skipped or fails. Per-run lookups try each repository - in order. A run from any other repository records it as `source_repository` in its - summary, and source, methodology, and run links use it; runs without the field link to - `SemiAnalysisAI/InferenceX`. The ingest CLI takes the repository from its `[repo]` - argument, then from a full run URL, then `INGEST_REPO`, and exits when none is set. - `GITHUB_TOKEN` must be able to - read Actions runs and artifacts in every listed repository. +- **Caching**: responses carry the `collectivex` CDN tag with the default one-day `s-maxage`. + The ingest workflow purges only that tag through `POST /api/v1/invalidate?scope=collectivex`; the + main dashboard's blob cache is untouched by CollectiveX operations. +- **Partial results**: a commit may hold results for only some pools (the first committed + results cover only `mi300x` and `mi325x`). Every runnable case in `cases.json` without a + result file stays in `coverage` with outcome `pending` (requested, not run), counts toward + `requested_cases` but not `measured_cases`, and adds no series, so the SKU controls list + only measured pools. A later commit that adds pools is a new run with a later `run_id`, + so it becomes the latest run while the earlier one stays listed. +- **Run list**: `/api/v1/collectivex/runs` always reports `discovery_complete: true`; the + field stays for existing clients. +- **No deletion or ingest in the app**: the app exposes no CollectiveX mutation. Runs are + added by the ingest workflow or CLI and removed by a `backfill` with `prune` after the + committed results change, or at once with `delete_run` (see the runbook). +- **Covered pools**: a run summary's `covered_skus` lists only pools with at least one + measured result (EP, KV or swap_blocks); pools that are requested but have no result yet + are not listed. +- **Vendor scope is explicit**: the shared reader accepts only `amd` and `nvidia` results + (case-insensitive). Other vendor values are omitted from chart series, coverage, and SKU + summaries; runs with no supported-vendor cases stay hidden. +- **Withheld tails**: an EP result may publish p90/p95/p99 as `null` when too many of a + series' samples were misaligned or host-stalled (p50 is always present). The reader keeps + the derived rates `null` there, the alpha-beta fit skips them, and the tooltip shows them + as unavailable. This is an API response-shape change: in `series[].points[]`, + `components.*.latency_us`, `components.*.activation_data_rate_gbps_at_latency_percentile`, + `components.*.payload_data_rate_gbps_at_latency_percentile` and + `roundtrip_token_rate_at_latency_percentile` may now carry `null` for `p90`, `p95` and + `p99`; the API documentation says so. +- **Links**: a run links to `collectivex/results/` and the methodology at its `source_sha`. ## Known-support matrix @@ -111,12 +162,13 @@ same shared transform across the wire. ## vLLM swap_blocks -Standalone `backend=swap-blocks` sweep runs are discovered through the same matrix/shard -artifact path. Their execution-only matrix (`include` with one or more swap-blocks cells) maps to -app contract version 1. The shared reader validates `collectivex-swap-blocks-v1` documents, -including correctness, payload arithmetic, finite positive ordered latency percentiles, -sample counts and the run's source SHA. EP/KV documents keep their existing reader path. -No database migration is needed: raw JSON documents remain the source of truth. +swap_blocks cases are requested in `cases.json` with `suite: "swap-blocks"` and answered by +`collectivex-swap-blocks-v1` documents at `/.json`. Each result must answer +exactly one requested swap_blocks case of its own GPU pool; requested swap_blocks counts come +from `cases.json`, and those cases never enter the EP ladder parser. The shared reader +validates correctness, payload arithmetic, finite positive ordered latency percentiles and +sample counts. EP/KV documents keep their existing reader path. No database migration is +needed: raw JSON documents remain the source of truth. The additive `swap_blocks` dataset field holds runtime provenance and measured points; `swap_cases` labels their run summaries separately from EP/KV. One verified artifact counts @@ -145,6 +197,6 @@ References: [H200 specifications](https://www.nvidia.com/en-us/data-center/h200/ [GB200 bidirectional C2C](https://developer.nvidia.com/blog/nvidia-gb200-nvl72-delivers-trillion-parameter-llm-training-and-real-time-inference/), [HGX CPU uplink topology](https://docs.nvidia.com/enterprise-reference-architectures/whitepaper/hgx-servers-and-spectrum-x.pdf). -Multi-pool artifacts must carry `runtime.sku` matching a requested matrix cell. Older -single-pool artifacts may omit it. Measurement identity includes SKU so equivalent grid -points from different GPU pools remain independent. +Every swap_blocks result carries `runtime.sku`, which must match its requested case. +Measurement identity includes SKU so equivalent grid points from different GPU pools remain +independent. diff --git a/docs/deployment-configuration.md b/docs/deployment-configuration.md index 900fab4d5..2b573c9f3 100644 --- a/docs/deployment-configuration.md +++ b/docs/deployment-configuration.md @@ -15,10 +15,10 @@ scripts. | Variable | Required | Effect | | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `INFERENCEX_SOURCE_REPOS` | optional | Comma-separated `owner/repo` list searched after `SemiAnalysisAI/InferenceX` for unofficial-run overlays, live GPU metrics, live eval samples, CollectiveX sweeps, and ETL workflow-run lookups. Entries are trimmed and deduplicated; a malformed entry is an error. | +| `INFERENCEX_SOURCE_REPOS` | optional | Comma-separated `owner/repo` list searched after `SemiAnalysisAI/InferenceX` for unofficial-run overlays, live GPU metrics, live eval samples, and ETL workflow-run lookups. Entries are trimmed and deduplicated; a malformed entry is an error. | | `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Sent as `x-vercel-protection-bypass` by the cache invalidate and warmup scripts, the H3 media warmer, and the PowerX publication verifier. | | `INFERENCEX_APP_URL` | `warm-video-results.ts`, and `verify-power-publication.ts` without an origin argument | Origin for those two scripts. The verifier's origin argument wins when given. The cache scripts keep their localhost default. | -| `INGEST_REPO` | ingest, backfill, and artifact-preparation scripts when no argument or run URL names the repository | Repository for those scripts. `ingest-ci-run`, `ingest-collectivex`, `ingest-operatorx`, and the backfills take a `[repo]` argument or a full run URL first. CI-mode `ingest-ci-run` and `prepare-ci-artifacts` always require it. | +| `INGEST_REPO` | ingest, backfill, and artifact-preparation scripts when no argument or run URL names the repository | Repository for those scripts. `ingest-ci-run`, `ingest-operatorx`, and the backfills take a `[repo]` argument or a full run URL first. CI-mode `ingest-ci-run` and `prepare-ci-artifacts` always require it. | | `DATABASE_READONLY_URL` | yes, for the app | Read-only connection for the dashboard API routes. Read-only admin scripts and the MCP server use it too. | | `DATABASE_DRIVER` | optional | `neon` (serverless HTTP) or `postgres` (TCP) for the app's database clients. When unset, hosts ending in `.neon.tech` use `neon` and all others use `postgres`. Admin scripts always use `postgres`. | | `DATABASE_SSL` | optional | `false` disables TLS, and `true` or `require` requires it. When unset, TLS is required except for loopback hosts. Read by the app, the admin scripts, and the MCP server. | @@ -26,21 +26,15 @@ scripts. | `BLOB_CACHE_PREFIX`, `BLOB_READ_WRITE_TOKEN` | optional | Vercel Blob storage. The blob response cache runs only when both are set. A full invalidation deletes every blob under the prefix, so deployments that share a Blob store need distinct prefixes. H3 video storage needs only `BLOB_READ_WRITE_TOKEN`. Both features turn themselves off when their variables are unset. | | `NEXT_PUBLIC_POSTHOG_KEY`, `NEXT_PUBLIC_POSTHOG_HOST` | optional | PostHog project key and host for browser analytics and server-side events. Without the key, analytics is off. Without the host, `https://us.i.posthog.com` is used. Both are inlined at build time (see [Vercel project](#vercel-project)). | | `POSTHOG_PERSONAL_API_KEY`, `POSTHOG_PROJECT_ID` | optional, at build time, set together | Production builds upload source maps to PostHog only when both are set. | -| `GITHUB_TOKEN` | yes, for GitHub-backed features | Reads Actions runs and artifacts. Without it, `/api/unofficial-run`, `/api/gpu-metrics` live telemetry, and `/api/v1/eval-samples-live` return 500, CollectiveX discovery reports `unavailable`, and framework release and GitHub star requests go out unauthenticated. | +| `GITHUB_TOKEN` | yes, for GitHub-backed features | Reads Actions runs and artifacts. Without it, `/api/unofficial-run`, `/api/gpu-metrics` live telemetry, and `/api/v1/eval-samples-live` return 500, the CollectiveX ingest reads GitHub unauthenticated, and framework release and GitHub star requests go out unauthenticated. | | `OPERATORX_SOURCE` | optional | `database`, the behavior when unset, reads `DATABASE_OPERATORX_READONLY_URL`. `local` reads `OPERATORX_LOCAL_ARTIFACT_DIR` and works only in development. Any other value makes `/api/v1/operatorx/*` return 503. | `GITHUB_TOKEN` must be able to read Actions runs and artifacts in every listed source repository. Artifacts from every listed repository are served to any caller through -`/api/unofficial-run`, `/api/gpu-metrics`, `/api/v1/eval-samples-live`, and -`/api/v1/collectivex/*`. +`/api/unofficial-run`, `/api/gpu-metrics`, and `/api/v1/eval-samples-live`. -CollectiveX discovery skips a listed repository whose sweep-workflow listing returns 404. -GitHub also returns 404 for a misspelled repository or one the token cannot see, so -check the token's access when a repository's sweeps do not appear. A repository that -fails to list for another reason is logged, and the pass counts as failed: the runs list -stays incomplete and retries after a short cooldown. Runs from a repository other than -`SemiAnalysisAI/InferenceX` record it as `source_repository` in the stored summary, and -their links use it. +The CollectiveX ingest reads only `SemiAnalysisAI/InferenceX`'s committed +`collectivex/results/`, never `INFERENCEX_SOURCE_REPOS`. ### Feature variables @@ -48,15 +42,14 @@ The production block of `.env.example` lists the core connection, cache, and ana variables. The deployed app also reads the variables below. Each one backs a single feature, which fails as described when the variable is unset. -| Variable | Read by | When unset | -| ---------------------------------- | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `INVALIDATE_SECRET` | `POST /api/v1/invalidate` | Requests return 500 with `INVALIDATE_SECRET is not set`, except requests to the shared `staging` preview that carry `x-vercel-protection-bypass`. When it is set, a missing or wrong bearer token returns 401. Set it to the value of the `VERCEL_INVALIDATE_SECRET` GitHub secret. | -| `FEEDBACK_SECRET` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | -| `DATABASE_WRITE_URL` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | -| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read and lazily ingest through this primary connection, so they return errors. | -| `COLLECTIVEX_ADMIN_SECRET` | `DELETE /api/v1/collectivex/runs/[runId]` | Every request returns 500 with `COLLECTIVEX_ADMIN_SECRET is not set`. When it is set, a missing or wrong bearer token returns 401. | -| `DATABASE_OPERATORX_READONLY_URL` | `/api/v1/operatorx/*` | The routes return 503 with `The OperatorX database is not configured`. | -| `DATABASE_PROXYTRACE_READONLY_URL` | `/api/v1/agentic-workload-explorer/*` | The routes return 500. | +| Variable | Read by | When unset | +| ---------------------------------- | ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `INVALIDATE_SECRET` | `POST /api/v1/invalidate` | Requests return 500 with `INVALIDATE_SECRET is not set`, except requests to the shared `staging` preview that carry `x-vercel-protection-bypass`. When it is set, a missing or wrong bearer token returns 401. Set it to the value of the `VERCEL_INVALIDATE_SECRET` GitHub secret. | +| `FEEDBACK_SECRET` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | +| `DATABASE_WRITE_URL` | `POST /api/v1/feedback` | Submissions return 500 with code `E_CRYPTO`. | +| `DATABASE_COLLECTIVEX_WRITE_URL` | `/api/v1/collectivex/*` | The routes read through this primary connection, so they return errors. | +| `DATABASE_OPERATORX_READONLY_URL` | `/api/v1/operatorx/*` | The routes return 503 with `The OperatorX database is not configured`. | +| `DATABASE_PROXYTRACE_READONLY_URL` | `/api/v1/agentic-workload-explorer/*` | The routes return 500. | `DATABASE_COLLECTIVEX_READONLY_URL` is not read by the app routes. Every CollectiveX route reads through `DATABASE_COLLECTIVEX_WRITE_URL`. @@ -79,34 +72,36 @@ project, redeploy for the new value to take effect. | Name | Required | Effect | | ----------------- | ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `APP_URL` | yes, by every workflow that uses it | Site targeted by cache invalidation and warmup, OperatorX invalidation, post-deploy OperatorX warmup, H3 media warmup, and PowerX verification on the `production` ingest target. | +| `APP_URL` | yes, by every workflow that uses it | Site targeted by cache invalidation and warmup, OperatorX and CollectiveX invalidation, post-deploy OperatorX warmup, H3 media warmup, and PowerX verification on the `production` ingest target. | | `BENCHMARK_REPO` | yes, by every workflow that uses it | Repository whose benchmark artifacts the ingest workflows download and that dispatches `stage-results`. When it is not `SemiAnalysisAI/InferenceX`, also list it in `INFERENCEX_SOURCE_REPOS` for the app's GitHub lookups. | | `STAGING_APP_URL` | `staging` target and `stage-results` | Site invalidated by the `staging` ingest target and linked from staged chart comments. | ### Secrets `GITHUB_TOKEN` is provided by GitHub and is not listed. `SLACK_WEBHOOK_URL` is the -failure alarm, so `ingest-results`, `ingest-operatorx`, and `stage-results` check it +failure alarm, so `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, and `stage-results` check it before doing any other work. `stage-results` also checks `PAT` up front. `ingest-operatorx` checks `APP_URL`, `VERCEL_INVALIDATE_SECRET`, -`DATABASE_OPERATORX_WRITE_URL`, and `INFX_MAIN_PAT` in its first step. +`DATABASE_OPERATORX_WRITE_URL`, and `INFX_MAIN_PAT` in its first step, and +`ingest-collectivex` checks the same with `DATABASE_COLLECTIVEX_WRITE_URL`. `backfill-server-log-files` checks `INFX_MAIN_PAT` before the backfill. `apply-run-overrides`, `backfill-server-log-files`, and `recompute-agentic-metrics` check `DATABASE_WRITE_URL` before writing to the production database. -| Secret | Required | Used for | Workflows | -| --------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `stage-results` (passed to `ingest-results`) | -| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | -| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | -| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | -| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`. | `ingest-results`, `ingest-operatorx`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | -| `PAT` | yes, by `stage-results`; optional for `ingest-results` | Sends the `stage-results-started`, `stage-results-completed` and `ingest-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results`, `ingest-results` (dispatched merge ingests only; a failed report never fails the ingest) | -| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | -| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | -| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | -| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | -| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | +| Secret | Required | Used for | Workflows | +| --------------------------------- | -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `SLACK_WEBHOOK_URL` | yes | Incoming webhook for ingest failure and unmapped-entity alerts. | `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, `stage-results` (passed to `ingest-results`) | +| `DATABASE_WRITE_URL` | yes, for writes to the production database | Production database write connection. | `apply-run-overrides`, `ingest-results` on the `production` target, `backfill-server-log-files` and `recompute-agentic-metrics` without `neon-branch` | +| `DATABASE_STAGING_WRITE_URL` | yes, for the `staging` target | Write connection for the Neon `staging` branch. | `ingest-results`, `stage-results` | +| `DATABASE_OPERATORX_WRITE_URL` | yes, by `ingest-operatorx` | OperatorX database write connection for migrations and ingest. | `ingest-operatorx` | +| `DATABASE_COLLECTIVEX_WRITE_URL` | yes, by `ingest-collectivex` | CollectiveX database write connection for migrations and ingest. | `ingest-collectivex` | +| `INFX_MAIN_PAT` | yes | Reads Actions runs and artifacts in `BENCHMARK_REPO`, and the committed CollectiveX results. | `ingest-results`, `ingest-operatorx`, `ingest-collectivex`, `stage-results`, `backfill-server-log-files`, `recompute-agentic-metrics` with `restore-missing-trace` | +| `PAT` | yes, by `stage-results`; optional for `ingest-results` | Sends the `stage-results-started`, `stage-results-completed` and `ingest-results-completed` repository dispatch events to `BENCHMARK_REPO`. | `stage-results`, `ingest-results` (dispatched merge ingests only; a failed report never fails the ingest) | +| `NEON_API_KEY` | yes, for Neon branch operations | Resolves named Neon child branches and restores or syncs the `staging` branch. | `ingest-results` on the `neon-branch` target, `backfill-server-log-files` and `recompute-agentic-metrics` with `neon-branch`, `stage-results` with `reset-staging-database`, `sync-staging-database` | +| `NEON_PROJECT_ID` | with `NEON_API_KEY` | Neon project that holds those branches. | Same as `NEON_API_KEY` | +| `VERCEL_INVALIDATE_SECRET` | yes, for cache invalidation outside the `staging` target | Bearer token for `/api/v1/invalidate`. It must equal the app's runtime `INVALIDATE_SECRET`. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `ingest-collectivex`, `ingest-operatorx`, `ingest-results`, `recompute-agentic-metrics` | +| `VERCEL_PROTECTION_BYPASS_SECRET` | optional | Deployment Protection bypass sent with requests to `APP_URL` and by the cache scripts. | `apply-run-overrides`, `backfill-server-log-files`, `cache-refresh`, `h3-media`, `ingest-collectivex`, `ingest-operatorx`, `ingest-results` on the `production` target, `recompute-agentic-metrics`, `warm-after-deploy` | +| `VERCEL_STAGING_BYPASS_SECRET` | yes, for the `staging` and `neon-branch` targets | Vercel automation bypass for the staging and preview deployments. | `ingest-results`, `stage-results`, `sync-staging-database` | Values reach scripts only through step `env:`, never by inline expression in a `run:` block. diff --git a/docs/index.md b/docs/index.md index bea8036af..5dd168434 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,7 +28,7 @@ Design rationale and non-obvious conventions. Repo-wide rules are in [AGENTS.md] - [Data Transforms](./data-transforms.md) covers power boundaries, prefix-cache tier shares, and hardware-key gotchas in the chart data pipeline. - [State Ownership](./state-ownership.md) explains where shared chart state lives and how URL state is restored across navigation. - [Blog](./blog.md) covers MDX content, frontmatter and scheduled publishing, captions, heading IDs, and math. -- [CollectiveX](./collectivex.md) covers the lazy ingest-on-read pipeline, its separate Neon database, tombstoned deletes, and the shared reader. +- [CollectiveX](./collectivex.md) covers the separate Neon database filled from committed results, run identity, the ingest route and CLI, the re-seed runbook, and the shared reader. - [Chinese Pages (/zh)](./i18n.md) explains why /zh pages are hand-authored and how hreflang and blog translation pairing work. - [Chinese Copy Editorial Guide](./chinese-copy.md) covers audience, register, terminology, the review workflow, and the PR checklist for Chinese copy. diff --git a/packages/app/cypress/e2e/collectivex.cy.ts b/packages/app/cypress/e2e/collectivex.cy.ts index eecbc8ed5..1e1a29e8f 100644 --- a/packages/app/cypress/e2e/collectivex.cy.ts +++ b/packages/app/cypress/e2e/collectivex.cy.ts @@ -30,7 +30,6 @@ const incompleteDataset = buildDataset({ meta: { run_id: '161', generated_at: '2026-07-09T12:20:00Z', - conclusion: 'failure', }, }); const kvDataset = buildDataset({ @@ -72,7 +71,6 @@ const kvOnlyDataset = buildDataset({ kv: [{}], meta: { run_id: '163', generated_at: '2026-08-08T12:20:00Z', source_sha: 'f'.repeat(40) }, }); -const ADMIN_TOKEN_KEY = 'collectivex-admin-token'; function installRuns(bodies: CollectiveXDataset[] = [dataset]) { cy.intercept('GET', '/api/v1/collectivex/runs?*', { @@ -103,9 +101,8 @@ describe('CollectiveX neutral run view', () => { }); it('shows the run header, coverage stats, and revision-pinned source links', () => { - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', `#${runId}`) - .and('contain.text', 'success'); + cy.get('[data-testid="collectivex-run-conclusion"]').should('not.exist'); + cy.get('[data-testid^="collectivex-delete"]').should('not.exist'); cy.get('[data-testid="collectivex-display"]') .should('contain.text', `${dataset.run.measured_cases}/${dataset.run.requested_cases}`) .and('contain.text', String(dataset.series.length)); @@ -114,19 +111,24 @@ describe('CollectiveX neutral run view', () => { cy.get('[data-testid="collectivex-source-link"]').should( 'have.attr', 'href', - `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/experimental/CollectiveX`, + `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/collectivex`, ); cy.get('[data-testid="collectivex-methodology-link"]') .should('contain.text', 'Methodology') .and( 'have.attr', 'href', - `https://github.com/SemiAnalysisAI/InferenceX/blob/${SOURCE_SHA}/experimental/CollectiveX/docs/methodology.md`, + `https://github.com/SemiAnalysisAI/InferenceX/blob/${SOURCE_SHA}/collectivex/docs/methodology.md`, + ); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`) + .should('contain.text', `#${runId}`) + .and( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/tree/${SOURCE_SHA}/collectivex/results`, ); cy.get(`[data-testid="collectivex-run-row-${runId}"] a[href*="/actions/runs/"]`).should( - 'have.attr', - 'href', - `https://github.com/SemiAnalysisAI/InferenceX/actions/runs/${runId}`, + 'not.exist', ); installRuns(); @@ -137,32 +139,6 @@ describe('CollectiveX neutral run view', () => { cy.get(`[data-testid="collectivex-run-suite-kv-${runId}"]`).should('not.exist'); }); - it('keeps loading bounded discovery batches until every run is listed', () => { - let requests = 0; - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - requests += 1; - request.reply({ - body: { - version: 1, - runs: (requests === 1 ? [] : [dataset, comparisonDataset]).map(buildRunSummary), - discovery_complete: requests > 1, - }, - }); - }).as('progressiveRuns'); - - cy.reload(); - cy.wait('@progressiveRuns'); - cy.wait('@progressiveRuns'); - cy.wait('@run'); - - cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( - 'be.visible', - ); - cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); - cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); - cy.then(() => expect(requests).to.be.gte(2)); - }); - it('selects the EP16 series through the identity controls', () => { cy.get('[data-testid="collectivex-ep-select"]').should('be.visible'); cy.get('[data-testid="collectivex-phase-toggle"]').should('be.visible'); @@ -369,9 +345,10 @@ describe('CollectiveX neutral run view', () => { }); cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).uncheck(); - cy.get('[data-testid="collectivex-run-conclusion"]').should( - 'contain.text', - `#${comparisonDataset.run.run_id}`, + cy.get('[data-testid="collectivex-source-link"]').should( + 'have.attr', + 'href', + `https://github.com/SemiAnalysisAI/InferenceX/tree/${'d'.repeat(40)}/collectivex`, ); cy.get(`[data-testid="collectivex-run-line-style-${runId}"]`).should('not.exist'); cy.get( @@ -442,7 +419,10 @@ describe('CollectiveX neutral run view', () => { 'not.be.checked', ); cy.get(`[data-testid="collectivex-run-visible-${runId}"]`).should('be.checked'); - cy.get('[data-testid="collectivex-run-conclusion"]').should('contain.text', `#${runId}`); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`).should('be.visible'); + cy.get(`[data-testid="collectivex-run-source-${incompleteDataset.run.run_id}"]`).should( + 'not.exist', + ); cy.get('[data-testid="collectivex-explorer-chart"] .line-path').should('have.length', 1); }); @@ -546,9 +526,11 @@ describe('CollectiveX neutral run view', () => { cy.wait('@runs'); cy.wait('@run'); - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', `#${runId}`) - .and('contain.text', '成功'); + cy.get(`[data-testid="collectivex-run-source-${runId}"]`).should( + 'have.attr', + 'aria-label', + `打开运行 #${runId} 对应的已提交结果`, + ); cy.get('[data-testid="collectivex-runs"]') .should('contain.text', '运行记录') .and('contain.text', '终态数据点'); @@ -566,21 +548,6 @@ describe('CollectiveX neutral run view', () => { .and('contain.text', '常规') .and('contain.text', '解码') .and('contain.text', '延迟 p50 / p90 / p95 / p99'); - - const cancelled = buildDataset({ - shards: [makeRawShard()], - meta: { run_id: '179', generated_at: '2026-08-29T12:20:00Z', conclusion: 'cancelled' }, - }); - installRuns([cancelled]); - installRun(cancelled, 'cancelledRun'); - cy.visit('/zh/collectivex'); - cy.wait('@runs'); - cy.wait('@cancelledRun'); - - cy.get('[data-testid="collectivex-run-conclusion"]') - .should('contain.text', '已取消') - .and('not.contain.text', '待处理') - .and('not.contain.text', 'cancelled'); }); it('keeps the Chinese explorer and runs table reachable at 375px', () => { @@ -597,121 +564,6 @@ describe('CollectiveX neutral run view', () => { }); }); -describe('CollectiveX run deletion', () => { - beforeEach(() => { - installRuns(); - installRun(); - openCollectiveX(); - }); - - it('deletes a table row after confirm + token prompt and remembers the token', () => { - let deleted = false; - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - request.reply({ - body: { - version: 1, - runs: deleted ? [] : [buildRunSummary(dataset)], - }, - }); - }).as('runsAfterDelete'); - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, (request) => { - expect(request.headers.authorization).to.eq('Bearer test-token'); - deleted = true; - request.reply({ deleted: true, runId }); - }).as('deleteRun'); - cy.window().then((win) => { - win.localStorage.removeItem(ADMIN_TOKEN_KEY); - cy.stub(win, 'confirm').returns(true); - cy.stub(win, 'prompt').returns('test-token'); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.wait('@deleteRun'); - cy.wait('@runsAfterDelete'); - cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('not.exist'); - cy.window().then((win) => { - expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq('test-token'); - }); - }); - - it('deletes every shown run with one confirmation and one token prompt', () => { - const deletedRunIds = new Set(); - cy.intercept('GET', '/api/v1/collectivex/runs?*', (request) => { - request.reply({ - body: { - version: 1, - runs: [dataset, comparisonDataset] - .filter((item) => !deletedRunIds.has(item.run.run_id)) - .map(buildRunSummary), - discovery_complete: true, - }, - }); - }).as('runsAfterBulkDelete'); - installRun(comparisonDataset, 'comparisonRunForDelete'); - cy.intercept('DELETE', '/api/v1/collectivex/runs/*', (request) => { - expect(request.headers.authorization).to.eq('Bearer bulk-test-token'); - deletedRunIds.add(request.url.split('/').at(-1) ?? ''); - request.reply({ deleted: true }); - }).as('deleteShownRun'); - - cy.reload(); - cy.wait('@runsAfterBulkDelete'); - cy.wait('@run'); - cy.get(`[data-testid="collectivex-run-visible-${comparisonDataset.run.run_id}"]`).check(); - cy.wait('@comparisonRunForDelete'); - cy.window().then((win) => { - win.localStorage.removeItem(ADMIN_TOKEN_KEY); - cy.stub(win, 'confirm').as('bulkDeleteConfirm').returns(true); - cy.stub(win, 'prompt').as('bulkDeletePrompt').returns('bulk-test-token'); - }); - - cy.get('[data-testid="collectivex-delete-shown-runs"]').click(); - cy.wait('@deleteShownRun'); - cy.wait('@deleteShownRun'); - cy.get(`[data-testid="collectivex-run-row-${runId}"]`).should('not.exist'); - cy.get(`[data-testid="collectivex-run-row-${comparisonDataset.run.run_id}"]`).should( - 'not.exist', - ); - cy.get('@bulkDeleteConfirm').should('have.been.calledOnce'); - cy.get('@bulkDeletePrompt').should('have.been.calledOnce'); - cy.then(() => { - expect(deletedRunIds).to.deep.eq(new Set([runId, comparisonDataset.run.run_id])); - }); - }); - - it('clears a stale stored token and reports unauthorized on 401', () => { - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, { statusCode: 401 }).as( - 'delete401', - ); - cy.window().then((win) => { - win.localStorage.setItem(ADMIN_TOKEN_KEY, 'stale-token'); - cy.stub(win, 'confirm').returns(true); - cy.stub(win, 'alert').as('unauthorizedAlert'); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.wait('@delete401'); - cy.get('@unauthorizedAlert').should('have.been.calledWith', 'Invalid admin token.'); - cy.window().then((win) => { - expect(win.localStorage.getItem(ADMIN_TOKEN_KEY)).to.eq(null); - }); - }); - - it('does nothing when the confirmation is declined', () => { - let deleteRequests = 0; - cy.intercept('DELETE', `/api/v1/collectivex/runs/${runId}`, () => { - deleteRequests += 1; - }); - cy.window().then((win) => { - cy.stub(win, 'confirm').returns(false); - }); - - cy.get(`[data-testid="collectivex-delete-run-${runId}"]`).click(); - cy.get('[data-testid="collectivex-display"]').should('be.visible'); - cy.then(() => expect(deleteRequests).to.eq(0)); - }); -}); - describe('CollectiveX availability states', () => { it('reports a missing run list', () => { cy.intercept('GET', '/api/v1/collectivex/runs?*', { @@ -815,17 +667,17 @@ describe('CollectiveX source repository links', () => { cy.get('[data-testid="collectivex-source-link"]').should( 'have.attr', 'href', - `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/experimental/CollectiveX`, + `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/collectivex`, ); cy.get('[data-testid="collectivex-methodology-link"]').should( 'have.attr', 'href', - `https://github.com/ExampleOrg/ExampleBench/blob/${SOURCE_SHA}/experimental/CollectiveX/docs/methodology.md`, + `https://github.com/ExampleOrg/ExampleBench/blob/${SOURCE_SHA}/collectivex/docs/methodology.md`, ); - cy.get('[data-testid="collectivex-run-row-166"] a[href*="/actions/runs/"]').should( + cy.get('[data-testid="collectivex-run-source-166"]').should( 'have.attr', 'href', - 'https://github.com/ExampleOrg/ExampleBench/actions/runs/166', + `https://github.com/ExampleOrg/ExampleBench/tree/${SOURCE_SHA}/collectivex/results`, ); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.test.ts b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts index fdd431c77..7381b79bf 100644 --- a/packages/app/src/app/api/v1/collectivex/latest/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/latest/route.test.ts @@ -2,11 +2,11 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockGetLatest, mockFromRow, mockGetWriteDb, mockEnsureLatest } = vi.hoisted(() => ({ +const { mockGetLatest, mockFromRow, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ mockGetLatest: vi.fn(), mockFromRow: vi.fn(), mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockEnsureLatest: vi.fn(), + mockCachedJson: vi.fn((data: unknown) => Response.json(data)), })); vi.mock('@semianalysisai/inferencex-db/connection', () => ({ @@ -19,21 +19,8 @@ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ collectiveXDatasetFromRow: mockFromRow, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureLatestCollectiveXRun: mockEnsureLatest, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', - cachedJson: (data: unknown) => Response.json(data), + cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', })); @@ -45,16 +32,11 @@ function req(url: string): NextRequest { return new NextRequest(new URL(url, 'http://localhost')); } -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - const dataset = makeCollectiveXDataset(); beforeEach(() => { vi.clearAllMocks(); vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureLatest.mockResolvedValue(undefined); mockGetLatest.mockResolvedValue({ run_id: dataset.run.run_id }); mockFromRow.mockReturnValue(dataset); }); @@ -65,32 +47,18 @@ describe('GET /api/v1/collectivex/latest', () => { const res = await GET(req(url)); expect(res.status).toBe(400); } - expect(mockEnsureLatest).not.toHaveBeenCalled(); + expect(mockGetLatest).not.toHaveBeenCalled(); }); - it('lazily ingests then serves the latest run assembled as a dataset', async () => { + it('serves the newest stored run assembled as a dataset under the CollectiveX tag', async () => { const res = await GET(req('/api/v1/collectivex/latest?version=1')); expect(res.status).toBe(200); expect(await res.json()).toEqual(dataset); - expect(mockEnsureLatest).toHaveBeenCalledWith(1); expect(mockGetLatest).toHaveBeenCalledWith('mock-write-sql', 1); + expect(mockCachedJson).toHaveBeenCalledWith(dataset, { tag: 'collectivex' }); }); - it('serves the stored run when GitHub discovery fails', async () => { - mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); - const res = await GET(req('/api/v1/collectivex/latest?version=1')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual(dataset); - }); - - it('returns 503 when discovery fails and nothing is stored', async () => { - mockEnsureLatest.mockRejectedValue(sweepError('unavailable')); - mockGetLatest.mockResolvedValue(null); - const res = await GET(req('/api/v1/collectivex/latest?version=1')); - expect(res.status).toBe(503); - }); - - it('returns 404 when neither GitHub nor the DB has a run', async () => { + it('returns 404 when nothing is stored for the version', async () => { mockGetLatest.mockResolvedValue(null); const res = await GET(req('/api/v1/collectivex/latest?version=1')); expect(res.status).toBe(404); diff --git a/packages/app/src/app/api/v1/collectivex/latest/route.ts b/packages/app/src/app/api/v1/collectivex/latest/route.ts index 08c220976..3bdaecd0f 100644 --- a/packages/app/src/app/api/v1/collectivex/latest/route.ts +++ b/packages/app/src/app/api/v1/collectivex/latest/route.ts @@ -7,11 +7,7 @@ import { getLatestCollectiveXRun, } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; -import { - collectiveXSweepErrorStatus, - ensureLatestCollectiveXRun, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; @@ -24,32 +20,11 @@ export async function GET(request: NextRequest) { } if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-latest')); - // Discovery failures must not take the page down — serve whatever the DB - // already holds and only surface the error when there is no fallback. - let ensureError: unknown = null; - try { - await ensureLatestCollectiveXRun(version); - } catch (error) { - ensureError = error; - } - try { - // Discovery may have persisted a run, so read from the same primary. + // The primary, so a response cached right after an ingest's purge is never a stale replica read. const row = await getLatestCollectiveXRun(getCollectiveXWriteDb(), version); - if (row !== null) { - if (ensureError) - console.error('CollectiveX discovery failed; serving stored run:', ensureError); - return cachedJson(collectiveXDatasetFromRow(row), { - tag: collectiveXCacheTag(), - cacheControl: COLLECTIVEX_CACHE_CONTROL, - }); - } - if (ensureError) { - console.error('CollectiveX discovery failed with no stored fallback:', ensureError); - const status = collectiveXSweepErrorStatus(ensureError) ?? 502; - return NextResponse.json({ error: 'Unavailable' }, { status }); - } - return NextResponse.json({ error: 'Not found' }, { status: 404 }); + if (row === null) return NextResponse.json({ error: 'Not found' }, { status: 404 }); + return cachedJson(collectiveXDatasetFromRow(row), { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error fetching CollectiveX latest run:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts index 1df319a8b..7ae876995 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.fixtures.test.ts @@ -17,20 +17,12 @@ vi.mock('@semianalysisai/inferencex-db/connection', () => ({ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ collectiveXDatasetFromRow: vi.fn(), - deleteCollectiveXRun: vi.fn(), getCollectiveXRun: vi.fn(), })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - collectiveXSweepErrorStatus: vi.fn(), - ensureCollectiveXRun: vi.fn(), -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', cachedJson: (data: unknown) => Response.json(data), collectiveXCacheTag: () => 'collectivex', - purgeCollectiveX: vi.fn(), })); vi.mock('@/lib/testing/test-fixtures', () => ({ diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts index 5ddf4be86..e58b1a452 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.test.ts @@ -1,16 +1,13 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockGetRun, mockDelete, mockFromRow, mockGetWriteDb, mockPurge, mockEnsureRun } = - vi.hoisted(() => ({ - mockGetRun: vi.fn(), - mockDelete: vi.fn(), - mockFromRow: vi.fn(), - mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockPurge: vi.fn(() => Promise.resolve(0)), - mockEnsureRun: vi.fn(), - })); +const { mockGetRun, mockFromRow, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ + mockGetRun: vi.fn(), + mockFromRow: vi.fn(), + mockGetWriteDb: vi.fn(() => 'mock-write-sql'), + mockCachedJson: vi.fn((data: unknown) => Response.json(data)), +})); vi.mock('@semianalysisai/inferencex-db/connection', () => ({ getCollectiveXWriteDb: mockGetWriteDb, @@ -19,151 +16,70 @@ vi.mock('@semianalysisai/inferencex-db/connection', () => ({ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ getCollectiveXRun: mockGetRun, - deleteCollectiveXRun: mockDelete, collectiveXDatasetFromRow: mockFromRow, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureCollectiveXRun: mockEnsureRun, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', - cachedJson: (data: unknown) => Response.json(data), + cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', - purgeCollectiveX: mockPurge, })); import { NextRequest } from 'next/server'; -import { DELETE, GET } from './route'; +import * as route from './route'; -const SECRET = 'test-admin-secret'; const dataset = makeCollectiveXDataset(); const runId = dataset.run.run_id; -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - function get(url: string, id: string) { - return GET(new NextRequest(new URL(url, 'http://localhost')), { + return route.GET(new NextRequest(new URL(url, 'http://localhost')), { params: Promise.resolve({ runId: id }), }); } -function del(id: string, token?: string) { - return DELETE( - new NextRequest(new URL(`/api/v1/collectivex/runs/${id}`, 'http://localhost'), { - method: 'DELETE', - headers: token ? { Authorization: `Bearer ${token}` } : {}, - }), - { params: Promise.resolve({ runId: id }) }, - ); -} - beforeEach(() => { vi.clearAllMocks(); - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', SECRET); - mockEnsureRun.mockResolvedValue(undefined); + vi.spyOn(console, 'error').mockImplementation(() => {}); mockGetRun.mockResolvedValue({ run_id: runId }); mockFromRow.mockReturnValue(dataset); - mockDelete.mockResolvedValue(true); -}); - -afterEach(() => { - vi.unstubAllEnvs(); }); describe('GET /api/v1/collectivex/runs/[runId]', () => { - it('returns 400 for malformed version or run id', async () => { - const badRunId = await get(`/x?version=1`, 'abc'); - const badVersion = await get(`/x?version=99`, runId); - expect(badRunId.status).toBe(400); - expect(badVersion.status).toBe(400); - expect(mockEnsureRun).not.toHaveBeenCalled(); + it('returns 400 for an unknown version or a malformed run id', async () => { + for (const [url, id] of [ + ['/x', runId], + ['/x?version=99', runId], + ['/x?version=1', '0'], + ['/x?version=1', 'abc'], + ]) { + const res = await get(url, id); + expect(res.status).toBe(400); + } + expect(mockGetRun).not.toHaveBeenCalled(); }); - it('lazily ingests then serves the run assembled as a dataset', async () => { - const res = await get(`/x?version=1`, runId); + it('serves a stored run assembled as a dataset under the CollectiveX tag', async () => { + const res = await get('/x?version=1', runId); expect(res.status).toBe(200); expect(await res.json()).toEqual(dataset); - expect(mockEnsureRun).toHaveBeenCalledWith(1, runId); expect(mockGetRun).toHaveBeenCalledWith('mock-write-sql', 1, runId); + expect(mockCachedJson).toHaveBeenCalledWith(dataset, { tag: 'collectivex' }); }); - it.each([ - ['not-found', 404], - ['unavailable', 503], - ['invalid', 502], - ] as const)('maps %s ingest failures without exposing details', async (code, status) => { - vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureRun.mockRejectedValue(sweepError(code)); + it('returns 404 for a run that is not stored', async () => { mockGetRun.mockResolvedValue(null); - const res = await get(`/x?version=1`, runId); - expect(res.status).toBe(status); - }); - - it('serves a stored run when its GitHub refresh fails', async () => { - vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureRun.mockRejectedValue(sweepError('unavailable')); - - const res = await get(`/x?version=1`, runId); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual(dataset); - }); - - it('returns 404 when the run is absent after a clean ensure', async () => { - mockGetRun.mockResolvedValue(null); - const res = await get(`/x?version=1`, runId); + const res = await get('/x?version=1', '1792089351'); expect(res.status).toBe(404); }); -}); - -describe('DELETE /api/v1/collectivex/runs/[runId]', () => { - it('rejects missing or wrong bearer tokens', async () => { - const missing = await del(runId); - const wrong = await del(runId, 'wrong-token'); - expect(missing.status).toBe(401); - expect(wrong.status).toBe(401); - expect(mockDelete).not.toHaveBeenCalled(); - }); - it('returns 500 when the secret is not configured', async () => { - vi.stubEnv('COLLECTIVEX_ADMIN_SECRET', ''); - const res = await del(runId, 'anything'); + it('returns 500 without leaking details on DB failure', async () => { + mockGetRun.mockRejectedValue(new Error('boom')); + const res = await get('/x?version=1', runId); expect(res.status).toBe(500); - expect(await res.text()).toBe('COLLECTIVEX_ADMIN_SECRET is not set'); - expect(mockDelete).not.toHaveBeenCalled(); - }); - - it('tombstones the run and purges only the CollectiveX cache scope', async () => { - const res = await del(runId, SECRET); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ deleted: true, runId }); - expect(mockDelete).toHaveBeenCalledWith('mock-write-sql', runId); - expect(mockPurge).toHaveBeenCalledTimes(1); - }); - - it('returns 404 without purging when the run does not exist', async () => { - mockDelete.mockResolvedValue(false); - const res = await del(runId, SECRET); - expect(res.status).toBe(404); - expect(mockPurge).not.toHaveBeenCalled(); + expect(await res.json()).toEqual({ error: 'Internal server error' }); }); - it('returns 400 for malformed run ids', async () => { - const res = await del('not-a-run-id', SECRET); - expect(res.status).toBe(400); - expect(mockDelete).not.toHaveBeenCalled(); + it('exposes no mutation handler', () => { + expect(Object.keys(route)).not.toContain('DELETE'); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts index b6277ea20..489774e94 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/[runId]/route.ts @@ -8,21 +8,10 @@ import { import { FIXTURES_MODE, getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; import { collectiveXDatasetFromRow, - deleteCollectiveXRun, getCollectiveXRun, } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { - COLLECTIVEX_CACHE_CONTROL, - collectiveXCacheTag, - cachedJson, - purgeCollectiveX, -} from '@/lib/cache/api-cache'; -import { bearerMatches } from '@/lib/api/bearer-auth'; -import { - collectiveXSweepErrorStatus, - ensureCollectiveXRun, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { collectiveXCacheTag, cachedJson } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; @@ -48,79 +37,12 @@ export async function GET(request: NextRequest, context: { params: Promise<{ run return cachedJson(loadFixture(`collectivex-run-${runId}`)); } - let ensureError: unknown = null; - try { - await ensureCollectiveXRun(version, runId); - } catch (error) { - ensureError = error; - } - try { - // Ensure may have inserted or refreshed this run, so read from the same primary. const row = await getCollectiveXRun(getCollectiveXWriteDb(), version, runId); - if (row === null) { - if (ensureError) { - const status = collectiveXSweepErrorStatus(ensureError); - if (status !== null) { - return NextResponse.json( - { error: status === 404 ? 'Not found' : 'Unavailable' }, - { status }, - ); - } - throw ensureError; - } - return NextResponse.json({ error: 'Not found' }, { status: 404 }); - } - if (ensureError) { - console.error('CollectiveX run refresh failed; serving stored run:', ensureError); - } - // Short window like the sibling routes: a GitHub re-run of failed shards - // refreshes this run's stored contents, and deletion must not linger. - return cachedJson(collectiveXDatasetFromRow(row), { - tag: collectiveXCacheTag(), - cacheControl: COLLECTIVEX_CACHE_CONTROL, - }); + if (row === null) return NextResponse.json({ error: 'Not found' }, { status: 404 }); + return cachedJson(collectiveXDatasetFromRow(row), { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error fetching CollectiveX run:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); } } - -/** - * Admin deletion of an ingested run. Authenticated with a dedicated Bearer - * secret — the token is remembered in browser localStorage, so it must not - * be the CI-held INVALIDATE_SECRET (scoped blast radius, independently - * rotatable). Deletion tombstones the run (lazy discovery must never - * re-ingest it) and purges the CollectiveX cache scope so the run table and - * latest views drop it immediately. - */ -export async function DELETE( - request: NextRequest, - context: { params: Promise<{ runId: string }> }, -) { - const secret = process.env.COLLECTIVEX_ADMIN_SECRET; - const authHeader = request.headers.get('Authorization') ?? ''; - if (!secret) { - return new NextResponse('COLLECTIVEX_ADMIN_SECRET is not set', { status: 500 }); - } - if (!bearerMatches(authHeader, secret)) { - return new NextResponse('Unauthorized', { status: 401 }); - } - - const { runId } = await context.params; - if (!RUN_ID.test(runId)) { - return NextResponse.json({ error: 'Unknown run id' }, { status: 400 }); - } - - try { - const deleted = await deleteCollectiveXRun(getCollectiveXWriteDb(), runId); - if (!deleted) { - return NextResponse.json({ error: 'Not found' }, { status: 404 }); - } - purgeCollectiveX(); - return NextResponse.json({ deleted: true, runId }); - } catch (error) { - console.error('Error deleting CollectiveX run:', error); - return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); - } -} diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.test.ts b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts index 47c482d8a..6ee7a8e56 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/route.test.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/route.test.ts @@ -3,10 +3,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { buildRunSummary } from '@semianalysisai/inferencex-db/collectivex/reader'; import { makeCollectiveXDataset } from '@semianalysisai/inferencex-db/collectivex/test-fixture'; -const { mockList, mockGetWriteDb, mockEnsureList, mockCachedJson } = vi.hoisted(() => ({ +const { mockList, mockGetWriteDb, mockCachedJson } = vi.hoisted(() => ({ mockList: vi.fn(), mockGetWriteDb: vi.fn(() => 'mock-write-sql'), - mockEnsureList: vi.fn(), mockCachedJson: vi.fn((data: unknown) => Response.json(data)), })); @@ -19,20 +18,7 @@ vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ listCollectiveXRuns: mockList, })); -vi.mock('@/lib/collectivex/collectivex-lazy-ingest', () => ({ - ensureCollectiveXRunsList: mockEnsureList, - collectiveXSweepErrorStatus: (error: unknown) => { - const code = error instanceof Error && 'code' in error ? (error.code as string) : null; - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; - }, -})); - vi.mock('@/lib/cache/api-cache', () => ({ - COLLECTIVEX_CACHE_SCOPE: 'collectivex', - COLLECTIVEX_CACHE_CONTROL: 'public, max-age=0, s-maxage=60', cachedJson: mockCachedJson, collectiveXCacheTag: () => 'collectivex', })); @@ -45,80 +31,33 @@ function req(url: string): NextRequest { return new NextRequest(new URL(url, 'http://localhost')); } -function sweepError(code: string): Error { - return Object.assign(new Error(code), { code }); -} - const summary = buildRunSummary(makeCollectiveXDataset()); beforeEach(() => { vi.clearAllMocks(); vi.spyOn(console, 'error').mockImplementation(() => {}); - mockEnsureList.mockResolvedValue(true); mockList.mockResolvedValue([summary]); }); describe('GET /api/v1/collectivex/runs', () => { it('returns 400 for a missing or unknown version', async () => { - for (const url of ['/api/v1/collectivex/runs', '/api/v1/collectivex/runs?version=abc']) { + for (const url of ['/api/v1/collectivex/runs', '/api/v1/collectivex/runs?version=99']) { const res = await GET(req(url)); expect(res.status).toBe(400); } - expect(mockEnsureList).not.toHaveBeenCalled(); + expect(mockList).not.toHaveBeenCalled(); }); - it('backfills recent runs then lists stored summaries newest first', async () => { + it('lists stored runs with a complete-discovery flag under the CollectiveX tag', async () => { const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: true, - }); - expect(mockEnsureList).toHaveBeenCalledWith(1); + const body = { version: 1, runs: [summary], discovery_complete: true }; + expect(await res.json()).toEqual(body); expect(mockList).toHaveBeenCalledWith('mock-write-sql', 1); + expect(mockCachedJson).toHaveBeenCalledWith(body, { tag: 'collectivex' }); }); - it('marks the list incomplete when another bounded discovery pass is needed', async () => { - mockEnsureList.mockResolvedValue(false); - - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: false, - }); - expect(mockCachedJson).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ cacheControl: 'private, no-store' }), - ); - }); - - it('serves the stored list as incomplete when the GitHub backfill fails', async () => { - mockEnsureList.mockRejectedValue(sweepError('unavailable')); - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - expect(res.status).toBe(200); - expect(await res.json()).toEqual({ - version: 1, - runs: [summary], - discovery_complete: false, - }); - expect(mockCachedJson).toHaveBeenCalledWith( - expect.anything(), - expect.objectContaining({ cacheControl: 'private, no-store' }), - ); - }); - - it('returns 503 when the backfill fails and nothing is stored', async () => { - mockEnsureList.mockRejectedValue(sweepError('unavailable')); - mockList.mockResolvedValue([]); - const res = await GET(req('/api/v1/collectivex/runs?version=1')); - expect(res.status).toBe(503); - }); - - it('returns an empty list when GitHub has no runs yet', async () => { + it('returns an empty list, not an error, when nothing is stored', async () => { mockList.mockResolvedValue([]); const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(200); @@ -129,5 +68,6 @@ describe('GET /api/v1/collectivex/runs', () => { mockList.mockRejectedValue(new Error('boom')); const res = await GET(req('/api/v1/collectivex/runs?version=1')); expect(res.status).toBe(500); + expect(await res.json()).toEqual({ error: 'Internal server error' }); }); }); diff --git a/packages/app/src/app/api/v1/collectivex/runs/route.ts b/packages/app/src/app/api/v1/collectivex/runs/route.ts index ff844a8db..ec69813b1 100644 --- a/packages/app/src/app/api/v1/collectivex/runs/route.ts +++ b/packages/app/src/app/api/v1/collectivex/runs/route.ts @@ -4,18 +4,12 @@ import { parseCollectiveXVersion } from '@semianalysisai/inferencex-db/collectiv import { FIXTURES_MODE, getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; import { listCollectiveXRuns } from '@semianalysisai/inferencex-db/queries/collectivex'; -import { COLLECTIVEX_CACHE_CONTROL, cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; -import { - collectiveXSweepErrorStatus, - ensureCollectiveXRunsList, -} from '@/lib/collectivex/collectivex-lazy-ingest'; +import { cachedJson, collectiveXCacheTag } from '@/lib/cache/api-cache'; import { loadFixture } from '@/lib/testing/test-fixtures'; export const dynamic = 'force-dynamic'; export const runtime = 'nodejs'; -const DISCOVERY_CACHE_CONTROL = 'private, no-store'; - export async function GET(request: NextRequest) { const version = parseCollectiveXVersion(request.nextUrl.searchParams.get('version') ?? ''); if (!version) { @@ -23,35 +17,10 @@ export async function GET(request: NextRequest) { } if (FIXTURES_MODE) return cachedJson(loadFixture('collectivex-runs')); - // Backfill failures must not take the run table down — serve the stored list - // and only surface the error when there is nothing at all to show. - let ensureError: unknown = null; - let discoveryComplete: boolean; - try { - discoveryComplete = await ensureCollectiveXRunsList(version); - } catch (error) { - ensureError = error; - discoveryComplete = false; - } - try { - // Backfill may have persisted runs, so read from the same primary. const runs = await listCollectiveXRuns(getCollectiveXWriteDb(), version); - if (runs.length === 0 && ensureError) { - console.error('CollectiveX run backfill failed with no stored fallback:', ensureError); - const status = collectiveXSweepErrorStatus(ensureError) ?? 502; - return NextResponse.json({ error: 'Unavailable' }, { status }); - } - if (ensureError) { - console.error('CollectiveX run backfill failed; serving stored list:', ensureError); - } - return cachedJson( - { version, runs, discovery_complete: discoveryComplete }, - { - tag: collectiveXCacheTag(), - cacheControl: discoveryComplete ? COLLECTIVEX_CACHE_CONTROL : DISCOVERY_CACHE_CONTROL, - }, - ); + // Every run is ingested ahead of time, so the stored list is always complete. + return cachedJson({ version, runs, discovery_complete: true }, { tag: collectiveXCacheTag() }); } catch (error) { console.error('Error listing CollectiveX runs:', error); return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); diff --git a/packages/app/src/components/collectivex/CollectiveXChart.tsx b/packages/app/src/components/collectivex/CollectiveXChart.tsx index b9fd045a4..2cea5d47e 100644 --- a/packages/app/src/components/collectivex/CollectiveXChart.tsx +++ b/packages/app/src/components/collectivex/CollectiveXChart.tsx @@ -121,7 +121,9 @@ function formatPercentiles( unavailable: string, ): string { if (value === null) return unavailable; - return `${value.latency_us.p50.toFixed(1)} / ${value.latency_us.p90.toFixed(1)} / ${value.latency_us.p95.toFixed(1)} / ${value.latency_us.p99.toFixed(1)} µs`; + const { p50, p90, p95, p99 } = value.latency_us; + const us = (v: number | null) => (v === null ? unavailable : v.toFixed(1)); + return `${us(p50)} / ${us(p90)} / ${us(p95)} / ${us(p99)} µs`; } function escapeHtml(value: string): string { diff --git a/packages/app/src/components/collectivex/CollectiveXDisplay.tsx b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx index 4f8902acf..4de0b6e60 100644 --- a/packages/app/src/components/collectivex/CollectiveXDisplay.tsx +++ b/packages/app/src/components/collectivex/CollectiveXDisplay.tsx @@ -1,6 +1,6 @@ 'use client'; -import { BookOpen, ExternalLink, Loader2, RefreshCw, Trash2 } from 'lucide-react'; +import { BookOpen, ExternalLink, Loader2, RefreshCw } from 'lucide-react'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; @@ -19,11 +19,7 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; -import { - useCollectiveXRunDatasets, - useCollectiveXRuns, - useDeleteCollectiveXRun, -} from '@/hooks/api/use-collectivex'; +import { useCollectiveXRunDatasets, useCollectiveXRuns } from '@/hooks/api/use-collectivex'; import { useThemeColors } from '@/hooks/useThemeColors'; import { track } from '@/lib/analytics/analytics'; import { useLocale } from '@/lib/i18n/use-locale'; @@ -34,7 +30,6 @@ import { CollectiveXKvSection } from './CollectiveXKvSection'; import { CollectiveXRunsTable } from './CollectiveXRunsTable'; import { CollectiveXSupportMatrices } from './CollectiveXSupportMatrices'; import { - collectiveXConclusionLabel, collectiveXColorKey, collectiveXLegendLabel, collectiveXRunDasharray, @@ -118,7 +113,6 @@ const STRINGS = { suiteAria: 'Filter CollectiveX runs by suite', allSuites: 'All', noSuiteRuns: 'No runs contain the selected suite.', - shownRunCount: (count: number) => `${count} runs shown`, selectRuns: 'Select one or more runs from the table to show their data.', selectedRunsFailed: 'One or more selected runs failed to load.', runControl: 'Run', @@ -146,18 +140,6 @@ const STRINGS = { 'Activation-data rate is derived at the selected latency percentile and is not physical link bandwidth.', payloadBandwidthNote: 'Payload bandwidth is the full logical payload (incl. FP8 scale bytes) ÷ latency, per chip — a derived rate over logical bytes, not physical link bandwidth. The tooltip β/α is a least-squares fit of latency vs bytes across the ladder (β = per-chip bandwidth term, α = fixed overhead).', - deleteRun: 'Delete run', - deleteShownRuns: 'Delete shown runs', - deletingShownRuns: 'Deleting shown runs…', - deleteConfirm: (id: string) => - `Delete run #${id} from the dashboard database? This cannot be undone.`, - deleteShownConfirm: (ids: readonly string[]) => - `Delete ${ids.length} shown ${ids.length === 1 ? 'run' : 'runs'} from the dashboard database? This cannot be undone.\n\n${ids.map((id) => `#${id}`).join('\n')}`, - deleteTokenPrompt: 'Admin token required to delete runs:', - deleteUnauthorized: 'Invalid admin token.', - deleteFailed: 'Deleting the run failed. Try again.', - deleteShownFailed: (deleted: number, total: number) => - `Deleted ${deleted} of ${total} shown runs before the operation failed. Try again.`, atLatency: (percentile: string) => `at ${percentile} latency`, chartControlsHeading: 'Chart filters', chartControlsDescription: @@ -203,7 +185,6 @@ const STRINGS = { runsHeading: '运行记录', runsDescription: '该基准测试版本下保存的全部运行记录。勾选一条或多条记录，即可在图表中对比。', runsShown: '已显示运行', - shownRunCount: (count: number) => `已显示 ${count} 个运行`, suiteControl: '测试套件', suiteAria: '按测试套件筛选 CollectiveX 运行', allSuites: '全部', @@ -234,29 +215,11 @@ const STRINGS = { payloadNote: '逻辑载荷速率按所选延迟分位点派生，不代表物理链路带宽。', payloadBandwidthNote: '载荷带宽为完整逻辑载荷（含 FP8 缩放字节）÷ 延迟（每芯片），是基于逻辑字节的派生速率，不代表物理链路带宽。工具提示中的 β/α 根据各 token 档位下的字节数与延迟做最小二乘拟合（β = 每芯片带宽项，α = 固定开销）。', - deleteRun: '删除运行', - deleteShownRuns: '删除已显示的运行', - deletingShownRuns: '正在删除已显示的运行……', - deleteConfirm: (id: string) => `从仪表板数据库中删除运行 #${id}？此操作无法撤销。`, - deleteShownConfirm: (ids: readonly string[]) => - `从仪表板数据库中删除当前显示的 ${ids.length} 个运行？此操作无法撤销。\n\n${ids.map((id) => `#${id}`).join('\n')}`, - deleteTokenPrompt: '删除运行需要管理员令牌：', - deleteUnauthorized: '管理员令牌无效。', - deleteFailed: '删除运行失败，请重试。', - deleteShownFailed: (deleted: number, total: number) => - `操作失败前已删除 ${total} 个已显示运行中的 ${deleted} 个，请重试。`, atLatency: (percentile: string) => `${percentile} 延迟分位点`, chartControlsHeading: '图表筛选', chartControlsDescription: '选择下方图表要展示的测量切片和指标。', }, } as const; -const CONCLUSION_CLASSES: Record = { - success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', -}; -const CONCLUSION_FALLBACK_CLASS = - 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; -const ADMIN_TOKEN_STORAGE_KEY = 'collectivex-admin-token'; function ControlGroup({ label, @@ -292,7 +255,6 @@ export default function CollectiveXDisplay() { const [version, setVersion] = useState(COLLECTIVEX_DEFAULT_VERSION); const [suiteFilter, setSuiteFilter] = useState('all'); const [visibleRunIds, setVisibleRunIds] = useState>(new Set()); - const [bulkDeletingRunIds, setBulkDeletingRunIds] = useState>(new Set()); const initializedVersionRef = useRef(null); const runsQuery = useCollectiveXRuns(version); const runList = runsQuery.data?.runs ?? []; @@ -373,7 +335,6 @@ export default function CollectiveXDisplay() { useEffect(() => { if (!runsQuery.data || initializedVersionRef.current === version) return; - if (runsQuery.data.runs.length === 0 && !runsQuery.data.discovery_complete) return; const initial = runsQuery.data.runs.find((run) => run.measured_cases > 0) ?? runsQuery.data.runs[0]; setVisibleRunIds(initial ? new Set([initial.run_id]) : new Set()); @@ -554,12 +515,6 @@ export default function CollectiveXDisplay() { void runsQuery.refetch(); for (const query of runQueries) void query.refetch(); }, [runQueries, runsQuery]); - const deleteRun = useDeleteCollectiveXRun(); - const deletingRunIds = useMemo(() => { - if (bulkDeletingRunIds.size > 0) return bulkDeletingRunIds; - const runId = deleteRun.isPending ? deleteRun.variables?.runId : undefined; - return new Set(runId ? [runId] : []); - }, [bulkDeletingRunIds, deleteRun.isPending, deleteRun.variables?.runId]); const handleVisibleRunChange = useCallback((runIds: string[], visible: boolean) => { setVisibleRunIds((previous) => { const next = new Set(previous); @@ -570,82 +525,6 @@ export default function CollectiveXDisplay() { return next; }); }, []); - const handleDeleteRun = useCallback( - async (runId: string) => { - track('collectivex_run_delete_prompted', { run: runId }); - if (!window.confirm(t.deleteConfirm(runId))) return; - const stored = localStorage.getItem(ADMIN_TOKEN_STORAGE_KEY) ?? ''; - const token = stored || (window.prompt(t.deleteTokenPrompt)?.trim() ?? ''); - if (!token) return; - try { - const deleted = await deleteRun.mutateAsync({ runId, token }); - if (!deleted) { - localStorage.removeItem(ADMIN_TOKEN_STORAGE_KEY); - track('collectivex_run_delete_failed', { run: runId, reason: 'unauthorized' }); - window.alert(t.deleteUnauthorized); - return; - } - localStorage.setItem(ADMIN_TOKEN_STORAGE_KEY, token); - setVisibleRunIds((previous) => { - const next = new Set(previous); - next.delete(runId); - return next; - }); - track('collectivex_run_delete_confirmed', { run: runId }); - } catch { - track('collectivex_run_delete_failed', { run: runId, reason: 'error' }); - window.alert(t.deleteFailed); - } - }, - [deleteRun, t], - ); - const handleDeleteShownRuns = useCallback(async () => { - const runIds = [...orderedVisibleRunIds]; - if (runIds.length === 0) return; - track('collectivex_shown_runs_delete_prompted', { count: runIds.length, runs: runIds }); - if (!window.confirm(t.deleteShownConfirm(runIds))) return; - const stored = localStorage.getItem(ADMIN_TOKEN_STORAGE_KEY) ?? ''; - const token = stored || (window.prompt(t.deleteTokenPrompt)?.trim() ?? ''); - if (!token) return; - - setBulkDeletingRunIds(new Set(runIds)); - let deletedCount = 0; - try { - for (const runId of runIds) { - const deleted = await deleteRun.mutateAsync({ runId, token }); - if (!deleted) { - localStorage.removeItem(ADMIN_TOKEN_STORAGE_KEY); - track('collectivex_shown_runs_delete_failed', { - count: runIds.length, - deleted: deletedCount, - reason: 'unauthorized', - }); - window.alert(t.deleteUnauthorized); - return; - } - deletedCount += 1; - setVisibleRunIds((previous) => { - const next = new Set(previous); - next.delete(runId); - return next; - }); - } - localStorage.setItem(ADMIN_TOKEN_STORAGE_KEY, token); - track('collectivex_shown_runs_delete_confirmed', { - count: runIds.length, - runs: runIds, - }); - } catch { - track('collectivex_shown_runs_delete_failed', { - count: runIds.length, - deleted: deletedCount, - reason: 'error', - }); - window.alert(t.deleteShownFailed(deletedCount, runIds.length)); - } finally { - setBulkDeletingRunIds(new Set()); - } - }, [deleteRun, orderedVisibleRunIds, t]); if (runsQuery.isLoading) { return ( @@ -701,30 +580,18 @@ export default function CollectiveXDisplay() { ), 0, ); - const singleConclusionClass = - (singleDataset?.run.conclusion && CONCLUSION_CLASSES[singleDataset.run.conclusion]) ?? - CONCLUSION_FALLBACK_CLASS; + const sourceShaByRunId = new Map( + datasets.map((dataset) => [dataset.run.run_id, dataset.run.source_sha]), + ); return (
-
- - CollectiveX - - - {singleDataset - ? `#${singleDataset.run.run_id} · ${collectiveXConclusionLabel(singleDataset.run.conclusion, locale)}` - : t.shownRunCount(datasets.length)} - -
+ + CollectiveX +

{t.description}

@@ -732,7 +599,7 @@ export default function CollectiveXDisplay() { <> @@ -746,7 +613,7 @@ export default function CollectiveXDisplay() { @@ -804,21 +671,6 @@ export default function CollectiveXDisplay() { testId="collectivex-suite-filter" /> -
void handleDeleteRun(runId)} emptyMessage={suiteFilter === 'all' ? undefined : t.noSuiteRuns} /> diff --git a/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx index dfbaa8152..2928ecd76 100644 --- a/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx +++ b/packages/app/src/components/collectivex/CollectiveXRunsTable.tsx @@ -1,16 +1,15 @@ 'use client'; -import { ExternalLink, Loader2, Trash2 } from 'lucide-react'; +import { ExternalLink, Loader2 } from 'lucide-react'; import { useRef } from 'react'; import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; -import { Button } from '@/components/ui/button'; import { track } from '@/lib/analytics/analytics'; import { useLocale } from '@/lib/i18n/use-locale'; import { cn } from '@/lib/shared/utils'; -import { collectiveXConclusionLabel, collectiveXRunDasharray, collectiveXSkuLabel } from './data'; +import { collectiveXRunDasharray, collectiveXSkuLabel } from './data'; import type { CollectiveXRunSummary } from './types'; interface CollectiveXRunsTableProps { @@ -18,9 +17,10 @@ interface CollectiveXRunsTableProps { selectedRunIndexById: ReadonlyMap; visibleRunIds: ReadonlySet; loadingRunIds: ReadonlySet; - deletingRunIds: ReadonlySet; + /** InferenceX commit of each loaded run; a run links to its committed results once known. */ + sourceShaByRunId: ReadonlyMap; + /** Applies one visibility state to every listed run (one id, or a shift-click range). */ onVisibleChange: (runIds: string[], visible: boolean) => void; - onDelete: (runId: string) => void; emptyMessage?: string; } @@ -29,18 +29,15 @@ const STRINGS = { shown: 'Shown', shownHint: 'Shift-click a checkbox to set every run between it and the last one clicked', run: 'Run', - result: 'Result', swapSuite: (m: number, n: number) => `swap_blocks: ${m}/${n} measured`, suites: 'Suites', cases: 'Measured cases', points: 'Terminal points', skus: 'SKUs', published: 'Published (UTC)', - actions: 'Actions', showRun: (id: string) => `Show run #${id}`, lineStyle: (id: string) => `Line style for run #${id}`, - openRun: (id: string) => `Open GitHub Actions run #${id}`, - deleteRun: (id: string) => `Delete run #${id}`, + openRun: (id: string) => `Open the results committed for run #${id}`, empty: 'No runs match this benchmark version.', epSuite: (measured: number, requested: number) => `EP: ${measured}/${requested} measured`, kvSuite: (measured: number, requested: number) => @@ -50,18 +47,15 @@ const STRINGS = { shown: '显示', shownHint: '按住 Shift 点击复选框，可一次性勾选或取消勾选它与上次点击行之间的全部运行', run: '运行', - result: '结果', swapSuite: (m: number, n: number) => `swap_blocks：已测量 ${m}/${n}`, suites: '测试套件', cases: '实测用例', points: '终态数据点', skus: 'SKUs', published: '发布时间（UTC）', - actions: '操作', showRun: (id: string) => `显示运行 #${id}`, lineStyle: (id: string) => `运行 #${id} 的线型`, - openRun: (id: string) => `打开 GitHub Actions 运行 #${id}`, - deleteRun: (id: string) => `删除运行 #${id}`, + openRun: (id: string) => `打开运行 #${id} 对应的已提交结果`, empty: '该基准测试版本下暂无运行记录。', epSuite: (measured: number, requested: number) => `EP：已测量 ${measured}/${requested}`, kvSuite: (measured: number, requested: number) => `KV 传输：已测量 ${measured}/${requested}`, @@ -73,13 +67,6 @@ const SUITE_BADGE_CLASSES = { kv: 'border-violet-600/40 bg-violet-500/10 text-violet-700 dark:text-violet-300', } as const; -const CONCLUSION_CLASSES: Record = { - success: 'border-emerald-600/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - failure: 'border-red-600/40 bg-red-500/10 text-red-700 dark:text-red-300', -}; -const CONCLUSION_FALLBACK_CLASS = - 'border-amber-600/40 bg-amber-500/10 text-amber-700 dark:text-amber-300'; - function formatDate(value: string, locale: 'en' | 'zh'): string { return new Intl.DateTimeFormat(locale === 'zh' ? 'zh-CN' : 'en', { dateStyle: 'medium', @@ -93,9 +80,8 @@ export function CollectiveXRunsTable({ selectedRunIndexById, visibleRunIds, loadingRunIds, - deletingRunIds, + sourceShaByRunId, onVisibleChange, - onDelete, emptyMessage, }: CollectiveXRunsTableProps) { const locale = useLocale(); @@ -120,21 +106,18 @@ export function CollectiveXRunsTable({ {t.shown} {t.run} - {t.result} {t.suites} {t.cases} {t.points} {t.skus} {t.published} - {t.actions} {runs.map((run, rowIndex) => { const visible = visibleRunIds.has(run.run_id); const loading = loadingRunIds.has(run.run_id); - const deleting = deletingRunIds.has(run.run_id); - const conclusion = run.conclusion; + const sourceSha = sourceShaByRunId.get(run.run_id); const selectedRunIndex = selectedRunIndexById.get(run.run_id); const kvRequested = run.kv_cases?.requested ?? 0; const kvMeasured = run.kv_cases?.measured ?? 0; @@ -158,7 +141,6 @@ export function CollectiveXRunsTable({ 0} aria-label={t.showRun(run.run_id)} data-testid={`collectivex-run-visible-${run.run_id}`} title={t.shownHint} @@ -219,28 +201,23 @@ export function CollectiveXRunsTable({ /> )} - track('collectivex_run_source_opened', { run: run.run_id })} - className="inline-flex items-center gap-1 hover:underline" - > - #{run.run_id} - - -
- - - #{run.run_id} + ) : ( + track('collectivex_run_source_opened', { run: run.run_id })} + className="inline-flex items-center gap-1 hover:underline" + > + #{run.run_id} + + )} - > - {collectiveXConclusionLabel(conclusion, locale)} - +
@@ -293,24 +270,6 @@ export function CollectiveXRunsTable({ {formatDate(run.generated_at, locale)} - - - ); })} diff --git a/packages/app/src/components/collectivex/data.test.ts b/packages/app/src/components/collectivex/data.test.ts index 9884c3a05..04b22e4f9 100644 --- a/packages/app/src/components/collectivex/data.test.ts +++ b/packages/app/src/components/collectivex/data.test.ts @@ -4,7 +4,6 @@ import { chartPoints, collectiveXCaseLabel, collectiveXColorKey, - collectiveXConclusionLabel, collectiveXLegendLabel, collectiveXRunDasharray, collectiveXSeriesForRun, @@ -31,33 +30,6 @@ import { makeCollectiveXDataset, makeCollectiveXSeries } from './test-fixture'; const dataset = makeCollectiveXDataset(); const [scaleUp, scaleOut] = dataset.series; -describe('collectiveXConclusionLabel', () => { - it.each([ - ['success', '成功'], - ['failure', '失败'], - ['cancelled', '已取消'], - ['skipped', '已跳过'], - ['timed_out', '超时'], - ['startup_failure', '启动失败'], - ['action_required', '需要处理'], - ['neutral', '中立'], - ['stale', '已过期'], - ])('localizes the %s workflow conclusion in Chinese', (conclusion, expected) => { - expect(collectiveXConclusionLabel(conclusion, 'zh')).toBe(expected); - }); - - it('derives pending only from a null workflow conclusion', () => { - expect(collectiveXConclusionLabel(null, 'zh')).toBe('待处理'); - expect(collectiveXConclusionLabel(null, 'en')).toBe('pending'); - }); - - it('preserves known English values and hides unknown values on Chinese pages', () => { - expect(collectiveXConclusionLabel('startup_failure', 'en')).toBe('startup_failure'); - expect(collectiveXConclusionLabel('future_status', 'en')).toBe('future_status'); - expect(collectiveXConclusionLabel('future_status', 'zh')).toBe('未知状态'); - }); -}); - describe('collectiveXTopologyLabel', () => { it('shows only the scale-up transport when there is no scale-out fabric', () => { expect(collectiveXTopologyLabel(scaleUp.system)).toBe( diff --git a/packages/app/src/components/collectivex/data.ts b/packages/app/src/components/collectivex/data.ts index 929958ac0..119d1f7ff 100644 --- a/packages/app/src/components/collectivex/data.ts +++ b/packages/app/src/components/collectivex/data.ts @@ -27,31 +27,6 @@ export interface CollectiveXSeriesSelection { const BASE_RUN_DASHARRAYS = ['none', '9 4', '3 3', '10 3 2 3', '2 3', '12 3 2 3'] as const; -const COLLECTIVEX_CONCLUSION_LABELS = { - en: { - action_required: 'action_required', - cancelled: 'cancelled', - failure: 'failure', - neutral: 'neutral', - skipped: 'skipped', - stale: 'stale', - startup_failure: 'startup_failure', - success: 'success', - timed_out: 'timed_out', - }, - zh: { - action_required: '需要处理', - cancelled: '已取消', - failure: '失败', - neutral: '中立', - skipped: '已跳过', - stale: '已过期', - startup_failure: '启动失败', - success: '成功', - timed_out: '超时', - }, -} as const; - const COLLECTIVEX_MODE_LABELS_ZH: Record = { normal: '常规', 'low-latency': '低延迟', @@ -62,16 +37,6 @@ const COLLECTIVEX_PHASE_LABELS_ZH: Record = { prefill: '预填充', }; -export function collectiveXConclusionLabel(conclusion: string | null, locale: Locale): string { - if (conclusion === null) return locale === 'zh' ? '待处理' : 'pending'; - const labels = COLLECTIVEX_CONCLUSION_LABELS[locale]; - return conclusion in labels - ? labels[conclusion as keyof typeof labels] - : locale === 'zh' - ? '未知状态' - : conclusion; -} - export function normalizeCollectiveXSku(sku: string): string { const normalized = sku.trim().toLowerCase(); const base = normalized.split(':').at(-1)?.split('-')[0] ?? normalized; @@ -211,7 +176,7 @@ export function fitAlphaBeta( const component = point.components[operation]; if (component === null || component.payload_bytes === null) continue; const latency = component.latency_us[percentile]; - if (latency <= 0) continue; + if (latency === null || latency <= 0) continue; bytesPerGpu.push(component.payload_bytes / ep); latencies.push(latency); } diff --git a/packages/app/src/hooks/api/use-collectivex.ts b/packages/app/src/hooks/api/use-collectivex.ts index bc51c4102..06c32eb6f 100644 --- a/packages/app/src/hooks/api/use-collectivex.ts +++ b/packages/app/src/hooks/api/use-collectivex.ts @@ -1,6 +1,6 @@ -import { useMutation, useQueries, useQuery, useQueryClient } from '@tanstack/react-query'; +import { useQueries, useQuery } from '@tanstack/react-query'; -import { deleteCollectiveXRun, fetchCollectiveXRun, fetchCollectiveXRunList } from '@/lib/api/api'; +import { fetchCollectiveXRun, fetchCollectiveXRunList } from '@/lib/api/api'; import { COLLECTIVEX_DEFAULT_VERSION, type CollectiveXVersion, @@ -10,9 +10,6 @@ export function useCollectiveXRuns(version: CollectiveXVersion = COLLECTIVEX_DEF return useQuery({ queryKey: ['collectivex-runs', version], queryFn: ({ signal }) => fetchCollectiveXRunList(version, signal), - staleTime: 0, - refetchOnMount: 'always', - refetchInterval: (query) => (query.state.data?.discovery_complete === false ? 1_000 : false), }); } @@ -21,22 +18,6 @@ export function useCollectiveXRunDatasets(version: CollectiveXVersion, runIds: r queries: runIds.map((runId) => ({ queryKey: ['collectivex-run', version, runId], queryFn: ({ signal }: { signal: AbortSignal }) => fetchCollectiveXRun(version, runId, signal), - staleTime: 0, - refetchOnMount: 'always' as const, })), }); } - -export function useDeleteCollectiveXRun() { - const queryClient = useQueryClient(); - return useMutation({ - mutationFn: ({ runId, token }: { runId: string; token: string }) => - deleteCollectiveXRun(runId, token), - onSuccess: (deleted) => { - if (!deleted) return; - for (const key of ['collectivex-runs', 'collectivex-run']) { - void queryClient.invalidateQueries({ queryKey: [key] }); - } - }, - }); -} diff --git a/packages/app/src/lib/api/api-documentation.ts b/packages/app/src/lib/api/api-documentation.ts index b76e6848f..8dd8da0a3 100644 --- a/packages/app/src/lib/api/api-documentation.ts +++ b/packages/app/src/lib/api/api-documentation.ts @@ -781,8 +781,8 @@ export const apiDocumentationGroups: readonly ApiDocumentationGroup[] = [ id: 'collectivex', title: text('CollectiveX', 'CollectiveX'), description: text( - 'Versioned collective communication sweep results and run discovery.', - '带版本的集合通信扫描结果与运行发现数据。', + 'Versioned collective communication benchmark results, one run per results commit.', + '带版本的集合通信基准测试结果，每个结果提交对应一次运行。', ), }, { @@ -1949,8 +1949,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/latest', summary: text('Read the latest CollectiveX dataset', '读取最新 CollectiveX 数据集'), description: text( - 'Discovers and ingests the latest sweep when needed, then returns its versioned neutral dataset. run.source_repository names the GitHub repository that ran the sweep when it is not SemiAnalysisAI/InferenceX. A stored run is served if refresh fails. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', - '按需发现并导入最新扫描，然后返回带版本的中立数据集。若扫描并非由 SemiAnalysisAI/InferenceX 执行，run.source_repository 会标明实际执行的 GitHub 仓库。若刷新失败，会返回已存储的运行。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', + 'Returns the versioned neutral dataset of the newest stored run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed collectivex/results/ and holds at least one measured result: run_id is that commit time in epoch seconds (raised past the previous results commit when two land in the same second), run_attempt is 1, and run.source_sha is the commit. run.covered_skus lists only pools with a measured result. Runs are ingested when results land on main, never on read. In series[].points[], components.*.latency_us, components.*.activation_data_rate_gbps_at_latency_percentile, components.*.payload_data_rate_gbps_at_latency_percentile and roundtrip_token_rate_at_latency_percentile carry p50, p90, p95 and p99; p90, p95 and p99 are null where the harness withheld a tail because too many samples were misaligned or host-stalled, while p50 is always a number. Optional swap_blocks results contain verified copy latency in microseconds and payload GB/s (bytes counted once), with block_bytes, num_blocks, runtime provenance and skipped-point counts. Multi-pool sweeps preserve each GPU pool as an independent result.', + '返回最新已存储运行的带版本中立数据集。每次运行对应 SemiAnalysisAI/InferenceX main 分支上一个修改了 collectivex/results/ 且至少包含一个已测量结果的提交：run_id 为该提交时间的 Unix 秒级时间戳（若两个结果提交落在同一秒，则递增至上一个结果提交之后），run_attempt 为 1，run.source_sha 为该提交。run.covered_skus 只列出有已测量结果的资源池。结果合入 main 时导入，读取时不会导入。 在 series[].points[] 中，components.*.latency_us、components.*.activation_data_rate_gbps_at_latency_percentile、components.*.payload_data_rate_gbps_at_latency_percentile 与 roundtrip_token_rate_at_latency_percentile 包含 p50、p90、p95 和 p99；若测试工具因过多样本未对齐或被主机阻塞而未发布尾部延迟，p90、p95 和 p99 为 null，p50 始终为数值。可选的 swap_blocks 结果包含校验通过的复制延迟（微秒）和有效载荷 GB/s（字节数仅计算一次），并保留 block_bytes、num_blocks、运行环境来源及未测量组合数。多池扫描分别保留各 GPU 池的独立结果。', ), audience: 'public', stability: 'beta', @@ -1974,11 +1974,11 @@ export const apiOperations: readonly ApiOperation[] = [ { version: 1, run: { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', - source_sha: '0123456789abcdef', + source_sha: 'e65c9045320c78f1fb86200802aceafa0c6310d8', requested_cases: 12, terminal_cases: 12, measured_cases: 10, @@ -2005,22 +2005,10 @@ export const apiOperations: readonly ApiOperation[] = [ ), errorResponse( '404', - 'No stored or discoverable run exists.', - '没有已存储或可发现的运行。', + 'No run is stored for the version.', + '该版本下没有已存储的运行。', 'Not found', ), - errorResponse( - '502', - 'Upstream sweep discovery is unavailable and no stored fallback exists.', - '上游扫描发现不可用，且没有已存储的回退数据。', - 'Unavailable', - ), - errorResponse( - '503', - 'Upstream sweep processing is temporarily unavailable.', - '上游扫描处理暂时不可用。', - 'Unavailable', - ), errorResponse( '500', 'The stored run query failed.', @@ -2038,8 +2026,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs', summary: text('List CollectiveX runs', '列出 CollectiveX 运行'), description: text( - 'Returns progressively discovered run summaries. discovery_complete=false means clients may poll while older runs are still being discovered.', - '返回逐步发现的运行汇总。discovery_complete=false 表示仍在发现更早的运行，客户端可以轮询。', + 'Returns every stored run summary, newest results commit first. Only commits with at least one measured result are runs, and covered_skus lists only pools with a measured result. Runs are ingested ahead of reads, so discovery_complete is always true; the field is kept for existing clients.', + '按结果提交从新到旧返回全部已存储的运行汇总。只有至少包含一个已测量结果的提交才构成运行，covered_skus 只列出有已测量结果的资源池。运行在读取前即已导入，因此 discovery_complete 始终为 true；保留该字段是为了兼容现有客户端。', ), audience: 'public', stability: 'beta', @@ -2057,8 +2045,8 @@ export const apiOperations: readonly ApiOperation[] = [ ], responses: [ success( - 'Version, run summaries, and discovery state.', - '版本、运行汇总和发现状态。', + 'Version, run summaries, and discovery_complete (always true).', + '版本、运行汇总以及 discovery_complete（始终为 true）。', objectSchema({ version: integerSchema, runs: arraySchema(collectiveRunSummarySchema), @@ -2068,9 +2056,9 @@ export const apiOperations: readonly ApiOperation[] = [ version: 1, runs: [ { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', covered_skus: ['h200_sxm'], requested_cases: 12, @@ -2090,18 +2078,6 @@ export const apiOperations: readonly ApiOperation[] = [ 'version 缺失或不受支持。', 'Unknown version', ), - errorResponse( - '502', - 'Discovery failed and no stored run list exists.', - '发现失败，且没有已存储的运行列表。', - 'Unavailable', - ), - errorResponse( - '503', - 'Discovery is temporarily unavailable.', - '发现暂时不可用。', - 'Unavailable', - ), errorResponse( '500', 'The stored run list query failed.', @@ -2119,8 +2095,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/collectivex/runs/{runId}', summary: text('Read a CollectiveX run', '读取 CollectiveX 运行'), description: text( - 'Returns one positive numeric run ID as a versioned CollectiveX dataset, discovering and ingesting it on demand when possible.', - '按正整数运行 ID 返回带版本的 CollectiveX 数据集，并在可能时按需发现和导入。', + 'Returns one stored run as a versioned CollectiveX dataset. The run ID is the results commit time in epoch seconds (raised past the previous results commit on a same-second tie), as listed by /api/v1/collectivex/runs. In series[].points[], components.*.latency_us, components.*.activation_data_rate_gbps_at_latency_percentile, components.*.payload_data_rate_gbps_at_latency_percentile and roundtrip_token_rate_at_latency_percentile carry p50, p90, p95 and p99; p90, p95 and p99 are null where the harness withheld a tail because too many samples were misaligned or host-stalled, while p50 is always a number.', + '以带版本的 CollectiveX 数据集返回一次已存储的运行。运行 ID 为结果提交时间的 Unix 秒级时间戳（同一秒出现多个结果提交时递增至上一个之后），与 /api/v1/collectivex/runs 列出的一致。 在 series[].points[] 中，components.*.latency_us、components.*.activation_data_rate_gbps_at_latency_percentile、components.*.payload_data_rate_gbps_at_latency_percentile 与 roundtrip_token_rate_at_latency_percentile 包含 p50、p90、p95 和 p99；若测试工具因过多样本未对齐或被主机阻塞而未发布尾部延迟，p90、p95 和 p99 为 null，p50 始终为数值。', ), audience: 'public', stability: 'beta', @@ -2130,10 +2106,10 @@ export const apiOperations: readonly ApiOperation[] = [ 'path', true, 'integer', - 'Positive GitHub Actions run ID.', - 'GitHub Actions 正整数运行 ID。', + 'Positive run ID: the results commit time in epoch seconds, below 10^10.', + '正整数运行 ID：结果提交时间的 Unix 秒级时间戳，小于 10^10。', positiveIdSchema, - 123456789, + 1792089351, ), parameter( 'version', @@ -2154,11 +2130,11 @@ export const apiOperations: readonly ApiOperation[] = [ { version: 1, run: { - run_id: '123456789', + run_id: '1792089351', run_attempt: 1, - generated_at: '2026-08-08T03:00:00Z', + generated_at: '2026-10-15T18:35:51Z', conclusion: 'success', - source_sha: '0123456789abcdef', + source_sha: 'e65c9045320c78f1fb86200802aceafa0c6310d8', requested_cases: 12, terminal_cases: 12, measured_cases: 10, @@ -2183,23 +2159,11 @@ export const apiOperations: readonly ApiOperation[] = [ 'version 或 runId 无效。', 'Unknown version or run id', ), - errorResponse('404', 'The run does not exist.', '该运行不存在。', 'Not found'), - errorResponse( - '502', - 'The run cannot be fetched from the upstream source.', - '无法从上游来源获取该运行。', - 'Unavailable', - ), - errorResponse( - '503', - 'Upstream processing is temporarily unavailable.', - '上游处理暂时不可用。', - 'Unavailable', - ), + errorResponse('404', 'The run is not stored.', '该运行未存储。', 'Not found'), errorResponse('500', 'The run query failed.', '运行查询失败。', 'Internal server error'), ], responseShapeName: 'CollectiveXDataset', - curlUrl: `${API_BASE_URL}/api/v1/collectivex/runs/123456789?version=1`, + curlUrl: `${API_BASE_URL}/api/v1/collectivex/runs/1792089351?version=1`, }, { id: 'get-agentic-aggregates', @@ -3245,8 +3209,8 @@ const overview = { id: 'cache', title: text('Caching', '缓存'), description: text( - 'Read endpoints may be served from shared caches. CollectiveX uses short refresh windows, and framework releases use a one-hour shared cache.', - '只读端点可能由共享缓存提供。CollectiveX 使用较短的刷新窗口，框架版本使用一小时共享缓存。', + 'Read endpoints may be served from shared caches. CollectiveX responses stay cached until a results ingest purges them, and framework releases use a one-hour shared cache.', + '只读端点可能由共享缓存提供。CollectiveX 响应会一直缓存，直到结果导入时被清除；框架版本使用一小时共享缓存。', ), }, { diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index 874bc5b9b..1d3e87f6c 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -685,7 +685,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'get-collectivex-latest', - sourceSha256: 'c60fff45af91fa38a071fee69b253983fe403738f7e35c56711708b5b3cb2d70', + sourceSha256: '1bd6b280006bc3ac40ac11382efb4e8e54bc0c7c22a97169791347525d2673c6', }, { source: 'src/app/api/v1/collectivex/runs/route.ts', @@ -693,7 +693,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'list-collectivex-runs', - sourceSha256: '32eaf00f56cb30db70b128f9aa8ec1132c56d9b02c2c6728072fd5bb4c465258', + sourceSha256: '17fbc05787bc21a9b7070914ba2457d197dc7954294d6014d494393b59f5c8db', }, { source: 'src/app/api/v1/collectivex/runs/[runId]/route.ts', @@ -701,18 +701,7 @@ export const apiRouteCatalog = [ method: 'GET', classification: 'published-read', operationId: 'get-collectivex-run', - sourceSha256: 'e242fb048f89d0e5eff200e63f6baac7307f066a3fbe5984007390a2095f3699', - }, - { - source: 'src/app/api/v1/collectivex/runs/[runId]/route.ts', - path: '/api/v1/collectivex/runs/{runId}', - method: 'DELETE', - classification: 'admin', - exclusionReason: { - en: 'Authenticated CollectiveX administration mutation; deletion is not part of the public read API.', - zh: '需要身份验证的 CollectiveX 管理写操作；删除不属于公开只读 API。', - }, - sourceSha256: 'e242fb048f89d0e5eff200e63f6baac7307f066a3fbe5984007390a2095f3699', + sourceSha256: '9de0597df0db1923b11347072226e6b79f932ed07bf9edaa5807af6e3e1bb69b', }, { source: 'src/app/api/v1/datasets/route.ts', @@ -1580,7 +1569,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/cache/api-cache.ts', - sourceSha256: 'a96667af1379f6d9281da92923455d194e22da84ef8d31e1690436a23e7c2eb9', + sourceSha256: '29c35918b58926ade3e28c81459e5938bdc761a28250b3d923fcac9141741a67', reviewArea: { en: 'Public CDN tags, cache lifetimes, Blob key dimensions, and purge behavior.', zh: '公开 CDN 标签、缓存时长、Blob 键维度和清除行为。', @@ -1652,7 +1641,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/api/api.ts', - sourceSha256: 'bceb0916e3643f90f637530c7928bdf67860aab812362e48715432c88d93cdcc', + sourceSha256: '03c00a746ff4fc59cf9b3b01e5f69eee6c936d3d6d11887beeace7ce978a1624', reviewArea: { en: 'Public API client parameter serialization and TypeScript response contracts.', zh: '公开 API 客户端的参数序列化和 TypeScript 响应契约。', @@ -1704,7 +1693,7 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/collectivex/types.ts', - sourceSha256: '006f87eb54bca94142d6c8f1aa57679ce96330286b30654e0c4bd37600cd0d00', + sourceSha256: '3108c08a60ff154577fb93b52d6008f21f5597090ab5e8de289e6977dc73c517', reviewArea: { en: 'CollectiveX version negotiation and versioned dataset/run response types.', zh: 'CollectiveX 版本协商以及带版本的数据集与运行响应类型。', @@ -1747,10 +1736,10 @@ export const apiContractSourceDigests = [ }, { source: '../db/src/queries/collectivex.ts', - sourceSha256: '5b4cd412a1b543a8f26b5daf72ab252bc6c95edd7ca975262eefc1bbddbed3c7', + sourceSha256: 'd330c47acd94331d815515287990a5e25ab76a0f0d49c57d47dea0852604c3cd', reviewArea: { - en: 'CollectiveX dataset projection, run summaries, coverage, series, and discovery state.', - zh: 'CollectiveX 数据集投影、运行汇总、覆盖范围、序列和发现状态。', + en: 'CollectiveX dataset projection, run summaries, coverage, series, and latest-run ordering.', + zh: 'CollectiveX 数据集投影、运行汇总、覆盖范围、序列和最新运行排序。', }, }, { diff --git a/packages/app/src/lib/api/api.test.ts b/packages/app/src/lib/api/api.test.ts index a51e21087..e5d605a73 100644 --- a/packages/app/src/lib/api/api.test.ts +++ b/packages/app/src/lib/api/api.test.ts @@ -4,7 +4,6 @@ import { fetchBenchmarks, fetchWorkflowInfo, fetchAvailability, - deleteCollectiveXRun, fetchCollectiveXRun, fetchCollectiveXRunList, fetchReliability, @@ -219,25 +218,4 @@ describe('CollectiveX API', () => { expect(response.runs).toHaveLength(1); expect(response.runs[0].run_id).toBe(dataset.run.run_id); }); - - it('sends the bearer token on delete and reports success', async () => { - mockFetch.mockResolvedValueOnce({ ok: true, status: 200 }); - - await expect(deleteCollectiveXRun('160', 'secret-token')).resolves.toBe(true); - - expect(mockFetch).toHaveBeenCalledWith('/api/v1/collectivex/runs/160', { - method: 'DELETE', - headers: { Authorization: 'Bearer secret-token' }, - }); - }); - - it('resolves false on 401 so callers can clear a stale token', async () => { - mockFetch.mockResolvedValueOnce({ ok: false, status: 401 }); - await expect(deleteCollectiveXRun('160', 'stale')).resolves.toBe(false); - }); - - it('throws on non-auth delete failures', async () => { - mockFetch.mockResolvedValueOnce({ ok: false, status: 500 }); - await expect(deleteCollectiveXRun('160', 'secret-token')).rejects.toThrow(/500/); - }); }); diff --git a/packages/app/src/lib/api/api.ts b/packages/app/src/lib/api/api.ts index 46888a2c7..7538a5265 100644 --- a/packages/app/src/lib/api/api.ts +++ b/packages/app/src/lib/api/api.ts @@ -265,7 +265,7 @@ export function fetchSubmissions(signal?: AbortSignal) { return fetchJson('/api/v1/submissions', signal); } -/** Stored sweep runs plus whether lazy discovery has reached the end of history. */ +/** Stored CollectiveX run summaries, one per results commit; `discovery_complete` is always true. */ export function fetchCollectiveXRunList( version: CollectiveXVersion = COLLECTIVEX_DEFAULT_VERSION, signal?: AbortSignal, @@ -277,7 +277,7 @@ export function fetchCollectiveXRunList( }>(`/api/v1/collectivex/runs?version=${version}`, signal); } -/** Resolve a specific ingested sweep run's neutral view dataset by run_id. */ +/** Resolve a CollectiveX run's neutral view dataset by run_id. */ export function fetchCollectiveXRun( version: CollectiveXVersion, runId: string, @@ -289,22 +289,6 @@ export function fetchCollectiveXRun( ); } -/** - * Admin deletion of an ingested run. Requires the dedicated CollectiveX admin - * Bearer secret (COLLECTIVEX_ADMIN_SECRET — deliberately not the CI-held - * INVALIDATE_SECRET); resolves false on 401 so callers can clear a stale - * stored token, throws on other failures. - */ -export async function deleteCollectiveXRun(runId: string, token: string): Promise { - const response = await fetch(`/api/v1/collectivex/runs/${runId}`, { - method: 'DELETE', - headers: { Authorization: `Bearer ${token}` }, - }); - if (response.status === 401) return false; - if (!response.ok) throw new Error(`CollectiveX delete failed (${response.status}).`); - return true; -} - export interface FeedbackListRow { id: string; created_at: string; diff --git a/packages/app/src/lib/cache/api-cache.ts b/packages/app/src/lib/cache/api-cache.ts index 775f1d1ed..e2840d248 100644 --- a/packages/app/src/lib/cache/api-cache.ts +++ b/packages/app/src/lib/cache/api-cache.ts @@ -6,9 +6,11 @@ import { blobGet, blobPurge, blobSet } from './blob-cache'; /** * CollectiveX data lives in its own database and gets its own cache scope: * every CollectiveX response carries this CDN/unstable_cache tag (via - * `cachedJson(..., { tag })`), so run deletion can purge the CollectiveX + * `cachedJson(..., { tag })`), so a CollectiveX ingest can purge the CollectiveX * cache without dropping the main dashboard's. CollectiveX routes read their - * DB directly — no blob layer, so the tag is the entire scope. + * DB directly — no blob layer, so the tag is the entire scope. Responses stay + * cached until the CollectiveX ingest workflow purges it + * (`/api/v1/invalidate?scope=collectivex`). */ export const COLLECTIVEX_CACHE_SCOPE = 'collectivex'; @@ -25,14 +27,6 @@ export function collectiveXCacheTag(): string { return namespace ? `${COLLECTIVEX_CACHE_SCOPE}:${namespace}` : COLLECTIVEX_CACHE_SCOPE; } -/** - * Short CDN window shared by the CollectiveX latest/runs routes: new sweep - * runs are discovered lazily at the origin, so freshness is bounded by this - * TTL rather than by an ingest-time purge. - */ -export const COLLECTIVEX_CACHE_CONTROL = - 'public, max-age=0, s-maxage=60, stale-while-revalidate=300'; - /** * OperatorX reads its own database and has its own scope, like CollectiveX: its * responses carry this tag; merged comparisons also use a revision-keyed Blob subtree. @@ -202,8 +196,7 @@ export async function purgeAll(): Promise { const deleted = await blobPurge('', [`${OPERATORX_CACHE_SCOPE}/`]); revalidateTag(cacheTag(), { expire: 0 }); // CollectiveX responses are cached only under their own tag (no blobs), so - // a full purge must drop that tag explicitly — this line is the sole - // mechanism that clears CollectiveX from the CDN. + // a full purge must drop that tag explicitly. purgeCollectiveX(); return deleted; } @@ -219,8 +212,8 @@ export function purgeCollectiveX(): void { /** * 1 day unless overridden. Purged on demand via revalidateTag with the matching - * tag. `tag` defaults to the environment-scoped database tag; CollectiveX routes - * pass their own scope (and a shorter cacheControl). + * tag. `tag` defaults to the environment-scoped database tag; CollectiveX and + * OperatorX routes pass their own scope. */ function cdnHeaders(tag: string = cacheTag(), cacheControl?: string): Record { return { diff --git a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts b/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts deleted file mode 100644 index 2ef6f9c29..000000000 --- a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.test.ts +++ /dev/null @@ -1,703 +0,0 @@ -import { - makeSwapDoc, - swapMatrix, -} from '@semianalysisai/inferencex-db/collectivex/swap-test-fixture'; -import AdmZip from 'adm-zip'; -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { makeRawMatrix, makeRawShard } from '@/components/collectivex/test-fixture'; - -const { mockGetStates, mockInsert, mockRefresh, mockGetWriteDb } = vi.hoisted(() => ({ - mockGetStates: vi.fn(), - mockInsert: vi.fn(), - mockRefresh: vi.fn(), - mockGetWriteDb: vi.fn(() => 'mock-write-sql'), -})); - -vi.mock('@semianalysisai/inferencex-db/connection', () => ({ - getCollectiveXWriteDb: mockGetWriteDb, -})); - -vi.mock('@semianalysisai/inferencex-db/queries/collectivex', () => ({ - getCollectiveXRunStates: mockGetStates, - insertCollectiveXRun: mockInsert, - refreshCollectiveXRunAttempt: mockRefresh, -})); - -import { - collectiveXSweepErrorCode, - ensureCollectiveXRun, - ensureCollectiveXRunsList, - ensureLatestCollectiveXRun, - resetCollectiveXDiscoveryCooldown, -} from './collectivex-lazy-ingest'; - -const mockFetch = vi.fn(); -vi.stubGlobal('fetch', mockFetch); - -const shardA = makeRawShard({ backend: 'deepep-v2', ep: 8 }); -const shardB = makeRawShard({ - sku: 'mi355x', - backend: 'mori', - implName: 'mori', - vendor: 'amd', - ep: 16, - scaleUpTransport: 'xgmi', - scaleOutTransport: 'rdma', - topologyClass: 'mi355x-xgmi-rdma', - nodes: 2, - gpusPerNode: 8, - scaleUpDomain: 8, -}); - -function requestedOf(shard: Record) { - const identity = shard.identity as Record; - const factors = identity.case_factors as Record; - return { - caseId: identity.case_id as string, - sku: factors.sku as string, - disposition: 'runnable' as const, - case: factors.case as Record, - }; -} - -const matrix = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)]); -const matrixV2 = makeRawMatrix([requestedOf(shardA), requestedOf(shardB)], 2); - -function zipDocs(...docs: unknown[]): ArrayBuffer { - const zip = new AdmZip(); - docs.forEach((doc, index) => zip.addFile(`doc-${index}.json`, Buffer.from(JSON.stringify(doc)))); - const bytes = zip.toBuffer(); - const archive = new ArrayBuffer(bytes.byteLength); - new Uint8Array(archive).set(bytes); - return archive; -} - -const matrixZip = zipDocs(matrix); -const matrixZipV2 = zipDocs(matrixV2); -const shardZip = zipDocs(shardA, shardB); -const shardZipV2 = zipDocs({ ...shardA, version: 2 }, { ...shardB, version: 2 }); - -function runObject(overrides: Record = {}) { - return { - id: 160, - name: 'CollectiveX Sweep', - path: '.github/workflows/collectivex-sweep.yml', - head_branch: 'collectivex-fp8-precision', - head_sha: 'a'.repeat(40), - status: 'completed', - conclusion: 'success', - run_attempt: 1, - updated_at: new Date().toISOString(), - ...overrides, - }; -} - -function runListing(...runs: ReturnType[]) { - return Response.json({ total_count: runs.length, workflow_runs: runs }); -} - -const twoRunListing = () => runListing(runObject({ id: 161 }), runObject({ id: 160 })); - -function artifactsBody(runId = 160, runAttempt = 1) { - return { - total_count: 2, - artifacts: [ - { - id: 1, - name: `cxsweep-matrix-${runId}`, - archive_download_url: 'https://example.test/matrix.zip', - expired: false, - }, - { - id: 2, - name: `cxshard-cases-${runId}-${runAttempt}`, - archive_download_url: 'https://example.test/shards.zip', - expired: false, - }, - ], - }; -} - -beforeEach(() => { - vi.restoreAllMocks(); - mockFetch.mockReset(); - mockGetStates.mockReset().mockResolvedValue({}); - mockInsert.mockReset().mockResolvedValue(true); - mockRefresh.mockReset().mockResolvedValue(true); - process.env.GITHUB_TOKEN = 'test-token'; - delete process.env.INFERENCEX_SOURCE_REPOS; - resetCollectiveXDiscoveryCooldown(); -}); - -afterAll(() => { - delete process.env.GITHUB_TOKEN; - delete process.env.INFERENCEX_SOURCE_REPOS; - vi.unstubAllGlobals(); -}); - -describe('ensureLatestCollectiveXRun', () => { - it('discovers a standalone swap artifact and persists its computed summary', async () => { - const doc = makeSwapDoc(); - doc.runtime.source_sha = 'a'.repeat(40); - mockFetch - .mockResolvedValueOnce(runListing(runObject())) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(zipDocs(swapMatrix))) - .mockResolvedValueOnce(new Response(zipDocs(doc))); - await ensureLatestCollectiveXRun(1); - expect(mockInsert.mock.calls[0][1].summary).toMatchObject({ - run_id: '160', - measured_cases: 1, - requested_points: 3, - swap_cases: { requested: 1, measured: 1 }, - kv_cases: { requested: 0, measured: 0 }, - }); - }); - - it('discovers the newest absent run and persists its raw documents', async () => { - mockFetch - .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - const [sql, run, docs] = mockInsert.mock.calls[0]; - expect(sql).toBe('mock-write-sql'); - expect(run).toMatchObject({ - run_id: '160', - run_attempt: 1, - version: 1, - source_branch: 'collectivex-fp8-precision', - conclusion: 'success', - matrix, - }); - expect(run.summary).toMatchObject({ run_id: '160', measured_cases: 2 }); - expect(run.summary.source_repository).toBeUndefined(); - expect(mockFetch.mock.calls[0][0]).toContain( - '/repos/SemiAnalysisAI/InferenceX/actions/workflows/collectivex-sweep.yml/runs?', - ); - expect(mockFetch.mock.calls[1][0]).toContain( - '/repos/SemiAnalysisAI/InferenceX/actions/runs/160/artifacts?', - ); - expect(docs).toEqual([shardA, shardB]); - }); - - it('stops without artifact downloads when the newest matching run is live', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject()] }), - ); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockFetch).toHaveBeenCalledTimes(1); - }); - - it('never re-ingests a tombstoned run — the next candidate wins', async () => { - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve( - ids[0] === '161' ? { '161': { state: 'deleted', version: 1, run_attempt: 1 } } : {}, - ), - ); - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('skips runs tagged for another version', async () => { - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody(161))) - .mockResolvedValueOnce(new Response(matrixZipV2)) - .mockResolvedValueOnce(new Response(shardZipV2)) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(2); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ version: 2, run_id: '161' }); - expect(mockInsert.mock.calls[1][1]).toMatchObject({ version: 1, run_id: '160' }); - }); - - it('refreshes a live run when GitHub reports a newer attempt', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch - .mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), - ) - .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockRefresh).toHaveBeenCalledTimes(1); - expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); - }); - - it('downloads only the highest usable attempt per shard', async () => { - const artifacts = { - total_count: 3, - artifacts: [ - { - id: 1, - name: 'cxsweep-matrix-160', - archive_download_url: 'https://example.test/matrix.zip', - expired: false, - }, - { - id: 2, - name: 'cxshard-cases-160-1', - archive_download_url: 'https://example.test/shard-attempt1.zip', - expired: false, - }, - { - id: 3, - name: 'cxshard-cases-160-2', - archive_download_url: 'https://example.test/shard-attempt2.zip', - expired: false, - }, - ], - }; - mockFetch - .mockResolvedValueOnce( - Response.json({ total_count: 1, workflow_runs: [runObject({ run_attempt: 2 })] }), - ) - .mockResolvedValueOnce(Response.json(artifacts)) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockFetch).toHaveBeenCalledTimes(4); - expect(mockFetch.mock.calls[3][0]).toBe('https://example.test/shard-attempt2.zip'); - }); - - it('classifies a matrix artifact without a matrix document as invalid', async () => { - mockFetch - .mockResolvedValueOnce(Response.json({ total_count: 1, workflow_runs: [runObject()] })) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('invalid'); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('reports GitHub outages as unavailable', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 500 })); - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); - - it('reports a missing GITHUB_TOKEN as unavailable', async () => { - delete process.env.GITHUB_TOKEN; - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); -}); - -describe('discovery cooldown', () => { - it('serves a warm target from the cooldown without calling GitHub again', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); - - await ensureLatestCollectiveXRun(1); - const afterFirst = mockFetch.mock.calls.length; - expect(afterFirst).toBeGreaterThan(0); - - await ensureLatestCollectiveXRun(1); - await ensureLatestCollectiveXRun(1); - expect(mockFetch.mock.calls.length).toBe(afterFirst); - }); - - it('walks again once the cooldown window has elapsed', async () => { - vi.useFakeTimers(); - try { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockImplementation(() => Promise.resolve(runListing(runObject()))); - - await ensureLatestCollectiveXRun(1); - const afterFirst = mockFetch.mock.calls.length; - - vi.advanceTimersByTime(61_000); - await ensureLatestCollectiveXRun(1); - expect(mockFetch.mock.calls.length).toBeGreaterThan(afterFirst); - } finally { - vi.useRealTimers(); - } - }); - - it('rethrows a remembered failure so an outage stays 502/503 rather than 404', async () => { - mockFetch.mockResolvedValue(new Response('nope', { status: 500 })); - - const first = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(first)).toBe('unavailable'); - const afterFirst = mockFetch.mock.calls.length; - - const second = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - expect(second).toBe(first); - expect(mockFetch.mock.calls.length).toBe(afterFirst); - }); -}); - -describe('ensureCollectiveXRunsList', () => { - it('bounds GitHub discovery to runs that could still have rerun artifacts', async () => { - vi.useFakeTimers(); - vi.setSystemTime('2026-07-29T00:00:00.000Z'); - try { - mockFetch.mockResolvedValueOnce(runListing()); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - const discoveryUrl = new URL(mockFetch.mock.calls[0][0] as string); - expect(discoveryUrl.searchParams.get('created')).toBe('>=2026-06-15T00:00:00.000Z'); - } finally { - vi.useRealTimers(); - } - }); - - it('does not probe artifacts for unknown runs beyond the retention window', async () => { - mockFetch.mockResolvedValueOnce(runListing(runObject({ updated_at: '2020-01-01T00:00:00Z' }))); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockGetStates).not.toHaveBeenCalled(); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('continues past more than eight known rows to backfill an older run', async () => { - const runs = Array.from({ length: 10 }, (_unused, index) => runObject({ id: 169 - index })); - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve( - ids[0] === '160' ? {} : { [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }, - ), - ); - mockFetch - .mockResolvedValueOnce(runListing(...runs)) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - const complete = await ensureCollectiveXRunsList(1); - - expect(complete).toBe(true); - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('continues past one malformed run instead of hiding later valid runs', async () => { - mockFetch - .mockResolvedValueOnce(twoRunListing()) - .mockResolvedValueOnce(Response.json(artifactsBody(161))) - .mockResolvedValueOnce(new Response(zipDocs({ record_type: 'samples' }))) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('reports an incomplete pass after changing eight runs', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - expect(mockInsert).toHaveBeenCalledTimes(8); - }); - - it('does not consume the mutation budget when concurrent inserts are no-ops', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => runObject({ id: 167 - index })); - mockInsert.mockResolvedValue(false); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - expect(mockInsert).toHaveBeenCalledTimes(8); - }); - - it('does not consume the mutation budget when guarded refreshes are no-ops', async () => { - const runs = Array.from({ length: 8 }, (_unused, index) => - runObject({ id: 167 - index, run_attempt: 2 }), - ); - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve({ [ids[0]]: { state: 'live', version: 1, run_attempt: 1 } }), - ); - mockRefresh.mockResolvedValue(false); - mockFetch.mockResolvedValueOnce(runListing(...runs)); - for (const run of runs) { - mockFetch - .mockResolvedValueOnce(Response.json(artifactsBody(run.id, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - } - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - expect(mockRefresh).toHaveBeenCalledTimes(8); - }); -}); - -describe('ensureCollectiveXRun', () => { - it('treats tombstoned runs as not found', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'deleted', version: 1, run_attempt: 1 } }); - const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).not.toHaveBeenCalled(); - }); - - it('checks the GitHub attempt for a live matching run without downloading artifacts', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch.mockResolvedValueOnce(Response.json(runObject())); - - await ensureCollectiveXRun(1, '160'); - expect(mockGetStates).toHaveBeenCalledWith('mock-write-sql', ['160']); - - expect(mockFetch).toHaveBeenCalledTimes(1); - expect(mockInsert).not.toHaveBeenCalled(); - expect(mockRefresh).not.toHaveBeenCalled(); - }); - - it('refreshes a live matching run when GitHub has a newer attempt', async () => { - mockGetStates.mockResolvedValue({ '160': { state: 'live', version: 1, run_attempt: 1 } }); - mockFetch - .mockResolvedValueOnce(Response.json(runObject({ run_attempt: 2 }))) - .mockResolvedValueOnce(Response.json(artifactsBody(160, 2))) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureCollectiveXRun(1, '160'); - - expect(mockRefresh).toHaveBeenCalledTimes(1); - expect(mockRefresh.mock.calls[0][1]).toMatchObject({ run_id: '160', run_attempt: 2 }); - }); - - it('persists an absent run fetched by id', async () => { - mockFetch - .mockResolvedValueOnce(Response.json(runObject())) - .mockResolvedValueOnce(Response.json(artifactsBody())) - .mockResolvedValueOnce(new Response(matrixZip)) - .mockResolvedValueOnce(new Response(shardZip)); - - await ensureCollectiveXRun(1, '160'); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('rejects runs from other workflows as not found', async () => { - mockFetch.mockResolvedValueOnce( - Response.json(runObject({ path: '.github/workflows/run-sweep.yml' })), - ); - const caught = await ensureCollectiveXRun(1, '160').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockInsert).not.toHaveBeenCalled(); - }); - - it('rejects malformed run ids without touching GitHub', async () => { - const caught = await ensureCollectiveXRun(1, 'abc').catch((error: unknown) => error); - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).not.toHaveBeenCalled(); - }); -}); - -describe('benchmark source repositories', () => { - const PRIMARY = 'SemiAnalysisAI/InferenceX'; - const EXTRA = 'ExampleOrg/ExampleBench'; - const PRIMARY_LISTING = `/repos/${PRIMARY}/actions/workflows/collectivex-sweep.yml/runs?`; - const EXTRA_LISTING = `/repos/${EXTRA}/actions/workflows/collectivex-sweep.yml/runs?`; - const downloads: [string, () => Response][] = [ - ['https://example.test/matrix.zip', () => new Response(matrixZip)], - ['https://example.test/shards.zip', () => new Response(shardZip)], - ]; - - // Answers each request from the first route whose fragment its URL contains, else 404. - function routeFetch(routes: [string, () => Response][]) { - mockFetch.mockImplementation((input: string) => { - const route = routes.find(([fragment]) => input.includes(fragment)); - return Promise.resolve(route ? route[1]() : new Response(null, { status: 404 })); - }); - } - - beforeEach(() => { - process.env.INFERENCEX_SOURCE_REPOS = EXTRA; - }); - - it('walks every repository newest-first by run id', async () => { - mockGetStates.mockImplementation((_sql: unknown, ids: string[]) => - Promise.resolve({ [ids[0]]: { state: 'live', version: 2, run_attempt: 1 } }), - ); - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 165 }), runObject({ id: 150 }))], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }), runObject({ id: 160 }))], - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(mockGetStates.mock.calls.map(([, ids]) => ids[0])).toEqual(['170', '165', '160', '150']); - }); - - it('reads artifacts from and records the repository that ran the newest sweep', async () => { - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 160 }))], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }))], - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - ...downloads, - ]); - - await ensureLatestCollectiveXRun(1); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ - run_id: '170', - summary: { source_repository: EXTRA }, - }); - expect( - mockFetch.mock.calls - .map(([url]) => String(url)) - .filter((url) => url.includes(`/repos/${PRIMARY}/actions/runs/`)), - ).toEqual([]); - }); - - it('keeps discovering the other repositories when one cannot be listed, but not as complete', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => new Response(null, { status: 500 })], - [EXTRA_LISTING, () => runListing(runObject({ id: 170 }))], - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - ...downloads, - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - - expect(mockInsert).toHaveBeenCalledTimes(1); - expect(mockInsert.mock.calls[0][1].run_id).toBe('170'); - }); - - it('skips another repository without a sweep workflow quietly', async () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([[PRIMARY_LISTING, () => runListing()]]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(true); - - expect(warn).not.toHaveBeenCalled(); - - expect( - mockFetch.mock.calls.map(([url]) => String(url)).some((url) => url.includes(EXTRA_LISTING)), - ).toBe(true); - }); - - it('keeps a failed list pass incomplete without walking again during the cooldown', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => runListing()], - [EXTRA_LISTING, () => new Response(null, { status: 403 })], - ]); - - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - const requests = mockFetch.mock.calls.length; - await expect(ensureCollectiveXRunsList(1)).resolves.toBe(false); - - expect(mockFetch.mock.calls.length).toBe(requests); - }); - - it('fails discovery when the only listable repository fails', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([[PRIMARY_LISTING, () => new Response(null, { status: 500 })]]); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); - - it('reports the latest pass as failed when another repository cannot be listed', async () => { - vi.spyOn(console, 'warn').mockImplementation(() => {}); - routeFetch([ - [PRIMARY_LISTING, () => runListing(runObject({ id: 160 }))], - [EXTRA_LISTING, () => new Response(null, { status: 500 })], - [`/repos/${PRIMARY}/actions/runs/160/artifacts?`, () => Response.json(artifactsBody(160))], - ...downloads, - ]); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - expect(mockInsert.mock.calls[0][1].run_id).toBe('160'); - }); - - it('fails discovery only when every repository fails', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 500 })); - - const caught = await ensureLatestCollectiveXRun(1).catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - expect( - mockFetch.mock.calls.map(([url]) => String(url)).some((url) => url.includes(EXTRA_LISTING)), - ).toBe(true); - }); - - it('looks a run up in each repository and records where it was found', async () => { - routeFetch([ - [`/repos/${EXTRA}/actions/runs/170/artifacts?`, () => Response.json(artifactsBody(170))], - [`/repos/${EXTRA}/actions/runs/170`, () => Response.json(runObject({ id: 170 }))], - ...downloads, - ]); - - await ensureCollectiveXRun(1, '170'); - - expect(mockFetch.mock.calls.map(([url]) => String(url))[0]).toContain( - `/repos/${PRIMARY}/actions/runs/170`, - ); - expect(mockInsert.mock.calls[0][1]).toMatchObject({ - run_id: '170', - summary: { source_repository: EXTRA }, - }); - }); - - it('reports a run absent from every repository as not found', async () => { - mockFetch.mockResolvedValue(new Response(null, { status: 404 })); - - const caught = await ensureCollectiveXRun(1, '170').catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('not-found'); - expect(mockFetch).toHaveBeenCalledTimes(2); - }); - - it('keeps a failed lookup in one repository from reading as not found', async () => { - routeFetch([[`/repos/${PRIMARY}/actions/runs/170`, () => new Response(null, { status: 502 })]]); - - const caught = await ensureCollectiveXRun(1, '170').catch((error: unknown) => error); - - expect(collectiveXSweepErrorCode(caught)).toBe('unavailable'); - }); -}); diff --git a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts b/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts deleted file mode 100644 index 0f717d9b3..000000000 --- a/packages/app/src/lib/collectivex/collectivex-lazy-ingest.ts +++ /dev/null @@ -1,624 +0,0 @@ -import AdmZip from 'adm-zip'; - -import { - benchmarkSourceRepos, - GITHUB_API_BASE, - GITHUB_REPO_FULL, -} from '@semianalysisai/inferencex-constants'; - -import { - buildDatasetFromNeutral, - buildRunSummary, - isMatrixDoc, - matrixVersion, -} from '@semianalysisai/inferencex-db/collectivex/reader'; -import { - matrixArtifactName, - selectShardArtifacts, -} from '@semianalysisai/inferencex-db/collectivex/artifact-selection'; -import type { CollectiveXVersion } from '@semianalysisai/inferencex-db/collectivex/types'; -import { getCollectiveXWriteDb } from '@semianalysisai/inferencex-db/connection'; -import { - getCollectiveXRunStates, - insertCollectiveXRun, - refreshCollectiveXRunAttempt, -} from '@semianalysisai/inferencex-db/queries/collectivex'; - -const WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; -const WORKFLOW_FILE = 'collectivex-sweep.yml'; -const WORKFLOW_NAME = 'CollectiveX Sweep'; -const RUNS_PER_PAGE = 100; -const ARTIFACTS_PER_PAGE = 100; - -const MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; -const MAX_RUN_BYTES = 256 * 1024 * 1024; -const REQUEST_TIMEOUT_MS = 30_000; -const MAX_REQUEST_ATTEMPTS = 3; -const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); -const ARTIFACT_RETENTION_MS = 14 * 24 * 60 * 60 * 1000; -const WORKFLOW_RERUN_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; -const DISCOVERY_LOOKBACK_MS = WORKFLOW_RERUN_WINDOW_MS + ARTIFACT_RETENTION_MS; -const MAX_CHANGED_RUNS_PER_PASS = 8; - -type SweepErrorCode = 'invalid' | 'not-found' | 'unavailable'; - -interface GithubWorkflowRun { - id: number; - name: string; - path: string; - head_branch: string | null; - head_sha: string; - status: string | null; - conclusion: string | null; - run_attempt: number; - run_started_at?: string | null; - updated_at?: string | null; - created_at?: string | null; -} - -interface WorkflowRun extends GithubWorkflowRun { - /** `owner/repo` the run was read from. */ - source_repository: string; -} - -interface GithubArtifact { - id: number; - name: string; - archive_download_url: string; - expired?: boolean; - size_in_bytes?: number; -} - -class CollectiveXSweepError extends Error { - readonly code: SweepErrorCode; - - constructor(code: SweepErrorCode, message: string, options?: ErrorOptions) { - super(message, options); - this.name = 'CollectiveXSweepError'; - this.code = code; - } -} - -export function collectiveXSweepErrorCode(error: unknown): SweepErrorCode | null { - return error instanceof CollectiveXSweepError ? error.code : null; -} - -export function collectiveXSweepErrorStatus(error: unknown): 404 | 502 | 503 | null { - const code = collectiveXSweepErrorCode(error); - if (code === 'not-found') return 404; - if (code === 'unavailable') return 503; - if (code === 'invalid') return 502; - return null; -} - -const inFlight = new Map>(); - -function dedupe(key: string, work: () => Promise): Promise { - const existing = inFlight.get(key) as Promise | undefined; - if (existing) return existing; - const promise = work().finally(() => inFlight.delete(key)); - inFlight.set(key, promise); - return promise; -} - -const DISCOVERY_COOLDOWN_MS = 60_000; -const DISCOVERY_FAILURE_COOLDOWN_MS = 10_000; - -interface DiscoveryOutcome { - until: number; - error: unknown; -} - -const discoveryCooldown = new Map(); - -function throttled(key: string, work: () => Promise): () => Promise { - return async () => { - const settled = discoveryCooldown.get(key); - if (settled && Date.now() < settled.until) { - if (settled.error !== null) throw settled.error; - return; - } - try { - await work(); - } catch (error) { - discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_FAILURE_COOLDOWN_MS, error }); - throw error; - } - discoveryCooldown.set(key, { until: Date.now() + DISCOVERY_COOLDOWN_MS, error: null }); - }; -} - -export function resetCollectiveXDiscoveryCooldown(): void { - discoveryCooldown.clear(); -} - -function githubHeaders(token: string) { - return { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${token}`, - 'X-GitHub-Api-Version': '2022-11-28', - }; -} - -async function waitBeforeRetry(attempt: number): Promise { - const delay = process.env.NODE_ENV === 'test' ? 0 : Math.min(250 * 2 ** (attempt - 1), 2000); - await new Promise((resolve) => { - setTimeout(resolve, delay); - }); -} - -async function githubFetch(url: string, token: string): Promise { - let lastError: unknown; - for (let attempt = 1; attempt <= MAX_REQUEST_ATTEMPTS; attempt++) { - try { - const response = await fetch(url, { - cache: 'no-store', - headers: githubHeaders(token), - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - if ( - response.ok || - !RETRYABLE_STATUSES.has(response.status) || - attempt === MAX_REQUEST_ATTEMPTS - ) { - return response; - } - lastError = new Error(`GitHub returned ${response.status}`); - } catch (error) { - lastError = error; - if (attempt === MAX_REQUEST_ATTEMPTS) break; - } - await waitBeforeRetry(attempt); - } - throw new CollectiveXSweepError('unavailable', 'GitHub request failed', { cause: lastError }); -} - -function requireToken(): string { - const token = process.env.GITHUB_TOKEN; - if (!token) throw new CollectiveXSweepError('unavailable', 'GITHUB_TOKEN is not configured'); - return token; -} - -function isSweepRun(run: GithubWorkflowRun): boolean { - return ( - run.name === WORKFLOW_NAME && - run.path === WORKFLOW_PATH && - Number.isSafeInteger(run.id) && - run.id > 0 && - Number.isSafeInteger(run.run_attempt) && - run.run_attempt > 0 - ); -} - -function runGeneratedAt(run: WorkflowRun): string { - return run.updated_at || run.run_started_at || run.created_at || ''; -} - -function artifactsMayBeAvailable(run: WorkflowRun): boolean { - const timestamp = Date.parse(runGeneratedAt(run)); - return !Number.isFinite(timestamp) || Date.now() - timestamp <= ARTIFACT_RETENTION_MS; -} - -async function* repositorySweepRuns( - token: string, - repository: string, -): AsyncGenerator { - let page = 1; - let visited = 0; - let total: number | null = null; - const createdSince = new Date(Date.now() - DISCOVERY_LOOKBACK_MS).toISOString(); - while (total === null || visited < total) { - const parameters = new URLSearchParams({ - status: 'completed', - created: `>=${createdSince}`, - per_page: String(RUNS_PER_PAGE), - page: String(page), - }); - const response = await githubFetch( - `${GITHUB_API_BASE}/repos/${repository}/actions/workflows/${WORKFLOW_FILE}/runs?${parameters}`, - token, - ); - // An extra source repository without the sweep workflow has no sweeps to list. - if (response.status === 404 && page === 1 && repository !== GITHUB_REPO_FULL) { - throw new CollectiveXSweepError('not-found', 'no sweep workflow'); - } - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub run discovery failed (${response.status})`, - ); - } - const payload = (await response.json()) as { - total_count?: number; - workflow_runs?: GithubWorkflowRun[]; - }; - const runs = payload.workflow_runs ?? []; - if ( - total === null && - Number.isSafeInteger(payload.total_count) && - (payload.total_count ?? -1) >= 0 - ) { - total = payload.total_count!; - } - if (runs.length === 0) break; - visited += runs.length; - for (const run of runs) { - if (isSweepRun(run)) yield { ...run, source_repository: repository }; - } - if (runs.length < RUNS_PER_PAGE || (total !== null && visited >= total)) break; - page += 1; - } -} - -interface SweepListing { - /** First error from a repository that has the sweep workflow but could not be fully listed. */ - failure: unknown; -} - -// Merges each source repository's newest-first runs by global run id, which is -// also how getLatestCollectiveXRun orders stored rows. An extra repository without -// the sweep workflow is skipped; one that fails is skipped and logged, and its error -// is recorded on the listing. Discovery throws when every repository is skipped or fails. -async function* sweepRuns(token: string, listing: SweepListing): AsyncGenerator { - const repositories = benchmarkSourceRepos(); - const sources = repositories.map((repository) => repositorySweepRuns(token, repository)); - const heads: (WorkflowRun | null)[] = sources.map(() => null); - const failures = new Map(); - let skipped = 0; - const advance = async (index: number): Promise => { - try { - const next = await sources[index].next(); - heads[index] = next.done ? null : next.value; - } catch (error) { - heads[index] = null; - if ( - repositories[index] !== GITHUB_REPO_FULL && - collectiveXSweepErrorCode(error) === 'not-found' - ) { - skipped += 1; - return; - } - failures.set(index, error); - listing.failure ??= error; - if (sources.length > 1) { - console.warn(`[collectivex] skipping ${repositories[index]} during discovery:`, error); - } - } - }; - await Promise.all(sources.map((_source, index) => advance(index))); - for (;;) { - let newest = -1; - for (const [index, run] of heads.entries()) { - if (run && (newest < 0 || run.id > heads[newest]!.id)) newest = index; - } - if (newest < 0) break; - yield heads[newest]!; - await advance(newest); - } - if (failures.size > 0 && failures.size + skipped === sources.length) throw failures.get(0); -} - -// Run ids are global, so at most one source repository holds a run. A failed -// lookup in any repository keeps a miss from reading as not found. -async function fetchRun(runId: number, token: string): Promise { - let failure: unknown = null; - for (const repository of benchmarkSourceRepos()) { - let response: Response; - try { - response = await githubFetch( - `${GITHUB_API_BASE}/repos/${repository}/actions/runs/${runId}`, - token, - ); - } catch (error) { - failure ??= error; - continue; - } - if (response.ok) { - const run = (await response.json()) as GithubWorkflowRun; - return { ...run, source_repository: repository }; - } - if (response.status !== 404) { - failure ??= new CollectiveXSweepError( - 'unavailable', - `GitHub run lookup failed (${response.status})`, - ); - } - } - throw failure ?? new CollectiveXSweepError('not-found', 'sweep run not found'); -} - -async function listArtifacts(run: WorkflowRun, token: string): Promise { - const artifacts: GithubArtifact[] = []; - let page = 1; - let total: number | null = null; - while (total === null || artifacts.length < total) { - const response = await githubFetch( - `${GITHUB_API_BASE}/repos/${run.source_repository}/actions/runs/${run.id}/artifacts?per_page=${ARTIFACTS_PER_PAGE}&page=${page}`, - token, - ); - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub artifact discovery failed (${response.status})`, - ); - } - const payload = (await response.json()) as { - total_count?: number; - artifacts?: GithubArtifact[]; - }; - const page_artifacts = payload.artifacts ?? []; - if ( - total === null && - Number.isSafeInteger(payload.total_count) && - (payload.total_count ?? -1) >= 0 - ) { - total = payload.total_count!; - } - if (page_artifacts.length === 0) break; - artifacts.push(...page_artifacts); - if (page_artifacts.length < ARTIFACTS_PER_PAGE) break; - page += 1; - } - return artifacts.filter((artifact) => !artifact.expired); -} - -function hasMatrixArtifact(artifacts: GithubArtifact[], run: WorkflowRun): boolean { - return artifacts.some((artifact) => artifact.name === matrixArtifactName(String(run.id))); -} - -async function collectDocs(artifact: GithubArtifact, token: string): Promise { - if (artifact.size_in_bytes !== undefined && artifact.size_in_bytes > MAX_ARTIFACT_BYTES) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is oversized`); - } - const response = await githubFetch(artifact.archive_download_url, token); - if (!response.ok) { - throw new CollectiveXSweepError( - 'unavailable', - `GitHub artifact download failed (${response.status})`, - ); - } - const archive = Buffer.from(await response.arrayBuffer()); - if (archive.byteLength > MAX_ARTIFACT_BYTES) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} archive is oversized`); - } - let zip: AdmZip; - try { - zip = new AdmZip(archive); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} is not a ZIP`, { - cause: error, - }); - } - const docs: unknown[] = []; - for (const entry of zip.getEntries()) { - if (entry.isDirectory || !entry.entryName.endsWith('.json')) continue; - let text: string; - try { - text = new TextDecoder('utf-8', { fatal: true }).decode(entry.getData()); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has non-UTF-8 entry`, { - cause: error, - }); - } - try { - docs.push(JSON.parse(text)); - } catch (error) { - throw new CollectiveXSweepError('invalid', `artifact ${artifact.name} has invalid JSON`, { - cause: error, - }); - } - } - return docs; -} - -interface MatrixCandidate { - matrixDoc: unknown; - version: number; - matrixArtifacts: GithubArtifact[]; - resultArtifacts: GithubArtifact[]; -} - -function resultArtifactsForRun(artifacts: GithubArtifact[], run: WorkflowRun): GithubArtifact[] { - return selectShardArtifacts(artifacts, String(run.id), run.run_attempt).toSorted((left, right) => - left.name.localeCompare(right.name), - ); -} - -async function loadMatrixCandidate( - artifacts: GithubArtifact[], - token: string, - run: WorkflowRun, -): Promise { - const matrixArtifacts = artifacts - .filter((artifact) => artifact.name === matrixArtifactName(String(run.id))) - .toSorted((left, right) => right.id - left.id) - .slice(0, 1); - if (matrixArtifacts.length === 0) { - throw new CollectiveXSweepError('not-found', 'sweep run has no matrix artifact'); - } - const matrixDocs: unknown[] = []; - for (const artifact of matrixArtifacts) matrixDocs.push(...(await collectDocs(artifact, token))); - const matrixCandidates = matrixDocs.filter((doc) => isMatrixDoc(doc)); - if (matrixCandidates.length !== 1) { - throw new CollectiveXSweepError('invalid', 'sweep run must carry exactly one matrix document'); - } - const version = matrixVersion(matrixCandidates[0]); - if (version === null) { - throw new CollectiveXSweepError('invalid', 'matrix document has no valid version tag'); - } - const resultArtifacts = resultArtifactsForRun(artifacts, run); - return { matrixDoc: matrixCandidates[0], version, matrixArtifacts, resultArtifacts }; -} - -async function persistRun( - run: WorkflowRun, - candidate: MatrixCandidate, - token: string, - refresh = false, -): Promise { - const generatedAt = runGeneratedAt(run); - if (!generatedAt) { - throw new CollectiveXSweepError('invalid', 'sweep run is missing a timestamp'); - } - - let totalBytes = 0; - for (const artifact of [...candidate.matrixArtifacts, ...candidate.resultArtifacts]) { - totalBytes += artifact.size_in_bytes ?? 0; - if (totalBytes > MAX_RUN_BYTES) { - throw new CollectiveXSweepError('invalid', 'sweep run artifacts exceed the size budget'); - } - } - - const docs: unknown[] = []; - for (const artifact of candidate.resultArtifacts) { - docs.push(...(await collectDocs(artifact, token))); - } - - const meta = { - run_id: String(run.id), - run_attempt: run.run_attempt, - generated_at: generatedAt, - conclusion: run.conclusion, - source_sha: run.head_sha, - ...(run.source_repository === GITHUB_REPO_FULL - ? {} - : { source_repository: run.source_repository }), - }; - - let summary; - try { - summary = buildRunSummary(buildDatasetFromNeutral(candidate.matrixDoc, docs, meta)); - } catch (error) { - throw new CollectiveXSweepError('invalid', 'sweep run artifacts failed validation', { - cause: error, - }); - } - - const row = { - ...meta, - version: candidate.version, - source_branch: run.head_branch, - matrix: candidate.matrixDoc, - summary, - }; - return refresh - ? refreshCollectiveXRunAttempt(getCollectiveXWriteDb(), row, docs) - : insertCollectiveXRun(getCollectiveXWriteDb(), row, docs); -} - -async function matrixCandidateFor( - run: WorkflowRun, - version: CollectiveXVersion, - token: string, -): Promise { - const artifacts = await listArtifacts(run, token); - if (!hasMatrixArtifact(artifacts, run)) return null; - const candidate = await loadMatrixCandidate(artifacts, token, run); - return candidate.version === version ? candidate : null; -} - -type CandidateResult = 'changed-match' | 'changed-other' | 'match' | 'skip'; - -async function considerCandidate( - run: WorkflowRun, - version: CollectiveXVersion, - token: string, -): Promise { - const states = await getCollectiveXRunStates(getCollectiveXWriteDb(), [String(run.id)]); - const known = states[String(run.id)]; - if (known) { - if (known.version !== version || known.state !== 'live') return 'skip'; - if (run.run_attempt > known.run_attempt) { - const candidate = await matrixCandidateFor(run, version, token); - if (candidate) { - const changed = await persistRun(run, candidate, token, true); - return changed ? 'changed-match' : 'match'; - } - } - return 'match'; - } - const artifacts = await listArtifacts(run, token); - if (!hasMatrixArtifact(artifacts, run)) return 'skip'; - const candidate = await loadMatrixCandidate(artifacts, token, run); - const changed = await persistRun(run, candidate, token); - if (!changed) return candidate.version === version ? 'match' : 'skip'; - return candidate.version === version ? 'changed-match' : 'changed-other'; -} - -export function ensureLatestCollectiveXRun(version: CollectiveXVersion): Promise { - const key = `latest:${version}`; - return dedupe( - key, - throttled(key, async () => { - const token = requireToken(); - const listing: SweepListing = { failure: undefined }; - for await (const run of sweepRuns(token, listing)) { - const result = await considerCandidate(run, version, token); - if (result === 'match' || result === 'changed-match') break; - } - // A repository that could not be listed may hold a newer sweep. - if (listing.failure !== undefined) throw listing.failure; - }), - ); -} - -export function ensureCollectiveXRunsList(version: CollectiveXVersion): Promise { - const key = `list:${version}`; - return dedupe(key, async () => { - // A source repository that recently failed to list keeps the pass incomplete - // without walking every repository again on each poll. - const settled = discoveryCooldown.get(key); - if (settled && Date.now() < settled.until) return false; - const token = requireToken(); - let changed = 0; - const listing: SweepListing = { failure: undefined }; - for await (const run of sweepRuns(token, listing)) { - if (!artifactsMayBeAvailable(run)) continue; - let result: CandidateResult; - try { - result = await considerCandidate(run, version, token); - } catch (error) { - const code = collectiveXSweepErrorCode(error); - if (code === 'invalid' || code === 'not-found') continue; - throw error; - } - if (result === 'changed-match' || result === 'changed-other') changed += 1; - if (changed >= MAX_CHANGED_RUNS_PER_PASS) return false; - } - if (listing.failure === undefined) return true; - discoveryCooldown.set(key, { - until: Date.now() + DISCOVERY_FAILURE_COOLDOWN_MS, - error: listing.failure, - }); - return false; - }); -} - -export function ensureCollectiveXRun(version: CollectiveXVersion, runId: string): Promise { - return dedupe(`run:${version}:${runId}`, async () => { - const numericId = Number(runId); - if (!Number.isSafeInteger(numericId) || numericId <= 0) { - throw new CollectiveXSweepError('not-found', 'invalid run id'); - } - const states = await getCollectiveXRunStates(getCollectiveXWriteDb(), [runId]); - const known = states[runId]; - if (known && (known.state !== 'live' || known.version !== version)) { - throw new CollectiveXSweepError('not-found', 'run is not available'); - } - const token = requireToken(); - const run = await fetchRun(numericId, token); - if (!isSweepRun(run)) { - throw new CollectiveXSweepError('not-found', 'run is not a CollectiveX sweep'); - } - if (run.status !== 'completed') { - throw new CollectiveXSweepError('not-found', 'sweep run has not completed'); - } - if (known && run.run_attempt <= known.run_attempt) return; - const artifacts = await listArtifacts(run, token); - const candidate = await loadMatrixCandidate(artifacts, token, run); - if (candidate.version !== version) { - throw new CollectiveXSweepError('not-found', 'run does not match the requested version'); - } - await persistRun(run, candidate, token, known !== undefined); - }); -} diff --git a/packages/db/migrations-collectivex/004_committed_results.sql b/packages/db/migrations-collectivex/004_committed_results.sql new file mode 100644 index 000000000..35dc97f02 --- /dev/null +++ b/packages/db/migrations-collectivex/004_committed_results.sql @@ -0,0 +1,18 @@ +-- CollectiveX runs are now the benchmark repo's committed results, one row per +-- InferenceX commit that touched collectivex/results/ on main. +-- +-- run_id is that commit's committer time in epoch seconds (still monotonic with +-- history, so "latest" keeps ordering by run_id desc), run_attempt is always 1, +-- source_sha is the results commit, matrix is collectivex/results/cases.json, and +-- cx_run_docs holds the commit's result documents verbatim. Re-ingesting a commit +-- replaces its row and documents. +-- +-- Lazy GitHub Actions discovery is gone, so a deleted run no longer needs a +-- tombstone to stay deleted: removing results is a commit to the benchmark repo. + +alter table cx_runs drop column if exists deleted_at; + +comment on table cx_runs is + 'One row per InferenceX commit that changed collectivex/results/ on main; run_id = commit time in epoch seconds.'; +comment on table cx_run_docs is + 'The committed collectivex/results//.json documents of each cx_runs row, verbatim.'; diff --git a/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql b/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql new file mode 100644 index 000000000..b9858c97d --- /dev/null +++ b/packages/db/migrations-collectivex/005_remove_lazy_ingest_runs.sql @@ -0,0 +1,22 @@ +-- Remove the runs the retired lazy GitHub Actions ingest stored, and keep them out. +-- +-- Those rows are keyed by GitHub Actions run id (>= 10^10), committed-results runs by +-- epoch seconds (< 10^10 until the year 2286). Left in place, every legacy row would +-- outrank every committed-results run under `ORDER BY run_id DESC`, so "latest" would +-- stay a stale CI sweep forever. Deleting cascades to their cx_run_docs. The archive of +-- the old contents is a pg_dump taken before this runs (see docs/collectivex.md). +-- +-- The CHECK makes the id range part of the schema, so a legacy-shaped id can never be +-- stored again. It matches MAX_COMMITTED_RUN_ID in +-- packages/db/src/collectivex/committed-results.ts. + +delete from cx_runs where run_id >= 10000000000 or run_id <= 0; + +-- Documents whose run is gone (none are expected: the foreign key cascades). +delete from cx_run_docs d where not exists (select 1 from cx_runs r where r.run_id = d.run_id); + +alter table cx_runs + add constraint cx_runs_committed_run_id check (run_id > 0 and run_id < 10000000000); + +comment on table cx_runs is + 'One row per InferenceX commit on main whose collectivex/results/ holds measured results; run_id = committer epoch second, raised past the previous results commit''s id on ties.'; diff --git a/packages/db/package.json b/packages/db/package.json index fd67436ef..a850e3a4b 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -17,7 +17,7 @@ "scripts": { "db:prepare:ci": "bun src/prepare-ci-artifacts.ts", "db:ingest:ci": "bun src/ingest-ci-run.ts", - "db:ingest:collectivex": "bun --env-file=../../.env src/ingest-collectivex.ts --download", + "db:ingest:collectivex": "bun --env-file=../../.env src/ingest-collectivex.ts", "db:ingest:run": "bun --env-file=../../.env src/ingest-ci-run.ts --download", "db:ingest:gcs": "bun --env-file=../../.env src/ingest-gcs-backup.ts", "db:ingest:supplemental": "bun --env-file=../../.env src/ingest-supplemental.ts", diff --git a/packages/db/src/collectivex/artifact-selection.test.ts b/packages/db/src/collectivex/artifact-selection.test.ts deleted file mode 100644 index 33e7392f1..000000000 --- a/packages/db/src/collectivex/artifact-selection.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { - matrixArtifactName, - selectShardArtifactNames, - selectShardArtifacts, -} from './artifact-selection'; - -describe('shard artifact selection', () => { - it('selects one artifact per cell, sorted by name', () => { - expect( - selectShardArtifactNames( - ['cxshard-b-160-1', 'cxshard-a-160-1', 'cxsweep-matrix-160'], - '160', - 1, - ), - ).toEqual(['cxshard-a-160-1', 'cxshard-b-160-1']); - }); - - it('prefers the highest attempt not above the run attempt', () => { - const names = ['cxshard-a-160-1', 'cxshard-a-160-2', 'cxshard-a-160-3', 'cxshard-b-160-1']; - expect(selectShardArtifactNames(names, '160', 2)).toEqual([ - 'cxshard-a-160-2', - 'cxshard-b-160-1', - ]); - }); - - it('keeps cells with hyphenated names intact across run-id collisions', () => { - const names = ['cxshard-h200-ep8-160-1', 'cxshard-h200-ep8-161-1']; - expect(selectShardArtifactNames(names, '160', 1)).toEqual(['cxshard-h200-ep8-160-1']); - }); - - it('ignores foreign names and zero attempts', () => { - expect( - selectShardArtifactNames(['cxshard-a-160-0', 'other-160-1', 'cxshard-a-999-1'], '160', 1), - ).toEqual([]); - }); - it('breaks same-attempt ties on the later artifact id', () => { - const artifacts = [ - { name: 'cxshard-a-160-1', id: 10 }, - { name: 'cxshard-a-160-1', id: 42 }, - ]; - expect(selectShardArtifacts(artifacts, '160', 1)).toEqual([ - { name: 'cxshard-a-160-1', id: 42 }, - ]); - }); - - it('keeps the first match when callers supply no ids', () => { - expect( - selectShardArtifacts([{ name: 'cxshard-a-160-1' }, { name: 'cxshard-a-160-1' }], '160', 1), - ).toHaveLength(1); - }); - - it('treats a run id with regex metacharacters literally', () => { - expect(selectShardArtifactNames(['cxshard-a-1x0-1'], '1.0', 1)).toEqual([]); - }); -}); - -describe('matrixArtifactName', () => { - it('derives the per-run matrix artifact name', () => { - expect(matrixArtifactName('160')).toBe('cxsweep-matrix-160'); - }); -}); diff --git a/packages/db/src/collectivex/artifact-selection.ts b/packages/db/src/collectivex/artifact-selection.ts deleted file mode 100644 index eba4fd705..000000000 --- a/packages/db/src/collectivex/artifact-selection.ts +++ /dev/null @@ -1,54 +0,0 @@ -const MATRIX_PREFIX = 'cxsweep-matrix-'; -const SHARD_PREFIX = 'cxshard-'; - -export function matrixArtifactName(runId: string): string { - return `${MATRIX_PREFIX}${runId}`; -} - -function escapeRegExp(value: string): string { - return value.replaceAll(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`); -} - -interface ShardArtifactRef { - name: string; - id?: number; -} - -export function selectShardArtifacts( - artifacts: readonly T[], - runId: string, - runAttempt: number, -): T[] { - const pattern = new RegExp( - `^${SHARD_PREFIX}(?.+)-${escapeRegExp(runId)}-(?[1-9][0-9]*)$`, - 'u', - ); - const selected = new Map(); - for (const artifact of artifacts) { - const match = pattern.exec(artifact.name); - if (!match) continue; - const attempt = Number(match.groups!.attempt); - if (attempt > runAttempt) continue; - const previous = selected.get(match.groups!.cell); - const supersedes = - !previous || - attempt > previous.attempt || - (attempt === previous.attempt && (artifact.id ?? -1) > (previous.artifact.id ?? -1)); - if (supersedes) selected.set(match.groups!.cell, { artifact, attempt }); - } - return [...selected.values()].map((entry) => entry.artifact); -} - -export function selectShardArtifactNames( - names: readonly string[], - runId: string, - runAttempt: number, -): string[] { - return selectShardArtifacts( - names.map((name) => ({ name })), - runId, - runAttempt, - ) - .map((entry) => entry.name) - .toSorted(); -} diff --git a/packages/db/src/collectivex/committed-results.test.ts b/packages/db/src/collectivex/committed-results.test.ts new file mode 100644 index 000000000..b1893833a --- /dev/null +++ b/packages/db/src/collectivex/committed-results.test.ts @@ -0,0 +1,427 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + assignRunIds, + committedRun, + CommittedResultsError, + fetchResultsSource, + listResultsCommits, + RESULTS_DIR, + resolveResultsCommit, + validateCommittedResults, + type ResultsSource, +} from './committed-results'; +import { makeRawMatrix, makeRawShard } from './test-fixture'; +import { makeSwapDoc, makeSwapRequestedCase, swapCaseId } from './swap-test-fixture'; + +const SHA = 'c'.repeat(40); +const OTHER_SHA = 'd'.repeat(40); +const COMMITTED_AT = '2026-10-15T18:35:51Z'; +const COMMITTED_SECONDS = String(Date.parse(COMMITTED_AT) / 1000); + +type Json = Record; + +function epResult(options: Parameters[0] = {}): Json { + return { + ...makeRawShard(options), + provenance: { source_sha: 'e'.repeat(40), source_dirty: false }, + }; +} + +function requestedOf(shard: Json) { + return { + caseId: shard.identity.case_id as string, + sku: shard.identity.case_factors.sku as string, + case: shard.identity.case_factors.case as Json, + }; +} + +function casesFor(epDocs: Json[], swapSkus: string[] = [], version = 1): Json { + const matrix = makeRawMatrix(epDocs.map(requestedOf), version); + matrix.requested_cases = [ + ...(matrix.requested_cases as unknown[]), + ...swapSkus.map((sku) => makeSwapRequestedCase(sku)), + ]; + return matrix; +} + +function encode(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)); +} + +function sourceOf(cases: unknown, files: Record): ResultsSource { + return { + sha: SHA, + committedAt: COMMITTED_AT, + runId: COMMITTED_SECONDS, + files: new Map([ + ['cases.json', encode(cases)], + ...Object.entries(files).map(([path, doc]): [string, Uint8Array] => [path, encode(doc)]), + ]), + }; +} + +function pathOf(doc: Json): string { + return `${doc.identity.case_factors.sku}/${doc.identity.case_id}.json`; +} + +describe('validateCommittedResults', () => { + it('accepts EP and swap_blocks results that answer runnable cases', () => { + const ep = epResult(); + const swap = makeSwapDoc('h200-dgxc'); + const results = validateCommittedResults( + sourceOf(casesFor([ep], ['h200-dgxc']), { + [pathOf(ep)]: ep, + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ); + expect(results.version).toBe(1); + expect(results.docs).toEqual([ep, swap]); + }); + + it('accepts a commit whose cases.json has no results yet', () => { + const ep = epResult(); + expect(validateCommittedResults(sourceOf(casesFor([ep]), {})).docs).toEqual([]); + }); + + it('rejects a file whose name does not match its case_id', () => { + const ep = epResult(); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { 'h200-dgxc/renamed.json': ep })), + ).toThrow( + `${RESULTS_DIR}/h200-dgxc/renamed.json holds case ${ep.identity.case_id} on h200-dgxc; the file must be h200-dgxc/${ep.identity.case_id}.json`, + ); + }); + + it('rejects a result filed under another SKU folder', () => { + const ep = epResult(); + expect(() => + validateCommittedResults( + sourceOf(casesFor([ep]), { [`b200-nscale/${ep.identity.case_id}.json`]: ep }), + ), + ).toThrow(/the file must be h200-dgxc\//u); + }); + + it('names the file whose version differs from cases.json', () => { + const ep = { ...epResult(), version: 2 }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + `${RESULTS_DIR}/${pathOf(ep)} has version 2, cases.json has 1`, + ); + }); + + it('recognizes swap_blocks documents by schema and checks their case and pool', () => { + const swap = makeSwapDoc('h200-dgxc'); + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc', 'random')}.json`]: swap, + }), + ), + ).toThrow(/the file must be h200-dgxc\/h200-dgxc-swap-blocks-smoke-contiguous\.json/u); + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['b200-nscale']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ), + ).toThrow(/is not a runnable case in cases\.json/u); + }); + + it('rejects documents that are not results, unparsable JSON, and stray paths', () => { + const ep = epResult(); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: { schema: 'other' } })), + ).toThrow(/is not a CollectiveX result document/u); + const broken = sourceOf(casesFor([ep]), {}); + broken.files.set(pathOf(ep), new TextEncoder().encode('{')); + expect(() => validateCommittedResults(broken)).toThrow( + `${RESULTS_DIR}/${pathOf(ep)} is not UTF-8 JSON`, + ); + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { 'nested/deeper/x.json': ep })), + ).toThrow(/is not at \/\.json/u); + }); + + it('rejects results measured from uncommitted code', () => { + const ep = { ...epResult(), provenance: { source_sha: 'e'.repeat(40), source_dirty: true } }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + /was not measured from committed collectivex\/ code/u, + ); + const unknown = { ...epResult(), provenance: { source_sha: 'e'.repeat(40) } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([unknown]), { [pathOf(unknown)]: unknown })), + ).toThrow(/was not measured from committed collectivex\/ code/u); + }); + + it('requires a full 40-hex source_sha on EP and swap_blocks results', () => { + for (const sourceSha of [undefined, 'abc123', 'E'.repeat(40), 'e'.repeat(41)]) { + const ep = { ...epResult(), provenance: { source_sha: sourceSha, source_dirty: false } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep })), + ).toThrow(`${RESULTS_DIR}/${pathOf(ep)} has no 40-hex source_sha`); + } + const swap = makeSwapDoc('h200-dgxc'); + swap.runtime.source_sha = 'abc123'; + expect(() => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ), + ).toThrow(/has no 40-hex source_sha/u); + }); + + it('requires the exact cases.json version, not a coercible one', () => { + const ep = { ...epResult(), version: '1' }; + expect(() => validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep }))).toThrow( + /has version 1, cases\.json has 1/u, + ); + }); + + it('rejects results that name machines by IPv4 address or host field', () => { + const withIp = { ...epResult(), notes: ['rendezvous at 10.0.12.7:29500'] }; + expect(() => + validateCommittedResults(sourceOf(casesFor([withIp]), { [pathOf(withIp)]: withIp })), + ).toThrow(`${RESULTS_DIR}/${pathOf(withIp)} contains an IP address`); + const withHost = { ...epResult(), launch: { SLURM_NODELIST: 'node[1-2]' } }; + expect(() => + validateCommittedResults(sourceOf(casesFor([withHost]), { [pathOf(withHost)]: withHost })), + ).toThrow(/contains the host-identifying field "SLURM_NODELIST"/u); + // Dotted versions that are not four octets, and host_* keys, are fine. + const fine = { ...epResult(), notes: ['vllm 0.25.1', 'v1.2.3.4.5'], host_memory: 'pinned' }; + expect( + validateCommittedResults(sourceOf(casesFor([fine]), { [pathOf(fine)]: fine })).docs, + ).toHaveLength(1); + }); + + it('requires cases.json with a version and requested cases', () => { + expect(() => + validateCommittedResults({ + sha: SHA, + committedAt: COMMITTED_AT, + runId: COMMITTED_SECONDS, + files: new Map(), + }), + ).toThrow(`${RESULTS_DIR}/cases.json is missing at ${SHA}`); + expect(() => validateCommittedResults(sourceOf({ requested_cases: [] }, {}))).toThrow( + /has no version and requested_cases/u, + ); + }); + + it('names the file the shared reader rejects', () => { + const swap = { ...makeSwapDoc('h200-dgxc'), iterations: 0 }; + const validate = () => + validateCommittedResults( + sourceOf(casesFor([], ['h200-dgxc']), { + [`h200-dgxc/${swapCaseId('h200-dgxc')}.json`]: swap, + }), + ); + expect(validate).toThrow(CommittedResultsError); + expect(validate).toThrow( + new RegExp(`^${RESULTS_DIR}/h200-dgxc/${swapCaseId('h200-dgxc')}\\.json: `, 'u'), + ); + expect(() => validate()).toThrowError(expect.objectContaining({ kind: 'invalid' })); + }); +}); + +describe('committedRun', () => { + it('identifies the run by the commit time in epoch seconds, one attempt per commit', () => { + const ep = epResult(); + const { run, docs } = committedRun( + validateCommittedResults(sourceOf(casesFor([ep]), { [pathOf(ep)]: ep })), + ); + expect(run).toMatchObject({ + run_id: COMMITTED_SECONDS, + run_attempt: 1, + generated_at: COMMITTED_AT, + source_sha: SHA, + source_branch: 'main', + conclusion: 'success', + version: 1, + }); + expect(run.matrix).toEqual(casesFor([ep])); + expect(run.summary).toMatchObject({ + run_id: COMMITTED_SECONDS, + measured_cases: 1, + requested_cases: 1, + }); + expect(docs).toEqual([ep]); + }); + + it('stores the run under the commit run id and keeps generated_at at the commit time', () => { + const ep = epResult(); + const bumped = { + ...sourceOf(casesFor([ep]), {}), + runId: String(Number(COMMITTED_SECONDS) + 1), + }; + const { run } = committedRun(validateCommittedResults(bumped)); + expect(run.run_id).toBe(bumped.runId); + expect(run.generated_at).toBe(COMMITTED_AT); + }); +}); + +describe('assignRunIds', () => { + it('is the committer second, bumped past ties and clocks behind history', () => { + const ids = assignRunIds([ + { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z' }, + { sha: 'b'.repeat(40), committedAt: '2026-10-15T00:00:00.900Z' }, + { sha: 'c'.repeat(40), committedAt: '2026-10-14T00:00:00Z' }, + { sha: 'd'.repeat(40), committedAt: '2026-10-16T00:00:00Z' }, + ]).map((commit) => Number(commit.runId)); + const base = Date.parse('2026-10-15T00:00:00Z') / 1000; + expect(ids).toEqual([base, base + 1, base + 2, Date.parse('2026-10-16T00:00:00Z') / 1000]); + }); + + it('gives a commit the same id whatever ingest computes it', () => { + const history = [ + { sha: 'a'.repeat(40), committedAt: COMMITTED_AT }, + { sha: 'b'.repeat(40), committedAt: COMMITTED_AT }, + ]; + expect(assignRunIds(history)).toEqual(assignRunIds(history)); + }); +}); + +function jsonResponse(body: unknown): Response { + return Response.json(body); +} + +function commitBody(sha: string, date: string) { + return { sha, commit: { committer: { date } } }; +} + +describe('GitHub reads', () => { + it('resolves a ref to its newest results commit when that commit is on main', async () => { + const fetch = vi.fn((url: string) => { + const query = new URL(url).searchParams; + expect(query.get('path')).toBe(RESULTS_DIR); + return query.get('sha') === 'main' + ? jsonResponse([ + commitBody(SHA, COMMITTED_AT), + commitBody(OTHER_SHA, '2026-10-01T00:00:00Z'), + ]) + : jsonResponse([commitBody(SHA, COMMITTED_AT)]); + }); + await expect( + resolveResultsCommit('feature', { fetch: fetch as unknown as typeof globalThis.fetch }), + ).resolves.toEqual({ sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }); + }); + + it('refuses a results commit that never landed on main', async () => { + const fetch = vi.fn((url: string) => + new URL(url).searchParams.get('sha') === 'main' + ? jsonResponse([commitBody(OTHER_SHA, COMMITTED_AT)]) + : jsonResponse([commitBody(SHA, COMMITTED_AT)]), + ); + await expect( + resolveResultsCommit(SHA, { fetch: fetch as unknown as typeof globalThis.fetch }), + ).rejects.toMatchObject({ kind: 'not-found' }); + }); + + it('lists main history oldest first across pages', async () => { + const page1 = Array.from({ length: 100 }, (_, index) => + commitBody(index.toString(16).padStart(40, '0'), '2026-10-15T00:00:00Z'), + ); + const fetch = vi.fn((url: string) => + jsonResponse( + new URL(url).searchParams.get('page') === '1' ? page1 : [commitBody(SHA, COMMITTED_AT)], + ), + ); + const commits = await listResultsCommits('main', { + fetch: fetch as unknown as typeof globalThis.fetch, + }); + expect(commits).toHaveLength(101); + expect(commits[0].sha).toBe(SHA); + // The 100 same-second commits after SHA get distinct, increasing ids. + const ids = commits.map((commit) => Number(commit.runId)); + expect(new Set(ids).size).toBe(101); + expect(ids).toEqual(ids.toSorted((a, b) => a - b)); + expect(fetch).toHaveBeenCalledTimes(2); + }); + + it('reads exactly the results tree at the commit and reuses cached blobs', async () => { + const urls: string[] = []; + const fetch = vi.fn((url: string, init?: RequestInit) => { + urls.push(url); + if (url.includes('/git/trees/')) { + return jsonResponse({ + truncated: false, + tree: [ + { path: 'cases.json', type: 'blob', sha: 'b1', size: 10 }, + { path: 'h200-dgxc', type: 'tree', sha: 't1' }, + { path: 'h200-dgxc/a.json', type: 'blob', sha: 'b2', size: 10 }, + ], + }); + } + expect(init?.headers).toEqual({ Authorization: 'Bearer token' }); + return new Response('{}'); + }); + const blobCache = new Map([['b2', new TextEncoder().encode('cached')]]); + const source = await fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: fetch as unknown as typeof globalThis.fetch, token: 'token', blobCache }, + ); + expect(urls[0]).toContain(`/git/trees/${SHA}:${RESULTS_DIR}?recursive=1`); + expect(urls.slice(1)).toEqual([ + `https://raw.githubusercontent.com/SemiAnalysisAI/InferenceX/${SHA}/collectivex/results/cases.json`, + ]); + expect([...source.files.keys()].toSorted()).toEqual(['cases.json', 'h200-dgxc/a.json']); + expect(new TextDecoder().decode(source.files.get('h200-dgxc/a.json'))).toBe('cached'); + expect(blobCache.has('b1')).toBe(true); + }); + + it('reports a commit that deleted the results directory as absent', async () => { + const fetch = vi.fn((url: string) => + url.includes(':collectivex/results') + ? new Response('', { status: 404 }) + : jsonResponse({ tree: [{ path: 'bench', type: 'tree', sha: 't1' }] }), + ); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: fetch as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ kind: 'absent' }); + }); + + it('retries a listed raw file that 404s and reports it unavailable, not missing', async () => { + vi.useFakeTimers(); + const fetch = vi.fn((url: string) => + new URL(url).hostname === 'raw.githubusercontent.com' + ? new Response('', { status: 404 }) + : jsonResponse({ tree: [{ path: 'cases.json', type: 'blob', sha: 'c1', size: 2 }] }), + ); + const read = fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: fetch as unknown as typeof globalThis.fetch }, + ); + const settled = expect(read).rejects.toMatchObject({ kind: 'unavailable' }); + await vi.runAllTimersAsync(); + await settled; + vi.useRealTimers(); + expect( + fetch.mock.calls.filter(([url]) => url.startsWith('https://raw.')).length, + ).toBeGreaterThan(1); + }); + + it('reports a commit without a results directory as not found and oversize files as invalid', async () => { + const missing = vi.fn(() => new Response('', { status: 404 })); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: missing as unknown as typeof globalThis.fetch }, + ), + ).rejects.toThrow(`${RESULTS_DIR} does not exist at ${SHA}`); + + const oversize = vi.fn(() => + jsonResponse({ + tree: [{ path: 'a/b.json', type: 'blob', sha: 'x', size: 64 * 1024 * 1024 }], + }), + ); + await expect( + fetchResultsSource( + { sha: SHA, committedAt: COMMITTED_AT, runId: COMMITTED_SECONDS }, + { fetch: oversize as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ kind: 'invalid' }); + }); +}); diff --git a/packages/db/src/collectivex/committed-results.ts b/packages/db/src/collectivex/committed-results.ts new file mode 100644 index 000000000..e92fe1bc5 --- /dev/null +++ b/packages/db/src/collectivex/committed-results.ts @@ -0,0 +1,543 @@ +/** + * CollectiveX results committed to the benchmark repo: `collectivex/results/cases.json` + * (the requested cases) plus one `collectivex/results//.json` per measured + * case. Each InferenceX commit on main that carries measured results becomes one stored + * run, so this module pins a ref to a commit, reads exactly those files at it, validates + * them, and turns them into a cx_runs row. Used by the ingest CLI (ingest-collectivex.ts), + * which the ingest-collectivex workflow runs; the app itself never ingests. + */ + +import { + GITHUB_API_BASE, + GITHUB_OWNER, + GITHUB_REPO, + GITHUB_REPO_FULL, +} from '@semianalysisai/inferencex-constants'; + +import type { CollectiveXRunInsert } from '../queries/collectivex'; +import { buildDatasetFromNeutral, buildRunSummary, type CollectiveXNeutralRunMeta } from './reader'; + +export const RESULTS_DIR = 'collectivex/results'; +export const RESULTS_BRANCH = 'main'; +const CASES_FILE = 'cases.json'; +const SWAP_SCHEMA = 'collectivex-swap-blocks-v1'; +// The benchmark repo's own test caps a result at 1 MiB; leave headroom without +// letting one commit exhaust a function's memory. +const MAX_FILE_BYTES = 4 * 1024 * 1024; +const MAX_TOTAL_BYTES = 256 * 1024 * 1024; +const FETCH_ATTEMPTS = 3; +const FETCH_TIMEOUT_MS = 30_000; +const FETCH_CONCURRENCY = 8; +const COMMITS_PER_PAGE = 100; +const SHA = /^[0-9a-f]{40}$/u; +const RESULT_PATH = /^(?[^/]+)\/(?[^/]+)\.json$/u; +// Mirrors collectivex/tests/test_results.py in the benchmark repo: results must not name +// machines, by IPv4 address in any string or by a host-identifying key anywhere. +const IPV4 = /(?; +} + +export interface CommittedResults extends ResultsCommit { + version: number; + matrix: unknown; + docs: unknown[]; +} + +/** + * `invalid`: the committed files break the results contract. `not-found`: the ref, + * the results directory, or a main-history results commit does not exist. `absent`: the + * commit is readable and has no results directory (it deleted them), so there is no run. + * `unavailable`: GitHub could not be read. + */ +export type CommittedResultsErrorKind = 'invalid' | 'not-found' | 'absent' | 'unavailable'; + +export class CommittedResultsError extends Error { + readonly kind: CommittedResultsErrorKind; + + constructor(kind: CommittedResultsErrorKind, message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'CommittedResultsError'; + this.kind = kind; + } +} + +function apiHeaders(token?: string): Record { + return { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + +async function request( + url: string, + headers: Record, + fetchImpl: typeof globalThis.fetch, +): Promise { + let failure: unknown; + for (let attempt = 1; attempt <= FETCH_ATTEMPTS; attempt++) { + try { + const response = await fetchImpl(url, { + headers, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (response.ok) return response; + failure = new Error(`${url} returned HTTP ${response.status}`); + // A raw file the trees API already listed exists, so its 404 is a CDN miss worth retrying; + // only the API's own 404/422 means the commit or tree is really absent. + const listedFile = new URL(url).hostname === 'raw.githubusercontent.com'; + if (!listedFile && (response.status === 404 || response.status === 422)) { + throw new CommittedResultsError('not-found', `${url} returned HTTP ${response.status}`); + } + if (listedFile && response.status === 404) { + if (attempt < FETCH_ATTEMPTS) { + await new Promise((settle) => { + setTimeout(settle, 1000 * attempt); + }); + } + continue; + } + if (response.status < 500 && response.status !== 429) break; + } catch (error) { + if (error instanceof CommittedResultsError) throw error; + failure = error; + } + if (attempt < FETCH_ATTEMPTS) { + await new Promise((settle) => { + setTimeout(settle, 1000 * attempt); + }); + } + } + throw new CommittedResultsError('unavailable', `Reading ${url} failed`, { cause: failure }); +} + +async function getJson(url: string, options: GithubReadOptions): Promise { + const response = await request(url, apiHeaders(options.token), options.fetch ?? fetch); + return response.json(); +} + +function toGithubCommit(body: unknown): GithubCommit { + const commit = body as { sha?: unknown; commit?: { committer?: { date?: unknown } } } | null; + const date = commit?.commit?.committer?.date; + if ( + typeof commit?.sha !== 'string' || + !SHA.test(commit.sha) || + typeof date !== 'string' || + !Number.isFinite(Date.parse(date)) + ) { + throw new CommittedResultsError('unavailable', 'GitHub returned a malformed commit'); + } + return { sha: commit.sha, committedAt: date }; +} + +async function resultsCommitsPage( + ref: string, + page: number, + perPage: number, + options: GithubReadOptions, +): Promise { + const query = new URLSearchParams({ + sha: ref, + path: RESULTS_DIR, + per_page: String(perPage), + page: String(page), + }); + const body = await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/commits?${query}`, + options, + ); + if (!Array.isArray(body)) { + throw new CommittedResultsError('unavailable', 'GitHub commit list is not an array'); + } + return body.map(toGithubCommit); +} + +/** + * Give each results commit of a history (oldest first) its run_id: its committer + * second, or one more than the previous id when that is not larger. + */ +export function assignRunIds(history: readonly GithubCommit[]): ResultsCommit[] { + let previous = 0; + return history.map((commit) => { + const id = Math.max(Math.floor(Date.parse(commit.committedAt) / 1000), previous + 1); + if (id >= MAX_COMMITTED_RUN_ID) { + throw new CommittedResultsError('unavailable', `${commit.sha} has an out-of-range time`); + } + previous = id; + return { ...commit, runId: String(id) }; + }); +} + +/** Every commit on `branch` that touched RESULTS_DIR, oldest first, with its run_id. */ +export async function listResultsCommits( + branch: string = RESULTS_BRANCH, + options: GithubReadOptions = {}, +): Promise { + const commits: GithubCommit[] = []; + for (let page = 1; ; page++) { + const batch = await resultsCommitsPage(branch, page, COMMITS_PER_PAGE, options); + commits.push(...batch); + if (batch.length < COMMITS_PER_PAGE) break; + } + return assignRunIds(commits.toReversed()); +} + +/** + * Pin a branch, tag, or commit to the newest RESULTS_DIR commit at or before it, + * and require that commit to be on RESULTS_BRANCH. A ref whose own commit did not + * touch the results (for example main's head after an unrelated merge) therefore + * resolves to the run already published for those files instead of a duplicate, + * and results that never landed on main are never published. + */ +export async function resolveResultsCommit( + ref: string, + options: GithubReadOptions = {}, +): Promise { + const [commit] = await resultsCommitsPage(ref, 1, 1, options); + if (!commit) { + throw new CommittedResultsError( + 'not-found', + `No commit at or before ${ref} touched ${RESULTS_DIR}`, + ); + } + const history = await listResultsCommits(RESULTS_BRANCH, options); + const entry = history.find((candidate) => candidate.sha === commit.sha); + if (!entry) { + throw new CommittedResultsError( + 'not-found', + `${commit.sha} (from ${ref}) is not a ${RESULTS_DIR} commit on ${RESULTS_BRANCH}`, + ); + } + return entry; +} + +interface TreeEntry { + path: string; + type: string; + sha: string; + size?: number; +} + +/** + * Read RESULTS_DIR at one commit: list it with the git trees API, then fetch each + * blob from raw.githubusercontent.com, which does not count against the REST rate + * limit. `blobCache` (keyed by blob sha) lets a backfill skip unchanged files. + */ +export async function fetchResultsSource( + commit: ResultsCommit, + options: GithubReadOptions & { blobCache?: Map } = {}, +): Promise { + const fetchImpl = options.fetch ?? fetch; + let tree: { tree?: TreeEntry[]; truncated?: boolean }; + try { + tree = (await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/git/trees/${commit.sha}:${RESULTS_DIR}?recursive=1`, + options, + )) as typeof tree; + } catch (error) { + if (error instanceof CommittedResultsError && error.kind === 'not-found') { + // A 404 alone can be a GitHub miss; only a readable parent tree without results/ proves + // the commit removed them. + const parent = RESULTS_DIR.slice(0, RESULTS_DIR.lastIndexOf('/')); + const leaf = RESULTS_DIR.slice(RESULTS_DIR.lastIndexOf('/') + 1); + const listing = (await getJson( + `${GITHUB_API_BASE}/repos/${GITHUB_OWNER}/${GITHUB_REPO}/git/trees/${commit.sha}:${parent}`, + options, + ).catch(() => null)) as { tree?: TreeEntry[] } | null; + const absent = + Array.isArray(listing?.tree) && !listing.tree.some((entry) => entry.path === leaf); + throw new CommittedResultsError( + absent ? 'absent' : 'not-found', + `${RESULTS_DIR} does not exist at ${commit.sha}`, + { + cause: error, + }, + ); + } + throw error; + } + if (tree.truncated || !Array.isArray(tree.tree)) { + throw new CommittedResultsError( + 'unavailable', + `GitHub truncated the ${RESULTS_DIR} tree at ${commit.sha}`, + ); + } + const blobs = tree.tree.filter((entry) => entry.type === 'blob' && entry.path.endsWith('.json')); + let total = 0; + for (const blob of blobs) { + const size = blob.size ?? 0; + if (size > MAX_FILE_BYTES) { + throw new CommittedResultsError( + 'invalid', + `${RESULTS_DIR}/${blob.path} exceeds ${MAX_FILE_BYTES} bytes`, + ); + } + total += size; + } + if (total > MAX_TOTAL_BYTES) { + throw new CommittedResultsError( + 'invalid', + `${RESULTS_DIR} at ${commit.sha} exceeds ${MAX_TOTAL_BYTES} bytes`, + ); + } + + const headers: Record = options.token + ? { Authorization: `Bearer ${options.token}` } + : {}; + const files = new Map(); + let next = 0; + const worker = async () => { + while (next < blobs.length) { + const blob = blobs[next++]; + let data = options.blobCache?.get(blob.sha); + if (!data) { + const path = `${RESULTS_DIR}/${blob.path}`.split('/').map(encodeURIComponent).join('/'); + const response = await request( + `https://raw.githubusercontent.com/${GITHUB_REPO_FULL}/${commit.sha}/${path}`, + headers, + fetchImpl, + ); + data = new Uint8Array(await response.arrayBuffer()); + options.blobCache?.set(blob.sha, data); + } + files.set(blob.path, data); + } + }; + await Promise.all(Array.from({ length: Math.min(FETCH_CONCURRENCY, blobs.length) }, worker)); + return { ...commit, files }; +} + +interface ResultDoc { + record_type?: unknown; + schema?: unknown; + case_id?: unknown; + version?: unknown; + identity?: { case_id?: unknown; case_factors?: { sku?: unknown } }; + provenance?: { source_sha?: unknown; source_dirty?: unknown }; + runtime?: { sku?: unknown; source_sha?: unknown; source_dirty?: unknown }; +} + +/** Identity fields of an EP case attempt or a swap_blocks result; null for anything else. */ +function resultIdentity(doc: ResultDoc | null) { + if (doc?.record_type === 'case-attempt') { + return { + caseId: doc.identity?.case_id, + sku: doc.identity?.case_factors?.sku, + sourceSha: doc.provenance?.source_sha, + sourceDirty: doc.provenance?.source_dirty, + }; + } + if (doc?.schema === SWAP_SCHEMA) { + return { + caseId: doc.case_id, + sku: doc.runtime?.sku, + sourceSha: doc.runtime?.source_sha, + sourceDirty: doc.runtime?.source_dirty, + }; + } + return null; +} + +/** The first machine identifier in a document: an IPv4 address or a host-naming key. */ +function machineIdentifier(value: unknown): string | null { + if (typeof value === 'string') return IPV4.test(value) ? 'an IP address' : null; + if (Array.isArray(value)) { + for (const item of value) { + const found = machineIdentifier(item); + if (found) return found; + } + return null; + } + if (value && typeof value === 'object') { + for (const [key, item] of Object.entries(value)) { + if (HOST_KEYS.has(key.toLowerCase())) return `the host-identifying field "${key}"`; + const found = machineIdentifier(item); + if (found) return found; + } + } + return null; +} + +function parseJson(path: string, data: Uint8Array): unknown { + try { + return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(data)); + } catch (error) { + throw new CommittedResultsError('invalid', `${path} is not UTF-8 JSON`, { cause: error }); + } +} + +function runMeta(source: ResultsCommit): CollectiveXNeutralRunMeta { + const seconds = Math.floor(Date.parse(source.committedAt) / 1000); + return { + run_id: source.runId, + run_attempt: 1, + generated_at: new Date(seconds * 1000).toISOString().replace(/\.\d{3}Z$/u, 'Z'), + conclusion: 'success', + source_sha: source.sha, + }; +} + +/** + * Check one commit's files the way the benchmark repo's own test does: every + * result sits at `/.json` matching its document, answers a + * runnable case in cases.json, was measured from committed code, and carries + * the cases.json version. Every failure names the offending file. + */ +export function validateCommittedResults(source: ResultsSource): CommittedResults { + const casesPath = `${RESULTS_DIR}/${CASES_FILE}`; + const casesBytes = source.files.get(CASES_FILE); + if (!casesBytes) + throw new CommittedResultsError('invalid', `${casesPath} is missing at ${source.sha}`); + const matrix = parseJson(casesPath, casesBytes) as { + version?: unknown; + requested_cases?: { sku?: unknown; disposition?: unknown; case?: { case_id?: unknown } }[]; + }; + if ( + typeof matrix?.version !== 'number' || + !Number.isSafeInteger(matrix.version) || + matrix.version <= 0 || + !Array.isArray(matrix.requested_cases) + ) { + throw new CommittedResultsError('invalid', `${casesPath} has no version and requested_cases`); + } + const { version } = matrix; + const runnable = new Set( + matrix.requested_cases + .filter((entry) => entry?.disposition === 'runnable') + .map((entry) => JSON.stringify([entry.sku, entry.case?.case_id])), + ); + + const results: { path: string; doc: unknown }[] = []; + for (const [file, data] of [...source.files].toSorted(([a], [b]) => a.localeCompare(b))) { + if (file === CASES_FILE) continue; + const path = `${RESULTS_DIR}/${file}`; + const location = RESULT_PATH.exec(file)?.groups; + if (!location) + throw new CommittedResultsError('invalid', `${path} is not at /.json`); + const doc = parseJson(path, data) as ResultDoc | null; + const identity = resultIdentity(doc); + if (!identity) + throw new CommittedResultsError('invalid', `${path} is not a CollectiveX result document`); + if (identity.caseId !== location.caseId || identity.sku !== location.sku) { + throw new CommittedResultsError( + 'invalid', + `${path} holds case ${String(identity.caseId)} on ${String(identity.sku)}; the file must be ${String(identity.sku)}/${String(identity.caseId)}.json`, + ); + } + if (!runnable.has(JSON.stringify([location.sku, location.caseId]))) { + throw new CommittedResultsError('invalid', `${path} is not a runnable case in ${CASES_FILE}`); + } + if (typeof identity.sourceSha !== 'string' || !SHA.test(identity.sourceSha)) { + throw new CommittedResultsError('invalid', `${path} has no 40-hex source_sha`); + } + if (identity.sourceDirty !== false) { + throw new CommittedResultsError( + 'invalid', + `${path} was not measured from committed collectivex/ code`, + ); + } + if (doc?.record_type === 'case-attempt' && doc.version !== version) { + throw new CommittedResultsError( + 'invalid', + `${path} has version ${String(doc.version)}, ${CASES_FILE} has ${version}`, + ); + } + const leaked = machineIdentifier(doc); + if (leaked) throw new CommittedResultsError('invalid', `${path} contains ${leaked}`); + results.push({ path, doc }); + } + + // Assemble each document alone first so a reader failure names its file. + const meta = runMeta(source); + for (const { path, doc } of results) { + try { + buildDatasetFromNeutral(matrix, [doc], meta); + } catch (error) { + throw new CommittedResultsError( + 'invalid', + `${path}: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + } + return { + sha: source.sha, + committedAt: source.committedAt, + runId: source.runId, + version, + matrix, + docs: results.map(({ doc }) => doc), + }; +} + +/** + * The cx_runs row for one results commit. run_id comes from assignRunIds, so it grows + * with main's history and `ORDER BY run_id DESC` still means "latest"; each commit is + * one attempt whose documents are replaced wholesale when it is ingested again. + */ +export function committedRun(results: CommittedResults): { + run: CollectiveXRunInsert; + docs: unknown[]; +} { + const meta = runMeta(results); + const summary = buildRunSummary(buildDatasetFromNeutral(results.matrix, results.docs, meta)); + return { + run: { + ...meta, + version: results.version, + source_branch: RESULTS_BRANCH, + matrix: results.matrix, + summary, + }, + docs: results.docs, + }; +} + +/** Fetch, validate, and assemble one main-history results commit into its cx_runs row. */ +export async function loadCommittedRun( + commit: ResultsCommit, + options: GithubReadOptions & { blobCache?: Map } = {}, +): Promise<{ run: CollectiveXRunInsert; docs: unknown[] }> { + return committedRun(validateCommittedResults(await fetchResultsSource(commit, options))); +} diff --git a/packages/db/src/collectivex/reader.test.ts b/packages/db/src/collectivex/reader.test.ts index 07b3e7eaf..1f5321623 100644 --- a/packages/db/src/collectivex/reader.test.ts +++ b/packages/db/src/collectivex/reader.test.ts @@ -238,13 +238,36 @@ describe('CollectiveX artifact assembly', () => { ).toBeNull(); }); - it('rejects malformed and cross-version artifacts', () => { + it('keeps a withheld tail null through the derived rates', () => { + const shard = makeRawShard({ rows: [{}] }); + const row = (shard.measurement as { rows: Record[] }).rows[0]; + row.components.dispatch.percentiles_us.p99 = null; + row.token_rate_at_latency_percentile.p99 = null; + const point = buildDatasetFromNeutral( + makeRawMatrix([requestedOf(shard)]), + [shard], + makeRunMeta(), + ).series[0].points[0]; + const dispatch = point.components.dispatch; + expect(dispatch?.latency_us.p99).toBeNull(); + expect(dispatch?.activation_data_rate_gbps_at_latency_percentile?.p99).toBeNull(); + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p99).toBeNull(); + expect(dispatch?.payload_data_rate_gbps_at_latency_percentile?.p50).toBeCloseTo( + (400000000 / 8 / 417) * 1e-3, + 3, + ); + expect(point.roundtrip_token_rate_at_latency_percentile.p99).toBeNull(); + }); + + it('rejects malformed and cross-version artifacts, naming the mismatched case', () => { expect(() => buildDatasetFromNeutral({}, [], makeRunMeta())).toThrow(/matrix/); const shard = makeRawShard(); shard.version = 2; expect(() => buildDatasetFromNeutral(makeRawMatrix([requestedOf(shard)]), [shard], makeRunMeta()), - ).toThrow(/version/); + ).toThrow( + 'CollectiveX version mismatch: h200-dgxc-deepep-v2-deepseek-v3-normal-decode-ep8-uniform-bf16 has version 2, cases.json has 1', + ); }); }); diff --git a/packages/db/src/collectivex/reader.ts b/packages/db/src/collectivex/reader.ts index fd62085ad..e7ae4148c 100644 --- a/packages/db/src/collectivex/reader.ts +++ b/packages/db/src/collectivex/reader.ts @@ -1,4 +1,4 @@ -import { readSwapResults, isSwapMatrix } from './swap-reader'; +import { readSwapResults } from './swap-reader'; import type { CollectiveXComponent, @@ -10,11 +10,11 @@ import type { CollectiveXKvRow, CollectiveXMode, CollectiveXOutcome, - CollectiveXPercentiles, CollectiveXPoint, CollectiveXPrecision, CollectiveXRunSummary, CollectiveXSeries, + CollectiveXTailPercentiles, CollectiveXTerminalStatus, } from './types'; @@ -38,13 +38,13 @@ interface RawCase { interface RawComponent { availability: string; - percentiles_us: CollectiveXPercentiles | null; + percentiles_us: CollectiveXTailPercentiles | null; } interface RawRow { tokens_per_rank: number; global_tokens: number; - token_rate_at_latency_percentile: CollectiveXPercentiles; + token_rate_at_latency_percentile: CollectiveXTailPercentiles; components: Record; byte_provenance: Record; wire_byte_provenance?: Record< @@ -115,7 +115,6 @@ function toSupportedVendor(raw: string): CollectiveXVendor | null { } function matrixOf(value: unknown): RawMatrix { - if (isSwapMatrix(value)) return { version: 1, requested_cases: [] }; const matrix = value as RawMatrix; if (!Number.isSafeInteger(matrix?.version) || !Array.isArray(matrix?.requested_cases)) { throw new TypeError('invalid CollectiveX matrix'); @@ -152,10 +151,11 @@ function toMode(raw: string | undefined): CollectiveXMode { function ratesFrom( bytes: number, - latency: CollectiveXPercentiles, + latency: CollectiveXTailPercentiles, divisor = 1, -): CollectiveXPercentiles { - const rate = (us: number) => (bytes / divisor / us) * 1e-3; +): CollectiveXTailPercentiles { + const rate = (us: number | null) => + us !== null && us > 0 ? (bytes / divisor / us) * 1e-3 : null; return { p50: rate(latency.p50), p90: rate(latency.p90), @@ -285,6 +285,11 @@ function isKvCase(kase: RawCase): boolean { return kase.suite === 'kv-transfer'; } +// swap_blocks cases have no EP ladder; their measurements come from swap-reader. +function isSwapCase(kase: RawCase): boolean { + return kase.suite === 'swap-blocks'; +} + function mapKvRow(row: RawKvRow): CollectiveXKvRow { return { kind: row.kind === 'bulk' ? 'bulk' : 'paged', @@ -365,12 +370,17 @@ export function buildDatasetFromNeutral( run: CollectiveXNeutralRunMeta, ): CollectiveXDataset { const matrix = matrixOf(matrixRaw); - const swap = readSwapResults(matrixRaw, docs, run.source_sha); + const swapRequested = matrix.requested_cases.filter((requested) => isSwapCase(requested.case)); + const swap = readSwapResults(matrix.requested_cases, docs); const swapPoints = swap.reduce((count, result) => count + result.points.length, 0); const shards = docs.flatMap((doc) => { const shard = shardOf(doc); if (!shard) return []; - if (Number(shard.version) !== matrix.version) throw new Error('CollectiveX version mismatch'); + if (Number(shard.version) !== matrix.version) { + throw new Error( + `CollectiveX version mismatch: ${shard.identity.case_id} has version ${String(shard.version)}, cases.json has ${matrix.version}`, + ); + } return [shard]; }); const supportedShards = shards.flatMap((shard): SupportedShard[] => { @@ -397,7 +407,7 @@ export function buildDatasetFromNeutral( const coverage: CollectiveXCoverage[] = matrix.requested_cases.flatMap((requested) => { const kase = requested.case; const caseId = kase.case_id; - if (isKvCase(kase) || !caseId || hiddenCaseIds.has(caseId)) return []; + if (isKvCase(kase) || isSwapCase(kase) || !caseId || hiddenCaseIds.has(caseId)) return []; const measured = successful.get(caseId)?.shard; const failed = terminal.get(caseId)?.shard; let outcome: CollectiveXOutcome; @@ -446,7 +456,7 @@ export function buildDatasetFromNeutral( version: matrix.version, run: { ...run, - requested_cases: coverage.length + kv.length + swap.length, + requested_cases: coverage.length + kv.length + swapRequested.length, terminal_cases: coverage.filter((item) => item.points.every((point) => point.terminal_status !== 'pending')) .length + @@ -468,14 +478,15 @@ export function buildDatasetFromNeutral( points.filter((point) => point.terminal_status !== 'pending').length + swapPoints, measured_points: points.filter((point) => point.terminal_status === 'measured').length + swapPoints, + // Pools with at least one measured result; requested-only pools are not covered. covered_skus: [ ...new Set([ - ...coverage.map((item) => item.sku), - ...kv.map((item) => item.sku), + ...coverage.filter((item) => item.outcome === 'success').map((item) => item.sku), + ...kv.filter((item) => item.outcome === 'success').map((item) => item.sku), ...swap.map((item) => item.sku), ]), ].toSorted(), - swap_requested_cases: swap.length, + swap_requested_cases: swapRequested.length, swap_measured_cases: swap.length, kv_requested_cases: kv.length, kv_measured_cases: kv.filter((item) => item.outcome === 'success').length, @@ -517,19 +528,3 @@ export function buildRunSummary(dataset: CollectiveXDataset): CollectiveXRunSumm }, }; } - -export function isMatrixDoc(doc: unknown): boolean { - if (isSwapMatrix(doc)) return true; - const candidate = doc as { requested_cases?: unknown; include?: unknown } | null; - return ( - Array.isArray(candidate?.requested_cases) && - Array.isArray(candidate?.include) && - matrixVersion(doc) !== null - ); -} - -export function matrixVersion(doc: unknown): number | null { - if (isSwapMatrix(doc)) return 1; - const value = (doc as { version?: unknown } | null)?.version; - return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : null; -} diff --git a/packages/db/src/collectivex/swap-reader.test.ts b/packages/db/src/collectivex/swap-reader.test.ts index bc3edb027..1d8ad5c74 100644 --- a/packages/db/src/collectivex/swap-reader.test.ts +++ b/packages/db/src/collectivex/swap-reader.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { buildDatasetFromNeutral, buildRunSummary, isMatrixDoc, matrixVersion } from './reader'; -import { makeSwapDoc, swapMatrix, swapMeta } from './swap-test-fixture'; +import { buildDatasetFromNeutral, buildRunSummary } from './reader'; +import { + makeSwapDoc, + makeSwapRequestedCase, + swapCaseId, + swapMatrix, + swapMeta, +} from './swap-test-fixture'; +import { makeRawMatrix, makeRawShard } from './test-fixture'; -describe('standalone swap_blocks artifacts', () => { +describe('swap_blocks results', () => { it('assembles verified copies through GiB and exposes the run without EP cases', () => { - expect(isMatrixDoc(swapMatrix)).toBe(true); - expect(matrixVersion(swapMatrix)).toBe(1); const dataset = buildDatasetFromNeutral( swapMatrix, [makeSwapDoc(), { log: 'unrelated' }], @@ -32,6 +37,7 @@ describe('standalone swap_blocks artifacts', () => { covered_skus: ['h200-dgxc'], }); }); + it.each(['correctness', 'payload', 'latency', 'samples', 'provenance', 'duplicate'])( 'rejects %s corruption instead of plotting success', (corruption) => { @@ -40,45 +46,105 @@ describe('standalone swap_blocks artifacts', () => { if (corruption === 'payload') doc.cases[0].payload_bytes = 2048; if (corruption === 'latency') doc.cases[0].latency.percentiles_us.p50 = NaN; if (corruption === 'samples') doc.cases[0].latency.sample_count = 0; - if (corruption === 'provenance') doc.runtime.source_sha = 'other'; + if (corruption === 'provenance') (doc.runtime as Record).source_sha = null; if (corruption === 'duplicate') doc.cases.push(doc.cases[0]); expect(() => buildDatasetFromNeutral(swapMatrix, [doc], swapMeta)).toThrow(/invalid/); }, ); - it('does not misclassify arbitrary execution matrices', () => { - const matrix = { include: [{ backend: 'other', sku: 'h200-dgxc' }] }; - expect(isMatrixDoc(matrix)).toBe(false); - expect(matrixVersion(matrix)).toBeNull(); - expect(() => buildDatasetFromNeutral(matrix, [], swapMeta)).toThrow( + + it('keeps the measuring revision instead of requiring the snapshot revision', () => { + const doc = makeSwapDoc(); + doc.runtime.source_sha = 'f'.repeat(40); + const dataset = buildDatasetFromNeutral(swapMatrix, [doc], swapMeta); + expect(dataset.run.source_sha).toBe('abc123'); + expect(dataset.swap_blocks?.[0].runtime.source_sha).toBe('f'.repeat(40)); + }); + + it('rejects the retired execution-only swap matrix', () => { + const matrix = { include: [{ backend: 'swap-blocks', sku: 'h200-dgxc' }] }; + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc()], swapMeta)).toThrow( 'invalid CollectiveX matrix', ); }); }); +describe('swap_blocks cases in a unified cases.json', () => { + it('skips ladder-free swap cases in EP coverage and keeps unanswered ones pending', () => { + const shard = makeRawShard(); + const identity = shard.identity as { + case_id: string; + case_factors: { sku: string; case: Record }; + }; + const ep = makeRawMatrix([ + { + caseId: identity.case_id, + sku: identity.case_factors.sku, + case: identity.case_factors.case, + }, + ]); + const matrix = { + ...ep, + requested_cases: [ + ...(ep.requested_cases as unknown[]), + makeSwapRequestedCase('h200-dgxc', 'contiguous'), + makeSwapRequestedCase('h200-dgxc', 'random'), + makeSwapRequestedCase('mi300x', 'contiguous'), + ], + }; + + const dataset = buildDatasetFromNeutral(matrix, [shard, makeSwapDoc()], swapMeta); + + expect(dataset.coverage.map((row) => row.case_id)).toEqual([identity.case_id]); + expect(dataset.series).toHaveLength(1); + expect(dataset.swap_blocks?.map((result) => result.sku)).toEqual(['h200-dgxc']); + expect(dataset.run).toMatchObject({ + requested_cases: 4, + measured_cases: 2, + terminal_cases: 2, + swap_requested_cases: 3, + swap_measured_cases: 1, + // mi300x is requested but has no result, so it is not a covered pool. + covered_skus: ['h200-dgxc'], + }); + }); + + it('assembles a snapshot with requested cases and no results', () => { + const dataset = buildDatasetFromNeutral( + { version: 1, requested_cases: [makeSwapRequestedCase()] }, + [], + swapMeta, + ); + expect(dataset.swap_blocks).toEqual([]); + expect(buildRunSummary(dataset)).toMatchObject({ + requested_cases: 1, + measured_cases: 0, + swap_cases: { requested: 1, measured: 0 }, + }); + }); +}); + it('keeps identical measurements from separate GPU pools and rejects ambiguous provenance', () => { const matrix = { - include: [ - { backend: 'swap-blocks', sku: 'h200-dgxc' }, - { backend: 'swap-blocks', sku: 'mi355x' }, - ], + version: 1, + requested_cases: [makeSwapRequestedCase('h200-dgxc'), makeSwapRequestedCase('mi355x')], }; - const first = makeSwapDoc(); - const docs = ['h200-dgxc', 'mi355x'].map((sku) => ({ - ...first, - runtime: { ...first.runtime, sku }, - })); + const docs = ['h200-dgxc', 'mi355x'].map((sku) => makeSwapDoc(sku)); const dataset = buildDatasetFromNeutral(matrix, docs, swapMeta); expect(dataset.swap_blocks?.map((r) => [r.sku, r.points[0].latency_us.p50])).toEqual([ ['h200-dgxc', 2], ['mi355x', 2], ]); expect(new Set(dataset.swap_blocks?.map((r) => r.result_id)).size).toBe(2); - expect(() => buildDatasetFromNeutral(matrix, [first], swapMeta)).toThrow('GPU pool'); - expect(() => - buildDatasetFromNeutral( - matrix, - [{ ...first, runtime: { ...first.runtime, sku: 'unknown' } }], - swapMeta, - ), - ).toThrow('GPU pool'); + + const unlabeled = makeSwapDoc(); + delete (unlabeled.runtime as { sku?: string }).sku; + expect(() => buildDatasetFromNeutral(matrix, [unlabeled], swapMeta)).toThrow('GPU pool'); + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc('unknown')], swapMeta)).toThrow( + 'GPU pool', + ); + const unrequested = { ...makeSwapDoc(), case_id: swapCaseId('h200-dgxc', 'random') }; + expect(() => buildDatasetFromNeutral(matrix, [unrequested], swapMeta)).toThrow('requested case'); + expect(() => buildDatasetFromNeutral(matrix, [makeSwapDoc(), makeSwapDoc()], swapMeta)).toThrow( + 'requested case', + ); }); diff --git a/packages/db/src/collectivex/swap-reader.ts b/packages/db/src/collectivex/swap-reader.ts index 2f6c17590..2067420a4 100644 --- a/packages/db/src/collectivex/swap-reader.ts +++ b/packages/db/src/collectivex/swap-reader.ts @@ -1,16 +1,9 @@ import type { CollectiveXPercentiles, CollectiveXSwapPoint, CollectiveXSwapResult } from './types'; -export function isSwapMatrix(value: unknown): boolean { - const matrix = value as { - version?: unknown; - include?: { backend?: unknown; sku?: unknown }[]; - } | null; - return ( - matrix?.version === undefined && - Array.isArray(matrix?.include) && - matrix.include.length > 0 && - matrix.include.every((cell) => cell?.backend === 'swap-blocks' && typeof cell?.sku === 'string') - ); +/** The cases.json entries this reader needs: swap_blocks cases carry `suite: 'swap-blocks'`. */ +export interface SwapRequestedCase { + sku: string; + case: { suite?: string; case_id?: string }; } function positive(value: unknown): value is number { @@ -32,16 +25,23 @@ function percentiles(value: unknown): value is CollectiveXPercentiles { } export function readSwapResults( - matrix: unknown, + requestedCases: readonly SwapRequestedCase[], docs: unknown[], - sourceSha: string, ): CollectiveXSwapResult[] { - if (!isSwapMatrix(matrix)) return []; - const skus = (matrix as { include: { sku: string }[] }).include.map((cell) => cell.sku); + // Each result must answer one requested case of its own GPU pool, exactly once. + const requested = new Map>(); + for (const entry of requestedCases) { + if (entry.case.suite !== 'swap-blocks' || !entry.case.case_id) continue; + const caseIds = requested.get(entry.sku) ?? new Set(); + caseIds.add(entry.case.case_id); + requested.set(entry.sku, caseIds); + } + const answered = new Set(); const seen = new Set(); return docs.flatMap((doc) => { const raw = doc as { schema?: string; + case_id?: string; operation?: string; timing?: string; warmup: number; @@ -66,15 +66,20 @@ export function readSwapResults( !Number.isSafeInteger(raw.warmup) || raw.warmup < 0 || !raw.runtime || - raw.runtime.source_sha !== sourceSha || - !['device', 'torch', 'vllm', 'image'].every( + !['device', 'torch', 'vllm', 'image', 'source_sha'].every( (key) => typeof raw.runtime[key as keyof typeof raw.runtime] === 'string', ) ) { throw new TypeError('invalid swap_blocks artifact or provenance'); } - const sku = raw.runtime.sku ?? (skus.length === 1 ? skus[0] : undefined); - if (!sku || !skus.includes(sku)) throw new TypeError('invalid swap_blocks GPU pool'); + const { sku } = raw.runtime; + const caseIds = sku ? requested.get(sku) : undefined; + if (!sku || !caseIds) throw new TypeError('invalid swap_blocks GPU pool'); + const caseKey = JSON.stringify([sku, raw.case_id]); + if (!raw.case_id || !caseIds.has(raw.case_id) || answered.has(caseKey)) { + throw new TypeError('swap_blocks result does not answer one requested case'); + } + answered.add(caseKey); const points = raw.cases.map((row): CollectiveXSwapPoint => { const key = JSON.stringify([ sku, diff --git a/packages/db/src/collectivex/swap-test-fixture.ts b/packages/db/src/collectivex/swap-test-fixture.ts index 146e1a11a..7d0c3d598 100644 --- a/packages/db/src/collectivex/swap-test-fixture.ts +++ b/packages/db/src/collectivex/swap-test-fixture.ts @@ -1,4 +1,36 @@ -export const swapMatrix = { include: [{ backend: 'swap-blocks', sku: 'h200-dgxc' }] }; +export function swapCaseId(sku: string, layout = 'contiguous'): string { + return `${sku}-swap-blocks-smoke-${layout}`; +} + +/** One cases.json entry: the swap_blocks suite requests one case per GPU pool and layout. */ +export function makeSwapRequestedCase(sku = 'h200-dgxc', layout = 'contiguous') { + return { + sku, + disposition: 'runnable' as const, + reason: null, + detail: null, + case: { + suite: 'swap-blocks', + backend: 'swap-blocks', + case_id: swapCaseId(sku, layout), + profile: 'smoke', + layout, + directions: 'h2d d2h d2d', + block_bytes: '1024 1073741824', + num_blocks: '1 4', + max_payload_bytes: 1073741824, + warmup: 4, + iterations: 20, + seed: 0, + device: 0, + nodes: 1, + gpus_per_node: 1, + scale_up_domain: 1, + }, + }; +} + +export const swapMatrix = { version: 1, requested_cases: [makeSwapRequestedCase()] }; export const swapMeta = { run_id: '170', run_attempt: 1, @@ -6,21 +38,26 @@ export const swapMeta = { conclusion: 'success', source_sha: 'abc123', }; -export function makeSwapDoc() { +export function makeSwapDoc(sku = 'h200-dgxc') { return { schema: 'collectivex-swap-blocks-v1', + case_id: swapCaseId(sku), + profile: 'smoke', + generated_at: '2026-09-16T19:58:41.123456+08:00', operation: 'vllm._custom_ops.swap_blocks', timing: 'drained-wall-clock-including-submission-and-synchronization', warmup: 4, iterations: 20, runtime: { + sku, device: 'NVIDIA H200', torch: '2.11', vllm: '0.25.1', cuda: '13.0', hip: null, image: 'vllm/vllm-openai:v0.25.1', - source_sha: 'abc123', + source_sha: 'f'.repeat(40), + source_dirty: false, }, selection: { max_payload_bytes: 1073741824, diff --git a/packages/db/src/collectivex/test-fixture.ts b/packages/db/src/collectivex/test-fixture.ts index b8fd56c6f..f26e9e6de 100644 --- a/packages/db/src/collectivex/test-fixture.ts +++ b/packages/db/src/collectivex/test-fixture.ts @@ -100,6 +100,7 @@ function makeRawCase(options: ShardOverrides, caseId: string): Json { const phase = options.phase === 'prefill' ? 'prefill' : 'decode'; return { case_id: caseId, + suite: 'ep-core', backend: options.backend ?? 'deepep-v2', ep: options.ep ?? 8, gpus_per_node: options.gpusPerNode ?? 8, @@ -171,10 +172,10 @@ function requestedFromShard(shard: Json): RequestedCaseSpec { }; } +/** The shape of the benchmark repo's committed `collectivex/results/cases.json`. */ export function makeRawMatrix(requested: RequestedCaseSpec[], version = 1): Json { return { version, - include: [], requested_cases: requested.map((entry) => ({ case: entry.case, sku: entry.sku, diff --git a/packages/db/src/collectivex/types.ts b/packages/db/src/collectivex/types.ts index 2d10a90fa..36b163ed8 100644 --- a/packages/db/src/collectivex/types.ts +++ b/packages/db/src/collectivex/types.ts @@ -1,6 +1,6 @@ /** - * CollectiveX neutral contract — the version-tagged shape shared by the sweep - * artifacts' reader, the ingest script, and the dashboard frontend. + * CollectiveX neutral contract — the version-tagged shape shared by the + * committed-results reader, the Neon ingest, and the dashboard frontend. * * This module is pure TypeScript (no Node/DB imports) so it is safe to import * from client components. UI-only types (chart points, axis modes, display @@ -34,10 +34,13 @@ export type CollectiveXOutcome = export type CollectiveXTerminalStatus = Exclude | 'measured'; export type CollectiveXPercentiles = Record; +// EP rows: null where the harness withheld a tail because too many of its samples were misaligned +// or host-stalled. p50 is always published. +export type CollectiveXTailPercentiles = Record; export interface CollectiveXComponent { - latency_us: CollectiveXPercentiles; - activation_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + latency_us: CollectiveXTailPercentiles; + activation_data_rate_gbps_at_latency_percentile: CollectiveXTailPercentiles | null; /** * Per-GPU bandwidth over the FULL logical payload (activation bytes plus any * FP8 scale bytes), i.e. `total_logical_bytes / ep_size / latency`. Distinct @@ -45,7 +48,7 @@ export interface CollectiveXComponent { * and excludes scale bytes. Null when the component carries no byte * provenance (e.g. an unavailable component). */ - payload_data_rate_gbps_at_latency_percentile: CollectiveXPercentiles | null; + payload_data_rate_gbps_at_latency_percentile: CollectiveXTailPercentiles | null; /** * Aggregate total logical payload bytes for this component at this point * (the numerator behind `payload_data_rate_*`). Carried raw so a consumer can @@ -59,7 +62,7 @@ export interface CollectiveXPoint { tokens_per_rank: number; global_tokens: number; components: Record; - roundtrip_token_rate_at_latency_percentile: CollectiveXPercentiles; + roundtrip_token_rate_at_latency_percentile: CollectiveXTailPercentiles; } export interface CollectiveXTopology { diff --git a/packages/db/src/ingest-collectivex.test.ts b/packages/db/src/ingest-collectivex.test.ts index f93aaa2ba..296f7013f 100644 --- a/packages/db/src/ingest-collectivex.test.ts +++ b/packages/db/src/ingest-collectivex.test.ts @@ -1,124 +1,165 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; - import { afterEach, describe, expect, it, vi } from 'vitest'; +import type * as CommittedResults from './collectivex/committed-results'; +import { CommittedResultsError } from './collectivex/committed-results'; + const mocks = vi.hoisted(() => ({ - fetchRunMeta: vi.fn(), - buildDataset: vi.fn(), - begin: vi.fn(), + loadCommittedRun: vi.fn(), + upsert: vi.fn(), })); -vi.mock('./lib/github-artifacts', () => ({ - fetchRunMeta: mocks.fetchRunMeta, - listRunArtifacts: () => [], - downloadArtifact: () => {}, +vi.mock('./collectivex/committed-results', async (importOriginal) => ({ + ...(await importOriginal()), + loadCommittedRun: mocks.loadCommittedRun, })); -vi.mock('./collectivex/reader', () => ({ - isMatrixDoc: () => true, - matrixVersion: () => 1, - buildDatasetFromNeutral: mocks.buildDataset, - buildRunSummary: () => ({ requested_cases: 0, measured_cases: 0, requested_points: 0 }), -})); -vi.mock('./etl/db-utils', () => ({ - createAdminSql: () => ({ begin: mocks.begin, end: () => Promise.resolve() }), +vi.mock('./queries/collectivex', () => ({ + upsertCollectiveXRun: mocks.upsert, + pruneCollectiveXRuns: vi.fn(), + deleteCollectiveXRun: vi.fn(), })); -const RUN_ID = '170'; -const originalArgv = process.argv; -const originalExitCode = process.exitCode; +import { ingestCommits, parseIngestArgs } from './ingest-collectivex'; +import type { Sql } from './etl/db-utils'; + +const sql = (() => Promise.resolve([])) as unknown as Sql; afterEach(() => { - process.argv = originalArgv; - process.exitCode = originalExitCode; - vi.unstubAllEnvs(); vi.restoreAllMocks(); vi.clearAllMocks(); - vi.resetModules(); }); -// The script runs main() on import, so it must load after argv and the environment are set. -async function runCli(argv: string[]): Promise { - vi.spyOn(console, 'log').mockImplementation(() => {}); - vi.spyOn(console, 'error').mockImplementation(() => {}); - process.argv = ['bun', 'ingest-collectivex.ts', ...argv]; - await import('./ingest-collectivex'); -} +describe('parseIngestArgs', () => { + it('defaults to main', () => { + expect(parseIngestArgs([])).toEqual({ mode: 'commit', ref: 'main' }); + }); -describe('download-mode repository selection', () => { - // A non-sweep workflow path stops the run right after the repository lookup. - const otherWorkflow = { path: '.github/workflows/other.yml' }; - - it.each([ - { - name: 'the [repo] argument', - argv: [ - '--download', - `https://github.com/example-org/url-repo/actions/runs/${RUN_ID}`, - 'example-org/arg-repo', - ], - expected: 'example-org/arg-repo', - }, - { - name: 'a full run URL', - argv: ['--download', `https://github.com/example-org/url-repo/actions/runs/${RUN_ID}`], - expected: 'example-org/url-repo', - }, - { name: 'INGEST_REPO', argv: ['--download', RUN_ID], expected: 'example-org/env-repo' }, - ])('prefers $name', async ({ argv, expected }) => { - vi.stubEnv('INGEST_REPO', 'example-org/env-repo'); - mocks.fetchRunMeta.mockReturnValue(otherWorkflow); - - await runCli(argv); - - await vi.waitFor(() => expect(mocks.fetchRunMeta).toHaveBeenCalledWith(expected, RUN_ID)); + it('takes one commit from --sha or --ref', () => { + expect(parseIngestArgs(['--sha', 'a'.repeat(40)])).toEqual({ + mode: 'commit', + ref: 'a'.repeat(40), + }); + expect(parseIngestArgs(['--ref', 'v1.2', '--yes'])).toEqual({ mode: 'commit', ref: 'v1.2' }); }); - it('exits when no argument, run URL, or INGEST_REPO names the repository', async () => { - vi.stubEnv('INGEST_REPO', ''); - const exit = vi.spyOn(process, 'exit').mockImplementation(() => { - throw new Error('process.exit'); + it('walks main for --backfill, optionally pruning', () => { + expect(parseIngestArgs(['--backfill', '-y', '--no-ssl'])).toEqual({ + mode: 'backfill', + prune: false, }); + expect(parseIngestArgs(['--backfill', '--prune'])).toEqual({ mode: 'backfill', prune: true }); + }); - await expect(runCli(['--download', RUN_ID])).rejects.toThrow('process.exit'); + it('deletes one named run', () => { + expect(parseIngestArgs(['--delete-run', '1792039236'])).toEqual({ + mode: 'delete', + runId: '1792039236', + }); + }); - expect(exit).toHaveBeenCalledWith(1); - expect(console.error).toHaveBeenCalledWith('INGEST_REPO is not set'); - expect(mocks.fetchRunMeta).not.toHaveBeenCalled(); + it('rejects conflicting, malformed, and unknown arguments', () => { + expect(() => parseIngestArgs(['--sha', 'a'.repeat(40), '--ref', 'main'])).toThrow( + /mutually exclusive/u, + ); + expect(() => parseIngestArgs(['--backfill', '--ref', 'main'])).toThrow(/--backfill/u); + expect(() => parseIngestArgs(['--prune'])).toThrow(/--prune needs --backfill/u); + expect(() => parseIngestArgs(['--delete-run', '5', '--backfill'])).toThrow(/--delete-run/u); + expect(() => parseIngestArgs(['--delete-run', '37459990486'])).toThrow(/Invalid run id/u); + expect(() => parseIngestArgs(['--delete-run', '0'])).toThrow(/Invalid run id/u); + expect(() => parseIngestArgs(['--sha', 'not-a-sha'])).toThrow(/Invalid commit sha/u); + expect(() => parseIngestArgs(['12345'])).toThrow(); }); }); -describe('recorded source repository', () => { - it.each([ - { repo: 'example-org/bench', recorded: 'example-org/bench' }, - { repo: 'semianalysisai/inferencex', recorded: undefined }, - ])('records $repo as $recorded', async ({ repo, recorded }) => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cx-ingest-test-')); - try { - fs.mkdirSync(path.join(dir, `cxsweep-matrix-${RUN_ID}`)); - fs.writeFileSync(path.join(dir, `cxsweep-matrix-${RUN_ID}`, 'matrix.json'), '{}'); - vi.stubEnv('INGEST_RUN_ID', RUN_ID); - vi.stubEnv('INGEST_ARTIFACTS_PATH', dir); - vi.stubEnv('INGEST_REPO', repo); - mocks.fetchRunMeta.mockReturnValue({ - path: '.github/workflows/collectivex-sweep.yml', - name: 'CollectiveX sweep', - run_attempt: 1, - head_sha: 'abc', - head_branch: 'main', - conclusion: 'success', - created_at: '2026-08-01T00:00:00Z', - }); - mocks.buildDataset.mockReturnValue({}); - mocks.begin.mockResolvedValue(undefined); - - await runCli([]); - - await vi.waitFor(() => expect(mocks.begin).toHaveBeenCalled()); - const meta = mocks.buildDataset.mock.calls[0][2] as Record; - expect(meta.source_repository).toBe(recorded); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } +function runFor(sha: string, measured = 1) { + return { + run: { run_id: sha.slice(0, 3), summary: { measured_cases: measured, requested_cases: 2 } }, + docs: measured > 0 ? [{}] : [], + }; +} + +describe('ingestCommits', () => { + const commits = [ + { sha: 'a'.repeat(40), committedAt: '2026-10-15T00:00:00Z', runId: '1760486400' }, + { sha: 'b'.repeat(40), committedAt: '2026-10-16T00:00:00Z', runId: '1760572800' }, + { sha: 'c'.repeat(40), committedAt: '2026-10-17T00:00:00Z', runId: '1760659200' }, + ]; + + it('stores commits in order and shares one blob cache across them', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('inserted'); + + const outcome = await ingestCommits(sql, commits, 'token'); + + expect(mocks.loadCommittedRun.mock.calls.map(([commit]) => commit.sha)).toEqual( + commits.map((commit) => commit.sha), + ); + const [first, second] = mocks.loadCommittedRun.mock.calls.map(([, options]) => options); + expect(first.token).toBe('token'); + expect(first.blobCache).toBe(second.blobCache); + expect(mocks.upsert).toHaveBeenCalledTimes(3); + expect(outcome).toEqual({ stored: commits, skipped: [], missing: [] }); + }); + + it('skips a commit with nothing measured instead of storing an empty run', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha, commit.sha === commits[1].sha ? 0 : 1)), + ); + mocks.upsert.mockResolvedValue('inserted'); + + const outcome = await ingestCommits(sql, commits, undefined); + + expect(mocks.upsert).toHaveBeenCalledTimes(2); + expect(outcome.skipped).toEqual([commits[1]]); + expect(outcome.stored).toEqual([commits[0], commits[2]]); + }); + + it('skips commits GitHub answers 404 for only when asked to', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + commit.sha === commits[0].sha + ? Promise.reject(new CommittedResultsError('not-found', 'gone')) + : Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('inserted'); + + await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow('gone'); + const outcome = await ingestCommits(sql, commits, undefined, { skipMissing: true }); + expect(outcome.skipped).toEqual([commits[0]]); + expect(outcome.missing).toEqual([commits[0]]); + expect(outcome.stored).toEqual(commits.slice(1)); + }); + + it('skips a commit that deleted its results without counting it as missing', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + commit.sha === commits[0].sha + ? Promise.reject(new CommittedResultsError('absent', 'deleted')) + : Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('inserted'); + const outcome = await ingestCommits(sql, commits, undefined); + expect(outcome.skipped).toEqual([commits[0]]); + expect(outcome.missing).toEqual([]); + expect(outcome.stored).toEqual(commits.slice(1)); + }); + + it('stops at invalid results and at a run id held by a different commit', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + mocks.loadCommittedRun.mockRejectedValueOnce(new CommittedResultsError('invalid', 'bad')); + await expect(ingestCommits(sql, commits, undefined, { skipMissing: true })).rejects.toThrow( + 'bad', + ); + + mocks.loadCommittedRun.mockImplementation((commit: { sha: string }) => + Promise.resolve(runFor(commit.sha)), + ); + mocks.upsert.mockResolvedValue('conflict'); + await expect(ingestCommits(sql, commits, undefined)).rejects.toThrow( + /already holds a different commit/u, + ); + expect(mocks.loadCommittedRun).toHaveBeenCalledTimes(2); }); }); diff --git a/packages/db/src/ingest-collectivex.ts b/packages/db/src/ingest-collectivex.ts index 69cf78028..bbda5a4f3 100644 --- a/packages/db/src/ingest-collectivex.ts +++ b/packages/db/src/ingest-collectivex.ts @@ -1,164 +1,183 @@ -import fs from 'fs'; -import os from 'os'; -import path from 'path'; +import { parseArgs } from 'node:util'; -import { GITHUB_REPO_FULL } from '@semianalysisai/inferencex-constants'; - -import { hasNoSslFlag, requireIngestRepo } from './cli-utils'; -import { createAdminSql } from './etl/db-utils'; +import { confirm, hasNoSslFlag, hasYesFlag } from './cli-utils'; +import { createAdminSql, type Sql } from './etl/db-utils'; import { - downloadArtifact, - fetchRunMeta, - listRunArtifacts, - type RunMeta, -} from './lib/github-artifacts'; -import { repositoryFromRunUrl } from './lib/runtime-metadata-artifacts'; -import { matrixArtifactName, selectShardArtifactNames } from './collectivex/artifact-selection'; + CommittedResultsError, + listResultsCommits, + loadCommittedRun, + MAX_COMMITTED_RUN_ID, + RESULTS_BRANCH, + RESULTS_DIR, + resolveResultsCommit, + type ResultsCommit, +} from './collectivex/committed-results'; import { - buildDatasetFromNeutral, - buildRunSummary, - isMatrixDoc, - matrixVersion, - type CollectiveXNeutralRunMeta, -} from './collectivex/reader'; - -const SWEEP_WORKFLOW_PATH = '.github/workflows/collectivex-sweep.yml'; -const DOCS_INSERT_CHUNK = 200; - -const isDownloadMode = process.argv[2] === '--download'; - -let artifactsDir: string; -let runIdStr: string; -let REPO: string; -let tempDir: string | null = null; - -if (isDownloadMode) { - const args = process.argv.slice(3).filter((a) => !a.startsWith('--')); - const input = args[0]; - if (!input) { - console.error('Usage: bun run admin:db:ingest:collectivex [repo]'); - process.exit(1); + deleteCollectiveXRun, + pruneCollectiveXRuns, + upsertCollectiveXRun, +} from './queries/collectivex'; + +const USAGE = + 'Usage: bun run admin:db:ingest:collectivex ' + + '[--sha | --ref | --backfill [--prune] | --delete-run ] [--yes] [--no-ssl]'; + +export type IngestCollectiveXArgs = + | { mode: 'commit'; ref: string } + | { mode: 'backfill'; prune: boolean } + | { mode: 'delete'; runId: string }; + +/** + * --sha and --ref name one commit (default: main); --backfill walks every results + * commit on main, and with --prune then deletes stored runs it did not store; + * --delete-run removes one stored run. + */ +export function parseIngestArgs(argv: string[]): IngestCollectiveXArgs { + const { values } = parseArgs({ + args: argv, + options: { + sha: { type: 'string' }, + ref: { type: 'string' }, + backfill: { type: 'boolean', default: false }, + prune: { type: 'boolean', default: false }, + 'delete-run': { type: 'string' }, + yes: { type: 'boolean', short: 'y', default: false }, + 'no-ssl': { type: 'boolean', default: false }, + }, + strict: true, + allowPositionals: false, + }); + const deleteRun = values['delete-run']; + if (values.sha !== undefined && values.ref !== undefined) { + throw new Error(`--sha and --ref are mutually exclusive\n${USAGE}`); + } + if (values.backfill && (values.sha !== undefined || values.ref !== undefined)) { + throw new Error(`--backfill reads every commit on ${RESULTS_BRANCH}\n${USAGE}`); } - const match = input.match(/\/runs\/(?\d+)/u); - const parsedId = match ? match.groups!.runId : /^\d+$/u.test(input) ? input : null; - if (!parsedId) { - console.error(`Could not parse run ID from: ${input}`); - process.exit(1); + if (values.prune && !values.backfill) { + throw new Error( + `--prune needs --backfill: it keeps exactly the runs a backfill stores\n${USAGE}`, + ); } - runIdStr = parsedId; - REPO = args[1] ?? repositoryFromRunUrl(input) ?? requireIngestRepo(); - tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cx-ingest-')); - artifactsDir = tempDir; -} else { - const runId = process.env.INGEST_RUN_ID; - const artifactsPath = process.env.INGEST_ARTIFACTS_PATH; - if (!runId || !artifactsPath) { - console.error('INGEST_RUN_ID and INGEST_ARTIFACTS_PATH are required without --download'); - process.exit(1); + if ( + deleteRun !== undefined && + (values.backfill || values.sha !== undefined || values.ref !== undefined) + ) { + throw new Error(`--delete-run takes no other target\n${USAGE}`); } - runIdStr = runId; - REPO = requireIngestRepo(); - artifactsDir = artifactsPath; + if (values.sha !== undefined && !/^[0-9a-f]{7,40}$/u.test(values.sha)) { + throw new Error(`Invalid commit sha: ${values.sha}`); + } + if (deleteRun !== undefined) { + if (!/^[1-9]\d{0,9}$/u.test(deleteRun) || Number(deleteRun) >= MAX_COMMITTED_RUN_ID) { + throw new Error(`Invalid run id: ${deleteRun}`); + } + return { mode: 'delete', runId: deleteRun }; + } + if (values.backfill) return { mode: 'backfill', prune: values.prune }; + return { mode: 'commit', ref: values.sha ?? values.ref ?? RESULTS_BRANCH }; } -if (!/^[\w.-]+\/[\w.-]+$/u.test(REPO)) { - console.error(`Invalid repo slug: ${REPO}`); - process.exit(1); -} -if (!/^\d+$/u.test(runIdStr)) { - console.error(`Invalid run id: ${runIdStr}`); - process.exit(1); +export interface IngestOutcome { + /** Commits whose run is stored (inserted or replaced). */ + stored: ResultsCommit[]; + /** Commits not stored: no results tree or file at them, or nothing measured. */ + skipped: ResultsCommit[]; + /** The skipped commits GitHub answered 404 for: a prune must not trust this walk. */ + missing: ResultsCommit[]; } -function readDocsDir(dir: string): unknown[] { - const docs: unknown[] = []; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - docs.push(...readDocsDir(full)); +/** + * Ingest each commit in order. A commit with no measured result (for example one + * that only changed cases.json) is skipped, never stored as an empty run. With + * `skipMissing` (the backfill), a commit GitHub answers 404 for is skipped too; + * any other failure stops the walk before later commits. + */ +export async function ingestCommits( + sql: Sql, + commits: readonly ResultsCommit[], + token: string | undefined, + { skipMissing = false }: { skipMissing?: boolean } = {}, +): Promise { + const blobCache = new Map(); + const outcome: IngestOutcome = { stored: [], skipped: [], missing: [] }; + for (const commit of commits) { + const label = `${commit.sha.slice(0, 12)} (${commit.committedAt})`; + let loaded: Awaited>; + try { + loaded = await loadCommittedRun(commit, { token, blobCache }); + } catch (error) { + if (error instanceof CommittedResultsError && error.kind === 'absent') { + console.log(` skipped ${label}: ${error.message}`); + outcome.skipped.push(commit); + continue; + } + if (skipMissing && error instanceof CommittedResultsError && error.kind === 'not-found') { + console.log(` skipped ${label}: ${error.message}`); + outcome.skipped.push(commit); + outcome.missing.push(commit); + continue; + } + throw error; + } + const { run, docs } = loaded; + if (run.summary.measured_cases === 0) { + console.log( + ` skipped ${label}: nothing measured (${run.summary.requested_cases} cases requested)`, + ); + outcome.skipped.push(commit); continue; } - if (!entry.name.endsWith('.json')) continue; - docs.push(JSON.parse(fs.readFileSync(full, 'utf8'))); + const result = await upsertCollectiveXRun(sql, run, docs); + if (result === 'conflict') { + throw new Error(`run ${run.run_id} already holds a different commit than ${commit.sha}`); + } + console.log( + ` ${result} run ${run.run_id} @ ${label}: ` + + `${run.summary.measured_cases}/${run.summary.requested_cases} cases measured, ${docs.length} docs`, + ); + outcome.stored.push(commit); } - return docs; + return outcome; } -function runGeneratedAt(run: RunMeta): string { - return run.updated_at || run.run_started_at || run.created_at || ''; +function describePlan(args: IngestCollectiveXArgs, commits: readonly ResultsCommit[]): string { + if (args.mode === 'delete') { + return `This will delete stored run ${args.runId} and its documents.\n`; + } + if (args.mode === 'commit') { + return ( + `This will store ${RESULTS_DIR} at ${commits[0].sha} (resolved from ${args.ref}) as run ` + + `${commits[0].runId}, replacing any run already stored for that commit.\n` + ); + } + const then = args.prune ? ', then DELETE every stored run it did not store' : ''; + return `This will store ${commits.length} ${RESULTS_DIR} commits from ${RESULTS_BRANCH}, oldest first, replacing any run already stored for the same commit${then}.\n`; } async function main(): Promise { - console.log(`=== db:ingest:collectivex — run ${runIdStr} (${REPO}) ===`); - - const run = fetchRunMeta(REPO, runIdStr); - if (run.path !== SWEEP_WORKFLOW_PATH) { - throw new Error(`run ${runIdStr} is not a CollectiveX sweep (workflow: ${run.path})`); - } - const generatedAt = runGeneratedAt(run); - if (!generatedAt) throw new Error(`run ${runIdStr} is missing a timestamp`); - console.log( - ` workflow: ${run.name} branch: ${run.head_branch ?? '?'} attempt: ${run.run_attempt} conclusion: ${run.conclusion ?? 'in-progress'}`, - ); - - const matrixDirName = matrixArtifactName(runIdStr); - - if (isDownloadMode) { - const artifacts = listRunArtifacts(REPO, runIdStr); - const byName = new Map(); - for (const artifact of artifacts) { - const existing = byName.get(artifact.name); - if (!existing || artifact.created_at > existing.created_at) { - byName.set(artifact.name, artifact); - } - } - const wanted = [ - matrixDirName, - ...selectShardArtifactNames([...byName.keys()], runIdStr, run.run_attempt), - ]; - for (const name of wanted) { - const artifact = byName.get(name); - if (!artifact) continue; - console.log(` downloading ${name}`); - downloadArtifact(artifact, artifactsDir); + const args = parseIngestArgs(process.argv.slice(2)); + const token = process.env.GITHUB_TOKEN || undefined; + console.log('=== db:ingest:collectivex ==='); + + let commits: ResultsCommit[] = []; + if (args.mode === 'backfill') { + commits = await listResultsCommits(RESULTS_BRANCH, { token }); + if (commits.length === 0 && !args.prune) { + console.log(`No commit on ${RESULTS_BRANCH} touched ${RESULTS_DIR}.`); + return; } + } else if (args.mode === 'commit') { + commits = [await resolveResultsCommit(args.ref, { token })]; } - - const availableDirs = fs - .readdirSync(artifactsDir, { withFileTypes: true }) - .filter((entry) => entry.isDirectory()) - .map((entry) => entry.name); - - if (!availableDirs.includes(matrixDirName)) { - throw new Error(`run ${runIdStr} has no ${matrixDirName} artifact`); - } - const matrixDocs = readDocsDir(path.join(artifactsDir, matrixDirName)).filter((doc) => - isMatrixDoc(doc), - ); - if (matrixDocs.length !== 1) { - throw new Error(`expected exactly one matrix document, found ${matrixDocs.length}`); + console.log(describePlan(args, commits)); + if (!hasYesFlag()) { + const ok = await confirm('Continue? (y/N) '); + if (!ok) { + console.log('Aborted.'); + return; + } } - const matrix = matrixDocs[0]; - const version = matrixVersion(matrix); - if (version === null) throw new Error('matrix document has no valid version tag'); - - const shardDirs = selectShardArtifactNames(availableDirs, runIdStr, run.run_attempt); - console.log(` matrix version: ${version} shard artifacts: ${shardDirs.length}`); - const docs = shardDirs.flatMap((dir) => readDocsDir(path.join(artifactsDir, dir))); - - const meta: CollectiveXNeutralRunMeta = { - run_id: runIdStr, - run_attempt: run.run_attempt, - generated_at: generatedAt, - conclusion: run.conclusion, - source_sha: run.head_sha, - ...(REPO.toLowerCase() === GITHUB_REPO_FULL.toLowerCase() ? {} : { source_repository: REPO }), - }; - const dataset = buildDatasetFromNeutral(matrix, docs, meta); - const summary = buildRunSummary(dataset); - console.log( - ` assembled: ${summary.requested_cases} cases (${summary.measured_cases} measured), ${summary.requested_points} points`, - ); const sql = createAdminSql({ envVar: 'DATABASE_COLLECTIVEX_WRITE_URL', @@ -167,37 +186,45 @@ async function main(): Promise { onnotice: () => {}, }); try { - await sql.begin(async (tx) => { - await tx`DELETE FROM cx_runs WHERE run_id = ${runIdStr}`; - await tx` - INSERT INTO cx_runs - (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) - VALUES - (${runIdStr}, ${run.run_attempt}, ${version}, ${generatedAt}, ${run.head_sha}, - ${run.head_branch}, ${run.conclusion}, ${matrix as never}::jsonb, - ${summary as never}::jsonb) - `; - for (let i = 0; i < docs.length; i += DOCS_INSERT_CHUNK) { - const chunk = docs.slice(i, i + DOCS_INSERT_CHUNK).map((doc) => JSON.stringify(doc)); - await tx` - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT ${runIdStr}, ${run.run_attempt}, unnest(${tx.array(chunk)}::jsonb[]) - `; + if (args.mode === 'delete') { + const removed = await deleteCollectiveXRun(sql, args.runId); + console.log( + removed + ? ` deleted run ${removed.run_id} @ ${removed.source_sha.slice(0, 12)}` + : ` no stored run ${args.runId}`, + ); + } else { + const { stored, skipped, missing } = await ingestCommits(sql, commits, token, { + skipMissing: args.mode === 'backfill', + }); + console.log(` ${stored.length} stored, ${skipped.length} skipped`); + if (args.mode === 'backfill' && args.prune && missing.length > 0) { + // A 404 can be a transient GitHub miss; pruning after it would delete a good run. + throw new Error( + `not pruning: ${missing.length} commit(s) could not be read ` + + `(${missing.map((commit) => commit.sha.slice(0, 12)).join(', ')}); rerun the backfill`, + ); } - }); - console.log(` stored run ${runIdStr} (version ${version}, ${docs.length} docs)`); + if (args.mode === 'backfill' && args.prune) { + const removed = await pruneCollectiveXRuns( + sql, + stored.map((commit) => commit.sha), + ); + for (const run of removed) { + console.log(` pruned run ${run.run_id} @ ${run.source_sha.slice(0, 12)}`); + } + console.log(` ${removed.length} pruned`); + } + } } finally { await sql.end(); } - console.log('=== db:ingest:collectivex complete ==='); } -main() - .catch((error) => { +if (import.meta.main) { + main().catch((error: unknown) => { console.error('db:ingest:collectivex failed:', error); process.exitCode = 1; - }) - .finally(() => { - if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); }); +} diff --git a/packages/db/src/queries/collectivex-db.test.ts b/packages/db/src/queries/collectivex-db.test.ts new file mode 100644 index 000000000..fb983f349 --- /dev/null +++ b/packages/db/src/queries/collectivex-db.test.ts @@ -0,0 +1,236 @@ +import fs from 'node:fs'; + +import { PGlite } from '@electric-sql/pglite'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { + assignRunIds, + committedRun, + validateCommittedResults, + type ResultsSource, +} from '../collectivex/committed-results'; +import { makeRawMatrix, makeRawShard } from '../collectivex/test-fixture'; +import { pgliteSql, type PgliteSql } from '../lib/test-pglite'; +import { + collectiveXDatasetFromRow, + deleteCollectiveXRun, + getCollectiveXRun, + getLatestCollectiveXRun, + listCollectiveXRuns, + pruneCollectiveXRuns, + upsertCollectiveXRun, +} from './collectivex'; + +type Json = Record; + +const FIRST_SHA = 'a'.repeat(40); +const SECOND_SHA = 'b'.repeat(40); +const FIRST_AT = '2026-10-28T00:00:00Z'; +const SECOND_AT = '2026-10-29T00:00:00Z'; + +const MIGRATIONS = new URL('../../migrations-collectivex/', import.meta.url); + +async function applyMigrations(db: PGlite, filter: (name: string) => boolean): Promise { + for (const name of fs.readdirSync(MIGRATIONS).toSorted()) { + if (name.endsWith('.sql') && filter(name)) { + await db.exec(fs.readFileSync(new URL(name, MIGRATIONS), 'utf8')); + } + } +} + +async function collectiveXDb(): Promise { + const db = await PGlite.create(); + await applyMigrations(db, () => true); + return pgliteSql(db); +} + +function result(sku: string, vendor: 'amd' | 'nvidia'): Json { + return { + ...makeRawShard({ sku, vendor }), + provenance: { source_sha: 'e'.repeat(40), source_dirty: false }, + }; +} + +// The first committed results cover two AMD pools; cases.json requests every pool. +const mi300x = result('mi300x', 'amd'); +const mi325x = result('mi325x', 'amd'); +const h200 = result('h200-dgxc', 'nvidia'); +const b200 = result('b200-nscale', 'nvidia'); +const cases = makeRawMatrix( + [mi300x, mi325x, h200, b200].map((doc) => ({ + caseId: doc.identity.case_id, + sku: doc.identity.case_factors.sku, + case: doc.identity.case_factors.case, + })), +); + +function encode(value: unknown): Uint8Array { + return new TextEncoder().encode(JSON.stringify(value)); +} + +function source(sha: string, committedAt: string, docs: Json[]): ResultsSource { + const [{ runId }] = assignRunIds([{ sha, committedAt }]); + return { + sha, + committedAt, + runId, + files: new Map([ + ['cases.json', encode(cases)], + ...docs.map((doc): [string, Uint8Array] => [ + `${doc.identity.case_factors.sku}/${doc.identity.case_id}.json`, + encode(doc), + ]), + ]), + }; +} + +let sql: PgliteSql; + +beforeEach(async () => { + sql = await collectiveXDb(); +}); + +describe('CollectiveX runs from committed results in PostgreSQL', () => { + it('serves a first commit holding only the mi300x and mi325x folders', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + await expect(upsertCollectiveXRun(sql, first.run, first.docs)).resolves.toBe('inserted'); + + const row = await getLatestCollectiveXRun(sql, 1); + expect(row).not.toBeNull(); + const dataset = collectiveXDatasetFromRow(row!); + expect(dataset.run).toMatchObject({ + run_id: first.run.run_id, + source_sha: FIRST_SHA, + requested_cases: 4, + measured_cases: 2, + }); + expect(dataset.series.map((series) => series.system.sku).toSorted()).toEqual([ + 'mi300x', + 'mi325x', + ]); + // Pools without a result file stay requested but not run, never an error. + expect(Object.fromEntries(dataset.coverage.map((item) => [item.sku, item.outcome]))).toEqual({ + mi300x: 'success', + mi325x: 'success', + 'h200-dgxc': 'pending', + 'b200-nscale': 'pending', + }); + }); + + it('stores a later commit that adds pools as a newer superset run and serves it as latest', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + const second = committedRun( + validateCommittedResults(source(SECOND_SHA, SECOND_AT, [mi300x, mi325x, h200, b200])), + ); + await upsertCollectiveXRun(sql, first.run, first.docs); + await expect(upsertCollectiveXRun(sql, second.run, second.docs)).resolves.toBe('inserted'); + + expect(Number(second.run.run_id)).toBeGreaterThan(Number(first.run.run_id)); + const latest = await getLatestCollectiveXRun(sql, 1); + expect(latest?.source_sha).toBe(SECOND_SHA); + expect(collectiveXDatasetFromRow(latest!).run.measured_cases).toBe(4); + + const runs = await listCollectiveXRuns(sql, 1); + expect(runs.map((run) => run.run_id)).toEqual([second.run.run_id, first.run.run_id]); + expect(runs.map((run) => run.measured_cases)).toEqual([4, 2]); + + const earlier = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(earlier?.docs).toHaveLength(2); + }); + + it('replaces a re-ingested commit in place and refuses a run id held by another commit', async () => { + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + await upsertCollectiveXRun(sql, first.run, first.docs); + const again = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + await expect(upsertCollectiveXRun(sql, again.run, again.docs)).resolves.toBe('replaced'); + const row = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(row?.docs).toHaveLength(2); + + const clash = committedRun(validateCommittedResults(source(SECOND_SHA, FIRST_AT, [h200]))); + await expect(upsertCollectiveXRun(sql, clash.run, clash.docs)).resolves.toBe('conflict'); + const kept = await getCollectiveXRun(sql, 1, first.run.run_id); + expect(kept?.source_sha).toBe(FIRST_SHA); + expect(await listCollectiveXRuns(sql, 1)).toHaveLength(1); + }); + + it('never serves a run with nothing measured', async () => { + const empty = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, []))); + expect(empty.run.summary.measured_cases).toBe(0); + await upsertCollectiveXRun(sql, empty.run, empty.docs); + expect(await getLatestCollectiveXRun(sql, 1)).toBeNull(); + expect(await getCollectiveXRun(sql, 1, empty.run.run_id)).toBeNull(); + expect(await listCollectiveXRuns(sql, 1)).toEqual([]); + }); + + it('stores each document once when two ingests of one commit overlap', async () => { + const first = committedRun( + validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x, mi325x])), + ); + const results = await Promise.all([ + upsertCollectiveXRun(sql, first.run, first.docs), + upsertCollectiveXRun(sql, first.run, first.docs), + upsertCollectiveXRun(sql, first.run, first.docs), + ]); + expect(results.toSorted()).toEqual(['inserted', 'replaced', 'replaced']); + const [{ count }] = await sql`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(2); + }); + + it('prunes runs outside the kept commits and deletes one named run', async () => { + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + const second = committedRun(validateCommittedResults(source(SECOND_SHA, SECOND_AT, [h200]))); + await upsertCollectiveXRun(sql, first.run, first.docs); + await upsertCollectiveXRun(sql, second.run, second.docs); + + await expect(pruneCollectiveXRuns(sql, [SECOND_SHA])).resolves.toEqual([ + { run_id: first.run.run_id, source_sha: FIRST_SHA }, + ]); + const kept = await listCollectiveXRuns(sql, 1); + expect(kept.map((run) => run.run_id)).toEqual([second.run.run_id]); + const [{ count }] = await sql`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(1); + + await expect(deleteCollectiveXRun(sql, second.run.run_id)).resolves.toEqual({ + run_id: second.run.run_id, + source_sha: SECOND_SHA, + }); + await expect(deleteCollectiveXRun(sql, second.run.run_id)).resolves.toBeNull(); + expect(await getLatestCollectiveXRun(sql, 1)).toBeNull(); + }); +}); + +describe('migration 005', () => { + it('drops lazy-ingest runs that would outrank committed runs, and forbids new ones', async () => { + const db = await PGlite.create(); + await applyMigrations(db, (name) => name < '005'); + const legacy = pgliteSql(db); + const first = committedRun(validateCommittedResults(source(FIRST_SHA, FIRST_AT, [mi300x]))); + await upsertCollectiveXRun(legacy, first.run, first.docs); + await upsertCollectiveXRun( + legacy, + { ...first.run, run_id: '37459990486', source_sha: SECOND_SHA }, + first.docs, + ); + const before = await getLatestCollectiveXRun(legacy, 1); + expect(before?.run_id).toBe('37459990486'); + + await applyMigrations(db, (name) => name >= '005'); + const after = await getLatestCollectiveXRun(legacy, 1); + expect(after?.run_id).toBe(first.run.run_id); + const [{ count }] = await legacy`SELECT count(*)::int AS count FROM cx_run_docs`; + expect(count).toBe(1); + await expect( + upsertCollectiveXRun( + legacy, + { ...first.run, run_id: '37459990487', source_sha: SECOND_SHA }, + first.docs, + ), + ).rejects.toThrow(/cx_runs_committed_run_id/u); + }); +}); diff --git a/packages/db/src/queries/collectivex.test.ts b/packages/db/src/queries/collectivex.test.ts index ddeb8eb56..e7372d036 100644 --- a/packages/db/src/queries/collectivex.test.ts +++ b/packages/db/src/queries/collectivex.test.ts @@ -2,15 +2,13 @@ import { describe, expect, it } from 'vitest'; import { makeRawMatrix, makeRawShard, makeRunMeta } from '../collectivex/test-fixture'; import type { DbClient } from '../connection.js'; +import type { Sql } from '../etl/db-utils'; import { collectiveXDatasetFromRow, - deleteCollectiveXRun, getCollectiveXRun, - getCollectiveXRunStates, getLatestCollectiveXRun, - insertCollectiveXRun, listCollectiveXRuns, - refreshCollectiveXRunAttempt, + upsertCollectiveXRun, type CollectiveXRunInsert, type CollectiveXRunRow, } from './collectivex'; @@ -22,25 +20,28 @@ interface Captured { function fakeSql(rowsQueue: Record[][]) { const calls: Captured[] = []; - const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { + const tag = ((strings: TemplateStringsArray, ...values: unknown[]) => { calls.push({ text: strings.join('?'), values }); return Promise.resolve(rowsQueue.shift() ?? []); }) as DbClient; + const sql = Object.assign(tag, { + begin: (fn: (tx: DbClient) => Promise) => fn(tag), + }) as unknown as Sql & DbClient; return { sql, calls }; } const runInsert: CollectiveXRunInsert = { - run_id: '160', - run_attempt: 2, + run_id: '1783513200', + run_attempt: 1, version: 1, generated_at: '2026-07-08T12:20:00Z', source_sha: 'a'.repeat(40), - source_branch: 'collectivex', + source_branch: 'main', conclusion: 'success', - matrix: { version: 1, requested_cases: [], include: [] }, + matrix: { version: 1, requested_cases: [] }, summary: { - run_id: '160', - run_attempt: 2, + run_id: '1783513200', + run_attempt: 1, generated_at: '2026-07-08T12:20:00Z', conclusion: 'success', covered_skus: [], @@ -52,37 +53,16 @@ const runInsert: CollectiveXRunInsert = { }, }; -describe('getCollectiveXRunStates', () => { - it('maps deleted flags to tombstone states with version and attempt', async () => { - const { sql } = fakeSql([ - [ - { run_id: '160', version: 1, run_attempt: 2, deleted: false }, - { run_id: '161', version: 2, run_attempt: 1, deleted: true }, - ], - ]); - const states = await getCollectiveXRunStates(sql, ['160', '161']); - expect(states).toEqual({ - '160': { state: 'live', version: 1, run_attempt: 2 }, - '161': { state: 'deleted', version: 2, run_attempt: 1 }, - }); - }); - - it('skips the query entirely for an empty id list', async () => { - const { sql, calls } = fakeSql([]); - expect(await getCollectiveXRunStates(sql, [])).toEqual({}); - expect(calls).toHaveLength(0); - }); -}); - describe('read queries', () => { - it('excludes tombstoned and unsupported-vendor-only rows', async () => { + it('excludes unsupported-vendor-only rows and orders newest commit first', async () => { const { sql, calls } = fakeSql([[], [], []]); await getLatestCollectiveXRun(sql, 1); await getCollectiveXRun(sql, 1, '160'); await listCollectiveXRuns(sql, 1); for (const call of calls) { - expect(call.text).toContain('deleted_at IS NULL'); + expect(call.text).not.toContain('deleted_at'); expect(call.text).toContain("summary->>'requested_cases'"); + expect(call.text).toContain("summary->>'measured_cases')::int, 0) > 0"); } expect(calls[0].text).toContain('ORDER BY r.run_id DESC'); expect(calls[2].text).toContain('ORDER BY run_id DESC'); @@ -99,68 +79,50 @@ describe('read queries', () => { }); }); -describe('insertCollectiveXRun', () => { - it('binds raw objects (never pre-stringified JSON) and reports insertion', async () => { - const { sql, calls } = fakeSql([[{ runs_inserted: 1 }]]); +describe('upsertCollectiveXRun', () => { + it('binds raw objects (never pre-stringified JSON) and reports a first insert', async () => { + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); const docs = [{ record_type: 'case-attempt' }]; - await expect(insertCollectiveXRun(sql, runInsert, docs)).resolves.toBe(true); + await expect(upsertCollectiveXRun(sql, runInsert, docs)).resolves.toBe('inserted'); - expect(calls[0].text).toContain('ON CONFLICT (run_id) DO NOTHING'); - expect(calls[0].values).toContainEqual(runInsert.matrix); - expect(calls[0].values).toContainEqual({ docs }); + expect(calls).toHaveLength(2); + expect(calls[1].values).toContainEqual(runInsert.matrix); + expect(calls[1].values).toContainEqual(runInsert.summary); + expect(calls[1].values).toContainEqual({ docs }); expect( - calls[0].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), + calls[1].values.every((value) => typeof value !== 'string' || !value.startsWith('{')), ).toBe(true); }); - it('reports a conflict no-op as not inserted', async () => { - const { sql } = fakeSql([[{ runs_inserted: 0 }]]); - await expect(insertCollectiveXRun(sql, runInsert, [])).resolves.toBe(false); + it('takes the CollectiveX write lock in the transaction before writing', async () => { + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); + await upsertCollectiveXRun(sql, runInsert, []); + expect(calls[0].text).toBe('SELECT pg_advisory_xact_lock(7384110219)'); }); -}); -describe('refreshCollectiveXRunAttempt', () => { - it('guards on a strictly newer attempt and reports replacement', async () => { - const { sql, calls } = fakeSql([[{ runs_updated: 1 }]]); - await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(true); - expect(calls[0].text).toContain('run_attempt < '); - expect(calls[0].text).toContain('deleted_at IS NULL'); - expect(calls[0].text).toContain('FOR UPDATE'); + it('replaces the row and its documents when the same commit is ingested again', async () => { + const { sql, calls } = fakeSql([[], [{ inserted: false }]]); + await expect(upsertCollectiveXRun(sql, runInsert, [])).resolves.toBe('replaced'); + const { text } = calls[1]; + expect(text).toContain('ON CONFLICT (run_id) DO UPDATE'); + expect(text).toContain('WHERE cx_runs.source_sha = EXCLUDED.source_sha'); + expect(text).toContain('DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted)'); + expect(text).toContain('INSERT INTO cx_run_docs'); }); - it('reports a guarded no-op (older or equal attempt, or tombstoned) as false', async () => { - const { sql } = fakeSql([[{ runs_updated: 0 }]]); - await expect(refreshCollectiveXRunAttempt(sql, runInsert, [])).resolves.toBe(false); - }); -}); - -describe('deleteCollectiveXRun', () => { - it('tombstones the run and frees its documents in one atomic statement', async () => { - const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); - await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(true); - expect(calls).toHaveLength(1); - expect(calls[0].text).toContain('SET deleted_at = now()'); - expect(calls[0].text).toContain('deleted_at IS NULL'); - expect(calls[0].text).toContain('DELETE FROM cx_run_docs'); + it('writes nothing when the run id already belongs to another commit', async () => { + const { sql } = fakeSql([[], []]); + await expect(upsertCollectiveXRun(sql, runInsert, [{}])).resolves.toBe('conflict'); }); - it('reports absent or already-tombstoned runs as false', async () => { - const { sql } = fakeSql([[{ runs_deleted: 0 }]]); - await expect(deleteCollectiveXRun(sql, '160')).resolves.toBe(false); - }); - - it('cannot reach anything but the one run it is given', async () => { - const { sql, calls } = fakeSql([[{ runs_deleted: 1 }]]); - await deleteCollectiveXRun(sql, '160'); - expect(calls[0].values).toEqual(['160']); - expect(calls[0].text).toContain( - 'DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned)', - ); - expect(new Set([...calls[0].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( + it('touches only the CollectiveX tables and never drops data wholesale', async () => { + const { sql, calls } = fakeSql([[], [{ inserted: true }]]); + await upsertCollectiveXRun(sql, runInsert, []); + expect(new Set([...calls[1].text.matchAll(/\bcx_[a-z_]+/gu)].map((match) => match[0]))).toEqual( new Set(['cx_runs', 'cx_run_docs']), ); - expect(calls[0].text).not.toMatch(/\b(?:DROP|TRUNCATE|ALTER)\b/iu); + expect(calls[1].text).not.toMatch(/\b(?:DROP|TRUNCATE|ALTER)\b/iu); }); }); @@ -178,7 +140,7 @@ describe('collectiveXDatasetFromRow', () => { version: 1, generated_at: meta.generated_at, source_sha: meta.source_sha, - source_branch: 'collectivex', + source_branch: 'main', source_repository: null, conclusion: meta.conclusion, matrix: makeRawMatrix([ diff --git a/packages/db/src/queries/collectivex.ts b/packages/db/src/queries/collectivex.ts index 756fb77dd..cb97b233a 100644 --- a/packages/db/src/queries/collectivex.ts +++ b/packages/db/src/queries/collectivex.ts @@ -1,4 +1,5 @@ import type { DbClient } from '../connection.js'; +import type { Sql } from '../etl/db-utils'; import { buildDatasetFromNeutral } from '../collectivex/reader'; import type { CollectiveXDataset, CollectiveXRunSummary } from '../collectivex/types'; @@ -17,7 +18,7 @@ export interface CollectiveXRunRow { docs: unknown[]; } -/** Row metadata + raw docs as written by the lazy ingest. */ +/** One results commit's cx_runs row (see collectivex/committed-results.ts). */ export interface CollectiveXRunInsert { run_id: string; run_attempt: number; @@ -30,34 +31,20 @@ export interface CollectiveXRunInsert { summary: CollectiveXRunSummary; } -/** - * Known-run state. Runs are discovered lazily from GitHub on read, so a - * deleted run keeps a tombstoned cx_runs row — otherwise the next discovery - * pass would re-ingest it. - */ -type CollectiveXRunState = 'live' | 'deleted'; - -interface CollectiveXRunStateRow { - state: CollectiveXRunState; - version: number; - run_attempt: number; -} - function toRunRow(row: Record): CollectiveXRunRow { const { docs, ...rest } = row as unknown as Omit & { docs: unknown }; return { ...rest, docs: Array.isArray(docs) ? docs : [] }; } /** - * The most recent visible live run for a version. Zero-case summaries contain - * no AMD/NVIDIA data and stay hidden. Ordered by run_id — GitHub run ids - * increase monotonically with creation, matching lazy discovery's newest-first - * walk (completion time would let a long-failing older run shadow a newer - * successful one). + * The most recent visible run for a version. A run is visible when its summary + * requests AMD/NVIDIA cases AND measured at least one of them; the ingest never + * stores an unmeasured commit, and this keeps any such row hidden regardless. + * Ordered by run_id, which grows with main's results history, so the newest + * results commit on main is the latest run. * * Row and documents come back in ONE query, with docs filtered to the row's - * CURRENT run_attempt: a reader can never observe one attempt's metadata with - * another attempt's documents, even while a refresh commits concurrently. + * run_attempt, so a reader never mixes one ingest's row with another's docs. */ export async function getLatestCollectiveXRun( sql: DbClient, @@ -75,8 +62,9 @@ export async function getLatestCollectiveXRun( FROM cx_run_docs WHERE run_id = r.run_id AND run_attempt = r.run_attempt ) d ON true - WHERE r.version = ${version} AND r.deleted_at IS NULL + WHERE r.version = ${version} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((r.summary->>'measured_cases')::int, 0) > 0 ORDER BY r.run_id DESC LIMIT 1 `; @@ -84,8 +72,8 @@ export async function getLatestCollectiveXRun( } /** - * One specific visible live run by id, or null when absent, tombstoned, on - * another version, or carrying no supported-vendor cases. + * One specific visible run by id, or null when absent, on another version, or + * carrying no measured supported-vendor cases. */ export async function getCollectiveXRun( sql: DbClient, @@ -104,16 +92,17 @@ export async function getCollectiveXRun( FROM cx_run_docs WHERE run_id = r.run_id AND run_attempt = r.run_attempt ) d ON true - WHERE r.version = ${version} AND r.run_id = ${runId} AND r.deleted_at IS NULL + WHERE r.version = ${version} AND r.run_id = ${runId} AND COALESCE((r.summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((r.summary->>'measured_cases')::int, 0) > 0 `; return rows.length === 0 ? null : toRunRow(rows[0]); } /** - * Every visible live run summary for a version, newest-first, straight from + * Every visible run summary for a version, newest-first, straight from * the precomputed `summary` column — no document loading. Runs whose reader - * summary contains no AMD/NVIDIA cases stay hidden. + * summary requests or measures no AMD/NVIDIA cases stay hidden. */ export async function listCollectiveXRuns( sql: DbClient, @@ -122,143 +111,118 @@ export async function listCollectiveXRuns( const rows = await sql` SELECT summary FROM cx_runs - WHERE version = ${version} AND deleted_at IS NULL + WHERE version = ${version} AND COALESCE((summary->>'requested_cases')::int, 0) > 0 + AND COALESCE((summary->>'measured_cases')::int, 0) > 0 ORDER BY run_id DESC `; return rows.map((row) => row.summary as CollectiveXRunSummary); } -/** State of each known run id (live or tombstoned); absent ids are omitted. */ -export async function getCollectiveXRunStates( - sql: DbClient, - runIds: readonly string[], -): Promise> { - if (runIds.length === 0) return {}; - const rows = await sql` - SELECT run_id::text, version, run_attempt, (deleted_at IS NOT NULL) AS deleted - FROM cx_runs - WHERE run_id = ANY(${runIds as string[]}::bigint[]) - `; - return Object.fromEntries( - rows.map((row) => [ - row.run_id as string, - { - state: row.deleted ? 'deleted' : 'live', - version: row.version as number, - run_attempt: row.run_attempt as number, - }, - ]), - ); -} +export type CollectiveXUpsertResult = 'inserted' | 'replaced' | 'conflict'; + +// Every CollectiveX write below first takes the transaction-scoped advisory lock +// `pg_advisory_xact_lock(7384110219)` (an arbitrary fixed key), so ingests, prunes and +// deletions run one at a time. Without it two concurrent ingests of one commit each +// delete only the documents visible to their snapshot and then both insert, storing +// every document twice. /** - * Atomically persist one run and its raw documents in a single statement - * (data-modifying CTEs), so concurrent lazy ingests can race safely: - * `ON CONFLICT DO NOTHING` turns the loser into a clean no-op, and a partial - * run can never become visible. Returns true when this call inserted the run. + * Store one results commit and its raw documents in one transaction: take the + * write lock, then one statement (data-modifying CTEs) upserts the row and + * replaces its documents. Readers see the previous contents or the new ones, + * never a partial run, and concurrent ingests of the same commit serialize on + * the lock, so the second one's DELETE sees and removes the first one's + * documents. Re-ingesting a commit replaces its row and documents. + * `conflict` means run_id already belongs to a different commit (main's + * results history was rewritten); nothing is written. * * The `::jsonb`-cast parameters must be raw objects: both drivers * JSON-serialize objects exactly once, while pre-stringified values get * double-encoded by postgres.js. `docs` is wrapped in an object because the * neon driver would serialize a bare JS array as a Postgres array literal. */ -export async function insertCollectiveXRun( - sql: DbClient, +export async function upsertCollectiveXRun( + sql: Sql, run: CollectiveXRunInsert, docs: unknown[], -): Promise { - const rows = await sql` - WITH new_run AS ( - INSERT INTO cx_runs - (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) - VALUES - (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, - ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, - ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) - ON CONFLICT (run_id) DO NOTHING - RETURNING run_id - ), - new_docs AS ( - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT new_run.run_id, ${run.run_attempt}, entries.value - FROM new_run, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) - RETURNING id - ) - SELECT (SELECT count(*)::int FROM new_run) AS runs_inserted - `; - return (rows[0]?.runs_inserted as number) > 0; +): Promise { + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx` + WITH upserted AS ( + INSERT INTO cx_runs + (run_id, run_attempt, version, generated_at, source_sha, source_branch, conclusion, matrix, summary) + VALUES + (${run.run_id}, ${run.run_attempt}, ${run.version}, ${run.generated_at}, + ${run.source_sha}, ${run.source_branch}, ${run.conclusion}, + ${run.matrix as never}::jsonb, ${run.summary as never}::jsonb) + ON CONFLICT (run_id) DO UPDATE SET + run_attempt = EXCLUDED.run_attempt, + version = EXCLUDED.version, + generated_at = EXCLUDED.generated_at, + source_branch = EXCLUDED.source_branch, + conclusion = EXCLUDED.conclusion, + matrix = EXCLUDED.matrix, + summary = EXCLUDED.summary, + ingested_at = now() + WHERE cx_runs.source_sha = EXCLUDED.source_sha + RETURNING run_id, (xmax = 0) AS inserted + ), + removed AS ( + DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM upserted) + ), + new_docs AS ( + INSERT INTO cx_run_docs (run_id, run_attempt, doc) + SELECT upserted.run_id, ${run.run_attempt}, entries.value + FROM upserted, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) + RETURNING id + ) + SELECT inserted FROM upserted + `; + }); + if (rows.length === 0) return 'conflict'; + return rows[0].inserted ? 'inserted' : 'replaced'; } -/** - * Replace a live run's contents when GitHub reports a NEWER attempt (a re-run - * of failed shards after the run was already ingested). Single statement with - * `FOR UPDATE` + an attempt guard: concurrent refreshers serialize on the row - * lock and the loser re-evaluates the guard to a no-op. Readers filter docs - * by the row's current run_attempt, so any superseded docs this statement's - * snapshot could not see (and therefore could not DELETE) stay invisible until - * the next refresh garbage-collects them. Tombstoned runs are never refreshed. - * Returns true when replaced. - */ -export async function refreshCollectiveXRunAttempt( - sql: DbClient, - run: CollectiveXRunInsert, - docs: unknown[], -): Promise { - const rows = await sql` - WITH target AS ( - SELECT run_id FROM cx_runs - WHERE run_id = ${run.run_id} AND deleted_at IS NULL AND run_attempt < ${run.run_attempt} - FOR UPDATE - ), - removed AS ( - DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM target) - ), - updated AS ( - UPDATE cx_runs SET - run_attempt = ${run.run_attempt}, - generated_at = ${run.generated_at}, - source_sha = ${run.source_sha}, - source_branch = ${run.source_branch}, - conclusion = ${run.conclusion}, - matrix = ${run.matrix as never}::jsonb, - summary = ${run.summary as never}::jsonb, - ingested_at = now() - WHERE run_id IN (SELECT run_id FROM target) - RETURNING run_id - ), - new_docs AS ( - INSERT INTO cx_run_docs (run_id, run_attempt, doc) - SELECT updated.run_id, ${run.run_attempt}, entries.value - FROM updated, jsonb_array_elements((${{ docs } as never}::jsonb)->'docs') AS entries(value) - RETURNING id - ) - SELECT (SELECT count(*)::int FROM updated) AS runs_updated - `; - return (rows[0]?.runs_updated as number) > 0; +/** A stored run removed by a prune or deletion. */ +export interface CollectiveXRemovedRun { + run_id: string; + source_sha: string; } /** - * Tombstone a run: mark it deleted (so lazy discovery never re-ingests it) - * and drop its documents to free space — one atomic statement, so a partial - * failure can never tombstone the run while leaving its documents orphaned - * behind an unretryable 404. Returns false when the run is absent or already - * tombstoned. Re-ingesting via the CLI intentionally clears the tombstone - * (operator override; its hard DELETE also cascades any leftover docs). + * Delete every stored run whose results commit is not in `keepShas` (with its + * documents, by cascade). The backfill passes the commits it just stored, so a + * run whose commit left main, stopped validating or measures nothing is retracted. */ -export async function deleteCollectiveXRun(sql: DbClient, runId: string): Promise { - const rows = await sql` - WITH tombstoned AS ( - UPDATE cx_runs SET deleted_at = now() - WHERE run_id = ${runId} AND deleted_at IS NULL - RETURNING run_id - ), - removed AS ( - DELETE FROM cx_run_docs WHERE run_id IN (SELECT run_id FROM tombstoned) - ) - SELECT (SELECT count(*)::int FROM tombstoned) AS runs_deleted - `; - return (rows[0]?.runs_deleted as number) > 0; +export async function pruneCollectiveXRuns( + sql: Sql, + keepShas: readonly string[], +): Promise { + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx` + DELETE FROM cx_runs + WHERE source_sha NOT IN ( + SELECT jsonb_array_elements_text((${{ shas: keepShas } as never}::jsonb)->'shas') + ) + RETURNING run_id::text, source_sha + `; + }); + return rows as unknown as CollectiveXRemovedRun[]; +} + +/** Delete one stored run and its documents; null when no such run is stored. */ +export async function deleteCollectiveXRun( + sql: Sql, + runId: string, +): Promise { + const rows = await sql.begin(async (tx) => { + await tx`SELECT pg_advisory_xact_lock(7384110219)`; + return tx`DELETE FROM cx_runs WHERE run_id = ${runId} RETURNING run_id::text, source_sha`; + }); + return rows.length === 0 ? null : (rows[0] as unknown as CollectiveXRemovedRun); } /** Assemble a stored run's raw documents into the dashboard dataset. */ diff --git a/packages/skills/skills/inferencex-api/integrity.json b/packages/skills/skills/inferencex-api/integrity.json index 83afdf01a..73335f5ef 100644 --- a/packages/skills/skills/inferencex-api/integrity.json +++ b/packages/skills/skills/inferencex-api/integrity.json @@ -8,7 +8,7 @@ "references/chart-templates.md": "aed6854627142019a4a5df4a2d07523b1a64e5814dd2bbdcc74ee18beb083912", "references/cli-contract.md": "4f070877eeefffad2860c58f79f68432fbe27aa21fc578a1c696f0309a3d7d7b", "references/cli.md": "11e1183f7ed70d669019e8435ac16a6a9f873de027507b8850a9943f6659bfd9", - "references/collectivex.md": "f79cf8e2779a9289ab69f79f4a4b338e0b9a5f76a3750a3e969f33a744e726a1", + "references/collectivex.md": "509b8cb7682de751a557c2d9f1db865d419998c62025c1bf140a3aec3fd612dd", "references/dashboard-views.md": "2095bef6178b7264db9e3c1e2bfcde1516fb190ddd5851dd5f8d104c8a0ab58b", "references/offline-exports.md": "95aa565dcd4c9e592159baa560a9a58217bc61e395de1c6daf0576795c7f4ec9", "references/pareto.md": "1b4d2d163f982e3f2d97addce310789eae9c1e5badd82dd7501708c9e0385027", diff --git a/packages/skills/skills/inferencex-api/references/collectivex.md b/packages/skills/skills/inferencex-api/references/collectivex.md index d6a6874e6..41d5e489a 100644 --- a/packages/skills/skills/inferencex-api/references/collectivex.md +++ b/packages/skills/skills/inferencex-api/references/collectivex.md @@ -2,9 +2,9 @@ Use this cookbook to discover public communication sweeps, compare two returned runs, and export the evidence. These are existing observations; no new benchmark -is run. Public GETs can populate the service's documented lazy cache from existing -GitHub artifacts. No database credentials, admin operations, or launch tools are -needed. +is run. Each run is one SemiAnalysisAI/InferenceX commit on main that changed +`collectivex/results/`; the service ingests it when it lands, never on read. No +database credentials, admin operations, or launch tools are needed. For a replayable contract 1 comparison: @@ -44,8 +44,12 @@ node .claude/skills/inferencex-api/scripts/inferencex.mjs collectivex compare \ Requires Node 24 or 26. The command checks the current OpenAPI operations, reads the run list **once**, and selects its two newest runs with `measured_cases > 0`, ordered by numeric run ID. The older selection is `left`; the newer is `right`. This is a -bounded example selection, not a representative sample. A cancelled or failed -workflow can still contain measured cases; its conclusion remains in the export. +bounded example selection, not a representative sample. The run ID is the results +commit time in epoch seconds (raised past the previous results commit on a same-second +tie) and the dataset's `run.source_sha` is that commit; the conclusion remains in the +export. EP tail percentiles (`p90`, `p95`, `p99`) of latencies and derived rates can be +`null` where the harness withheld them; `p50` is always a number. Report a `null` tail as +unavailable, never as zero. Run-list counts identify candidate pairs; exact EP and KV identities in the returned details establish whether they have matched groups. For a requested pair, supply @@ -67,31 +71,29 @@ The public operations require `version=1`: | Operation | Meaning | | ------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------- | -| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest available run by run ID; it need not contain measurements. | +| [`/collectivex/latest`](https://inferencex.semianalysis.com/api/v1/collectivex/latest?version=1) | Newest stored run by run ID; only runs with a measured result are stored. | | [`/collectivex/runs`](https://inferencex.semianalysis.com/api/v1/collectivex/runs?version=1) | Retained summaries and `discovery_complete`; the helper uses this for selection. | -| `/collectivex/runs/{runId}` | One run dataset, possibly refreshed to a newer attempt. | +| `/collectivex/runs/{runId}` | One run dataset; a re-ingest of the same commit can replace its contents. | All paths above are under `/api/v1`. Consult the [public API reference](https://inferencex.semianalysis.com/api) and [OpenAPI document](https://inferencex.semianalysis.com/api/openapi.json) for the -current contract. A missing/unsupported version is HTTP 400. HTTP 404, an upstream -502/503, malformed data, a redirect, or an interrupted response is a failed read, +current contract. A missing/unsupported version is HTTP 400. HTTP 404, a 5xx, +malformed data, a redirect, or an interrupted response is a failed read, not an empty comparison. The command exits nonzero without a completed manifest on those failures; it never substitutes a different run. -`discovery_complete=false` means further bounded discovery passes may reveal more -runs. Report that flag even when two selections were available. The route has no +The service now always reports `discovery_complete=true`; a `false` value would mean +further passes may reveal more runs. Report the flag as returned. The route has no documented `limit`, `offset`, or cursor: do not invent pagination parameters. If fewer than two measured summaries were returned, the helper exports `fewer_than_two_measured_runs` and makes no detail requests. Further discovery, if needed, should have an explicit small request budget and retain each response. -Even `discovery_complete=true` does **not** mean complete workflow history. Recent -discovery has a bounded upstream window, artifacts expire, and retained runs can -outlive their artifacts. The export therefore always sets `history_complete=false`. -Stored fallback data can also be served when upstream refresh fails. Compare each -detail's returned `run_attempt` with its discovery summary; a later attempt is a -different snapshot, even under the same run ID. Equal attempt numbers do not prove -identical snapshots; data can refresh within an attempt. +Even `discovery_complete=true` does **not** prove complete history: the stored runs +are whatever results commits the service has ingested. The export therefore always +sets `history_complete=false`. Compare each detail's returned `run_attempt` with its +summary; equal attempt numbers do not prove identical snapshots, because re-ingesting +a commit replaces its contents under the same run ID and attempt. ## 2. Check comparability before interpreting differences @@ -183,6 +185,11 @@ effect. Equal SHAs also do not establish identical runtime conditions. | Swap `swap_blocks[].points[].latency_us` | Copy latency in microseconds at the named percentile. | | Swap `swap_blocks[].points[].payload_gbps_at_latency_percentile` | Payload GB/s at the named latency percentile, with bytes counted once. | +EP p90, p95 and p99 are null where the harness withheld that tail: too many of the +series' samples were misaligned across ranks or host-stalled. The rates at a +withheld tail are null too; p50 is always present. Report a null tail as withheld, +not as zero or as a missing measurement. + The saved OpenAPI operation description defines the optional swap metrics even when their property schemas omit descriptions. Swap points remain in the raw datasets, outside the formal EP/KV comparison groups. For supplemental summaries,