From 6716d8cb5dbef72f293b9d0a077be2f476fbaf59 Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Tue, 13 Oct 2026 15:53:04 +0000 Subject: [PATCH] =?UTF-8?q?fix(pricing):=20never=20price=20missing,=20part?= =?UTF-8?q?ial=20or=20invalid=20ATOM=20cache=20observations=20as=20actual?= =?UTF-8?q?=20reuse=20/=20=E4=B8=8D=E5=B0=86=E7=BC=BA=E5=A4=B1=E3=80=81?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=88=96=E6=97=A0=E6=95=88=E7=9A=84=20ATOM?= =?UTF-8?q?=20=E7=BC=93=E5=AD=98=E8=A7=82=E6=B5=8B=E6=8C=89=E5=AE=9E?= =?UTF-8?q?=E9=99=85=E5=A4=8D=E7=94=A8=E8=AE=A1=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ATOM rows could be priced in actual mode with a missing cache rate treated as zero reuse, or with a GPU-only reading treated as the full hit rate. Published rows of run 36734492117 carry known-wrong rates. - ETL carries the producer's server_metrics.cache_validity as numeric columns (validated with runtime narrowing; unknown or conflicting blocks are not eligible) and registers them in METRIC_KEYS. - actualCacheRateUnavailable(): the eligibility flag, status code and the row's own rates must agree; used by the calculator, curve interpolation, per-point revenue and trend revenue. - Explicit theoretical mode prices an invalid actual observation only when it swapped in a valid theoretical rate. - ATOM P/D rows no longer use the doubled prompt_total as a token mix. - Default actual/theoretical mode, prices and hardware fallbacks unchanged. Refs: run 36734492117, SemiAnalysisAI/InferenceX#3543 --- .../calculator/ProfitEstimatorDisplay.tsx | 2 + .../calculator/atom-cache-validity.test.ts | 141 ++++++++++++++++++ .../components/calculator/interpolation.ts | 6 + .../calculator/profit-estimator.test.ts | 42 ++++++ .../components/calculator/profit-estimator.ts | 27 +++- .../components/calculator/throughput-data.ts | 23 ++- .../app/src/components/calculator/types.ts | 6 + .../hooks/interpolated-trend-core.ts | 8 +- .../inference/token-revenue.test.ts | 38 +++++ .../src/components/inference/token-revenue.ts | 4 + .../app/src/components/inference/types.ts | 4 + .../src/lib/benchmarks/benchmark-transform.ts | 2 + .../src/lib/calculator/cache-pricing.test.ts | 86 +++++++++++ .../app/src/lib/calculator/cache-pricing.ts | 80 ++++++++++ packages/constants/src/metric-keys.ts | 3 + .../db/src/etl/agentic-v3-flatten.test.ts | 69 +++++++++ packages/db/src/etl/agentic-v3-flatten.ts | 56 +++++++ packages/db/src/etl/benchmark-mapper.test.ts | 35 +++++ 18 files changed, 626 insertions(+), 6 deletions(-) create mode 100644 packages/app/src/components/calculator/atom-cache-validity.test.ts create mode 100644 packages/db/src/etl/agentic-v3-flatten.test.ts diff --git a/packages/app/src/components/calculator/ProfitEstimatorDisplay.tsx b/packages/app/src/components/calculator/ProfitEstimatorDisplay.tsx index 35637b560..28d29d24b 100644 --- a/packages/app/src/components/calculator/ProfitEstimatorDisplay.tsx +++ b/packages/app/src/components/calculator/ProfitEstimatorDisplay.tsx @@ -315,6 +315,7 @@ const STRINGS = { 'outside-power-model': 'these benchmark points are outside the supported power model', 'no-cost': 'no TCO for this tier', 'no-token-mix': 'no input/output token mix recorded', + 'invalid-cache-observation': 'cache observation missing, partial or not validated', } satisfies Record, compareHistory: 'Compare history', gpuConfig: 'Chip Config', @@ -439,6 +440,7 @@ const STRINGS = { 'outside-power-model': '这些基准测试数据点超出功耗模型的适用范围', 'no-cost': '该层级无 TCO 数据', 'no-token-mix': '未记录输入/输出 token 比例', + 'invalid-cache-observation': '缓存观测缺失、不完整或未通过有效性检查', } satisfies Record, compareHistory: '对比历史趋势', gpuConfig: '芯片配置', diff --git a/packages/app/src/components/calculator/atom-cache-validity.test.ts b/packages/app/src/components/calculator/atom-cache-validity.test.ts new file mode 100644 index 000000000..4c266e8e6 --- /dev/null +++ b/packages/app/src/components/calculator/atom-cache-validity.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; + +import { NORMALIZED_TOKEN_REVENUE_PRICING } from '@/components/inference/token-revenue'; +import type { BenchmarkRow } from '@/lib/api/api'; +import { Percentile, Sequence } from '@/lib/catalog/data-mappings'; + +import { interpolateForGPU } from './interpolation'; +import { estimateSkuProfit } from './profit-estimator'; +import { buildGpuGroups } from './throughput-data'; + +const RUN = 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/36734492117/attempts/1'; +const SPECS = { powerKwPerGpu: 2, costPerGpuHour: 10 }; +const ASSUMPTIONS = { utilizationPct: 60, labCutPct: 30, basis: 'chip-hour' as const }; +const COMPLETE = { + server_gpu_cache_hit_rate: 0.97161, + server_cpu_cache_hit_rate: 0, + server_external_cache_hit_rate: 0, + server_cache_pricing_eligible: 1, + server_cache_validity_code: 1, +}; + +function atomRow( + conc: number, + intvty: number, + tput: number, + metrics: Record, +): BenchmarkRow { + return { + id: conc, + hardware: 'mi355x', + framework: 'atom-disagg', + model: 'dsv4', + precision: 'fp4', + spec_method: 'none', + disagg: true, + is_multinode: true, + prefill_tp: 8, + prefill_ep: 1, + prefill_dp_attention: false, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 1, + decode_dp_attention: false, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + conc, + offload_mode: 'none', + image: 'rocm/atom-dev:nightly', + metrics: { + // benchmark-transform derives *_intvty as 1 / *_itl and drops it when itl is absent. + p90_itl: 1 / intvty, + median_itl: 1 / intvty, + tput_per_gpu: tput, + input_tput_per_gpu: tput * 0.99, + output_tput_per_gpu: tput * 0.01, + input_tput_tps: tput * 16 * 0.99, + output_tput_tps: tput * 16 * 0.01, + ...metrics, + }, + date: '2026-09-30', + run_url: RUN, + } as BenchmarkRow; +} + +function price(rows: BenchmarkRow[], target?: number) { + const { grouped } = buildGpuGroups(rows, { + sequence: Sequence.AgenticTraces, + precisions: ['fp4'], + percentile: Percentile.P90, + classify: () => ({ key: 'curve', meta: { hwKey: 'curve' } }), + }); + const points = grouped.curve ?? []; + const at = target ?? points[0]!.interactivity; + const result = interpolateForGPU(points, at, 'interactivity_to_throughput', 'costh')!; + return { + points, + result, + row: estimateSkuProfit(result, SPECS, NORMALIZED_TOKEN_REVENUE_PRICING, ASSUMPTIONS), + }; +} + +describe('ATOM cache validity through buildGpuGroups, interpolateForGPU and estimateSkuProfit', () => { + it('prices an eligible complete observation', () => { + const { points, row } = price([atomRow(1, 190, 840, COMPLETE)]); + expect(points[0]!.cacheHitRate).toBeCloseTo(0.97161, 12); + expect(row).toHaveProperty('revenue'); + }); + + it('keeps performance but refuses actual pricing for a partial observation', () => { + const partial = { + ...COMPLETE, + server_cpu_cache_hit_rate: undefined, + server_external_cache_hit_rate: undefined, + }; + const { points, row } = price([atomRow(1, 190, 840, partial as Record)]); + expect(points[0]!.throughput).toBe(840); + expect(points[0]!.cacheHitRateUnavailable).toBe('PARTIAL_COMPONENT'); + expect(row).toMatchObject({ reason: 'invalid-cache-observation' }); + }); + + it('quarantines the known-bad legacy row and restores a re-derived one', () => { + const legacy = { + server_gpu_cache_hit_rate: 0.48352, + server_cpu_cache_hit_rate: 0, + server_external_cache_hit_rate: 0, + }; + expect(price([atomRow(1, 190, 840, legacy)]).row).toMatchObject({ + reason: 'invalid-cache-observation', + }); + expect(price([atomRow(1, 190, 840, COMPLETE)]).row).toHaveProperty('revenue'); + }); + + it('does not restore a mixed curve whose frontier holds an unresolved point', () => { + const unresolved = { + ...COMPLETE, + server_cache_pricing_eligible: 0, + server_cache_validity_code: 14, + }; + const valid = [atomRow(1, 190, 840, COMPLETE), atomRow(16, 120, 4_000, COMPLETE)]; + expect(price(valid, 150).row).toHaveProperty('revenue'); + const mixed = [...valid, atomRow(256, 40, 48_000, unresolved)]; + const { result, row } = price(mixed, 150); + expect(result.cacheHitRateUnavailable).toBe('REQUEST_SCOPE_UNRESOLVED'); + expect(row).toMatchObject({ reason: 'invalid-cache-observation' }); + }); + + it('takes the ATOM P/D token mix from client rates, not the doubled prompt_total', () => { + const inconsistent = { + ...COMPLETE, + input_tput_per_gpu: 10, + total_prompt_tokens: 2, + total_generation_tokens: 1, + }; + const { points } = price([atomRow(1, 190, 840, inconsistent)]); + expect(points[0]!.inputTokenShare).toBeCloseTo(0.99, 12); + }); +}); diff --git a/packages/app/src/components/calculator/interpolation.ts b/packages/app/src/components/calculator/interpolation.ts index dbb0217bd..8a94b7e5a 100644 --- a/packages/app/src/components/calculator/interpolation.ts +++ b/packages/app/src/components/calculator/interpolation.ts @@ -390,6 +390,7 @@ export function interpolateForGPU( inputTpPerMw: sorted[0].inputTpPerMw, outputTpPerMw: sorted[0].outputTpPerMw, cacheHitRate: sorted[0].cacheHitRate, + cacheHitRateUnavailable: sorted[0].cacheHitRateUnavailable, theoreticalCacheHitRate: sorted[0].theoreticalCacheHitRate, inputTokenShare: sorted[0].inputTokenShare, concurrency: sorted[0].concurrency, @@ -453,6 +454,10 @@ export function interpolateForGPU( // would invent a dip in the cached fraction, so the whole sweep opts out and // its input tokens bill at full price. That avoids inventing a cache curve, // but overstates revenue and margin if the missing point actually had hits. + // An invalid actual cache rate on a frontier point leaves the curve's rate unknown: the + // all-or-nothing spline below cannot exclude it, so it must not be priced as zero reuse. + const unavailableAnchor = sorted.find((p) => p.cacheHitRateUnavailable); + const cacheHitRateUnavailable = unavailableAnchor?.cacheHitRateUnavailable; const cacheHitRate = sorted.every((p) => typeof p.cacheHitRate === 'number') ? buildMetric((p) => p.cacheHitRate!) : undefined; @@ -495,6 +500,7 @@ export function interpolateForGPU( inputTpPerMw, outputTpPerMw, cacheHitRate, + cacheHitRateUnavailable, theoreticalCacheHitRate, inputTokenShare, concurrency, diff --git a/packages/app/src/components/calculator/profit-estimator.test.ts b/packages/app/src/components/calculator/profit-estimator.test.ts index e1f0364dd..21b261142 100644 --- a/packages/app/src/components/calculator/profit-estimator.test.ts +++ b/packages/app/src/components/calculator/profit-estimator.test.ts @@ -531,3 +531,45 @@ describe('applyCacheHitMode', () => { expect(theoreticalRow.revenue).toBeLessThan(actualRow.revenue); }); }); + +describe('explicit theoretical pricing of an invalid actual observation', () => { + const pricing = { inputPerMillion: 1, cachedInputPerMillion: 0.1, outputPerMillion: 1 }; + const specs = { powerKwPerGpu: 2, costPerGpuHour: 10 }; + const assumptions = { utilizationPct: 60, labCutPct: 30, basis: 'chip-hour' as const }; + const invalid = { + hwKey: 'mi355x_atom', + resultKey: 'mi355x_atom', + value: 10_000, + inputTokenShare: 0.75, + clamped: false, + cacheHitRateUnavailable: 'REQUEST_SCOPE_UNRESOLVED', + theoreticalCacheHitRate: 0.96797, + }; + + it('is skipped in actual mode even when a number is present', () => { + const estimate = estimateSkuProfit( + { ...invalid, cacheHitRate: 0.95 }, + specs, + pricing, + assumptions, + ); + expect(estimate).toMatchObject({ reason: 'invalid-cache-observation' }); + }); + + it('is priced only after the explicit theoretical mode swaps in a valid rate', () => { + const [theoretical] = applyCacheHitMode([invalid], 'theoretical'); + expect(theoretical).toMatchObject({ cacheHitRate: 0.96797, cacheHitRateSource: 'theoretical' }); + expect(estimateSkuProfit(theoretical, specs, pricing, assumptions)).toHaveProperty('revenue'); + }); + + it('is not authorised by a non-finite theoretical rate', () => { + const [swapped] = applyCacheHitMode( + [{ ...invalid, theoreticalCacheHitRate: Number.NaN }], + 'theoretical', + ); + expect(swapped).not.toHaveProperty('cacheHitRateSource'); + expect(estimateSkuProfit(swapped, specs, pricing, assumptions)).toMatchObject({ + reason: 'invalid-cache-observation', + }); + }); +}); diff --git a/packages/app/src/components/calculator/profit-estimator.ts b/packages/app/src/components/calculator/profit-estimator.ts index d7d958ee5..d8670ad78 100644 --- a/packages/app/src/components/calculator/profit-estimator.ts +++ b/packages/app/src/components/calculator/profit-estimator.ts @@ -245,7 +245,8 @@ export type ProfitEstimatorSkipReason = | 'outside-measured-range' | 'no-power' | 'no-cost' - | 'no-token-mix'; + | 'no-token-mix' + | 'invalid-cache-observation'; export interface ProfitEstimatorSkipped { hwKey: string; @@ -291,7 +292,15 @@ export interface ProfitEstimatorSpecs { export function estimateSkuProfit( result: Pick< InterpolatedResult, - 'hwKey' | 'resultKey' | 'precision' | 'value' | 'inputTokenShare' | 'cacheHitRate' | 'clamped' + | 'hwKey' + | 'resultKey' + | 'precision' + | 'value' + | 'inputTokenShare' + | 'cacheHitRate' + | 'cacheHitRateUnavailable' + | 'cacheHitRateSource' + | 'clamped' > & { date?: string }, specs: ProfitEstimatorSpecs, pricing: TokenRevenuePricing, @@ -308,6 +317,10 @@ export function estimateSkuProfit( const gpuHours = assumptions.basis === 'chip-hour' ? 1 : gpuHoursPerGwYear(specs.powerKwPerGpu); if (gpuHours === null) return { ...base, reason: 'no-power' }; if (!(specs.costPerGpuHour > 0)) return { ...base, reason: 'no-cost' }; + // Only a rate swapped in by an explicit theoretical mode may price an invalid actual observation. + if (result.cacheHitRateUnavailable && result.cacheHitRateSource !== 'theoretical') { + return { ...base, reason: 'invalid-cache-observation' }; + } const revenuePerGpuHour = tokenRevenueFromRatesPerGpuHour( result.value, @@ -428,7 +441,15 @@ export function applyCacheHitMode< if (mode === 'actual') return results as T[]; return results.map((r) => typeof r.theoreticalCacheHitRate === 'number' - ? { ...r, cacheHitRate: r.theoreticalCacheHitRate } + ? { + ...r, + cacheHitRate: r.theoreticalCacheHitRate, + ...(Number.isFinite(r.theoreticalCacheHitRate) && + r.theoreticalCacheHitRate >= 0 && + r.theoreticalCacheHitRate <= 1 + ? { cacheHitRateSource: 'theoretical' as const } + : {}), + } : r, ); } diff --git a/packages/app/src/components/calculator/throughput-data.ts b/packages/app/src/components/calculator/throughput-data.ts index f0d0450ad..476282ca8 100644 --- a/packages/app/src/components/calculator/throughput-data.ts +++ b/packages/app/src/components/calculator/throughput-data.ts @@ -3,7 +3,7 @@ import { rowToSequence } from '@semianalysisai/inferencex-constants'; import type { AggDataEntry, HardwareConfig } from '@/components/inference/types'; import type { BenchmarkRow } from '@/lib/api/api'; import { rowToAggDataEntry } from '@/lib/benchmarks/benchmark-transform'; -import { pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; +import { actualCacheRateUnavailable, pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; import { getHardwareKey } from '@/lib/charts/chart-utils'; import { DEFAULT_TCO_BASIS, @@ -152,6 +152,16 @@ function inputTokenShare(row: BenchmarkRow, inputTput: number, outputTput: numbe if (typeof isl === 'number' && typeof osl === 'number' && isl + osl > 0) { return isl / (isl + osl); } + // ATOM P/D prompt_total sums the prefill and decode prompt counters (decode re-counts each + // prompt), so it is not a token mix; use the client cohort's own input/output rates. + if (row.framework?.toLowerCase().startsWith('atom') && row.disagg) { + const clientIn = row.metrics.input_tput_tps; + const clientOut = row.metrics.output_tput_tps; + if (typeof clientIn === 'number' && typeof clientOut === 'number' && clientIn + clientOut > 0) { + return clientIn / (clientIn + clientOut); + } + return null; + } const prompt = row.metrics.total_prompt_tokens; const generated = row.metrics.total_generation_tokens; if (typeof prompt === 'number' && typeof generated === 'number' && prompt + generated > 0) { @@ -221,6 +231,11 @@ export function buildGpuGroups( const outputTput = m.output_tput_per_gpu ?? tput; const inputTput = m.input_tput_per_gpu ?? 0; const cacheHitRate = pricingCacheHitRate({ ...m, hw: row.hardware }); + const cacheUnavailable = actualCacheRateUnavailable({ + ...m, + framework: row.framework, + run_url: row.run_url, + }); const theoreticalRate = typeof m.theoretical_cache_hit_rate === 'number' && Number.isFinite(m.theoretical_cache_hit_rate) @@ -252,7 +267,11 @@ export function buildGpuGroups( throughput: tput, outputThroughput: outputTput, inputThroughput: inputTput, - ...(cacheHitRate === null ? {} : { cacheHitRate }), + ...(cacheUnavailable === null + ? cacheHitRate === null + ? {} + : { cacheHitRate } + : { cacheHitRateUnavailable: cacheUnavailable }), ...(theoreticalRate === null ? {} : { theoreticalCacheHitRate: theoreticalRate }), ...(tokenShare === null ? {} : { inputTokenShare: tokenShare }), concurrency: row.conc, diff --git a/packages/app/src/components/calculator/types.ts b/packages/app/src/components/calculator/types.ts index 5ef917c12..f25b0dcb9 100644 --- a/packages/app/src/components/calculator/types.ts +++ b/packages/app/src/components/calculator/types.ts @@ -44,6 +44,8 @@ export interface GPUDataPoint { * absence is what makes cached-input billing a no-op outside agentic traces. */ cacheHitRate?: number; + /** Reason code when the actual cache rate must not be priced. */ + cacheHitRateUnavailable?: string; /** * Infinite-cache theoretical hit rate (0..1) computed from the trace itself, * representing the maximum reuse potential of the workload with unbounded @@ -100,6 +102,10 @@ export interface InterpolatedResult { * fixed-sequence frontier). See {@link GPUDataPoint.cacheHitRate}. */ cacheHitRate?: number; + /** Reason code when the actual cache rate must not be priced. */ + cacheHitRateUnavailable?: string; + /** Set by an explicit theoretical cache-hit mode that swapped in a valid rate. */ + cacheHitRateSource?: 'theoretical'; /** * Infinite-cache theoretical hit rate at that operating point, or undefined * when the frontier did not carry a theoretical rate on every point. diff --git a/packages/app/src/components/inference/hooks/interpolated-trend-core.ts b/packages/app/src/components/inference/hooks/interpolated-trend-core.ts index 861ed210b..c9eea2e19 100644 --- a/packages/app/src/components/inference/hooks/interpolated-trend-core.ts +++ b/packages/app/src/components/inference/hooks/interpolated-trend-core.ts @@ -27,7 +27,7 @@ import { dedupeAgenticHistoryRuns, } from '@/lib/benchmarks/benchmark-run-selection'; import { rowToAggDataEntry } from '@/lib/benchmarks/benchmark-transform'; -import { pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; +import { actualCacheRateUnavailable, pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; import { buildDerivedChartFields, getHardwareKey, @@ -85,6 +85,11 @@ export function rowToLightweightPoint( server_external_cache_hit_rate: entry.server_external_cache_hit_rate, server_cpu_cache_hit_rate: entry.server_cpu_cache_hit_rate, theoretical_cache_hit_rate: entry.theoretical_cache_hit_rate, + framework: entry.framework, + disagg: entry.disagg, + run_url: entry.run_url, + server_cache_pricing_eligible: entry.server_cache_pricing_eligible, + server_cache_validity_code: entry.server_cache_validity_code, ...buildDerivedChartFields(derivedEntry, hwKey, requestedMetrics, tcoBasis), } as InferenceData; @@ -238,6 +243,7 @@ export function interpolateMetricAtInteractivity( }; const throughputYs = sorted.map((p) => extractMetric(p, 'tpPerGpu')!); const inputShares = sorted.map(inputTokenShareForRevenue); + if (sorted.some((p) => actualCacheRateUnavailable(p) !== null)) return null; const cacheHitRates = sorted.map(pricingCacheHitRate); const inputShare = inputShares.every((share): share is number => share !== null) ? interpolateBounded(inputShares) diff --git a/packages/app/src/components/inference/token-revenue.test.ts b/packages/app/src/components/inference/token-revenue.test.ts index 6f160a654..d4b84e012 100644 --- a/packages/app/src/components/inference/token-revenue.test.ts +++ b/packages/app/src/components/inference/token-revenue.test.ts @@ -241,3 +241,41 @@ describe('GB300 cache hit-rate fallback', () => { ); }); }); + +const atomPoint = (fields: Record) => + ({ + framework: 'atom-disagg', + disagg: true, + run_url: 'https://github.com/x/y/actions/runs/1/attempts/1', + tput_per_gpu: 10_000, + input_tput_per_gpu: 7_500, + output_tput_per_gpu: 2_500, + ...fields, + }) as unknown as InferenceData; + +describe('ATOM actual-pricing guard on inference points', () => { + const pricing = { inputPerMillion: 1, cachedInputPerMillion: 0.1, outputPerMillion: 1 }; + + it('returns no revenue rather than zero-hit revenue for an invalid observation', () => { + const eligibleNoRates = atomPoint({ + server_cache_pricing_eligible: 1, + server_cache_validity_code: 1, + }); + expect(tokenRevenuePerGpuHour(eligibleNoRates, pricing)).toBeNull(); + const complete = atomPoint({ server_gpu_cache_hit_rate: 0.6, server_cpu_cache_hit_rate: 0.1 }); + expect(tokenRevenuePerGpuHour(complete, pricing)).toBeCloseTo(18.99, 10); + }); + + it('never uses the doubled ATOM P/D prompt_total as a token mix', () => { + const inconsistent = { + input_tput_per_gpu: 10, + output_tput_per_gpu: 10, + total_prompt_tokens: 2, + total_generation_tokens: 1, + }; + expect(inputTokenShareForRevenue(atomPoint(inconsistent))).toBeNull(); + expect( + inputTokenShareForRevenue(atomPoint({ ...inconsistent, framework: 'vllm' })), + ).toBeCloseTo(2 / 3, 12); + }); +}); diff --git a/packages/app/src/components/inference/token-revenue.ts b/packages/app/src/components/inference/token-revenue.ts index 0dfd9a479..003da5176 100644 --- a/packages/app/src/components/inference/token-revenue.ts +++ b/packages/app/src/components/inference/token-revenue.ts @@ -1,6 +1,7 @@ import type { InferenceData, TokenRevenuePricing } from './types'; import { DEFAULT_CACHED_INPUT_PRICE_RATIO, + actualCacheRateUnavailable, pricingCacheHitRate, } from '@/lib/calculator/cache-pricing'; @@ -52,6 +53,8 @@ export function inputTokenShareForRevenue(point: InferenceData): number | null { return isl / (isl + osl); } + // ATOM P/D prompt_total sums prefill and decode prompt counters; it is not a token mix. + if (point.framework?.toLowerCase().startsWith('atom') && point.disagg) return null; const prompt = point.total_prompt_tokens; const generation = point.total_generation_tokens; if (typeof prompt === 'number' && typeof generation === 'number' && prompt + generation > 0) { @@ -96,6 +99,7 @@ export function tokenRevenuePerGpuHour( point: InferenceData, pricing: TokenRevenuePricing, ): number | null { + if (actualCacheRateUnavailable(point) !== null) return null; const total = point.tput_per_gpu ?? 0; const cacheHitRate = pricingCacheHitRate(point); const inputShare = inputTokenShareForRevenue(point); diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 6df8a1d2f..bb7fb0c31 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -235,6 +235,10 @@ export interface AggDataEntry { server_cpu_cache_hit_rate?: number; /** Infinite-cache theoretical hit rate (0..1) computed from trace. */ theoretical_cache_hit_rate?: number; + /** 1 when the producer marked the cache rates a complete actual observation, else 0. */ + server_cache_pricing_eligible?: number; + /** Producer validity status code; see CACHE_VALIDITY_REASONS. */ + server_cache_validity_code?: number; /** Total requests attempted during the window. */ num_requests_total?: number; /** Requests that completed successfully. */ diff --git a/packages/app/src/lib/benchmarks/benchmark-transform.ts b/packages/app/src/lib/benchmarks/benchmark-transform.ts index 850562d78..f6673deb7 100644 --- a/packages/app/src/lib/benchmarks/benchmark-transform.ts +++ b/packages/app/src/lib/benchmarks/benchmark-transform.ts @@ -327,6 +327,8 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { server_external_cache_hit_rate: m.server_external_cache_hit_rate, server_cpu_cache_hit_rate: m.server_cpu_cache_hit_rate, theoretical_cache_hit_rate: m.theoretical_cache_hit_rate, + server_cache_pricing_eligible: m.server_cache_pricing_eligible, + server_cache_validity_code: m.server_cache_validity_code, num_requests_total: m.num_requests_total, num_requests_successful: m.num_requests_successful, total_prompt_tokens: m.total_prompt_tokens, diff --git a/packages/app/src/lib/calculator/cache-pricing.test.ts b/packages/app/src/lib/calculator/cache-pricing.test.ts index b646430c6..82afc0ae7 100644 --- a/packages/app/src/lib/calculator/cache-pricing.test.ts +++ b/packages/app/src/lib/calculator/cache-pricing.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from 'vitest'; import { + actualCacheRateUnavailable, CACHE_HIT_RATE_FALLBACK_HARDWARE, measuredCacheHitRate, pricingCacheHitRate, + SERIALIZED_RATE_SUM_TOLERANCE, } from './cache-pricing'; describe('measuredCacheHitRate', () => { @@ -75,3 +77,87 @@ describe('pricingCacheHitRate', () => { expect([...CACHE_HIT_RATE_FALLBACK_HARDWARE]).toEqual(['gb300']); }); }); + +describe('actualCacheRateUnavailable (ATOM actual pricing guard)', () => { + const RUN = 'https://github.com/SemiAnalysisAI/InferenceX/actions/runs/1/attempts/1'; + const atom = (fields: Record) => ({ + framework: 'atom-disagg', + run_url: RUN, + ...fields, + }); + const complete = { server_gpu_cache_hit_rate: 0.6, server_cpu_cache_hit_rate: 0.1 }; + const eligible = { server_cache_pricing_eligible: 1, server_cache_validity_code: 1 }; + + it('prices a complete eligible observation and a complete legacy one', () => { + expect(actualCacheRateUnavailable(atom({ ...complete, ...eligible }))).toBeNull(); + expect(actualCacheRateUnavailable(atom(complete))).toBeNull(); + }); + + it.each([ + ['eligible but no rates', { ...eligible }, 'MISSING_OBSERVATION'], + ['eligible but GPU only', { ...eligible, server_gpu_cache_hit_rate: 0.6 }, 'PARTIAL_COMPONENT'], + [ + 'eligible flag with a window code', + { ...complete, ...eligible, server_cache_validity_code: 13 }, + 'VALIDITY_CONFLICT', + ], + [ + 'eligible with GPU above 1', + { ...eligible, server_gpu_cache_hit_rate: 1.2, server_cpu_cache_hit_rate: 0 }, + 'INVALID_RATE', + ], + [ + 'legacy non-finite GPU', + { server_gpu_cache_hit_rate: Number.NaN, server_cpu_cache_hit_rate: 0 }, + 'INVALID_RATE', + ], + ['legacy CPU only', { server_cpu_cache_hit_rate: 0.1 }, 'PARTIAL_COMPONENT'], + ['legacy without rates', {}, 'MISSING_OBSERVATION'], + [ + 'an unknown flag value', + { ...complete, server_cache_pricing_eligible: 2 }, + 'UNKNOWN_VALIDITY', + ], + [ + 'CPU and external disagreeing', + { ...complete, ...eligible, server_external_cache_hit_rate: 0.2 }, + 'CPU_EXTERNAL_CONFLICT', + ], + [ + 'GPU plus CPU above 1', + { ...eligible, server_gpu_cache_hit_rate: 0.95, server_cpu_cache_hit_rate: 0.1 }, + 'INVALID_RATE', + ], + ])('refuses %s', (_, fields, reason) => { + expect(actualCacheRateUnavailable(atom(fields))).toBe(reason); + }); + + it('accepts a validated zero and a sum within the 5-decimal serialization tolerance', () => { + const zero = { server_gpu_cache_hit_rate: 0, server_cpu_cache_hit_rate: 0 }; + expect( + actualCacheRateUnavailable(atom({ ...zero, ...eligible, server_cache_validity_code: 2 })), + ).toBeNull(); + const rounded = { + server_gpu_cache_hit_rate: 0.99999, + server_cpu_cache_hit_rate: SERIALIZED_RATE_SUM_TOLERANCE, + }; + expect(actualCacheRateUnavailable(atom({ ...rounded, ...eligible }))).toBeNull(); + }); + + it('reports the producer reason for an ineligible row and quarantines the known-bad run', () => { + const scope = { ...complete, server_cache_pricing_eligible: 0, server_cache_validity_code: 14 }; + expect(actualCacheRateUnavailable(atom(scope))).toBe('REQUEST_SCOPE_UNRESOLVED'); + const legacy = { + ...atom(complete), + run_url: 'https://github.com/x/y/actions/runs/36734492117/attempts/1', + }; + expect(actualCacheRateUnavailable(legacy)).toBe('KNOWN_BAD_HISTORICAL_RESULT'); + expect(actualCacheRateUnavailable({ ...legacy, ...eligible })).toBeNull(); + }); + + it('leaves other frameworks unchanged', () => { + expect( + actualCacheRateUnavailable({ framework: 'vllm', server_gpu_cache_hit_rate: 1.2 }), + ).toBeNull(); + }); +}); diff --git a/packages/app/src/lib/calculator/cache-pricing.ts b/packages/app/src/lib/calculator/cache-pricing.ts index ffff0e6ac..cda03c99c 100644 --- a/packages/app/src/lib/calculator/cache-pricing.ts +++ b/packages/app/src/lib/calculator/cache-pricing.ts @@ -60,3 +60,83 @@ export function pricingCacheHitRate(point: PricingCacheHitPoint): number | null if (typeof theoretical !== 'number' || !Number.isFinite(theoretical)) return null; return Math.max(0, Math.min(1, theoretical)); } + +/** Decoding of `server_cache_validity_code` (mirrors the ETL table in agentic-v3-flatten.ts). */ +export const CACHE_VALIDITY_REASONS: Readonly> = { + 1: 'VALID_COMPLETE', + 2: 'VALID_ZERO', + 10: 'PARTIAL_COMPONENT', + 11: 'MISSING_OBSERVATION', + 12: 'ROLE_OR_COVERAGE_UNRESOLVED', + 13: 'WINDOW_UNRESOLVED', + 14: 'REQUEST_SCOPE_UNRESOLVED', + 15: 'INVALID_COUNTER', + 98: 'VALIDITY_CONFLICT', + 99: 'UNKNOWN_VALIDITY', +}; + +/** Validity codes the producer marks as a complete actual observation. */ +const PRICEABLE_VALIDITY_CODES: ReadonlySet = new Set([1, 2]); + +/** + * The producer serialises each rate to 5 decimal places, so GPU + CPU of a valid observation can + * exceed 1 by at most two half-units of the fifth decimal. + */ +export const SERIALIZED_RATE_SUM_TOLERANCE = 1e-5; + +/** + * ATOM runs whose published cache rates are confirmed wrong without validity metadata: run + * 36734492117 summed prefill and decode denominators and priced LMCache transfer volume as CPU + * reuse. A re-derived row that carries producer validity is judged by that metadata instead. + */ +export const KNOWN_BAD_ATOM_CACHE_RUNS: ReadonlySet = new Set(['36734492117']); + +export interface CacheRateValidityPoint extends CacheHitMetrics { + framework?: string; + run_url?: string | null; + server_cache_pricing_eligible?: number; + server_cache_validity_code?: number; +} + +const isUnitRate = (value: unknown): value is number => + typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1; + +const present = (value: unknown) => value !== undefined && value !== null; + +/** Why this row's own rates are not a complete actual observation, or null when they are. */ +function atomRateDefect(point: CacheRateValidityPoint): string | null { + const gpu: unknown = point.server_gpu_cache_hit_rate; + const cpu: unknown = point.server_cpu_cache_hit_rate; + const external: unknown = point.server_external_cache_hit_rate; + if (!present(gpu) && !present(cpu) && !present(external)) return 'MISSING_OBSERVATION'; + if (!present(gpu) || (!present(cpu) && !present(external))) return 'PARTIAL_COMPONENT'; + if (!isUnitRate(gpu)) return 'INVALID_RATE'; + if (present(cpu) && !isUnitRate(cpu)) return 'INVALID_RATE'; + if (present(external) && !isUnitRate(external)) return 'INVALID_RATE'; + // External is an alias of the CPU reuse for ATOM; two different values are not one observation. + if (present(cpu) && present(external) && cpu !== external) return 'CPU_EXTERNAL_CONFLICT'; + const secondary = (present(external) ? external : cpu) as number; + return gpu + secondary > 1 + SERIALIZED_RATE_SUM_TOLERANCE ? 'INVALID_RATE' : null; +} + +/** + * Why an ATOM row's measured cache rate must not be priced as a complete actual observation, or + * null when it may be. The producer flag, its status code and the row's own rates must agree; + * missing is not zero and a partial reading is not the full hit rate. Other frameworks keep their + * existing behaviour. + */ +export function actualCacheRateUnavailable(point: CacheRateValidityPoint): string | null { + if (!point.framework?.toLowerCase().startsWith('atom')) return null; + const eligible = point.server_cache_pricing_eligible; + const code = point.server_cache_validity_code; + if (eligible !== undefined || code !== undefined) { + if (eligible === 0) return CACHE_VALIDITY_REASONS[code ?? 99] ?? 'UNKNOWN_VALIDITY'; + if (eligible !== 1) return 'UNKNOWN_VALIDITY'; + if (code === undefined || !PRICEABLE_VALIDITY_CODES.has(code)) return 'VALIDITY_CONFLICT'; + } else { + const run = /\/runs\/(?\d+)/.exec(point.run_url ?? '')?.groups?.run; + if (run !== undefined && KNOWN_BAD_ATOM_CACHE_RUNS.has(run)) + return 'KNOWN_BAD_HISTORICAL_RESULT'; + } + return atomRateDefect(point); +} diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 5e7e44217..06538552e 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -80,6 +80,9 @@ export const POWER_METRIC_KEYS = [ * carry median/mean/p99/std for latency only. */ export const METRIC_KEYS = new Set([ + // ATOM cache-rate validity (agentic-v3-flatten cacheValidityColumns) + 'server_cache_pricing_eligible', + 'server_cache_validity_code', 'dp', // throughput (tokens/sec/GPU) 'tput_per_gpu', diff --git a/packages/db/src/etl/agentic-v3-flatten.test.ts b/packages/db/src/etl/agentic-v3-flatten.test.ts new file mode 100644 index 000000000..dada93ba2 --- /dev/null +++ b/packages/db/src/etl/agentic-v3-flatten.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; + +import { + CACHE_VALIDITY_CONFLICT, + CACHE_VALIDITY_UNKNOWN, + cacheValidityColumns, + flattenAgenticAggRow, +} from './agentic-v3-flatten'; + +const valid = { schema: 'atom-cache-validity/1', status: 'VALID_COMPLETE', pricing_eligible: true }; +const notEligible = (code: number) => ({ + server_cache_pricing_eligible: 0, + server_cache_validity_code: code, +}); + +describe('cacheValidityColumns', () => { + it('adds no columns for legacy rows and non-ATOM adapters', () => { + expect(cacheValidityColumns(undefined)).toBeNull(); + expect(cacheValidityColumns(null)).toBeNull(); + }); + + it('encodes eligible and ineligible well-formed blocks', () => { + expect(cacheValidityColumns(valid)).toEqual({ + server_cache_pricing_eligible: 1, + server_cache_validity_code: 1, + }); + expect( + cacheValidityColumns({ ...valid, status: 'WINDOW_UNRESOLVED', pricing_eligible: false }), + ).toEqual(notEligible(13)); + }); + + it.each([ + ['a string', 'VALID_COMPLETE'], + ['an array', [valid]], + ['a block without a flag', { schema: valid.schema, status: valid.status }], + ['a string boolean', { ...valid, pricing_eligible: 'true' }], + ['an unknown status', { ...valid, status: 'MOSTLY_VALID' }], + ['an unknown schema', { ...valid, schema: 'atom-cache-validity/0' }], + ])('records %s as unknown and not eligible', (_, value) => { + expect(cacheValidityColumns(value)).toEqual(notEligible(CACHE_VALIDITY_UNKNOWN)); + }); + + it.each([ + ['eligible with an unresolved status', { ...valid, status: 'WINDOW_UNRESOLVED' }], + ['not eligible with a complete status', { ...valid, pricing_eligible: false }], + ])('records a flag %s as a conflict', (_, value) => { + expect(cacheValidityColumns(value)).toEqual(notEligible(CACHE_VALIDITY_CONFLICT)); + }); +}); + +describe('flattenAgenticAggRow cache validity', () => { + it('flattens the validity block next to the cache rates', () => { + const flat = flattenAgenticAggRow({ + request_metrics: {}, + server_metrics: { cache: { gpu_cache_hit_rate: 0.6 }, cache_validity: valid }, + }); + expect(flat.server_gpu_cache_hit_rate).toBe(0.6); + expect(flat.server_cache_pricing_eligible).toBe(1); + expect(flat.server_cache_validity_code).toBe(1); + }); + + it('leaves a row whose adapter emitted no validity without validity columns', () => { + const flat = flattenAgenticAggRow({ + request_metrics: {}, + server_metrics: { cache_validity: null }, + }); + expect('server_cache_pricing_eligible' in flat).toBe(false); + }); +}); diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index 1349e2263..66dbf893d 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -113,6 +113,60 @@ function flattenStatBlock(block: unknown, suffix: string, out: Record = { + VALID_COMPLETE: 1, + VALID_ZERO: 2, + PARTIAL_COMPONENT: 10, + MISSING_OBSERVATION: 11, + ROLE_OR_COVERAGE_UNRESOLVED: 12, + WINDOW_UNRESOLVED: 13, + REQUEST_SCOPE_UNRESOLVED: 14, + INVALID_COUNTER: 15, +}; + +const CACHE_VALIDITY_SCHEMA = 'atom-cache-validity/1'; +const PRICING_STATUSES: ReadonlySet = new Set(['VALID_COMPLETE', 'VALID_ZERO']); +/** Code for a validity block whose `pricing_eligible` contradicts its status. */ +export const CACHE_VALIDITY_CONFLICT = 98; +/** Code for a malformed validity block or an unknown schema or status. */ +export const CACHE_VALIDITY_UNKNOWN = 99; + +export interface CacheValidityColumns { + server_cache_pricing_eligible: number; + server_cache_validity_code: number; +} + +/** + * Numeric columns for a producer `cache_validity` block. Absent or null (legacy rows, non-ATOM + * adapters) yields none. A present block that is not a well-formed known schema, or whose flag + * contradicts its status, is recorded as not eligible. + */ +const notEligible = (code: number): CacheValidityColumns => ({ + server_cache_pricing_eligible: 0, + server_cache_validity_code: code, +}); + +export function cacheValidityColumns(value: unknown): CacheValidityColumns | null { + if (value === undefined || value === null) return null; + if (typeof value !== 'object' || Array.isArray(value)) return notEligible(CACHE_VALIDITY_UNKNOWN); + if (!('schema' in value) || !('status' in value) || !('pricing_eligible' in value)) { + return notEligible(CACHE_VALIDITY_UNKNOWN); + } + const { schema, status, pricing_eligible: eligible } = value; + if (schema !== CACHE_VALIDITY_SCHEMA || typeof status !== 'string') { + return notEligible(CACHE_VALIDITY_UNKNOWN); + } + const code = CACHE_VALIDITY_CODES[status]; + if (code === undefined || typeof eligible !== 'boolean') + return notEligible(CACHE_VALIDITY_UNKNOWN); + if (eligible !== PRICING_STATUSES.has(status)) return notEligible(CACHE_VALIDITY_CONFLICT); + return { server_cache_pricing_eligible: eligible ? 1 : 0, server_cache_validity_code: code }; +} + export function flattenAgenticAggRow(row: Record): Record { const rm = row.request_metrics; if (!rm || typeof rm !== 'object' || Array.isArray(rm)) return row; @@ -135,5 +189,7 @@ export function flattenAgenticAggRow(row: Record): Record { expect(result!.offloadMode).toBe('on'); }); }); + +const atomCacheValidity = (status: string, eligible: unknown) => ({ + schema: 'atom-cache-validity/1', + status, + pricing_eligible: eligible, +}); + +describe('ATOM cache validity through mapBenchmarkRow', () => { + const run = 36734492117; + const atomRow = (cacheValidity?: unknown) => { + const base = makeV3AgenticRow({ framework: 'atom-disagg' }); + const serverMetrics = { ...base.server_metrics, cache: { gpu_cache_hit_rate: 0.95207 } }; + if (cacheValidity !== undefined) serverMetrics.cache_validity = cacheValidity; + return { ...base, server_metrics: serverMetrics }; + }; + + it.each([ + ['an eligible block', atomCacheValidity('VALID_COMPLETE', true), 1, 1], + ['an ineligible block', atomCacheValidity('REQUEST_SCOPE_UNRESOLVED', false), 0, 14], + ['an unknown block', { status: 'VALID_COMPLETE' }, 0, 99], + ])('keeps %s in the stored metrics', (_, block, eligible, code) => { + const result = mapBenchmarkRow(atomRow(block), createSkipTracker(), null, run); + expect(result).not.toBeNull(); + expect(result!.metrics.server_cache_pricing_eligible).toBe(eligible); + expect(result!.metrics.server_cache_validity_code).toBe(code); + expect(result!.metrics.server_gpu_cache_hit_rate).toBe(0.95207); + }); + + it('stores no validity for a legacy re-import, so it cannot inherit eligibility', () => { + const result = mapBenchmarkRow(atomRow(), createSkipTracker(), null, run); + expect(result).not.toBeNull(); + expect('server_cache_pricing_eligible' in result!.metrics).toBe(false); + expect('server_cache_validity_code' in result!.metrics).toBe(false); + }); +});