Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 121 additions & 0 deletions packages/app/cypress/e2e/inference-chart.cy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
} from '../support/e2e';
import {
interceptOverlayRun,
AGENTIC_DATE,
OVERLAY_RUN_ID,
OVERLAY_RUN_URL,
SINGLE_TURN_DATE,
Expand Down Expand Up @@ -154,6 +155,7 @@ describe('Inference Chart', () => {
.and('not.contain.text', 'Workload:')
.and('not.contain.text', 'Precision:')
.and('not.contain.text', 'Metric:');
cy.get('[data-testid="deprecated-watermark"]').should('not.exist');
});

it('plots OpenRouter-priced token revenue for official and unofficial runs', () => {
Expand Down Expand Up @@ -388,6 +390,125 @@ describe('Inference Chart', () => {
});
});

describe('Deprecated model background watermark', () => {
it('marks every deprecated model and clears the mark for an active model', () => {
cy.viewport(1280, 900);
for (const [display, dbKey, deprecated] of [
['MiniMax-M3', 'minimaxm3', true],
['MiniMax-M2.5', 'minimaxm2.5', true],
['Kimi-K2.5', 'kimik2.5', true],
['GLM-5', 'glm5', true],
['gpt-oss-120b', 'gptoss120b', true],
['Llama-3.3-70B-Instruct-FP8', 'llama70b', true],
['MiniMax-M3.1-Flash', 'minimaxm31flash', false],
] as const) {
const rows = b300Rows(null).map((row) => ({ ...row, model: dbKey }));
cy.intercept('GET', '/api/v1/availability', {
body: rows.map(({ metrics: _metrics, ...row }) => row),
});
cy.intercept('GET', '/api/v1/benchmarks*', { body: rows });
cy.visit(`/inference?g_model=${display}&i_seq=agentic-traces`, {
onBeforeLoad: unlockAgenticGate,
});
cy.get('#chart-0 svg[data-testid="d3-chart-svg"]').should('be.visible');
if (deprecated) {
cy.get('#chart-0 .deprecated-watermark')
.should('have.length', 2)
.each(($mark) => {
cy.wrap($mark).should('have.text', 'Deprecated').and('have.attr', 'opacity', '0.3');
});
} else {
cy.get('#chart-0 .deprecated-watermark').should('not.exist');
}
}
});

it('keeps a large plot watermark in PNG exports, overlays, and the Chinese mobile chart', () => {
const model = 'minimaxm3';
const officialRows = b300Rows(null).map((row) => ({ ...row, model }));
const overlayRows = b300Rows(OVERLAY_RUN_URL).map((row) => ({ ...row, model }));
cy.intercept('GET', '/api/v1/availability', {
body: officialRows.map(({ metrics: _metrics, ...row }) => row),
});
cy.intercept('GET', '/api/v1/benchmarks*', { body: officialRows });
cy.intercept('GET', '/api/unofficial-run*', {
body: {
runInfos: [
{
id: OVERLAY_RUN_ID,
name: 'historical-m3-overlay',
branch: 'historical-m3-overlay',
sha: 'abc000',
createdAt: `${AGENTIC_DATE}T00:00:00Z`,
url: OVERLAY_RUN_URL,
conclusion: 'success',
status: 'completed',
isNonMainBranch: true,
},
],
benchmarks: overlayRows,
evaluations: [],
},
}).as('historicalM3Overlay');

cy.visit('/inference/minimax-m3?i_seq=agentic-traces', {
onBeforeLoad: unlockAgenticGate,
});
cy.get('#chart-0 [data-testid="result-context"]')
.should('contain.text', 'Source: SemiAnalysis InferenceX™')
.and('not.contain.text', 'Historical benchmark — submissions closed');
cy.get('#chart-0 svg [data-testid="deprecated-watermark"]').should('be.visible');
let exportedPng = '';
let exportedLabels: string[] = [];
cy.window().then((win) => {
win.HTMLAnchorElement.prototype.click = function () {
if (this.download.endsWith('.png')) {
exportedPng = this.href;
exportedLabels = [
...win.document.querySelectorAll('#chart-0-export .deprecated-watermark'),
].map((mark) => mark.textContent ?? '');
}
};
});
cy.get('[data-testid="chart-figure"] [data-testid="export-button"]').first().click();
cy.get('[data-testid="export-png-button"]').click();
cy.window()
.should(() => expect(exportedPng).to.match(/^data:image\/png;base64,/u))
.then(() => {
expect(exportedLabels).to.deep.equal(['Deprecated', 'Deprecated']);
cy.writeFile(
'cypress/downloads/deprecated-model-watermark.png',
exportedPng.split(',')[1],
'base64',
);
});

cy.visit(`/inference/minimax-m3?i_seq=agentic-traces&unofficialrun=${OVERLAY_RUN_ID}`, {
onBeforeLoad: unlockAgenticGate,
});
cy.wait('@historicalM3Overlay');
cy.get('#chart-0 .unofficial-overlay-pt').should('have.length.greaterThan', 0);
cy.get('#chart-0 .deprecated-watermark').should('contain.text', 'Deprecated');
cy.get('#chart-0 .unofficial-watermark-image').should('exist');
cy.get('[aria-label="Dismiss historical-m3-overlay"]').click();
cy.get('#chart-0 .unofficial-overlay-pt').should('not.exist');
cy.get('#chart-0 svg [data-testid="deprecated-watermark"]').should('be.visible');

cy.viewport(375, 900);
cy.visit(`/zh/inference/minimax-m3?i_seq=agentic-traces&unofficialrun=${OVERLAY_RUN_ID}`, {
onBeforeLoad: unlockAgenticGate,
});
cy.wait('@historicalM3Overlay');
cy.get('#chart-0 svg [data-testid="deprecated-watermark"]')
.should('be.visible')
.and('have.length', 2)
.each(($mark) => cy.wrap($mark).should('have.text', '已弃用'));
cy.document().then((doc) => {
expect(doc.documentElement.scrollWidth).to.be.at.most(doc.documentElement.clientWidth);
});
});
});

describe('Inference Chart — Simplified Chinese mobile path', () => {
beforeEach(() => {
cy.viewport(375, 900);
Expand Down
7 changes: 6 additions & 1 deletion packages/app/src/components/inference/ui/GPUGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ import {
getInferenceRunLabel,
getOverlayLineLabel,
} from '@/lib/benchmarks/inference-labels';
import { getChartWatermark, Sequence } from '@/lib/catalog/data-mappings';
import {
getChartWatermark,
getDeprecatedChartWatermark,
Sequence,
} from '@/lib/catalog/data-mappings';
import { useLocale } from '@/lib/i18n/use-locale';
import { formatNumber, getDisplayLabel, updateRepoUrl } from '@/lib/shared/utils';
import { perfRulerAxisMetricKey, usePerfRulerAxisReset } from '@/hooks/usePerfRulerAxisReset';
Expand Down Expand Up @@ -1310,6 +1314,7 @@ const GPUGraph = React.memo(
data={officialPoints}
margin={CHART_MARGIN}
watermark={getChartWatermark()}
backgroundWatermark={getDeprecatedChartWatermark(modelLabel, locale)}
testId="gpu-graph"
grabCursor={true}
caption={caption}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import { lttbDownsample } from '@/lib/d3-chart/downsample';
import { CHART_FONT_SANS, CHART_TYPE, px } from '@/lib/d3-chart/typography';
import { overlayRunColor, overlayRunIndex } from '@/lib/charts/overlay-run-style';
import { useLocale } from '@/lib/i18n/use-locale';
import { getDeprecatedChartWatermark } from '@/lib/catalog/data-mappings';
import { getDisplayLabel } from '@/lib/shared/utils';

import {
Expand Down Expand Up @@ -628,7 +629,8 @@ export default function PowerTimeline({
const locale = useLocale();
const t = STRINGS[locale];
const { hardwareConfig, hwTypesWithData } = useInferenceData();
const { activeHwTypes, selectedPrecisions, quickFilters, activeDates } = useInferenceFilters();
const { activeHwTypes, selectedPrecisions, quickFilters, activeDates, selectedModel } =
useInferenceFilters();
const { isLegendExpanded, highContrast } = useInferenceDisplay();
const { setBestPerSku, toggleHwType, setIsLegendExpanded, toggleActiveDate } =
useInferenceActions();
Expand Down Expand Up @@ -1423,6 +1425,7 @@ export default function PowerTimeline({
height={CHART_HEIGHT}
margin={MARGIN}
watermark={overlayData && overlayPoints.length > 0 ? 'unofficial' : 'logo'}
backgroundWatermark={getDeprecatedChartWatermark(selectedModel, locale)}
testId="power-timeline-chart-svg"
grabCursor
instructions={t.instructions}
Expand Down
2 changes: 2 additions & 0 deletions packages/app/src/components/inference/ui/ScatterGraph.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
} from '@/lib/catalog/constants';
import {
getChartWatermark,
getDeprecatedChartWatermark,
getPrecisionLabel,
getSequenceLabel,
type Precision,
Expand Down Expand Up @@ -3723,6 +3724,7 @@ const ScatterGraph = React.memo(
metricIdentity={metricIdentity}
margin={CHART_MARGIN}
watermark={getChartWatermark(isUnofficialRun)}
backgroundWatermark={getDeprecatedChartWatermark(modelLabel, locale)}
testId="scatter-graph"
grabCursor={true}
caption={caption}
Expand Down
22 changes: 22 additions & 0 deletions packages/app/src/lib/catalog/data-mappings.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
getModelAndSequence,
getModelAndSequenceFromArtifact,
getModelLabel,
getDeprecatedChartWatermark,
getOpenRouterModelId,
MODEL_OPTIONS,
SEQUENCE_OPTIONS,
Expand Down Expand Up @@ -167,6 +168,27 @@ describe('getModelAndSequenceFromArtifact', () => {
});

describe('isModelDeprecated', () => {
it.each([
Model.MiniMax_M3,
Model.MiniMax_M2_5,
Model.Kimi_K2_5,
Model.GLM_5,
Model.GptOss,
Model.Llama3_3_70B,
])('adds a localized Deprecated watermark to %s', (model) => {
expect(getDeprecatedChartWatermark(model, 'en')).toBe('Deprecated');
expect(getDeprecatedChartWatermark(model, 'zh')).toBe('已弃用');
});

it.each([
Model.MiniMax_M3_1_Flash,
Model.DeepSeek_R1,
Model.Qwen3_8_27B,
Model.Llama3_1_70B,
undefined,
])('does not mark the non-deprecated model %s', (model) => {
expect(getDeprecatedChartWatermark(model, 'en')).toBeUndefined();
});
it('returns true for deprecated model Llama3_3_70B', () => {
expect(isModelDeprecated(Model.Llama3_3_70B)).toBe(true);
});
Expand Down
8 changes: 8 additions & 0 deletions packages/app/src/lib/catalog/data-mappings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,14 @@ export function getChartWatermark(isUnofficialRun = false): 'logo' | 'unofficial
return isUnofficialRun ? 'unofficial' : 'logo';
}

export function getDeprecatedChartWatermark(
model: string | undefined,
locale: 'en' | 'zh',
): string | undefined {
if (!model || !isModelDeprecated(model as Model)) return undefined;
return locale === 'zh' ? '已弃用' : 'Deprecated';
}

export const MODEL_PREFIX_MAPPING: Record<string, Model> = Object.fromEntries(
(Object.entries(MODEL_CONFIG) as [Model, (typeof MODEL_CONFIG)[Model]][])
.filter(([, c]) => c.prefix)
Expand Down
2 changes: 2 additions & 0 deletions packages/app/src/lib/d3-chart/D3Chart/D3Chart.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ function D3ChartInner<T>(
height = 600,
margin,
watermark,
backgroundWatermark,
testId,
grabCursor = true,
instructions,
Expand Down Expand Up @@ -107,6 +108,7 @@ function D3ChartInner<T>(
height,
margin,
watermark,
backgroundWatermark,
testId,
grabCursor,
instructions,
Expand Down
1 change: 1 addition & 0 deletions packages/app/src/lib/d3-chart/D3Chart/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ export interface D3ChartProps<T = any> {
height?: number;
margin?: ChartMargin;
watermark?: 'logo' | 'unofficial' | 'none';
backgroundWatermark?: string;
testId?: string;
grabCursor?: boolean;
instructions?: string;
Expand Down
3 changes: 3 additions & 0 deletions packages/app/src/lib/d3-chart/D3Chart/useD3ChartRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ export function useD3ChartRenderer<T>(props: D3ChartProps<T>, deps: RendererDeps
data,
margin = { top: 24, right: 10, bottom: 40, left: 60 },
watermark = 'logo',
backgroundWatermark,
clipContent = true,
dataIdentity,
metricIdentity,
Expand Down Expand Up @@ -317,6 +318,7 @@ export function useD3ChartRenderer<T>(props: D3ChartProps<T>, deps: RendererDeps
containerHeight: dimensions.height,
margin,
watermark: getDomainAwareChartWatermark(watermark, window.location.hostname),
backgroundWatermark,
xLabel: xAxisConfig?.label,
yLabel: yAxisConfig?.label,
clipContent,
Expand All @@ -331,6 +333,7 @@ export function useD3ChartRenderer<T>(props: D3ChartProps<T>, deps: RendererDeps
margin.bottom,
margin.left,
watermark,
backgroundWatermark,
xAxisConfig?.label,
yAxisConfig?.label,
clipContent,
Expand Down
71 changes: 71 additions & 0 deletions packages/app/src/lib/d3-chart/chart-setup.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,77 @@ describe('setupChartStructure', () => {
});

describe('watermark variants', () => {
it.each([
[400, 600],
[1200, 600],
[800, 300],
])(
'preserves the SemiAnalysis logo and places Deprecated above and below it at %s×%s',
(containerWidth, containerHeight) => {
const svgEl = makeSvgEl();
const config = defaultConfig({ watermark: 'logo', containerWidth, containerHeight });
setupChartStructure(svgEl, config);
const originalLogo = svgEl.querySelector('#logo-pattern-test-chart')!.outerHTML;
const originalRect = svgEl.querySelector('.watermark-rect')!.outerHTML;

setupChartStructure(svgEl, { ...config, backgroundWatermark: 'Deprecated' });
expect(svgEl.querySelector('#logo-pattern-test-chart')!.outerHTML).toBe(originalLogo);
expect(svgEl.querySelector('.watermark-rect')!.outerHTML).toBe(originalRect);
const logo = svgEl.querySelector('#logo-pattern-test-chart image')!;
const above = svgEl.querySelector('.deprecated-watermark[data-position="above"]')!;
const status = svgEl.querySelector('.deprecated-watermark[data-position="below"]')!;
expect(svgEl.querySelectorAll('.deprecated-watermark')).toHaveLength(2);
expect(Number(above.getAttribute('y'))).toBeGreaterThanOrEqual(config.margin.top);
expect(Number(above.getAttribute('y')) + Number(above.getAttribute('height'))).toBeLessThan(
Number(logo.getAttribute('y')),
);
expect(above.getAttribute('opacity')).toBe('0.3');
expect(status.getAttribute('opacity')).toBe('0.3');
const logoBottom = Number(logo.getAttribute('y')) + Number(logo.getAttribute('height'));
expect(Number(status.getAttribute('y'))).toBeGreaterThan(logoBottom);
expect(
Number(status.getAttribute('y')) + Number(status.getAttribute('height')),
).toBeLessThanOrEqual(containerHeight - config.margin.bottom);
},
);
it('keeps a large deprecated watermark behind the plot alongside unofficial marks through resize and removal', () => {
const svgEl = document.createElementNS('http://www.w3.org/2000/svg', 'svg');
setupChartStructure(
svgEl,
defaultConfig({
watermark: 'unofficial',
backgroundWatermark: 'Deprecated',
}),
);
const mark = svgEl.querySelector('.deprecated-watermark')!;
expect(mark.parentElement).toBe(svgEl);
expect([...svgEl.children].indexOf(mark)).toBeLessThan(
[...svgEl.children].indexOf(svgEl.querySelector('.chart-root')!),
);
expect(mark.getAttribute('viewBox')).toBe('0 0 1000 200');
expect(mark.getAttribute('pointer-events')).toBe('none');
expect([...mark.querySelectorAll('text')].map((node) => node.textContent)).toEqual([
'Deprecated',
]);
expect(svgEl.querySelector('.unofficial-watermark-image')).not.toBeNull();

const resized = defaultConfig({
watermark: 'unofficial',
backgroundWatermark: '已弃用',
containerWidth: 400,
});
setupChartStructure(svgEl, resized);
expect(svgEl.querySelectorAll('.deprecated-watermark')).toHaveLength(2);
expect(svgEl.querySelector('.deprecated-watermark')?.getAttribute('width')).toBe(
String((400 - resized.margin.left - resized.margin.right) * 0.8),
);
expect(svgEl.querySelector('.deprecated-watermark')?.textContent).toBe('已弃用');

setupChartStructure(svgEl, defaultConfig({ watermark: 'logo' }));
expect(svgEl.querySelector('.deprecated-watermark')).toBeNull();
expect(svgEl.querySelector('.unofficial-watermark-image')).toBeNull();
expect(svgEl.querySelector('.watermark-rect')).not.toBeNull();
});
it('creates logo watermark when watermark is "logo"', () => {
const svgEl = makeSvgEl();
const config = defaultConfig({ watermark: 'logo' });
Expand Down
Loading
Loading