Skip to content

Commit d2792c2

Browse files
committed
ci: balance desktop smoke shards by observed file durations
1 parent 7992b56 commit d2792c2

5 files changed

Lines changed: 167 additions & 3 deletions

File tree

‎.github/workflows/desktop-e2e.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: desktop-e2e
22

33
# Smoke coverage of the real Electron shell on desktop changes. Three isolated runners
4-
# partition the complete suite by Playwright test-file groups; workers remain serial per runner.
4+
# partition complete test files by measured duration; workers remain serial per runner.
55
# The weekly
66
# advisory canary against electron@latest so Chromium-cadence breakage surfaces
77
# before an upgrade is attempted.
@@ -47,6 +47,8 @@ on:
4747
- 'packages/utils/**'
4848
- 'bun.lock'
4949
- 'package.json'
50+
- 'scripts/desktop-e2e.ts'
51+
- 'scripts/duration-shards.ts'
5052
schedule:
5153
- cron: '23 9 * * 1'
5254
workflow_dispatch:
@@ -108,7 +110,7 @@ jobs:
108110

109111
- name: Run Playwright _electron smoke suite
110112
working-directory: apps/desktop
111-
run: bunx playwright test --shard=${{ matrix.shard }}/3 --reporter=line,json
113+
run: bun run ../../scripts/desktop-e2e.ts ${{ matrix.shard }}/3
112114
env:
113115
PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/desktop-smoke.json
114116
BACKGROUND_EXECUTOR_REPORT_PATH: test-results/background-executor-report.json
@@ -127,7 +129,10 @@ jobs:
127129
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
128130
with:
129131
name: desktop-smoke-reports-${{ matrix.shard }}
130-
path: apps/desktop/test-results/desktop-smoke.json
132+
path: |
133+
apps/desktop/test-results/desktop-smoke.json
134+
apps/desktop/test-results/desktop-shard-plan.json
135+
apps/desktop/test-results/desktop-discovery.json
131136
if-no-files-found: ignore
132137
retention-days: 7
133138

‎apps/desktop/e2e-durations.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"background-executor.spec.ts": 131.12,
3+
"browser-chrome.spec.ts": 15.28,
4+
"browser-focus.spec.ts": 6.53,
5+
"browser-page-dialogs.spec.ts": 11.81,
6+
"browser-tools.spec.ts": 94.66,
7+
"chat-panel.spec.ts": 31.08,
8+
"check-report.spec.ts": 3.78,
9+
"computer-use.spec.ts": 2.07,
10+
"desktop-tools-live-sim.spec.ts": 0.0,
11+
"local-files.spec.ts": 27.68,
12+
"packaged-smoke.spec.ts": 0.0,
13+
"password-autofill.spec.ts": 46.56,
14+
"session-cookies.spec.ts": 11.45,
15+
"smoke.spec.ts": 25.56,
16+
"source-connect.spec.ts": 41.49,
17+
"terminal-cancel.spec.ts": 199.76,
18+
"update-notification.spec.ts": 19.55,
19+
"updater.spec.ts": 2.48
20+
}

‎scripts/desktop-e2e.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import { spawnSync } from 'node:child_process'
2+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
3+
import { resolve } from 'node:path'
4+
import { createLogger } from '@sim/logger'
5+
import { escapeRegExp } from '@sim/utils/string'
6+
import { partitionFiles } from './duration-shards'
7+
8+
interface Suite {
9+
specs?: { file: string }[]
10+
suites?: Suite[]
11+
}
12+
13+
interface DiscoveryReport {
14+
config: { rootDir: string }
15+
suites: Suite[]
16+
errors: unknown[]
17+
}
18+
19+
const logger = createLogger('DesktopE2E')
20+
const match = /^(\d+)\/(\d+)$/.exec(process.argv[2] ?? '')
21+
if (!match) throw new Error('Usage: bun run ../../scripts/desktop-e2e.ts <shard>/<total>')
22+
const index = Number(match[1])
23+
const count = Number(match[2])
24+
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index < 1 || index > count) {
25+
throw new Error('Invalid desktop shard index or count')
26+
}
27+
28+
mkdirSync('test-results', { recursive: true })
29+
const discoveryPath = resolve('test-results/desktop-discovery.json')
30+
const discovery = spawnSync('bunx', ['playwright', 'test', '--list', '--reporter=json'], {
31+
stdio: 'inherit',
32+
env: { ...process.env, PLAYWRIGHT_JSON_OUTPUT_FILE: discoveryPath },
33+
})
34+
if (discovery.error) throw discovery.error
35+
if (discovery.status !== 0) process.exit(discovery.status ?? 1)
36+
const discoveryJson = readFileSync(discoveryPath, 'utf8')
37+
const report = JSON.parse(discoveryJson) as DiscoveryReport
38+
if (report.errors.length > 0) throw new Error('Playwright discovery reported errors')
39+
const files = new Set<string>()
40+
function visit(suite: Suite): void {
41+
for (const spec of suite.specs ?? []) files.add(spec.file)
42+
for (const child of suite.suites ?? []) visit(child)
43+
}
44+
for (const suite of report.suites) visit(suite)
45+
if (count > files.size) throw new Error('More desktop shards than discovered test files')
46+
const durations = JSON.parse(
47+
readFileSync(new URL('../apps/desktop/e2e-durations.json', import.meta.url), 'utf8')
48+
) as Record<string, number>
49+
const shards = partitionFiles([...files], durations, count)
50+
const selected = shards[index - 1]
51+
if (selected.length === 0) throw new Error('Desktop shard selected no test files')
52+
logger.info(`Running desktop shard ${index}/${count}: ${selected.join(', ')}`)
53+
const run = spawnSync(
54+
'bunx',
55+
[
56+
'playwright',
57+
'test',
58+
...selected.map((file) => `^${escapeRegExp(resolve(report.config.rootDir, file))}$`),
59+
'--reporter=line,json',
60+
],
61+
{ stdio: 'inherit', env: process.env }
62+
)
63+
mkdirSync('test-results', { recursive: true })
64+
writeFileSync(discoveryPath, discoveryJson)
65+
writeFileSync(
66+
'test-results/desktop-shard-plan.json',
67+
`${JSON.stringify({ index, count, discoveredFiles: files.size, shards }, null, 2)}\n`
68+
)
69+
if (run.error) throw run.error
70+
process.exit(run.status ?? 1)

‎scripts/duration-shards.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { partitionFiles } from '@scripts/duration-shards'
2+
import { describe, expect, it } from 'vitest'
3+
4+
describe('duration-balanced file partitions', () => {
5+
it('keeps whole files and assigns every discovered file exactly once, including unmeasured files', () => {
6+
const files = ['a.spec.ts', 'b.spec.ts', 'new/nested.test.ts', 'unmeasured.spec.ts']
7+
const shards = partitionFiles(
8+
files,
9+
{ 'a.spec.ts': 100, 'b.spec.ts': 1, 'deleted.spec.ts': 999 },
10+
3
11+
)
12+
expect(shards.flat().sort()).toEqual([...files].sort())
13+
expect(shards.every((shard) => shard.length > 0)).toBe(true)
14+
})
15+
16+
it('separates long files instead of packing them into the same shard', () => {
17+
const weights = { a: 100, b: 90, c: 80, d: 20, e: 10 }
18+
const shards = partitionFiles(Object.keys(weights), weights, 3)
19+
const loads = shards.map((files) =>
20+
files.reduce((sum, file) => sum + weights[file as keyof typeof weights], 0)
21+
)
22+
expect(Math.max(...loads)).toBe(100)
23+
})
24+
25+
it('gives all runners the same partition regardless of discovery order and equal weights', () => {
26+
const files = ['z', 'b', 'a', 'nested/c', 'x']
27+
expect(partitionFiles(files, {}, 3)).toEqual(partitionFiles([...files].reverse(), {}, 3))
28+
})
29+
30+
it('preserves zero-duration files without placing every skipped file in one empty shard', () => {
31+
const shards = partitionFiles(['a', 'b', 'c'], { a: 0, b: 0, c: 0 }, 3)
32+
expect(shards.map((files) => files.length)).toEqual([1, 1, 1])
33+
})
34+
35+
it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
36+
'rejects invalid shard count %s',
37+
(count) => {
38+
expect(() => partitionFiles(['a'], {}, count)).toThrow('shard count')
39+
}
40+
)
41+
})

‎scripts/duration-shards.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { compareStrings } from '@sim/utils/string'
2+
3+
/** Partitions discovered files without splitting their serial fixtures or omitting unmeasured files. */
4+
export function partitionFiles(
5+
files: readonly string[],
6+
durations: Record<string, number>,
7+
count: number
8+
): string[][] {
9+
if (!Number.isSafeInteger(count) || count < 1) throw new Error('Invalid shard count')
10+
const measured = files
11+
.map((file) => durations[file])
12+
.filter((duration) => Number.isFinite(duration) && duration > 0)
13+
.sort((a, b) => a - b)
14+
const fallback = measured[Math.floor(measured.length / 2)] ?? 1
15+
const weight = (file: string) =>
16+
Number.isFinite(durations[file]) && durations[file] >= 0
17+
? Math.max(1, durations[file])
18+
: fallback
19+
const shards = Array.from({ length: count }, () => ({ files: [] as string[], seconds: 0 }))
20+
for (const file of [...files].sort((a, b) => weight(b) - weight(a) || compareStrings(a, b))) {
21+
const target = shards.reduce((smallest, shard) =>
22+
shard.seconds < smallest.seconds ? shard : smallest
23+
)
24+
target.files.push(file)
25+
target.seconds += weight(file)
26+
}
27+
return shards.map((shard) => shard.files.sort(compareStrings))
28+
}

0 commit comments

Comments
 (0)