Skip to content

Commit 65aa259

Browse files
ALikesToCodeALikesToCode
committed
fix(release): fail closed on browser smoke retries
1 parent 6c7791e commit 65aa259

7 files changed

Lines changed: 338 additions & 55 deletions

‎apps/web/e2e/qualification-reporter.ts‎

Lines changed: 76 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ interface QualificationReporterOptions {
114114
baseUrl: string;
115115
outputFile: string;
116116
qualificationRequested: boolean;
117+
requireCleanTelemetry: boolean;
117118
releaseBinding: z.infer<typeof PublicationReleaseBindingSchema> | null;
118119
repositoryRoot: string;
119120
runtimeRoot: string;
@@ -399,6 +400,65 @@ function exactViewportCoverage(journeys: readonly CapturedJourney[]): boolean {
399400
);
400401
}
401402

403+
export function hasCleanAttemptZeroTelemetry(
404+
journey: CapturedJourney,
405+
expectedAuthority: "CLOAK_CDP_ENDPOINT" | "stock-chromium-design-review",
406+
): boolean {
407+
const expectedHttpErrors =
408+
REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID[
409+
journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID
410+
];
411+
const expectedRequestFailures =
412+
REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID[
413+
journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID
414+
];
415+
return (
416+
expectedHttpErrors !== undefined &&
417+
expectedRequestFailures !== undefined &&
418+
journey.status === "passed" &&
419+
journey.expectedStatus === "passed" &&
420+
journey.attempt === 0 &&
421+
journey.durationMs > 0 &&
422+
journey.assertionCount > 0 &&
423+
journey.unexpectedConsoleErrors === 0 &&
424+
JSON.stringify(journey.expectedHttpErrorResponses) ===
425+
JSON.stringify(expectedHttpErrors) &&
426+
JSON.stringify(journey.observedHttpErrorResponses) ===
427+
JSON.stringify(journey.expectedHttpErrorResponses) &&
428+
journey.unexpectedHttpErrorResponses === 0 &&
429+
JSON.stringify(journey.expectedRequestFailures) ===
430+
JSON.stringify(expectedRequestFailures) &&
431+
journey.expectedRequestFailures.length === journey.expectedFailedRequests &&
432+
JSON.stringify(journey.observedRequestFailures) ===
433+
JSON.stringify(journey.expectedRequestFailures) &&
434+
journey.observedFailedRequests === journey.expectedFailedRequests &&
435+
journey.unexpectedFailedRequests === 0 &&
436+
journey.browserVersion.trim() !== "" &&
437+
journey.browserVersion !== "unavailable" &&
438+
journey.browserAuthority === expectedAuthority &&
439+
journey.telemetryValid
440+
);
441+
}
442+
443+
export function hasCleanProductionLane(input: {
444+
authority: RawRunAuthority;
445+
journeys: readonly CapturedJourney[];
446+
playwrightStatus: FullResult["status"];
447+
rootErrors: number;
448+
}): boolean {
449+
return (
450+
input.playwrightStatus === "passed" &&
451+
input.rootErrors === 0 &&
452+
input.journeys.length === 1 &&
453+
hasCleanAttemptZeroTelemetry(
454+
input.journeys[0]!,
455+
input.authority === "CLOAKBROWSER"
456+
? "CLOAK_CDP_ENDPOINT"
457+
: "stock-chromium-design-review",
458+
)
459+
);
460+
}
461+
402462
export function buildCloakBrowserRawRun(input: BuildRawRunInput) {
403463
const expectedJourneyAuthority =
404464
input.authority === "CLOAKBROWSER"
@@ -414,39 +474,8 @@ export function buildCloakBrowserRawRun(input: BuildRawRunInput) {
414474
(input.rootErrors ?? 0) === 0 &&
415475
exactJourneyRegistry(input.journeys) &&
416476
exactViewportCoverage(input.journeys) &&
417-
input.journeys.every(
418-
(journey) =>
419-
journey.status === "passed" &&
420-
journey.expectedStatus === "passed" &&
421-
journey.attempt === 0 &&
422-
journey.durationMs > 0 &&
423-
journey.assertionCount > 0 &&
424-
journey.unexpectedConsoleErrors === 0 &&
425-
JSON.stringify(journey.expectedHttpErrorResponses) ===
426-
JSON.stringify(
427-
REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID[
428-
journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_HTTP_ERRORS_BY_ID
429-
],
430-
) &&
431-
JSON.stringify(journey.observedHttpErrorResponses) ===
432-
JSON.stringify(journey.expectedHttpErrorResponses) &&
433-
journey.unexpectedHttpErrorResponses === 0 &&
434-
JSON.stringify(journey.expectedRequestFailures) ===
435-
JSON.stringify(
436-
REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID[
437-
journey.id as keyof typeof REQUIRED_CLOAK_EXPECTED_REQUEST_FAILURES_BY_ID
438-
],
439-
) &&
440-
journey.expectedRequestFailures.length ===
441-
journey.expectedFailedRequests &&
442-
JSON.stringify(journey.observedRequestFailures) ===
443-
JSON.stringify(journey.expectedRequestFailures) &&
444-
journey.observedFailedRequests === journey.expectedFailedRequests &&
445-
journey.unexpectedFailedRequests === 0 &&
446-
journey.browserVersion.trim() !== "" &&
447-
journey.browserVersion !== "unavailable" &&
448-
journey.browserAuthority === expectedJourneyAuthority &&
449-
journey.telemetryValid,
477+
input.journeys.every((journey) =>
478+
hasCleanAttemptZeroTelemetry(journey, expectedJourneyAuthority),
450479
) &&
451480
new Set(input.journeys.map((journey) => journey.browserVersion)).size === 1;
452481

@@ -562,7 +591,8 @@ export default class CounterLabQualificationReporter implements Reporter {
562591
): Promise<{ status?: FullResult["status"] } | undefined> {
563592
if (
564593
this.capturedJourneys.length === 0 &&
565-
!this.options.qualificationRequested
594+
!this.options.qualificationRequested &&
595+
!this.options.requireCleanTelemetry
566596
) {
567597
return undefined;
568598
}
@@ -593,7 +623,18 @@ export default class CounterLabQualificationReporter implements Reporter {
593623
flag: "wx",
594624
mode: 0o600,
595625
});
596-
if (this.options.qualificationRequested && report.status !== "PASSED") {
626+
const cleanTelemetryPassed =
627+
!this.options.requireCleanTelemetry ||
628+
hasCleanProductionLane({
629+
authority: this.options.authority,
630+
journeys: this.capturedJourneys,
631+
playwrightStatus: result.status,
632+
rootErrors: this.rootErrors,
633+
});
634+
if (
635+
(this.options.qualificationRequested && report.status !== "PASSED") ||
636+
!cleanTelemetryPassed
637+
) {
597638
return { status: "failed" };
598639
}
599640
return undefined;

‎apps/web/playwright.config.ts‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,14 @@ function assertNoSymlinkTraversal(candidate: string, label: string): void {
5252
}
5353
}
5454

55+
function exactBooleanSetting(name: string): boolean {
56+
const setting = process.env[name];
57+
if (setting !== undefined && setting !== "true" && setting !== "false") {
58+
throw new Error(`${name} must be exactly true or false`);
59+
}
60+
return setting === "true";
61+
}
62+
5563
if (
5664
!isContained(repositoryRoot, runtimeRoot, false) ||
5765
!isContained(runtimeParent, runtimeRoot, false)
@@ -114,17 +122,12 @@ const staticDesignReview =
114122
const localAdmissionKey =
115123
process.env.COUNTERLAB_ADMISSION_KEY?.trim() ||
116124
["counterlab", "local", "e2e", "admission", "only", "000000"].join("-");
117-
const qualificationSetting = process.env.COUNTERLAB_BROWSER_QUALIFICATION;
118-
if (
119-
qualificationSetting !== undefined &&
120-
qualificationSetting !== "true" &&
121-
qualificationSetting !== "false"
122-
) {
123-
throw new Error(
124-
"COUNTERLAB_BROWSER_QUALIFICATION must be exactly true or false",
125-
);
126-
}
127-
const qualificationRequested = qualificationSetting === "true";
125+
const qualificationRequested = exactBooleanSetting(
126+
"COUNTERLAB_BROWSER_QUALIFICATION",
127+
);
128+
const requireCleanTelemetry = exactBooleanSetting(
129+
"COUNTERLAB_E2E_REQUIRE_CLEAN_TELEMETRY",
130+
);
128131
if (
129132
qualificationRequested &&
130133
(browserAuthority.kind !== "cloak" ||
@@ -134,6 +137,11 @@ if (
134137
"CloakBrowser qualification requires the exact public CounterLab origin",
135138
);
136139
}
140+
if (requireCleanTelemetry && browserAuthority.kind !== "cloak") {
141+
throw new Error(
142+
"Clean browser telemetry enforcement requires CloakBrowser authority",
143+
);
144+
}
137145
const deploymentReceiptSetting = process.env.COUNTERLAB_E2E_DEPLOYMENT_RECEIPT;
138146
let qualificationReleaseBinding: ReturnType<
139147
typeof PublicationReleaseBindingSchema.parse
@@ -201,7 +209,12 @@ export default defineConfig({
201209
fullyParallel: false,
202210
workers: 1,
203211
forbidOnly: true,
204-
retries: process.env.CI ? 1 : 0,
212+
retries:
213+
qualificationRequested || requireCleanTelemetry
214+
? 0
215+
: process.env.CI
216+
? 1
217+
: 0,
205218
outputDir,
206219
reporter: [
207220
["line"],
@@ -216,6 +229,7 @@ export default defineConfig({
216229
baseUrl: baseURL,
217230
outputFile: qualificationRunFile,
218231
qualificationRequested,
232+
requireCleanTelemetry,
219233
releaseBinding: qualificationReleaseBinding,
220234
repositoryRoot,
221235
runtimeRoot,

‎apps/web/src/test/playwrightConfig.test.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import { afterEach, describe, expect, it, vi } from "vitest";
2+
3+
const preservedEnvironment = {
4+
CLOAK_CDP_ENDPOINT: process.env.CLOAK_CDP_ENDPOINT,
5+
COUNTERLAB_E2E_DEPLOYMENT_RECEIPT:
6+
process.env.COUNTERLAB_E2E_DEPLOYMENT_RECEIPT,
7+
};
8+
9+
afterEach(() => {
10+
vi.unstubAllEnvs();
11+
for (const [name, value] of Object.entries(preservedEnvironment)) {
12+
if (value === undefined) delete process.env[name];
13+
else process.env[name] = value;
14+
}
15+
vi.resetModules();
16+
});
17+
18+
describe("Playwright release-smoke configuration", () => {
19+
it("forces attempt zero when clean CloakBrowser telemetry is required", async () => {
20+
vi.stubEnv("CI", "true");
21+
vi.stubEnv("CLOAK_CDP_ENDPOINT", "http://127.0.0.1:9");
22+
vi.stubEnv("COUNTERLAB_BROWSER_QUALIFICATION", "false");
23+
vi.stubEnv("COUNTERLAB_E2E_REQUIRE_CLEAN_TELEMETRY", "true");
24+
delete process.env.COUNTERLAB_E2E_DEPLOYMENT_RECEIPT;
25+
26+
const { default: config } = await import("../../playwright.config");
27+
28+
expect(config.retries).toBe(0);
29+
expect(config.reporter).toContainEqual([
30+
"./e2e/qualification-reporter.ts",
31+
expect.objectContaining({
32+
qualificationRequested: false,
33+
requireCleanTelemetry: true,
34+
}),
35+
]);
36+
});
37+
38+
it("rejects clean-telemetry enforcement without CloakBrowser", async () => {
39+
vi.stubEnv("COUNTERLAB_BROWSER_AUTHORITY", "stock-chromium-design-review");
40+
vi.stubEnv("COUNTERLAB_BROWSER_QUALIFICATION", "false");
41+
vi.stubEnv("COUNTERLAB_E2E_REQUIRE_CLEAN_TELEMETRY", "true");
42+
delete process.env.CLOAK_CDP_ENDPOINT;
43+
delete process.env.COUNTERLAB_E2E_DEPLOYMENT_RECEIPT;
44+
45+
await expect(import("../../playwright.config")).rejects.toThrow(
46+
/requires CloakBrowser authority/i,
47+
);
48+
});
49+
});

‎apps/web/src/test/qualificationReporter.test.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import {
88
classifyExpectedHttpResourceConsoleError,
99
classifyExpectedHttpResourceConsoleObservation,
1010
countPlaywrightAssertions,
11+
hasCleanAttemptZeroTelemetry,
12+
hasCleanProductionLane,
1113
journeyIdForTest,
1214
readJourneyObservation,
1315
summarizeExpectedHttpErrors,
@@ -511,6 +513,75 @@ describe("CloakBrowser qualification reporter", () => {
511513
);
512514
});
513515

516+
it("requires clean attempt-zero telemetry for an individual production lane", () => {
517+
const journey = capturedJourneys()[0]!;
518+
519+
expect(hasCleanAttemptZeroTelemetry(journey, "CLOAK_CDP_ENDPOINT")).toBe(
520+
true,
521+
);
522+
expect(
523+
hasCleanAttemptZeroTelemetry(
524+
{ ...journey, attempt: 1 },
525+
"CLOAK_CDP_ENDPOINT",
526+
),
527+
).toBe(false);
528+
expect(
529+
hasCleanAttemptZeroTelemetry(
530+
{ ...journey, telemetryValid: false },
531+
"CLOAK_CDP_ENDPOINT",
532+
),
533+
).toBe(false);
534+
expect(
535+
hasCleanAttemptZeroTelemetry(
536+
{ ...journey, unexpectedConsoleErrors: 1 },
537+
"CLOAK_CDP_ENDPOINT",
538+
),
539+
).toBe(false);
540+
expect(
541+
hasCleanAttemptZeroTelemetry(journey, "stock-chromium-design-review"),
542+
).toBe(false);
543+
});
544+
545+
it("fails a production lane closed for retries, extra matches, root errors, and bad telemetry", () => {
546+
const journey = capturedJourneys()[0]!;
547+
const cleanLane = {
548+
authority: "CLOAKBROWSER" as const,
549+
journeys: [journey],
550+
playwrightStatus: "passed" as const,
551+
rootErrors: 0,
552+
};
553+
554+
expect(hasCleanProductionLane(cleanLane)).toBe(true);
555+
expect(
556+
hasCleanProductionLane({
557+
...cleanLane,
558+
journeys: [],
559+
}),
560+
).toBe(false);
561+
expect(
562+
hasCleanProductionLane({
563+
...cleanLane,
564+
journeys: [{ ...journey, attempt: 1 }],
565+
}),
566+
).toBe(false);
567+
expect(
568+
hasCleanProductionLane({
569+
...cleanLane,
570+
journeys: [
571+
{ ...journey, status: "failed" },
572+
{ ...journey, attempt: 1 },
573+
],
574+
}),
575+
).toBe(false);
576+
expect(hasCleanProductionLane({ ...cleanLane, rootErrors: 1 })).toBe(false);
577+
expect(
578+
hasCleanProductionLane({
579+
...cleanLane,
580+
journeys: [{ ...journey, telemetryValid: false }],
581+
}),
582+
).toBe(false);
583+
});
584+
514585
it("accepts one response-bound Chromium resource error without hiding it", () => {
515586
const journeys = capturedJourneys().map((journey) =>
516587
journey.id ===

‎scripts/production-smoke-policy.test.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { readFileSync } from "node:fs";
2+
import { resolve } from "node:path";
3+
4+
import { describe, expect, it } from "vitest";
5+
6+
const source = readFileSync(
7+
resolve(process.cwd(), "scripts/production-smoke.sh"),
8+
"utf8",
9+
);
10+
11+
describe("production smoke policy", () => {
12+
it("requires clean attempt-zero telemetry for every browser lane", () => {
13+
expect(
14+
source.match(/COUNTERLAB_E2E_REQUIRE_CLEAN_TELEMETRY=true/gu),
15+
).toHaveLength(6);
16+
expect(source).not.toContain("COUNTERLAB_E2E_REQUIRE_CLEAN_TEMETRY");
17+
});
18+
19+
it("requires two consecutive deep-readiness passes before browser work", () => {
20+
expect(source).toContain("HEALTH_ATTEMPT < 12 && HEALTH_CONSECUTIVE < 2");
21+
expect(source).toContain("HEALTH_CONSECUTIVE != 2");
22+
expect(source).toContain(
23+
'"generationFilesystemReadIsolation": "OS_ENFORCED"',
24+
);
25+
expect(source).toContain(
26+
'"${HEALTH_REQUEST_ID}" == "${HEALTH_PREVIOUS_REQUEST_ID}"',
27+
);
28+
expect(source.indexOf("HEALTH_CONSECUTIVE != 2")).toBeLessThan(
29+
source.indexOf("BROWSER_RUNTIME_PARENT="),
30+
);
31+
});
32+
});

0 commit comments

Comments
 (0)