From 158e30143c6f0151cbadd35d99f937c84c4d83a2 Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:17:11 +0800 Subject: [PATCH 1/6] feat(http): readJson validates every body a test reads and returns it typed Tests no longer read the untyped value res.json() returns. readJson parses, checks the contract and throws a ContractError listing every broken field path, so every body read doubles as a contract check. The matcher shares the same issue formatting. Co-Authored-By: Claude Opus 5.5 --- src/http/readJson.ts | 36 +++++++++++++++++++++++++++++++++ src/matchers/schema.ts | 6 ++---- tests/auth.spec.ts | 5 +++-- tests/booking.contract.spec.ts | 13 +++++++++--- tests/booking.crud.spec.ts | 37 +++++++++++++++++++++++++--------- tests/booking.negative.spec.ts | 34 ++++++++++++++++++++++++------- 6 files changed, 105 insertions(+), 26 deletions(-) create mode 100644 src/http/readJson.ts diff --git a/src/http/readJson.ts b/src/http/readJson.ts new file mode 100644 index 0000000..97c6496 --- /dev/null +++ b/src/http/readJson.ts @@ -0,0 +1,36 @@ +import type { APIResponse } from '@playwright/test'; +import type { z } from 'zod'; + +/** One line per violated field, e.g. ` • booking.totalprice: Invalid input: expected number`. */ +export function formatIssues(issues: readonly z.core.$ZodIssue[]): string { + return issues + .map((issue) => ` • ${issue.path.map(String).join('.') || '(root)'}: ${issue.message}`) + .join('\n'); +} + +/** Thrown when a response body doesn't match its contract. Lists every violated field path. */ +export class ContractError extends Error { + constructor( + readonly status: number, + readonly url: string, + readonly issues: readonly z.core.$ZodIssue[], + ) { + super(`${status} ${url} broke its contract:\n${formatIssues(issues)}`); + this.name = 'ContractError'; + } +} + +/** + * Parse a response body and validate it against its schema. Returns a typed value, so tests never + * touch the `any` that `res.json()` returns, and every body a test reads is also a contract check. + * + * const { bookingid } = await readJson(await bookingClient.create(payload), CreatedBookingSchema); + */ +export async function readJson( + res: APIResponse, + schema: S, +): Promise> { + const result = schema.safeParse(await res.json()); + if (!result.success) throw new ContractError(res.status(), res.url(), result.error.issues); + return result.data; +} diff --git a/src/matchers/schema.ts b/src/matchers/schema.ts index 6340f74..46a638b 100644 --- a/src/matchers/schema.ts +++ b/src/matchers/schema.ts @@ -1,5 +1,6 @@ import { expect as baseExpect, type APIResponse } from '@playwright/test'; import type { z } from 'zod'; +import { formatIssues } from '../http/readJson.js'; function isAPIResponse(value: unknown): value is APIResponse { return ( @@ -31,13 +32,10 @@ export const expect = baseExpect.extend({ message: () => 'expected value NOT to match the schema, but it did', }; } - const issues = result.error.issues - .map((issue) => ` • ${issue.path.map(String).join('.') || '(root)'}: ${issue.message}`) - .join('\n'); return { pass: false, name: 'toMatchSchema', - message: () => `schema validation failed:\n${issues}`, + message: () => `schema validation failed:\n${formatIssues(result.error.issues)}`, }; }, }); diff --git a/tests/auth.spec.ts b/tests/auth.spec.ts index b8bab49..f2cecc1 100644 --- a/tests/auth.spec.ts +++ b/tests/auth.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from '../src/fixtures/api.js'; import { AuthFailureSchema, AuthTokenSchema } from '../src/schemas/auth.schema.js'; import { config } from '../src/config/env.js'; +import { readJson } from '../src/http/readJson.js'; test.describe('Auth', () => { test('valid credentials return a token @smoke @contract', async ({ authClient }) => { @@ -17,13 +18,13 @@ test.describe('Auth', () => { // Documented quirk: the API answers 200 (not 401) for bad credentials. expect(res.status()).toBe(200); await expect(res).toMatchSchema(AuthFailureSchema); - expect(await res.json()).not.toHaveProperty('token'); + expect(await readJson(res, AuthFailureSchema)).not.toHaveProperty('token'); }); test('missing credentials do not return a token @negative', async ({ authClient }) => { const res = await authClient.createToken({}); expect(res.status()).toBe(200); await expect(res).toMatchSchema(AuthFailureSchema); - expect(await res.json()).not.toHaveProperty('token'); + expect(await readJson(res, AuthFailureSchema)).not.toHaveProperty('token'); }); }); diff --git a/tests/booking.contract.spec.ts b/tests/booking.contract.spec.ts index 7012de8..32fe9e4 100644 --- a/tests/booking.contract.spec.ts +++ b/tests/booking.contract.spec.ts @@ -5,14 +5,18 @@ import { CreatedBookingSchema, } from '../src/schemas/booking.schema.js'; import { buildBooking } from '../src/data/bookingFactory.js'; +import { readJson } from '../src/http/readJson.js'; test.describe('Booking contract @contract', () => { test('GET /booking lists the id of a booking just created', async ({ bookingClient }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); const res = await bookingClient.list(); expect(res.status()).toBe(200); await expect(res).toMatchSchema(BookingIdListSchema); - expect(await res.json()).toContainEqual({ bookingid }); + expect(await readJson(res, BookingIdListSchema)).toContainEqual({ bookingid }); }); test('POST /booking returns the created-booking envelope', async ({ bookingClient }) => { @@ -22,7 +26,10 @@ test.describe('Booking contract @contract', () => { }); test('GET /booking/:id returns a booking matching the schema', async ({ bookingClient }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); const res = await bookingClient.get(bookingid); expect(res.status()).toBe(200); await expect(res).toMatchSchema(BookingSchema); diff --git a/tests/booking.crud.spec.ts b/tests/booking.crud.spec.ts index 5174fca..008c6ab 100644 --- a/tests/booking.crud.spec.ts +++ b/tests/booking.crud.spec.ts @@ -1,42 +1,56 @@ import { test, expect } from '../src/fixtures/api.js'; import { BookingClient } from '../src/clients/BookingClient.js'; import { buildBooking } from '../src/data/bookingFactory.js'; +import { readJson } from '../src/http/readJson.js'; +import { BookingSchema, CreatedBookingSchema } from '../src/schemas/booking.schema.js'; test.describe('Booking CRUD', () => { test('create then read returns the same booking @smoke', async ({ bookingClient }) => { const payload = buildBooking(); - const { bookingid, booking } = await (await bookingClient.create(payload)).json(); + const { bookingid, booking } = await readJson( + await bookingClient.create(payload), + CreatedBookingSchema, + ); expect(booking).toEqual(payload); const res = await bookingClient.get(bookingid); expect(res.status()).toBe(200); - expect(await res.json()).toEqual(payload); + expect(await readJson(res, BookingSchema)).toEqual(payload); }); test('PUT replaces the whole booking', async ({ bookingClient }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); const replacement = buildBooking({ firstname: 'Updated' }); const res = await bookingClient.update(bookingid, replacement); expect(res.status()).toBe(200); - expect(await res.json()).toEqual(replacement); - expect(await (await bookingClient.get(bookingid)).json()).toEqual(replacement); + expect(await readJson(res, BookingSchema)).toEqual(replacement); + expect(await readJson(await bookingClient.get(bookingid), BookingSchema)).toEqual(replacement); }); test('PATCH changes only the given fields', async ({ bookingClient }) => { const original = buildBooking(); - const { bookingid } = await (await bookingClient.create(original)).json(); + const { bookingid } = await readJson( + await bookingClient.create(original), + CreatedBookingSchema, + ); const patched = { ...original, firstname: 'Patched' }; const res = await bookingClient.patch(bookingid, { firstname: 'Patched' }); expect(res.status()).toBe(200); - expect(await res.json()).toEqual(patched); - expect(await (await bookingClient.get(bookingid)).json()).toEqual(patched); + expect(await readJson(res, BookingSchema)).toEqual(patched); + expect(await readJson(await bookingClient.get(bookingid), BookingSchema)).toEqual(patched); }); test('DELETE removes the booking', async ({ bookingClient }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); // Documented quirk: successful DELETE returns 201, not 200/204. expect((await bookingClient.delete(bookingid)).status()).toBe(201); @@ -48,7 +62,10 @@ test.describe('Booking CRUD', () => { token, bookingClient, }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); // Delete it behind this client's back, the way the demo API's periodic reset does. expect((await new BookingClient(request, token).delete(bookingid)).ok()).toBe(true); diff --git a/tests/booking.negative.spec.ts b/tests/booking.negative.spec.ts index 89f58af..2c6791b 100644 --- a/tests/booking.negative.spec.ts +++ b/tests/booking.negative.spec.ts @@ -1,6 +1,12 @@ import { test, expect } from '../src/fixtures/api.js'; import { BookingClient } from '../src/clients/BookingClient.js'; import { buildBooking } from '../src/data/bookingFactory.js'; +import { readJson } from '../src/http/readJson.js'; +import { + BookingIdListSchema, + BookingSchema, + CreatedBookingSchema, +} from '../src/schemas/booking.schema.js'; test.describe('Booking negative & access control @negative', () => { test('an unknown id returns 404', async ({ bookingClient }) => { @@ -9,18 +15,24 @@ test.describe('Booking negative & access control @negative', () => { test('PUT without auth is rejected and changes nothing', async ({ bookingClient }) => { const original = buildBooking(); - const { bookingid } = await (await bookingClient.create(original)).json(); + const { bookingid } = await readJson( + await bookingClient.create(original), + CreatedBookingSchema, + ); const res = await bookingClient.update(bookingid, buildBooking({ firstname: 'Hijack' }), { auth: false, }); expect(res.status()).toBe(403); // Side effect: the record is untouched, not just "a 403 came back". - expect(await (await bookingClient.get(bookingid)).json()).toEqual(original); + expect(await readJson(await bookingClient.get(bookingid), BookingSchema)).toEqual(original); }); test('DELETE without auth is rejected and the booking survives', async ({ bookingClient }) => { - const { bookingid } = await (await bookingClient.create(buildBooking())).json(); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking()), + CreatedBookingSchema, + ); expect((await bookingClient.delete(bookingid, { auth: false })).status()).toBe(403); expect((await bookingClient.get(bookingid)).status()).toBe(200); @@ -28,12 +40,15 @@ test.describe('Booking negative & access control @negative', () => { test('an invalid token is rejected and changes nothing', async ({ request, bookingClient }) => { const original = buildBooking(); - const { bookingid } = await (await bookingClient.create(original)).json(); + const { bookingid } = await readJson( + await bookingClient.create(original), + CreatedBookingSchema, + ); const intruder = new BookingClient(request, 'not-a-real-token'); const res = await intruder.update(bookingid, buildBooking({ firstname: 'Hijack' })); expect(res.status()).toBe(403); - expect(await (await bookingClient.get(bookingid)).json()).toEqual(original); + expect(await readJson(await bookingClient.get(bookingid), BookingSchema)).toEqual(original); }); test('a malformed create is rejected and stores nothing', async ({ bookingClient }) => { @@ -47,7 +62,12 @@ test.describe('Booking negative & access control @negative', () => { // A valid booking under the same name proves the firstname filter finds matches, so the // exact list below shows the malformed body was not stored. - const { bookingid } = await (await bookingClient.create(buildBooking({ firstname }))).json(); - expect(await (await bookingClient.list({ firstname })).json()).toEqual([{ bookingid }]); + const { bookingid } = await readJson( + await bookingClient.create(buildBooking({ firstname })), + CreatedBookingSchema, + ); + expect(await readJson(await bookingClient.list({ firstname }), BookingIdListSchema)).toEqual([ + { bookingid }, + ]); }); }); From 959f73aa5145051c56d6327ef7f4f555b5663237 Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:17:53 +0800 Subject: [PATCH 2/6] feat(report): attach every API call to the report when a test fails The request fixture records method, URL, params, status, duration and both bodies (token and password redacted, long bodies truncated) and attaches them as api-calls.json on failure, or always with API_LOG=always. Co-Authored-By: Claude Opus 5.5 --- src/fixtures/api.ts | 16 ++++++++ src/http/recorder.ts | 97 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 src/http/recorder.ts diff --git a/src/fixtures/api.ts b/src/fixtures/api.ts index da911b9..ed14be1 100644 --- a/src/fixtures/api.ts +++ b/src/fixtures/api.ts @@ -2,6 +2,7 @@ import { test as base } from '@playwright/test'; import { AuthClient } from '../clients/AuthClient.js'; import { BookingClient } from '../clients/BookingClient.js'; import { config, DEFAULT_HEADERS } from '../config/env.js'; +import { recordCalls, type ApiCall } from '../http/recorder.js'; type TestFixtures = { /** Client for /auth. */ @@ -16,6 +17,21 @@ type WorkerFixtures = { }; export const test = base.extend({ + // Every call through `request` (and so through every client) is recorded. When a test fails, the + // calls are attached to the report as api-calls.json, secrets redacted. API_LOG=always attaches + // them for passing tests too. + request: async ({ request }, use, testInfo) => { + const calls: ApiCall[] = []; + await use(recordCalls(request, calls)); + const failed = testInfo.status !== testInfo.expectedStatus; + if (calls.length > 0 && (failed || process.env.API_LOG === 'always')) { + await testInfo.attach('api-calls.json', { + body: JSON.stringify(calls, null, 2), + contentType: 'application/json', + }); + } + }, + token: [ async ({ playwright }, use) => { const ctx = await playwright.request.newContext({ diff --git a/src/http/recorder.ts b/src/http/recorder.ts new file mode 100644 index 0000000..73f54eb --- /dev/null +++ b/src/http/recorder.ts @@ -0,0 +1,97 @@ +import type { APIRequestContext, APIResponse } from '@playwright/test'; + +/** One API call as it appears in the `api-calls.json` attachment. Secrets are already redacted. */ +export interface ApiCall { + method: string; + url: string; + params?: unknown; + requestHeaders?: Record; + requestBody?: unknown; + status?: number; + durationMs: number; + responseBody?: unknown; + error?: string; +} + +const METHODS = ['get', 'post', 'put', 'patch', 'delete', 'head', 'fetch'] as const; +const SECRET_KEYS = new Set(['token', 'password']); +const MAX_BODY_CHARS = 2_000; +export const REDACTED = '[redacted]'; + +/** Replaces token/password values in a JSON-like value, at any depth. */ +export function redact(value: unknown): unknown { + if (Array.isArray(value)) return value.map(redact); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, v]) => [key, SECRET_KEYS.has(key) ? REDACTED : redact(v)]), + ); + } + return value; +} + +function redactHeaders(headers: Record = {}): Record { + return Object.fromEntries( + Object.entries(headers).map(([name, value]) => { + const lower = name.toLowerCase(); + if (lower === 'authorization') return [name, REDACTED]; + if (lower === 'cookie') return [name, value.replace(/token=[^;]*/g, `token=${REDACTED}`)]; + return [name, value]; + }), + ); +} + +async function readBody(res: APIResponse): Promise { + const text = await res.text().catch(() => ''); + if (text === '') return undefined; + try { + return redact(JSON.parse(text)); + } catch { + return text.length > MAX_BODY_CHARS ? `${text.slice(0, MAX_BODY_CHARS)}… (truncated)` : text; + } +} + +type Options = { + data?: unknown; + params?: unknown; + headers?: Record; + method?: string; +}; + +/** + * Wraps a request context so every call is appended to `calls`. Clients and tests use it exactly + * like the original; only the seven request methods are intercepted. + */ +export function recordCalls(ctx: APIRequestContext, calls: ApiCall[]): APIRequestContext { + return new Proxy(ctx, { + get(target, prop, receiver) { + const original = Reflect.get(target, prop, receiver); + if (!(METHODS as readonly PropertyKey[]).includes(prop) || typeof original !== 'function') { + return typeof original === 'function' ? original.bind(target) : original; + } + return async (url: string, options: Options = {}) => { + const call: ApiCall = { + method: (prop === 'fetch' ? (options.method ?? 'GET') : String(prop)).toUpperCase(), + url, + params: options.params, + requestHeaders: options.headers ? redactHeaders(options.headers) : undefined, + requestBody: options.data === undefined ? undefined : redact(options.data), + durationMs: 0, + }; + calls.push(call); + const started = performance.now(); + try { + const res: APIResponse = await original.call(target, url, options); + call.durationMs = Math.round(performance.now() - started); + call.url = res.url(); + call.status = res.status(); + call.responseBody = await readBody(res); + return res; + } catch (err) { + call.durationMs = Math.round(performance.now() - started); + call.error = err instanceof Error ? err.message : String(err); + throw err; + } + }; + }, + }); +} From 02dffc12bc046b0351cc543d979ae5a48c1760d9 Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:19:28 +0800 Subject: [PATCH 3/6] test(selftest): prove the matcher, readJson, cleanup and log redaction can fail A new offline 'selftest' project runs against a local stub server, so it needs no API. Each planted bug was checked: a matcher that always passes, a cleanup that ignores leaks and a recorder that leaks passwords each fail exactly one self-test. The API suite is now the 'api' project. Co-Authored-By: Claude Opus 5.5 --- package.json | 10 +- playwright.config.ts | 7 ++ tests/selftest/gates.selftest.spec.ts | 125 +++++++++++++++++++++++ tests/selftest/recorder.selftest.spec.ts | 56 ++++++++++ tests/selftest/stub.ts | 29 ++++++ 5 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 tests/selftest/gates.selftest.spec.ts create mode 100644 tests/selftest/recorder.selftest.spec.ts create mode 100644 tests/selftest/stub.ts diff --git a/package.json b/package.json index dcc70c4..85db1b7 100644 --- a/package.json +++ b/package.json @@ -1,13 +1,15 @@ { "name": "api-testing-framework", "version": "1.0.0", - "description": "A reusable, typed API test framework: API clients, Zod schema/contract validation, auth fixtures, data factories, and automatic cleanup — on Playwright's request API.", + "description": "A reusable, typed API test framework: API clients, Zod schema/contract validation, auth fixtures, data factories, and automatic cleanup \u2014 on Playwright's request API.", "type": "module", "scripts": { "test": "playwright test", - "test:smoke": "playwright test --grep @smoke", - "test:contract": "playwright test --grep @contract", - "test:negative": "playwright test --grep @negative", + "test:api": "playwright test --project=api", + "test:self": "playwright test --project=selftest", + "test:smoke": "playwright test --project=api --grep @smoke", + "test:contract": "playwright test --project=api --grep @contract", + "test:negative": "playwright test --project=api --grep @negative", "report": "playwright show-report", "typecheck": "tsc --noEmit", "lint": "eslint . --max-warnings 0", diff --git a/playwright.config.ts b/playwright.config.ts index 8caddfd..c708233 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -20,4 +20,11 @@ export default defineConfig({ baseURL: config.baseURL, extraHTTPHeaders: DEFAULT_HEADERS, }, + + projects: [ + // Offline: proves the framework's own gates can fail. Needs no API, so CI runs it first. + { name: 'selftest', testDir: './tests/selftest' }, + // The API suite, against BASE_URL. + { name: 'api', testIgnore: 'selftest/**' }, + ], }); diff --git a/tests/selftest/gates.selftest.spec.ts b/tests/selftest/gates.selftest.spec.ts new file mode 100644 index 0000000..4001e8c --- /dev/null +++ b/tests/selftest/gates.selftest.spec.ts @@ -0,0 +1,125 @@ +import { test } from '@playwright/test'; +import { expect } from '../../src/matchers/schema.js'; +import { BookingClient } from '../../src/clients/BookingClient.js'; +import { ContractError, readJson } from '../../src/http/readJson.js'; +import { BookingSchema, CreatedBookingSchema } from '../../src/schemas/booking.schema.js'; +import { buildBooking } from '../../src/data/bookingFactory.js'; +import { json, startStub } from './stub.js'; + +/** + * A gate that can't fail isn't a gate. These run offline against a local stub and prove the + * contract matcher, readJson and cleanup() each fail when they should, with a useful message. + */ + +const drifted = { + firstname: 'Ada', + lastname: '', + totalprice: '120', + depositpaid: true, + bookingdates: { checkin: '2026-13-01', checkout: '2026-11-03' }, + extra: 'not in the contract', +}; + +test.describe('toMatchSchema', () => { + test('passes a valid booking', async () => { + await expect(buildBooking()).toMatchSchema(BookingSchema); + }); + + test('fails on drift and names every broken field', async () => { + const error = await expect(drifted) + .toMatchSchema(BookingSchema) + .then( + () => undefined, + (e: Error) => e, + ); + expect(error?.message).toContain('schema validation failed'); + for (const path of ['lastname', 'totalprice', 'bookingdates.checkin', '(root)']) { + expect(error?.message).toContain(`• ${path}:`); + } + }); +}); + +test.describe('readJson', () => { + test('returns the typed body when it matches', async ({ playwright }) => { + const booking = buildBooking(); + const stub = await startStub({ + 'GET /booking/1': (_req, res) => json(res, 200, booking), + }); + const ctx = await playwright.request.newContext({ baseURL: stub.url }); + try { + expect(await readJson(await ctx.get('/booking/1'), BookingSchema)).toEqual(booking); + } finally { + await ctx.dispose(); + await stub.close(); + } + }); + + test('throws a ContractError with status, URL and field paths on drift', async ({ + playwright, + }) => { + const stub = await startStub({ 'GET /booking/1': (_req, res) => json(res, 200, drifted) }); + const ctx = await playwright.request.newContext({ baseURL: stub.url }); + try { + const error = await readJson(await ctx.get('/booking/1'), BookingSchema).then( + () => undefined, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(ContractError); + const { message, status } = error as ContractError; + expect(status).toBe(200); + expect(message).toContain(`${stub.url}/booking/1 broke its contract`); + expect(message).toContain('• totalprice:'); + } finally { + await ctx.dispose(); + await stub.close(); + } + }); +}); + +test.describe('BookingClient.cleanup', () => { + /** + * Creates booking 7 through a client pointed at a stub whose DELETE and GET answer with the given + * statuses, then runs cleanup(). Resolves to cleanup's outcome: undefined, or the error it threw. + */ + async function cleanupAgainst( + playwright: import('@playwright/test').PlaywrightWorkerArgs['playwright'], + deleteStatus: number, + getStatus: number, + ): Promise { + const booking = buildBooking(); + const stub = await startStub({ + 'POST /booking': (_req, res) => json(res, 200, { bookingid: 7, booking }), + 'DELETE /booking/7': (_req, res) => { + res.writeHead(deleteStatus); + res.end(); + }, + 'GET /booking/7': (_req, res) => json(res, getStatus, booking), + }); + const ctx = await playwright.request.newContext({ baseURL: stub.url }); + try { + const client = new BookingClient(ctx, 'token'); + await readJson(await client.create(booking), CreatedBookingSchema); + return await client.cleanup().then( + () => undefined, + (e: Error) => e, + ); + } finally { + await ctx.dispose(); + await stub.close(); + } + } + + test('deletes what the client created', async ({ playwright }) => { + expect(await cleanupAgainst(playwright, 201, 404)).toBeUndefined(); + }); + + test('tolerates a booking that is already gone', async ({ playwright }) => { + // The demo API answers 405 to DELETE on a missing booking; GET's 404 is the real answer. + expect(await cleanupAgainst(playwright, 405, 404)).toBeUndefined(); + }); + + test('fails when a booking survives', async ({ playwright }) => { + const error = await cleanupAgainst(playwright, 405, 200); + expect(error?.message).toBe('cleanup could not delete bookings: 7'); + }); +}); diff --git a/tests/selftest/recorder.selftest.spec.ts b/tests/selftest/recorder.selftest.spec.ts new file mode 100644 index 0000000..3400103 --- /dev/null +++ b/tests/selftest/recorder.selftest.spec.ts @@ -0,0 +1,56 @@ +import { test, expect } from '@playwright/test'; +import { recordCalls, REDACTED, type ApiCall } from '../../src/http/recorder.js'; +import { json, startStub } from './stub.js'; + +/** The request log must record every call and must never write a secret into the report. */ +test.describe('recordCalls', () => { + test('records calls and redacts tokens and passwords everywhere', async ({ playwright }) => { + const stub = await startStub({ + 'POST /auth': (_req, res) => json(res, 200, { token: 'abc123' }), + 'PUT /booking/1': (_req, res) => json(res, 403, { error: 'Forbidden' }), + }); + const ctx = await playwright.request.newContext({ baseURL: stub.url }); + const calls: ApiCall[] = []; + const recorded = recordCalls(ctx, calls); + try { + await recorded.post('/auth', { data: { username: 'admin', password: 'password123' } }); + await recorded.put('/booking/1', { + data: { firstname: 'Ada' }, + headers: { Cookie: 'token=abc123; theme=dark', Authorization: 'Bearer abc123' }, + }); + + expect(calls.map(({ method, status }) => `${method} ${status}`)).toEqual([ + 'POST 200', + 'PUT 403', + ]); + expect(calls[0]).toMatchObject({ + url: `${stub.url}/auth`, + requestBody: { username: 'admin', password: REDACTED }, + responseBody: { token: REDACTED }, + }); + expect(calls[1]?.requestHeaders).toEqual({ + Cookie: `token=${REDACTED}; theme=dark`, + Authorization: REDACTED, + }); + expect(JSON.stringify(calls)).not.toMatch(/abc123|password123/); + } finally { + await ctx.dispose(); + await stub.close(); + } + }); + + test('records a call that never got a response', async ({ playwright }) => { + const stub = await startStub({}); + await stub.close(); // nothing listens on this port any more + const ctx = await playwright.request.newContext({ baseURL: stub.url }); + const calls: ApiCall[] = []; + try { + await expect(recordCalls(ctx, calls).get('/ping')).rejects.toThrow(); + expect(calls).toHaveLength(1); + expect(calls[0]?.status).toBeUndefined(); + expect(calls[0]?.error).toBeTruthy(); + } finally { + await ctx.dispose(); + } + }); +}); diff --git a/tests/selftest/stub.ts b/tests/selftest/stub.ts new file mode 100644 index 0000000..a63acd6 --- /dev/null +++ b/tests/selftest/stub.ts @@ -0,0 +1,29 @@ +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export type Handler = (req: IncomingMessage, res: ServerResponse) => void; + +/** Reply with JSON and a status code. */ +export function json(res: ServerResponse, status: number, body: unknown): void { + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); +} + +/** + * A throwaway local HTTP server for self-tests. Routes are keyed "METHOD /path"; anything else + * answers 404. Nothing leaves the machine. + */ +export async function startStub(routes: Record) { + const server = createServer((req, res) => { + const path = new URL(req.url ?? '/', 'http://stub').pathname; + const handler = routes[`${req.method} ${path}`]; + if (handler) handler(req, res); + else json(res, 404, { error: 'no stub route' }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} From 3fb67593e3db58945d150c5e14b8b922d5d02aef Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:21:04 +0800 Subject: [PATCH 4/6] feat(contracts): publish the Zod contracts as JSON Schema in docs/schemas A self-test fails when a contract changes and its published file doesn't (compared as parsed JSON, so formatting can't break it), or when a file outlives its contract. npm run schemas:update regenerates them. Co-Authored-By: Claude Opus 5.5 --- docs/schemas/auth-failure.schema.json | 13 +++++ docs/schemas/auth-token.schema.json | 13 +++++ docs/schemas/booking-id-list.schema.json | 17 ++++++ docs/schemas/booking.schema.json | 45 ++++++++++++++++ docs/schemas/created-booking.schema.json | 57 ++++++++++++++++++++ package.json | 1 + src/schemas/index.ts | 15 ++++++ tests/selftest/json-schemas.selftest.spec.ts | 50 +++++++++++++++++ 8 files changed, 211 insertions(+) create mode 100644 docs/schemas/auth-failure.schema.json create mode 100644 docs/schemas/auth-token.schema.json create mode 100644 docs/schemas/booking-id-list.schema.json create mode 100644 docs/schemas/booking.schema.json create mode 100644 docs/schemas/created-booking.schema.json create mode 100644 src/schemas/index.ts create mode 100644 tests/selftest/json-schemas.selftest.spec.ts diff --git a/docs/schemas/auth-failure.schema.json b/docs/schemas/auth-failure.schema.json new file mode 100644 index 0000000..8d84c53 --- /dev/null +++ b/docs/schemas/auth-failure.schema.json @@ -0,0 +1,13 @@ +{ + "title": "auth-failure", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "reason": { + "type": "string", + "minLength": 1 + } + }, + "required": ["reason"], + "additionalProperties": false +} diff --git a/docs/schemas/auth-token.schema.json b/docs/schemas/auth-token.schema.json new file mode 100644 index 0000000..801e5f4 --- /dev/null +++ b/docs/schemas/auth-token.schema.json @@ -0,0 +1,13 @@ +{ + "title": "auth-token", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "token": { + "type": "string", + "minLength": 1 + } + }, + "required": ["token"], + "additionalProperties": false +} diff --git a/docs/schemas/booking-id-list.schema.json b/docs/schemas/booking-id-list.schema.json new file mode 100644 index 0000000..8194d19 --- /dev/null +++ b/docs/schemas/booking-id-list.schema.json @@ -0,0 +1,17 @@ +{ + "title": "booking-id-list", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "bookingid": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": ["bookingid"], + "additionalProperties": false + } +} diff --git a/docs/schemas/booking.schema.json b/docs/schemas/booking.schema.json new file mode 100644 index 0000000..725d50c --- /dev/null +++ b/docs/schemas/booking.schema.json @@ -0,0 +1,45 @@ +{ + "title": "booking", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "firstname": { + "type": "string", + "minLength": 1 + }, + "lastname": { + "type": "string", + "minLength": 1 + }, + "totalprice": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "depositpaid": { + "type": "boolean" + }, + "bookingdates": { + "type": "object", + "properties": { + "checkin": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "checkout": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + } + }, + "required": ["checkin", "checkout"], + "additionalProperties": false + }, + "additionalneeds": { + "type": "string" + } + }, + "required": ["firstname", "lastname", "totalprice", "depositpaid", "bookingdates"], + "additionalProperties": false +} diff --git a/docs/schemas/created-booking.schema.json b/docs/schemas/created-booking.schema.json new file mode 100644 index 0000000..c3031c5 --- /dev/null +++ b/docs/schemas/created-booking.schema.json @@ -0,0 +1,57 @@ +{ + "title": "created-booking", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "bookingid": { + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "booking": { + "type": "object", + "properties": { + "firstname": { + "type": "string", + "minLength": 1 + }, + "lastname": { + "type": "string", + "minLength": 1 + }, + "totalprice": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "depositpaid": { + "type": "boolean" + }, + "bookingdates": { + "type": "object", + "properties": { + "checkin": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + }, + "checkout": { + "type": "string", + "format": "date", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$" + } + }, + "required": ["checkin", "checkout"], + "additionalProperties": false + }, + "additionalneeds": { + "type": "string" + } + }, + "required": ["firstname", "lastname", "totalprice", "depositpaid", "bookingdates"], + "additionalProperties": false + } + }, + "required": ["bookingid", "booking"], + "additionalProperties": false +} diff --git a/package.json b/package.json index 85db1b7..cf5044e 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "test": "playwright test", "test:api": "playwright test --project=api", "test:self": "playwright test --project=selftest", + "schemas:update": "UPDATE_SCHEMAS=1 playwright test --project=selftest json-schemas && prettier --write docs/schemas", "test:smoke": "playwright test --project=api --grep @smoke", "test:contract": "playwright test --project=api --grep @contract", "test:negative": "playwright test --project=api --grep @negative", diff --git a/src/schemas/index.ts b/src/schemas/index.ts new file mode 100644 index 0000000..2c97127 --- /dev/null +++ b/src/schemas/index.ts @@ -0,0 +1,15 @@ +import type { z } from 'zod'; +import { AuthFailureSchema, AuthTokenSchema } from './auth.schema.js'; +import { BookingIdListSchema, BookingSchema, CreatedBookingSchema } from './booking.schema.js'; + +/** + * Every response contract, by the file name it's published under in docs/schemas/. Teams that + * don't use TypeScript can validate against those JSON Schema files instead of the Zod source. + */ +export const CONTRACTS: Record = { + 'auth-token': AuthTokenSchema, + 'auth-failure': AuthFailureSchema, + booking: BookingSchema, + 'created-booking': CreatedBookingSchema, + 'booking-id-list': BookingIdListSchema, +}; diff --git a/tests/selftest/json-schemas.selftest.spec.ts b/tests/selftest/json-schemas.selftest.spec.ts new file mode 100644 index 0000000..dd319da --- /dev/null +++ b/tests/selftest/json-schemas.selftest.spec.ts @@ -0,0 +1,50 @@ +import { test, expect } from '@playwright/test'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { z } from 'zod'; +import { CONTRACTS } from '../../src/schemas/index.js'; + +/** + * docs/schemas/ holds the Zod contracts exported as JSON Schema. This fails when a contract changes + * and the published file doesn't, so the two can't drift. Regenerate with `npm run schemas:update`. + */ +const DIR = 'docs/schemas'; +const UPDATE = process.env.UPDATE_SCHEMAS === '1'; + +function render(name: string, schema: z.ZodType): object { + return { title: name, ...z.toJSONSchema(schema, { target: 'draft-2020-12' }) }; +} + +test.describe('published JSON Schemas', () => { + test.beforeAll(() => { + if (!UPDATE) return; + mkdirSync(DIR, { recursive: true }); + for (const [name, schema] of Object.entries(CONTRACTS)) { + writeFileSync( + `${DIR}/${name}.schema.json`, + `${JSON.stringify(render(name, schema), null, 2)}\n`, + ); + } + }); + + for (const [name, schema] of Object.entries(CONTRACTS)) { + test(`${name}.schema.json matches its Zod contract`, () => { + const file = `${DIR}/${name}.schema.json`; + const expected = render(name, schema); + expect(existsSync(file), `${file} is missing; run npm run schemas:update`).toBe(true); + // Compare content, not bytes: Prettier is free to reformat the published files. + expect( + JSON.parse(readFileSync(file, 'utf8')), + `${file} is stale; run npm run schemas:update`, + ).toEqual(expected); + }); + } + + test('no file is published for a contract that no longer exists', () => { + const published = readdirSync(DIR).filter((f) => f.endsWith('.schema.json')); + expect(published.sort()).toEqual( + Object.keys(CONTRACTS) + .map((n) => `${n}.schema.json`) + .sort(), + ); + }); +}); From 2a3f24f6f2ba3fb05ee62f1aedf85354e7b777cf Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:27:37 +0800 Subject: [PATCH 5/6] ci: run PRs against Restful-Booker in Docker; open an issue on weekly drift Pushes and PRs run the API suite against mwinteringham/restfulbooker as a service container, so results don't depend on the shared public demo. The weekly schedule (or a manual run with live: true) tests the live API, and a failed weekly run opens or comments on a 'Contract drift' issue. The quality job now runs the offline self-tests. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 80 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 341a820..a74cc94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,10 +8,15 @@ on: schedule: - cron: '0 6 * * 1' # weekly run against the live API to catch contract drift workflow_dispatch: + inputs: + live: + description: 'Also run the suite against the live public API' + type: boolean + default: false jobs: quality: - name: Lint · format · typecheck + name: Lint · format · typecheck · self-tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -23,11 +28,22 @@ jobs: - run: npm run lint - run: npm run format:check - run: npm run typecheck + # Offline: proves the matcher, readJson, cleanup, log redaction and published schemas can fail. + - run: npm run test:self - api-tests: - name: API tests + api-local: + name: API tests (local Restful-Booker) needs: quality runs-on: ubuntu-latest + # Hermetic: every push and PR runs against the same image, so a red build means our change, + # not a cold start or a reset on the shared public demo. + services: + restful-booker: + image: mwinteringham/restfulbooker:latest + ports: + - 3001:3001 + env: + BASE_URL: http://localhost:3001 steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -35,10 +51,64 @@ jobs: node-version: 22 cache: npm - run: npm ci - - run: npx playwright test # request API only — no browser install needed + - name: Wait for Restful-Booker + run: | + for i in $(seq 1 30); do + [ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/ping")" = 201 ] && exit 0 + sleep 2 + done + echo "Restful-Booker did not answer /ping within 60s" >&2 + exit 1 + - run: npm run test:api # request API only, no browser install needed - uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} with: - name: playwright-report + name: playwright-report-local path: playwright-report/ retention-days: 14 + + api-live: + name: API tests (live public API) + needs: quality + if: ${{ github.event_name == 'schedule' || inputs.live }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run test:api + - uses: actions/upload-artifact@v7 + if: ${{ !cancelled() }} + with: + name: playwright-report-live + path: playwright-report/ + retention-days: 14 + + drift-alert: + name: Open or update the contract drift issue + needs: api-live + # Only the unattended weekly run raises an issue; a manual run reports in the Actions tab. + if: ${{ failure() && github.event_name == 'schedule' }} + runs-on: ubuntu-latest + permissions: + issues: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Open the issue, or comment on the open one + run: | + title="Contract drift: weekly run against the live API failed" + existing=$(gh issue list --state open --search "in:title \"$title\"" --json number --jq '.[0].number // empty') + body="The weekly run against the live Restful-Booker API failed: $RUN_URL + + The HTML report (playwright-report-live) has an api-calls.json attachment on every failed test with each request and response." + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "Failed again: $RUN_URL" + else + gh issue create --title "$title" --body "$body" + fi From 2b25753f7b5cd93c8312b18916e7d8907418cba7 Mon Sep 17 00:00:00 2001 From: Vincent Jerico Alcuran Date: Fri, 6 Nov 2026 21:28:27 +0800 Subject: [PATCH 6/6] docs: document typed reads, the request log, self-tests, schemas and CI layers Co-Authored-By: Claude Opus 5.5 --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e557de5..4b75675 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,16 @@ handled by fixtures, realistic test data, and cleanup that fails the test if any | -------------------- | -------------------------------- | -------------------------------------------------------------------------- | | Typed API clients | `src/clients/` | Endpoints live in one place; tests read like behavior, not URLs | | Schemas as contracts | `src/schemas/` | Zod checks responses, and the TS types come from the same schema | +| Typed reads | `src/http/readJson.ts` | Every body a test reads is validated and typed; no `any` from `res.json()` | | Custom matcher | `src/matchers/schema.ts` | `await expect(res).toMatchSchema(Schema)` with readable field-level errors | +| Request log | `src/http/recorder.ts` | A failed test's report shows every call it made, secrets redacted | +| Published contracts | `docs/schemas/` | The Zod contracts as JSON Schema, for teams that don't use TypeScript | +| Self-tests | `tests/selftest/` | Offline proof that the matcher, cleanup and redaction can fail | | Auth fixture | `src/fixtures/api.ts` | The token is fetched once per worker and passed in for you | | Auto cleanup | `BookingClient.cleanup()` | Every booking a test creates is deleted, and a leftover fails the test | | Data factories | `src/data/bookingFactory.ts` | Faker data with valid, ordered dates; override any field | | Tags | `@smoke` `@contract` `@negative` | Run focused subsets | -| Quality gates | ESLint, Prettier, `tsc`, CI | Lint, format, and typecheck all gate the build | +| Quality gates | ESLint, Prettier, `tsc`, CI | Lint, format, typecheck and self-tests all gate the build | ## What a contract failure looks like @@ -42,31 +46,66 @@ schema validation failed: ```bash npm install npm test # everything (no browser needed) +npm run test:self # offline self-tests: does each gate still fail when it should? +npm run test:api # the API suite only npm run test:smoke # @smoke subset npm run test:contract # schema/contract checks npm run test:negative # negative + access-control checks +npm run schemas:update # regenerate docs/schemas after changing a contract npm run report # open the HTML report ``` Configure via env vars (see `.env.example`): `BASE_URL`, `API_USERNAME`, `API_PASSWORD`. +To run against your own copy of the API instead of the shared demo (this is what CI does): + +```bash +docker run --rm -p 3001:3001 mwinteringham/restfulbooker +BASE_URL=http://localhost:3001 npm run test:api +``` + ## Writing a test ```ts import { test, expect } from '../src/fixtures/api.js'; -import { BookingSchema } from '../src/schemas/booking.schema.js'; +import { readJson } from '../src/http/readJson.js'; +import { BookingSchema, CreatedBookingSchema } from '../src/schemas/booking.schema.js'; import { buildBooking } from '../src/data/bookingFactory.js'; test('a created booking can be read back', async ({ bookingClient }) => { const payload = buildBooking({ firstname: 'Vincent' }); - const { bookingid } = await (await bookingClient.create(payload)).json(); + const { bookingid } = await readJson(await bookingClient.create(payload), CreatedBookingSchema); const res = await bookingClient.get(bookingid); - await expect(res).toMatchSchema(BookingSchema); - expect(await res.json()).toEqual(payload); + expect(res.status()).toBe(200); + expect(await readJson(res, BookingSchema)).toEqual(payload); }); // the booking is deleted automatically after the test ``` +`readJson` returns a typed value and throws a `ContractError` if the body breaks its contract, so +every read is also a contract check. Use `toMatchSchema` when the contract is what the test is about. + +## Debugging a failure + +Every call a test makes goes through a recording wrapper on the `request` fixture. When a test +fails, its report entry gets an `api-calls.json` attachment with the method, URL, query, status, +duration and both bodies of each call, including the cleanup after the test. Tokens and passwords +are replaced with `[redacted]`. Set `API_LOG=always` to attach it to passing tests too. + +```json +{ + "method": "PUT", + "url": ".../booking/3620", + "status": 403, + "durationMs": 257, + "requestHeaders": { "Cookie": "token=[redacted]" }, + "responseBody": "Forbidden" +} +``` + +That entry is from the "invalid token is rejected" test against the live API (request body left out +here). + ## Structure ``` @@ -75,11 +114,15 @@ api-testing-framework/ ├── src/ │ ├── config/env.ts # env-driven config │ ├── clients/ # BaseClient · AuthClient · BookingClient -│ ├── schemas/ # Zod contracts (+ inferred types) +│ ├── schemas/ # Zod contracts (+ inferred types), index.ts lists them all +│ ├── http/readJson.ts # parse + validate + type a response body +│ ├── http/recorder.ts # request log behind api-calls.json │ ├── matchers/schema.ts # toMatchSchema custom matcher -│ ├── fixtures/api.ts # token (worker) · authClient · bookingClient (auto-cleanup) +│ ├── fixtures/api.ts # request (recorded) · token (worker) · authClient · bookingClient │ └── data/bookingFactory.ts # faker data builder +├── docs/schemas/ # the contracts as JSON Schema (generated, checked by a self-test) └── tests/ + ├── selftest/ # offline: stub server, gates must fail, schemas not stale ├── health.spec.ts # @smoke ├── auth.spec.ts # token contract + bad/missing credentials ├── booking.crud.spec.ts # create · read · put · patch · delete @@ -100,10 +143,15 @@ The tests pin the demo API's **actual** behavior and flag where a well-behaved A ## CI -[`.github/workflows/ci.yml`](.github/workflows/ci.yml) runs **lint · format · typecheck** first and -starts the API suite only if they pass, so a broken build never hits the live API. It runs on every -push/PR, plus a **weekly scheduled run** against the live API to catch contract drift even when the -code hasn't changed. +[`.github/workflows/ci.yml`](.github/workflows/ci.yml) has three layers: + +1. **Quality:** lint, format, typecheck and the offline self-tests. Nothing else runs unless they + pass. +2. **Every push and PR** runs the API suite against Restful-Booker in a Docker service container. + The result depends on this repo's code, not on the shared public demo being up or freshly reset. +3. **Weekly** (and on a manual run with `live: true`) the suite runs against the live public API to + catch contract drift when the code hasn't changed. If the weekly run fails, CI opens a + "Contract drift" issue, or comments on the open one, with a link to the run and its report. ## Design note