Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 75 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@ on:
schedule:
- cron: '0 6 * * 1' # weekly run against the live API to catch contract drift
workflow_dispatch:
inputs:
live:
description: 'Also run the suite against the live public API'
type: boolean
default: false

jobs:
quality:
name: Lint · format · typecheck
name: Lint · format · typecheck · self-tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
Expand All @@ -23,22 +28,87 @@ jobs:
- run: npm run lint
- run: npm run format:check
- run: npm run typecheck
# Offline: proves the matcher, readJson, cleanup, log redaction and published schemas can fail.
- run: npm run test:self

api-tests:
name: API tests
api-local:
name: API tests (local Restful-Booker)
needs: quality
runs-on: ubuntu-latest
# Hermetic: every push and PR runs against the same image, so a red build means our change,
# not a cold start or a reset on the shared public demo.
services:
restful-booker:
image: mwinteringham/restfulbooker:latest
ports:
- 3001:3001
env:
BASE_URL: http://localhost:3001
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 22
cache: npm
- run: npm ci
- run: npx playwright test # request API only — no browser install needed
- name: Wait for Restful-Booker
run: |
for i in $(seq 1 30); do
[ "$(curl -s -o /dev/null -w '%{http_code}' "$BASE_URL/ping")" = 201 ] && exit 0
sleep 2
done
echo "Restful-Booker did not answer /ping within 60s" >&2
exit 1
- run: npm run test:api # request API only, no browser install needed
- uses: actions/upload-artifact@v7
if: ${{ !cancelled() }}
with:
name: playwright-report
name: playwright-report-local
path: playwright-report/
retention-days: 14

api-live:
name: API tests (live public API)
needs: quality
if: ${{ github.event_name == 'schedule' || inputs.live }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/setup-node@v7
with:
node-version: 22
cache: npm
- run: npm ci
- run: npm run test:api
- uses: actions/upload-artifact@v7
if: ${{ !cancelled() }}
with:
name: playwright-report-live
path: playwright-report/
retention-days: 14

drift-alert:
name: Open or update the contract drift issue
needs: api-live
# Only the unattended weekly run raises an issue; a manual run reports in the Actions tab.
if: ${{ failure() && github.event_name == 'schedule' }}
runs-on: ubuntu-latest
permissions:
issues: write
env:
GH_TOKEN: ${{ github.token }}
GH_REPO: ${{ github.repository }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- name: Open the issue, or comment on the open one
run: |
title="Contract drift: weekly run against the live API failed"
existing=$(gh issue list --state open --search "in:title \"$title\"" --json number --jq '.[0].number // empty')
body="The weekly run against the live Restful-Booker API failed: $RUN_URL

The HTML report (playwright-report-live) has an api-calls.json attachment on every failed test with each request and response."
if [ -n "$existing" ]; then
gh issue comment "$existing" --body "Failed again: $RUN_URL"
else
gh issue create --title "$title" --body "$body"
fi
70 changes: 59 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@ handled by fixtures, realistic test data, and cleanup that fails the test if any
| -------------------- | -------------------------------- | -------------------------------------------------------------------------- |
| Typed API clients | `src/clients/` | Endpoints live in one place; tests read like behavior, not URLs |
| Schemas as contracts | `src/schemas/` | Zod checks responses, and the TS types come from the same schema |
| Typed reads | `src/http/readJson.ts` | Every body a test reads is validated and typed; no `any` from `res.json()` |
| Custom matcher | `src/matchers/schema.ts` | `await expect(res).toMatchSchema(Schema)` with readable field-level errors |
| Request log | `src/http/recorder.ts` | A failed test's report shows every call it made, secrets redacted |
| Published contracts | `docs/schemas/` | The Zod contracts as JSON Schema, for teams that don't use TypeScript |
| Self-tests | `tests/selftest/` | Offline proof that the matcher, cleanup and redaction can fail |
| Auth fixture | `src/fixtures/api.ts` | The token is fetched once per worker and passed in for you |
| Auto cleanup | `BookingClient.cleanup()` | Every booking a test creates is deleted, and a leftover fails the test |
| Data factories | `src/data/bookingFactory.ts` | Faker data with valid, ordered dates; override any field |
| Tags | `@smoke` `@contract` `@negative` | Run focused subsets |
| Quality gates | ESLint, Prettier, `tsc`, CI | Lint, format, and typecheck all gate the build |
| Quality gates | ESLint, Prettier, `tsc`, CI | Lint, format, typecheck and self-tests all gate the build |

## What a contract failure looks like

Expand All @@ -42,31 +46,66 @@ schema validation failed:
```bash
npm install
npm test # everything (no browser needed)
npm run test:self # offline self-tests: does each gate still fail when it should?
npm run test:api # the API suite only
npm run test:smoke # @smoke subset
npm run test:contract # schema/contract checks
npm run test:negative # negative + access-control checks
npm run schemas:update # regenerate docs/schemas after changing a contract
npm run report # open the HTML report
```

Configure via env vars (see `.env.example`): `BASE_URL`, `API_USERNAME`, `API_PASSWORD`.

To run against your own copy of the API instead of the shared demo (this is what CI does):

```bash
docker run --rm -p 3001:3001 mwinteringham/restfulbooker
BASE_URL=http://localhost:3001 npm run test:api
```

## Writing a test

```ts
import { test, expect } from '../src/fixtures/api.js';
import { BookingSchema } from '../src/schemas/booking.schema.js';
import { readJson } from '../src/http/readJson.js';
import { BookingSchema, CreatedBookingSchema } from '../src/schemas/booking.schema.js';
import { buildBooking } from '../src/data/bookingFactory.js';

test('a created booking can be read back', async ({ bookingClient }) => {
const payload = buildBooking({ firstname: 'Vincent' });
const { bookingid } = await (await bookingClient.create(payload)).json();
const { bookingid } = await readJson(await bookingClient.create(payload), CreatedBookingSchema);

const res = await bookingClient.get(bookingid);
await expect(res).toMatchSchema(BookingSchema);
expect(await res.json()).toEqual(payload);
expect(res.status()).toBe(200);
expect(await readJson(res, BookingSchema)).toEqual(payload);
}); // the booking is deleted automatically after the test
```

`readJson` returns a typed value and throws a `ContractError` if the body breaks its contract, so
every read is also a contract check. Use `toMatchSchema` when the contract is what the test is about.

## Debugging a failure

Every call a test makes goes through a recording wrapper on the `request` fixture. When a test
fails, its report entry gets an `api-calls.json` attachment with the method, URL, query, status,
duration and both bodies of each call, including the cleanup after the test. Tokens and passwords
are replaced with `[redacted]`. Set `API_LOG=always` to attach it to passing tests too.

```json
{
"method": "PUT",
"url": ".../booking/3620",
"status": 403,
"durationMs": 257,
"requestHeaders": { "Cookie": "token=[redacted]" },
"responseBody": "Forbidden"
}
```

That entry is from the "invalid token is rejected" test against the live API (request body left out
here).

## Structure

```
Expand All @@ -75,11 +114,15 @@ api-testing-framework/
├── src/
│ ├── config/env.ts # env-driven config
│ ├── clients/ # BaseClient · AuthClient · BookingClient
│ ├── schemas/ # Zod contracts (+ inferred types)
│ ├── schemas/ # Zod contracts (+ inferred types), index.ts lists them all
│ ├── http/readJson.ts # parse + validate + type a response body
│ ├── http/recorder.ts # request log behind api-calls.json
│ ├── matchers/schema.ts # toMatchSchema custom matcher
│ ├── fixtures/api.ts # token (worker) · authClient · bookingClient (auto-cleanup)
│ ├── fixtures/api.ts # request (recorded) · token (worker) · authClient · bookingClient
│ └── data/bookingFactory.ts # faker data builder
├── docs/schemas/ # the contracts as JSON Schema (generated, checked by a self-test)
└── tests/
├── selftest/ # offline: stub server, gates must fail, schemas not stale
├── health.spec.ts # @smoke
├── auth.spec.ts # token contract + bad/missing credentials
├── booking.crud.spec.ts # create · read · put · patch · delete
Expand All @@ -100,10 +143,15 @@ The tests pin the demo API's **actual** behavior and flag where a well-behaved A

## CI

[`.github/workflows/ci.yml`](.github/workflows/ci.yml) runs **lint · format · typecheck** first and
starts the API suite only if they pass, so a broken build never hits the live API. It runs on every
push/PR, plus a **weekly scheduled run** against the live API to catch contract drift even when the
code hasn't changed.
[`.github/workflows/ci.yml`](.github/workflows/ci.yml) has three layers:

1. **Quality:** lint, format, typecheck and the offline self-tests. Nothing else runs unless they
pass.
2. **Every push and PR** runs the API suite against Restful-Booker in a Docker service container.
The result depends on this repo's code, not on the shared public demo being up or freshly reset.
3. **Weekly** (and on a manual run with `live: true`) the suite runs against the live public API to
catch contract drift when the code hasn't changed. If the weekly run fails, CI opens a
"Contract drift" issue, or comments on the open one, with a link to the run and its report.

## Design note

Expand Down
13 changes: 13 additions & 0 deletions docs/schemas/auth-failure.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"title": "auth-failure",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"reason": {
"type": "string",
"minLength": 1
}
},
"required": ["reason"],
"additionalProperties": false
}
13 changes: 13 additions & 0 deletions docs/schemas/auth-token.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"title": "auth-token",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"token": {
"type": "string",
"minLength": 1
}
},
"required": ["token"],
"additionalProperties": false
}
17 changes: 17 additions & 0 deletions docs/schemas/booking-id-list.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"title": "booking-id-list",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "array",
"items": {
"type": "object",
"properties": {
"bookingid": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
}
},
"required": ["bookingid"],
"additionalProperties": false
}
}
45 changes: 45 additions & 0 deletions docs/schemas/booking.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"title": "booking",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"firstname": {
"type": "string",
"minLength": 1
},
"lastname": {
"type": "string",
"minLength": 1
},
"totalprice": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"depositpaid": {
"type": "boolean"
},
"bookingdates": {
"type": "object",
"properties": {
"checkin": {
"type": "string",
"format": "date",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$"
},
"checkout": {
"type": "string",
"format": "date",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$"
}
},
"required": ["checkin", "checkout"],
"additionalProperties": false
},
"additionalneeds": {
"type": "string"
}
},
"required": ["firstname", "lastname", "totalprice", "depositpaid", "bookingdates"],
"additionalProperties": false
}
57 changes: 57 additions & 0 deletions docs/schemas/created-booking.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
{
"title": "created-booking",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"bookingid": {
"type": "integer",
"exclusiveMinimum": 0,
"maximum": 9007199254740991
},
"booking": {
"type": "object",
"properties": {
"firstname": {
"type": "string",
"minLength": 1
},
"lastname": {
"type": "string",
"minLength": 1
},
"totalprice": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"depositpaid": {
"type": "boolean"
},
"bookingdates": {
"type": "object",
"properties": {
"checkin": {
"type": "string",
"format": "date",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$"
},
"checkout": {
"type": "string",
"format": "date",
"pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))$"
}
},
"required": ["checkin", "checkout"],
"additionalProperties": false
},
"additionalneeds": {
"type": "string"
}
},
"required": ["firstname", "lastname", "totalprice", "depositpaid", "bookingdates"],
"additionalProperties": false
}
},
"required": ["bookingid", "booking"],
"additionalProperties": false
}
Loading
Loading