diff --git a/.gitignore b/.gitignore index 6abffe65..d521e689 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ npm-debug.log* playwright-report/ test-results/ playwright/.auth/ +crawl-report/ diff --git a/e2e/crawl/crawl.spec.ts b/e2e/crawl/crawl.spec.ts new file mode 100644 index 00000000..e0827d58 --- /dev/null +++ b/e2e/crawl/crawl.spec.ts @@ -0,0 +1,215 @@ +import AxeBuilder from '@axe-core/playwright'; +import { type Page, expect, test } from '@playwright/test'; +import { origin } from '../support/env'; +import { ParamResolver } from './params'; +import { + type ConsoleError, + type DuplicateRequest, + type FailedRequest, + type ScreenReport, + type Theme, + type Viewport, + crawlAccount, + screenName, + screenPath, + writeScreenReport, +} from './report'; +import { crawlRoutes } from './routes'; + +// Opens every screen of the application as the account of the run and records +// what the page did: its title, whether it crashed into the error boundary, the +// console errors, uncaught errors and failed or repeated requests it caused, +// the accessibility findings of axe, and a screenshot. The crawl only +// navigates and reads. It never clicks, types or submits anything and sends no +// request of its own besides loading the pages, so it changes nothing on the +// deployment. Findings never fail a test; only a screen that cannot be crawled +// does. + +const API = '/pulp/api/'; +const SETTLE_MS = 15_000; +const AXE_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']; +const account = crawlAccount(); + +test.describe.configure({ mode: 'parallel' }); + +// Resolves the parameters of a route with GET list calls; one per worker. +const resolver = new ParamResolver(); +test.afterAll(() => resolver.close()); + +// An answer the crawl expects: the anonymous probe of the session endpoint +// says 401, and a read-only account is refused what it may not read. +function isExpected(method: string, url: URL, status: number): boolean { + if (method === 'GET' && url.pathname.endsWith(`${API}v3/login/`)) { + return status === 401; + } + return account === 'viewer' && status === 403; +} + +async function uiBuild(page: Page): Promise { + const stamp = await page + .locator('#pulp-ui-version') + .evaluate((element) => element.innerHTML) + .catch(() => null); + const json = stamp?.match(/pulp-ui\s*(\{[\s\S]*\})/)?.[1]; + try { + return json ? JSON.parse(json) : null; + } catch { + return null; + } +} + +for (const route of crawlRoutes()) { + test(route.pattern, async ({ page }, testInfo) => { + test.setTimeout(90_000); + const { theme, viewport } = testInfo.project.metadata as { + theme: Theme; + viewport: Viewport; + }; + const { path, params } = await resolver.resolve(route); + const here = origin(); + + const consoleErrors: (ConsoleError & { source: string })[] = []; + const pageErrors: string[] = []; + const failedRequests: FailedRequest[] = []; + const seen = new Map(); + + page.on('console', (message) => { + if (message.type() !== 'error') { + return; + } + const { url, lineNumber, columnNumber } = message.location(); + consoleErrors.push({ + text: message.text(), + location: url ? `${url}:${lineNumber}:${columnNumber}` : '', + source: url, + expected: false, + }); + }); + page.on('pageerror', (error) => pageErrors.push(String(error))); + page.on('request', (request) => { + const url = new URL(request.url()); + if (url.origin === here && url.pathname.includes(API)) { + const key = `${request.method()} ${url.pathname}${url.search}`; + seen.set(key, (seen.get(key) ?? 0) + 1); + } + }); + page.on('response', (response) => { + const url = new URL(response.url()); + if (url.origin !== here || response.status() < 400) { + return; + } + const method = response.request().method(); + failedRequests.push({ + method, + url: response.url(), + status: response.status(), + expected: isExpected(method, url, response.status()), + }); + }); + page.on('requestfailed', (request) => { + const url = new URL(request.url()); + if (url.origin !== here) { + return; + } + const failure = request.failure()?.errorText ?? 'failed'; + failedRequests.push({ + method: request.method(), + url: request.url(), + status: 0, + // A request the page cancels itself, for example when it navigates + // away, is not a fault of the server. + expected: failure === 'net::ERR_ABORTED', + failure, + }); + }); + + const started = Date.now(); + await page.goto(path); + let settled = true; + await page + .waitForLoadState('networkidle', { timeout: SETTLE_MS }) + .catch(() => (settled = false)); + await page + .waitForFunction( + () => !document.querySelector('.pf-v5-c-spinner'), + null, + { + timeout: SETTLE_MS, + }, + ) + .catch(() => (settled = false)); + const loadMs = Date.now() - started; + + const dark = await page.evaluate(() => + document.documentElement.classList.contains('pf-v5-theme-dark'), + ); + expect(dark, `the page is in the ${theme} theme`).toBe(theme === 'dark'); + + // A page that crashed shows no title; the lookup must not wait for one. + const heading = await page + .locator('.pf-v5-c-title') + .first() + .textContent({ timeout: 1000 }) + .catch(() => null); + const errorBoundary = + (await page.getByText('The stack trace is:').count()) > 0 || + (await page.getByText('Something went horribly wrong!').count()) > 0; + + const axe = await new AxeBuilder({ page }).withTags(AXE_TAGS).analyze(); + + const screenshot = screenName(route.slug, theme, viewport, 'png'); + await page.screenshot({ + path: screenPath(route.slug, theme, viewport, 'png'), + fullPage: true, + }); + + // A console line the browser wrote for an expected answer is not an error + // of the page. + const expectedUrls = new Set( + failedRequests.filter((f) => f.expected).map((f) => f.url), + ); + const duplicateRequests: DuplicateRequest[] = [...seen] + .filter(([, count]) => count > 1) + .map(([key, count]) => { + const [method = '', url = ''] = key.split(' '); + return { method, url, count }; + }); + + writeScreenReport(route.slug, { + route: route.pattern, + url: page.url(), + params, + theme, + viewport, + account, + title: heading?.trim() || (await page.title()), + errorBoundary, + loadMs, + settled, + screenshot, + consoleErrors: consoleErrors.map(({ source, ...error }) => ({ + ...error, + expected: + /^Failed to load resource/.test(error.text) && + expectedUrls.has(source), + })), + pageErrors, + failedRequests, + duplicateRequests, + axe: { + violations: axe.violations.map((violation) => ({ + id: violation.id, + impact: violation.impact ?? null, + help: violation.help, + nodes: violation.nodes.length, + targets: violation.nodes + .slice(0, 5) + .map((node) => node.target.join(' ')), + })), + }, + uiBuild: await uiBuild(page), + run: process.env['PULP_UI_CRAWL_RUN'] ?? '', + capturedAt: new Date().toISOString(), + }); + }); +} diff --git a/e2e/crawl/global-setup.ts b/e2e/crawl/global-setup.ts new file mode 100644 index 00000000..9ae1765d --- /dev/null +++ b/e2e/crawl/global-setup.ts @@ -0,0 +1,8 @@ +import { randomBytes } from 'node:crypto'; + +// Names the crawl run before any worker starts. Every screen report carries the +// name, so the teardown tells the reports of this run from files an earlier run +// left in the output directory. The workers inherit the variable. +export default function globalSetup(): void { + process.env['PULP_UI_CRAWL_RUN'] = randomBytes(6).toString('hex'); +} diff --git a/e2e/crawl/params.ts b/e2e/crawl/params.ts new file mode 100644 index 00000000..b7f3659a --- /dev/null +++ b/e2e/crawl/params.ts @@ -0,0 +1,180 @@ +import { type APIRequestContext } from '@playwright/test'; +import { API_BASE, apiContext } from '../support/api'; +import { + type CrawlRoute, + PLACEHOLDER, + UNKNOWN_PATH, + fillPattern, +} from './routes'; + +// Turns a route with :parameters into a path the deployment can answer. Each +// parameter has a resolver that asks the API, as the admin account and with +// GET list calls only, for the first object of the matching list. A parameter +// whose list is empty, or that has no source on a plain Pulp, is filled with a +// placeholder and the screen is recorded as params: placeholder, so a page that +// then crashes on it is reported and not mistaken for a healthy one. A +// parameter that has no resolver at all fails the crawl, so a route with a new +// parameter can never be visited with a made-up value without anyone deciding +// so. + +type First = (path: string) => Promise | null>; + +interface Context { + route: CrawlRoute; + first: First; +} + +type Resolver = (context: Context) => Promise; + +// The last path segment of an object's href is its id. +function idOf(href: unknown): string | null { + return typeof href === 'string' + ? (href.replace(/\/$/, '').split('/').pop() ?? null) + : null; +} + +function text(value: unknown): string | null { + return typeof value === 'string' && value ? value : null; +} + +// Repository and remote detail and edit pages are addressed by the object's +// name; the list to ask depends on the plugin the route belongs to. +const NAMED_LISTS: Record = { + '/deb/repositories': 'repositories/deb/apt/', + '/deb/remotes': 'remotes/deb/apt/', + '/file/repositories': 'repositories/file/file/', + '/file/remotes': 'remotes/file/file/', + '/ansible/repositories': 'repositories/ansible/ansible/', + '/ansible/remotes': 'remotes/ansible/collection/', +}; + +function isCollectionRoute(route: CrawlRoute): boolean { + return route.pattern.startsWith('/ansible/collections/'); +} + +// The collection screens need a namespace, a collection name and the +// distribution that serves them, which only an uploaded collection provides. +async function collection( + { route, first }: Context, + field: 'namespace' | 'name' | 'repo', +): Promise { + if (!isCollectionRoute(route)) { + return null; + } + const version = await first('content/ansible/collection_versions/'); + if (!version) { + return null; + } + if (field === 'repo') { + const distribution = await first('distributions/ansible/ansible/'); + return text(distribution?.['base_path']); + } + return text(version[field]); +} + +const RESOLVERS: Record = { + task: async ({ first }) => idOf((await first('tasks/'))?.['pulp_href']), + group: async ({ first }) => { + const group = await first('groups/'); + return group?.['id'] === undefined ? null : String(group['id']); + }, + role: async ({ first }) => idOf((await first('roles/'))?.['pulp_href']), + user_id: async ({ first }) => { + const user = await first('users/'); + return user?.['id'] === undefined ? null : String(user['id']); + }, + name: async ({ route, first }) => { + const list = NAMED_LISTS[route.pattern.split('/').slice(0, 3).join('/')]; + return list ? text((await first(list))?.['name']) : null; + }, + namespace: (context) => collection(context, 'namespace'), + collection: (context) => collection(context, 'name'), + repo: (context) => collection(context, 'repo'), + // No source on a plain Pulp: the container plugin keeps its own lists, which + // the lab does not install, and the content documentation pages address a + // page of a collection's own documentation. + container: async () => null, + digest: async () => null, + type: async () => null, + page: async () => null, +}; + +export interface ResolvedPath { + // Relative to the UI root. + path: string; + params: 'resolved' | 'placeholder' | 'none'; +} + +export class ParamResolver { + private api: APIRequestContext | undefined; + private readonly answers = new Map< + string, + Promise | null> + >(); + + private readonly first: First = (path) => { + let answer = this.answers.get(path); + if (!answer) { + answer = this.fetchFirst(path); + this.answers.set(path, answer); + } + return answer; + }; + + private async fetchFirst(path: string) { + this.api ??= await apiContext(); + const response = await this.api.get(`${API_BASE}${path}`, { + params: { limit: '1' }, + }); + // A list the deployment does not have (a plugin that is not installed) + // simply offers nothing to resolve from. + if (response.status() === 404) { + return null; + } + if (!response.ok()) { + throw new Error(`GET ${path} answered ${response.status()}`); + } + const body = (await response.json()) as { + results?: Record[]; + }; + return body.results?.[0] ?? null; + } + + async resolve(route: CrawlRoute): Promise { + if (route.pattern === '/') { + return { path: '', params: 'none' }; + } + if (route.pattern === '*') { + return { path: UNKNOWN_PATH, params: 'none' }; + } + + const values: Record = {}; + let placeholder = false; + for (const name of route.params) { + const resolver = RESOLVERS[name]; + if (!resolver) { + throw new Error( + `${route.pattern}: the parameter :${name} has no resolver in e2e/crawl/params.ts`, + ); + } + const value = await resolver({ route, first: this.first }); + if (value === null) { + placeholder = true; + } + values[name] = value ?? PLACEHOLDER; + } + + return { + path: fillPattern(route.pattern, values), + params: route.params.length + ? placeholder + ? 'placeholder' + : 'resolved' + : 'none', + }; + } + + async close(): Promise { + await this.api?.dispose(); + } +} diff --git a/e2e/crawl/report.ts b/e2e/crawl/report.ts new file mode 100644 index 00000000..234bee45 --- /dev/null +++ b/e2e/crawl/report.ts @@ -0,0 +1,123 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { type Role } from '../support/accounts'; + +// The record the crawler keeps of every screen it visits, and where it is +// written. One screen is one route pattern in one theme at one width; its +// report is ...json next to ...png. +// The reports show the deployment's data and URLs: the output directory is +// ignored by git and is never uploaded anywhere. + +export type Theme = 'light' | 'dark'; +export type Viewport = 'desktop' | 'phone'; + +export const THEMES: Theme[] = ['light', 'dark']; +export const VIEWPORTS: Viewport[] = ['desktop', 'phone']; + +export const SCREEN_SIZES: Record = + { + desktop: { width: 1440, height: 900 }, + phone: { width: 390, height: 844 }, + }; + +export interface ConsoleError { + text: string; + location: string; + // The browser logs every failed request as a console error; the line that + // belongs to an answer the crawl expects is marked and not counted. + expected: boolean; +} + +export interface FailedRequest { + method: string; + url: string; + // 0 when the request never got an answer. + status: number; + expected: boolean; + failure?: string; +} + +export interface DuplicateRequest { + method: string; + url: string; + count: number; +} + +export interface AxeViolation { + id: string; + impact: string | null; + help: string; + nodes: number; + targets: string[]; +} + +export interface ScreenReport { + route: string; + url: string; + params: 'resolved' | 'placeholder' | 'none'; + theme: Theme; + viewport: Viewport; + account: Role; + title: string; + errorBoundary: boolean; + loadMs: number; + // False when the page still showed a spinner or kept the network busy at + // the end of the wait. + settled: boolean; + screenshot: string; + consoleErrors: ConsoleError[]; + pageErrors: string[]; + failedRequests: FailedRequest[]; + duplicateRequests: DuplicateRequest[]; + axe: { violations: AxeViolation[] }; + // The build stamp the application carries, null when the page has none. + uiBuild: { hash?: string; date?: string | null; version?: string } | null; + // Identifies the crawl run that wrote the file, so a report left by an + // earlier run is never mistaken for part of this one. + run: string; + capturedAt: string; +} + +export function crawlAccount(): Role { + const value = process.env['PULP_UI_CRAWL_ACCOUNT'] ?? 'admin'; + if (value !== 'admin' && value !== 'viewer') { + throw new Error( + `PULP_UI_CRAWL_ACCOUNT must be "admin" or "viewer", got "${value}"`, + ); + } + return value; +} + +// PULP_UI_CRAWL_OUT, relative to the directory the run starts in. +export function outDir(): string { + return resolve(process.env['PULP_UI_CRAWL_OUT'] || 'crawl-report'); +} + +export function screenName( + slug: string, + theme: Theme, + viewport: Viewport, + extension: 'json' | 'png', +): string { + return `${slug}.${theme}.${viewport}.${extension}`; +} + +// Matches the files a crawl writes and nothing else that may share the directory. +export const SCREEN_FILE = /\.(light|dark)\.(desktop|phone)\.(json|png)$/; + +export function screenPath( + slug: string, + theme: Theme, + viewport: Viewport, + extension: 'json' | 'png', +): string { + return join(outDir(), screenName(slug, theme, viewport, extension)); +} + +export function writeScreenReport(slug: string, report: ScreenReport): void { + mkdirSync(outDir(), { recursive: true }); + writeFileSync( + screenPath(slug, report.theme, report.viewport, 'json'), + `${JSON.stringify(report, null, 2)}\n`, + ); +} diff --git a/e2e/crawl/routes.test.mjs b/e2e/crawl/routes.test.mjs new file mode 100644 index 00000000..bd2d7ef1 --- /dev/null +++ b/e2e/crawl/routes.test.mjs @@ -0,0 +1,99 @@ +// @vitest-environment node +// Tests of the route list the crawler builds from the application source. +import { describe, expect, it } from 'vitest'; +import { crawlRoutes, fillPattern, parseRoutes, slugOf } from './routes.ts'; + +const PATHS = ` +export const Paths = { + core: { status: '/status', task: { detail: '/tasks/detail/:task' } }, + meta: { login: '/login' }, + container: { detail: '/container/containers/detail/:namespace?/:container' }, +}; +`; + +const routesModule = (entries) => ` +const routes: IRouteConfig[] = [ +${entries} +]; +`; + +describe('parseRoutes', () => { + it('reads paths and noAuth, and adds the index and the catch-all', () => { + const routes = parseRoutes( + routesModule(` + { component: A, path: Paths.core.status, noAuth: true }, + { component: B, path: Paths.core.task.detail, beta: true }, +`), + PATHS, + ); + expect(routes.map((r) => r.pattern)).toEqual([ + '/status', + '/tasks/detail/:task', + '/', + '*', + ]); + expect(routes[0].noAuth).toBe(true); + expect(routes[1].noAuth).toBe(false); + expect(routes[1].params).toEqual(['task']); + }); + + it('fails on an entry whose path is not a Paths reference', () => { + expect(() => + parseRoutes(routesModule(` { component: A, path: '/literal' },`), PATHS), + ).toThrow(/cannot be crawled/); + }); + + it('fails on an entry without a path', () => { + expect(() => + parseRoutes(routesModule(' { component: A },'), PATHS), + ).toThrow(/cannot be crawled/); + }); + + it('fails on a Paths reference that does not exist', () => { + expect(() => + parseRoutes( + routesModule(' { component: A, path: Paths.core.nope },'), + PATHS, + ), + ).toThrow(/not a path string/); + }); + + it('fails when two routes would share a report name', () => { + expect(() => + parseRoutes( + routesModule(` + { component: A, path: Paths.core.status }, + { component: B, path: Paths.core.status }, +`), + PATHS, + ), + ).toThrow(/share the report name/); + }); +}); + +describe('slugOf', () => { + it('derives distinct names for an optional and a required parameter', () => { + expect(slugOf('/c/:namespace?/:container')).not.toBe( + slugOf('/c/:namespace/:container'), + ); + }); +}); + +describe('fillPattern', () => { + it('fills parameters and drops an optional one without a value', () => { + expect(fillPattern('/a/:x?/:y', { y: 'v w' })).toBe('a/v%20w'); + expect(fillPattern('/a/:x?/:y', { x: '1', y: '2' })).toBe('a/1/2'); + }); + + it('fails on a required parameter without a value', () => { + expect(() => fillPattern('/a/:x', {})).toThrow(/no value/); + }); +}); + +describe('the application', () => { + it('has routes that all parse, with unique report names', () => { + const routes = crawlRoutes(); + expect(routes.length).toBeGreaterThan(40); + expect(new Set(routes.map((r) => r.slug)).size).toBe(routes.length); + }); +}); diff --git a/e2e/crawl/routes.ts b/e2e/crawl/routes.ts new file mode 100644 index 00000000..c4c45b0c --- /dev/null +++ b/e2e/crawl/routes.ts @@ -0,0 +1,220 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { runInNewContext } from 'node:vm'; + +// The screens the crawler visits: every route of the application plus the +// index and one path no route matches. The routes are read from the source of +// src/app-routes.tsx as text, because importing that module would load every +// container of the application into the test runner, and each route is +// resolved against the Paths object of src/paths.ts, whose literal is read the +// same way so the end-to-end code does not compile the application. An entry +// that cannot be read stops the crawl, so a route added to the application can +// never be skipped without anyone noticing. + +export interface CrawlRoute { + // The route as declared, with :parameters (for example /deb/remotes/detail/:name). + // The index is '/' and the catch-all is '*'. + pattern: string; + // A file-name-safe name derived from the pattern, unique among all routes. + slug: string; + // The route is reachable without a session. + noAuth: boolean; + // Names of the parameters in the pattern, optional ones included. + params: string[]; +} + +// What a parameter is replaced with when the server has no object to name. +export const PLACEHOLDER = 'zz-crawl-placeholder'; + +// The path the catch-all route is asked for. +export const UNKNOWN_PATH = 'zz-crawl-no-such-page'; + +const ROUTES_DECLARATION = /const routes\s*:[^=]*=\s*\[/; + +// The text between the brackets of the routes array. +function routesBlock(source: string): string { + const start = ROUTES_DECLARATION.exec(source); + if (!start) { + throw new Error( + 'src/app-routes.tsx: the "const routes = [" declaration was not found', + ); + } + + let depth = 1; + for (let i = start.index + start[0].length; i < source.length; i++) { + const character = source[i]; + if (character === '[') { + depth++; + } else if (character === ']') { + depth--; + if (depth === 0) { + return source.slice(start.index + start[0].length, i); + } + } + } + throw new Error('src/app-routes.tsx: the routes array is never closed'); +} + +// The top-level { ... } entries of the array text. +function entries(block: string): string[] { + const found: string[] = []; + let depth = 0; + let begin = -1; + for (let i = 0; i < block.length; i++) { + const character = block[i]; + if (character === '{') { + if (depth === 0) { + begin = i; + } + depth++; + } else if (character === '}') { + depth--; + if (depth === 0) { + found.push(block.slice(begin, i + 1)); + } + } + } + return found; +} + +// The value of the exported Paths object literal of src/paths.ts. The literal +// holds only strings and nested objects; anything else fails to evaluate. +function readPaths(source: string): Record { + const start = /export const Paths\s*=\s*\{/.exec(source); + if (!start) { + throw new Error('src/paths.ts: "export const Paths = {" was not found'); + } + let depth = 0; + for (let i = start.index + start[0].length - 1; i < source.length; i++) { + if (source[i] === '{') { + depth++; + } else if (source[i] === '}') { + depth--; + if (depth === 0) { + const literal = source.slice(start.index + start[0].length - 1, i + 1); + return runInNewContext(`(${literal})`, Object.create(null)) as Record< + string, + unknown + >; + } + } + } + throw new Error('src/paths.ts: the Paths object is never closed'); +} + +function lookup(paths: Record, dotted: string): string { + let node: unknown = paths; + for (const key of dotted.split('.')) { + node = (node as Record | undefined)?.[key]; + } + if (typeof node !== 'string') { + throw new Error(`Paths.${dotted} is not a path string`); + } + return node; +} + +export function slugOf(pattern: string): string { + if (pattern === '/') { + return 'root'; + } + if (pattern === '*') { + return 'unknown'; + } + return pattern + .replace(/^\//, '') + .split('/') + .map((segment) => { + const parameter = segment.match(/^:(\w+)(\?)?$/); + return parameter + ? `by-${parameter[1]}${parameter[2] ? '-opt' : ''}` + : segment; + }) + .join('-') + .replace(/[^A-Za-z0-9_-]/g, '_'); +} + +export function paramsOf(pattern: string): string[] { + return [...pattern.matchAll(/\/:(\w+)\??(?=\/|$)/g)].map( + (match) => match[1] as string, + ); +} + +// Parses the text of the routes module. Exported for the tests of the parser. +export function parseRoutes(source: string, pathsSource: string): CrawlRoute[] { + const paths = readPaths(pathsSource); + const routes: CrawlRoute[] = []; + + for (const entry of entries(routesBlock(source))) { + const declared = [...entry.matchAll(/\bpath\s*:/g)]; + const path = entry.match(/\bpath\s*:\s*Paths\.([\w.]+)\s*(?:,|\}|$)/m); + if (declared.length !== 1 || !path) { + throw new Error( + 'src/app-routes.tsx: a route entry has no path of the form ' + + `"path: Paths..", so it cannot be crawled:\n${entry}`, + ); + } + + const pattern = lookup(paths, path[1] as string); + routes.push({ + pattern, + slug: slugOf(pattern), + noAuth: /\bnoAuth\s*:\s*true\b/.test(entry), + params: paramsOf(pattern), + }); + } + + if (routes.length === 0) { + throw new Error('src/app-routes.tsx: no route entries were found'); + } + + routes.push( + { pattern: '/', slug: slugOf('/'), noAuth: false, params: [] }, + { pattern: '*', slug: slugOf('*'), noAuth: false, params: [] }, + ); + + const owner = new Map(); + for (const route of routes) { + const other = owner.get(route.slug); + if (other !== undefined) { + throw new Error( + `routes ${other} and ${route.pattern} share the report name ${route.slug}`, + ); + } + owner.set(route.slug, route.pattern); + } + return routes; +} + +export function crawlRoutes(): CrawlRoute[] { + return parseRoutes( + readFileSync(resolve('src', 'app-routes.tsx'), 'utf8'), + readFileSync(resolve('src', 'paths.ts'), 'utf8'), + ); +} + +// The path of a pattern relative to the UI root with each :parameter replaced +// by its value; an optional parameter without a value drops out. +export function fillPattern( + pattern: string, + values: Record, +): string { + return pattern + .split('/') + .map((segment) => { + const parameter = segment.match(/^:(\w+)(\?)?$/); + if (!parameter) { + return segment; + } + const value = values[parameter[1] as string]; + if (value === undefined) { + if (parameter[2]) { + return ''; + } + throw new Error(`no value for :${parameter[1]} in ${pattern}`); + } + return encodeURIComponent(value); + }) + .join('/') + .replace(/\/{2,}/g, '/') + .replace(/^\//, ''); +} diff --git a/e2e/crawl/summarize.test.mjs b/e2e/crawl/summarize.test.mjs new file mode 100644 index 00000000..58d0d81b --- /dev/null +++ b/e2e/crawl/summarize.test.mjs @@ -0,0 +1,177 @@ +// @vitest-environment node +// Tests of the run summary: the order of the screens and when index.json is +// written. Fixtures use only reserved example names. +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { orderScreens, scoreOf, summarizeRun } from './summarize.ts'; +import { verifyReport } from './verify-report.mjs'; + +const ROUTES = [ + { pattern: '/a', slug: 'a', params: [] }, + { pattern: '/b', slug: 'b', params: [] }, +]; +const COMBINATIONS = [ + ['light', 'desktop'], + ['dark', 'desktop'], + ['light', 'phone'], + ['dark', 'phone'], +]; + +function report(route, slug, theme, viewport, run, extra = {}) { + return { + route, + url: 'https://pulp.example.org/ui/x', + params: 'none', + theme, + viewport, + account: 'admin', + title: 'T', + errorBoundary: false, + loadMs: 5, + settled: true, + screenshot: `${slug}.${theme}.${viewport}.png`, + consoleErrors: [], + pageErrors: [], + failedRequests: [], + duplicateRequests: [], + axe: { violations: [] }, + uiBuild: { hash: 'abc', version: '1' }, + run, + capturedAt: '2000-01-01T00:00:00.000Z', + ...extra, + }; +} + +function write(dir, item) { + const base = `${item.screenshot.replace(/\.png$/, '')}`; + writeFileSync(join(dir, `${base}.json`), JSON.stringify(item)); + writeFileSync(join(dir, item.screenshot), 'png'); +} + +function writeRun(dir, run, skip = () => false) { + for (const { pattern, slug } of ROUTES) { + for (const [theme, viewport] of COMBINATIONS) { + if (!skip(slug, theme, viewport)) { + write(dir, report(pattern, slug, theme, viewport, run)); + } + } + } +} + +let dir; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'summarize-')); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +describe('order', () => { + it('puts the worst screen first and breaks ties by route, theme, width', () => { + const crashed = report('/z', 'z', 'light', 'desktop', 'r', { + errorBoundary: true, + }); + const quiet = [ + report('/b', 'b', 'dark', 'phone', 'r'), + report('/b', 'b', 'light', 'phone', 'r'), + report('/b', 'b', 'light', 'desktop', 'r'), + report('/a', 'a', 'dark', 'desktop', 'r'), + ]; + const ordered = orderScreens([...quiet, crashed]).map( + (r) => `${r.route} ${r.theme} ${r.viewport}`, + ); + expect(ordered).toEqual([ + '/z light desktop', + '/a dark desktop', + '/b dark phone', + '/b light desktop', + '/b light phone', + ]); + }); + + it('gives the same order for the same data in any input order', () => { + const items = COMBINATIONS.map(([t, v]) => report('/a', 'a', t, v, 'r')); + const forward = orderScreens(items).map((r) => `${r.theme}${r.viewport}`); + const backward = orderScreens([...items].reverse()).map( + (r) => `${r.theme}${r.viewport}`, + ); + expect(forward).toEqual(backward); + }); + + it('does not count expected answers as findings', () => { + const expected = report('/a', 'a', 'light', 'desktop', 'r', { + failedRequests: [ + { method: 'GET', url: 'x', status: 401, expected: true }, + ], + consoleErrors: [{ text: 't', location: '', expected: true }], + }); + expect(scoreOf(expected)).toBe(0); + }); +}); + +describe('summarizeRun', () => { + it('writes the summary and the index of a complete run', () => { + writeRun(dir, 'one'); + expect(summarizeRun(dir, ROUTES, 'one')).toBe(true); + expect(existsSync(join(dir, 'index.json'))).toBe(true); + expect(readFileSync(join(dir, 'summary.md'), 'utf8')).toContain( + 'Complete: yes', + ); + expect(verifyReport(dir).filter((c) => !c.ok)).toEqual([]); + }); + + it('writes no index for a run that lacks a screen', () => { + writeRun( + dir, + 'one', + (slug, theme, viewport) => + slug === 'b' && theme === 'dark' && viewport === 'phone', + ); + expect(summarizeRun(dir, ROUTES, 'one')).toBe(true); + expect(existsSync(join(dir, 'index.json'))).toBe(false); + expect(readFileSync(join(dir, 'summary.md'), 'utf8')).toContain( + 'Complete: no', + ); + expect(verifyReport(dir).filter((c) => !c.ok).length).toBeGreaterThan(0); + }); + + it('removes the index and the files of an earlier run when a new run is incomplete', () => { + writeRun(dir, 'old'); + summarizeRun(dir, ROUTES, 'old'); + writeRun(dir, 'new', (slug) => slug === 'b'); + summarizeRun(dir, ROUTES, 'new'); + expect(existsSync(join(dir, 'index.json'))).toBe(false); + expect(existsSync(join(dir, 'b.light.desktop.json'))).toBe(false); + expect(existsSync(join(dir, 'a.light.desktop.json'))).toBe(true); + }); + + it('touches nothing when no report belongs to the run', () => { + writeRun(dir, 'old'); + summarizeRun(dir, ROUTES, 'old'); + const before = readFileSync(join(dir, 'summary.md'), 'utf8'); + expect(summarizeRun(dir, ROUTES, 'other')).toBe(false); + expect(readFileSync(join(dir, 'summary.md'), 'utf8')).toBe(before); + expect(existsSync(join(dir, 'index.json'))).toBe(true); + }); + + it('leaves files that are not crawl output alone', () => { + writeRun(dir, 'one'); + writeFileSync(join(dir, 'notes.txt'), 'keep'); + summarizeRun(dir, ROUTES, 'one'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('keep'); + }); + + it('writes no host into the summary', () => { + writeRun(dir, 'one'); + summarizeRun(dir, ROUTES, 'one'); + expect(readFileSync(join(dir, 'summary.md'), 'utf8')).not.toContain( + 'pulp.example.org', + ); + }); +}); diff --git a/e2e/crawl/summarize.ts b/e2e/crawl/summarize.ts new file mode 100644 index 00000000..2fac81bc --- /dev/null +++ b/e2e/crawl/summarize.ts @@ -0,0 +1,493 @@ +import { + existsSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { join } from 'node:path'; +import { + SCREEN_FILE, + type ScreenReport, + THEMES, + VIEWPORTS, + outDir, + screenName, +} from './report'; +import { type CrawlRoute, crawlRoutes } from './routes'; + +// Turns the per-screen reports of a crawl into index.json (what was crawled and +// the totals) and summary.md (one row per screen, worst first, followed by the +// findings grouped across screens), for a person or an agent to act on without +// opening every report. Both are written by the global teardown, after every +// project has finished. index.json is written only when every screen of every +// route was captured, so a crawl that was interrupted, or that crashed on one +// screen, leaves none and the completeness check refuses the output. +// +// The summary and the index name no host and carry no session data: request +// addresses are reduced to their path. + +const IMPACT_WEIGHT: Record = { + critical: 10, + serious: 5, + moderate: 2, + minor: 1, +}; + +const unexpected = (items: T[]): T[] => + items.filter((item) => !item.expected); + +// How badly a screen is broken, as one number: a crash outweighs an uncaught +// error, which outweighs a failed request, a console error, an accessibility +// finding and a repeated request, in that order. +export function scoreOf(report: ScreenReport): number { + return ( + (report.errorBoundary ? 1000 : 0) + + report.pageErrors.length * 100 + + unexpected(report.failedRequests).length * 50 + + unexpected(report.consoleErrors).length * 20 + + (report.settled === false ? 30 : 0) + + report.axe.violations.reduce( + (sum, violation) => sum + (IMPACT_WEIGHT[violation.impact ?? ''] ?? 1), + 0, + ) + + report.duplicateRequests.length + ); +} + +const compare = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + +// Worst first; screens with the same score are ordered by route pattern, then +// theme, then width, so two crawls of the same data give the same summary. +export function orderScreens(reports: ScreenReport[]): ScreenReport[] { + return [...reports].sort( + (a, b) => + scoreOf(b) - scoreOf(a) || + compare(a.route, b.route) || + compare(a.theme, b.theme) || + compare(a.viewport, b.viewport), + ); +} + +// Printable ASCII only, so the files read the same everywhere. +function ascii(value: string): string { + return value.replace(/[^\x20-\x7e]/g, '?'); +} + +function asciiJson(value: unknown): string { + return JSON.stringify(value, null, 2).replace( + /[^\x20-\x7e\n]/g, + (character) => + `\\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`, + ); +} + +function cell(value: string, limit = 140): string { + const flat = ascii(value.replace(/\s+/g, ' ')).replace(/\|/g, '/').trim(); + return flat.length > limit ? `${flat.slice(0, limit - 3)}...` : flat; +} + +// The path of an address, with ids that differ between objects folded together. +function pathOf(address: string): string { + try { + const { pathname } = new URL(address); + return pathname.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f-]{23}/g, ''); + } catch { + return cell(address); + } +} + +function tally(items: T[], key: (item: T) => string): Map { + const counts = new Map(); + for (const item of items) { + counts.set(key(item), (counts.get(key(item)) ?? 0) + 1); + } + return counts; +} + +function byCount(counts: Map): [string, number][] { + return [...counts].sort((a, b) => b[1] - a[1] || compare(a[0], b[0])); +} + +export interface Totals { + screens: number; + errorBoundary: number; + pageErrors: number; + failedRequests: number; + expectedFailedRequests: number; + consoleErrors: number; + expectedConsoleErrors: number; + duplicateRequests: number; + axeViolations: number; + unsettled: number; + placeholderParams: number; +} + +export function totalsOf(reports: ScreenReport[]): Totals { + const sum = (pick: (report: ScreenReport) => number) => + reports.reduce((total, report) => total + pick(report), 0); + return { + screens: reports.length, + errorBoundary: sum((r) => (r.errorBoundary ? 1 : 0)), + pageErrors: sum((r) => r.pageErrors.length), + failedRequests: sum((r) => unexpected(r.failedRequests).length), + expectedFailedRequests: sum( + (r) => r.failedRequests.filter((f) => f.expected).length, + ), + consoleErrors: sum((r) => unexpected(r.consoleErrors).length), + expectedConsoleErrors: sum( + (r) => r.consoleErrors.filter((c) => c.expected).length, + ), + duplicateRequests: sum((r) => r.duplicateRequests.length), + axeViolations: sum((r) => r.axe.violations.length), + unsettled: sum((r) => (r.settled === false ? 1 : 0)), + placeholderParams: sum((r) => (r.params === 'placeholder' ? 1 : 0)), + }; +} + +function hostOf(reports: ScreenReport[]): string { + try { + return new URL(reports[0]?.url ?? '').host; + } catch { + return ''; + } +} + +export interface Index { + run: string; + complete: boolean; + account: string; + baseUrlHost: string; + uiBuild: ScreenReport['uiBuild']; + firstCapture: string; + lastCapture: string; + routes: { pattern: string; slug: string; params: string[] }[]; + totals: Totals; + screens: { + route: string; + theme: string; + viewport: string; + file: string; + screenshot: string; + score: number; + }[]; +} + +export function buildIndex( + reports: ScreenReport[], + routes: CrawlRoute[], + run: string, + complete: boolean, +): Index { + const slugs = new Map(routes.map((route) => [route.pattern, route.slug])); + const captured = reports.map((report) => report.capturedAt).sort(); + return { + run, + complete, + account: reports[0]?.account ?? '', + baseUrlHost: hostOf(reports), + uiBuild: reports.find((report) => report.uiBuild)?.uiBuild ?? null, + firstCapture: captured[0] ?? '', + lastCapture: captured[captured.length - 1] ?? '', + routes: routes.map(({ pattern, slug, params }) => ({ + pattern, + slug, + params, + })), + totals: totalsOf(reports), + screens: orderScreens(reports).map((report) => ({ + route: report.route, + theme: report.theme, + viewport: report.viewport, + file: screenName( + slugs.get(report.route) ?? '', + report.theme, + report.viewport, + 'json', + ), + screenshot: report.screenshot, + score: scoreOf(report), + })), + }; +} + +export function renderSummary( + reports: ScreenReport[], + routes: CrawlRoute[], + complete: boolean, +): string { + const ordered = orderScreens(reports); + const totals = totalsOf(reports); + const host = hostOf(reports); + const hide = (text: string) => + host ? text.split(host).join('') : text; + const slugs = new Map(routes.map((route) => [route.pattern, route.slug])); + const build = reports.find((report) => report.uiBuild)?.uiBuild; + const captured = reports.map((report) => report.capturedAt).sort(); + + const lines: string[] = [ + '# Screen crawl', + '', + `- Account: ${reports[0]?.account ?? 'none'}`, + `- UI build: ${build?.hash ?? 'unknown'} (version ${build?.version ?? 'unknown'})`, + `- Captured: ${captured[0] ?? ''} to ${captured[captured.length - 1] ?? ''}`, + `- Screens: ${totals.screens} of ${routes.length * THEMES.length * VIEWPORTS.length} (${routes.length} routes, ${THEMES.length} themes, ${VIEWPORTS.length} widths)`, + `- Complete: ${complete ? 'yes' : 'no'}`, + `- Screens are ordered worst first; equal scores by route, theme, width. The score weighs an error page 1000, an uncaught error 100, a failed request 50, a console error 20, a screen that never settled 30, an accessibility rule by impact (critical 10, serious 5, moderate 2, minor 1) and a repeated request 1.`, + '', + '## Totals', + '', + '| Finding | Count |', + '|---|---|', + `| Screens that crashed into the error page | ${totals.errorBoundary} |`, + `| Uncaught errors | ${totals.pageErrors} |`, + `| Failed requests | ${totals.failedRequests} |`, + `| Console errors | ${totals.consoleErrors} |`, + `| Accessibility rule violations | ${totals.axeViolations} |`, + `| Repeated requests | ${totals.duplicateRequests} |`, + `| Screens that did not settle | ${totals.unsettled} |`, + `| Screens visited with a placeholder parameter | ${totals.placeholderParams} |`, + `| Expected failed requests (not counted above) | ${totals.expectedFailedRequests} |`, + `| Console lines of expected failed requests (not counted above) | ${totals.expectedConsoleErrors} |`, + '', + '## Screens', + '', + '| # | Route | Theme | Width | Params | Score | Error page | Uncaught | Failed | Console | Axe | Repeated | Settled | Load ms | Report |', + '|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|', + ]; + + ordered.forEach((report, position) => { + lines.push( + [ + position + 1, + cell(report.route), + report.theme, + report.viewport, + report.params, + scoreOf(report), + report.errorBoundary ? 'yes' : 'no', + report.pageErrors.length, + unexpected(report.failedRequests).length, + unexpected(report.consoleErrors).length, + report.axe.violations.length, + report.duplicateRequests.length, + report.settled === false ? 'no' : 'yes', + report.loadMs, + screenName( + slugs.get(report.route) ?? '', + report.theme, + report.viewport, + 'json', + ), + ] + .join(' | ') + .replace(/^/, '| ') + .replace(/$/, ' |'), + ); + }); + + const rules = new Map< + string, + { impact: string; help: string; screens: number; nodes: number } + >(); + for (const report of reports) { + for (const violation of report.axe.violations) { + const rule = rules.get(violation.id) ?? { + impact: violation.impact ?? 'unknown', + help: violation.help, + screens: 0, + nodes: 0, + }; + rule.screens += 1; + rule.nodes += violation.nodes; + rules.set(violation.id, rule); + } + } + lines.push('', '## Accessibility rules', ''); + if (rules.size === 0) { + lines.push('None.'); + } else { + lines.push( + '| Rule | Impact | Screens | Nodes | What it says |', + '|---|---|---|---|---|', + ); + for (const [id, rule] of [...rules].sort( + (a, b) => + (IMPACT_WEIGHT[b[1].impact] ?? 0) - (IMPACT_WEIGHT[a[1].impact] ?? 0) || + b[1].screens - a[1].screens || + compare(a[0], b[0]), + )) { + lines.push( + `| ${id} | ${rule.impact} | ${rule.screens} | ${rule.nodes} | ${cell(rule.help)} |`, + ); + } + } + + const failed = tally( + reports.flatMap((report) => + unexpected(report.failedRequests).map((request) => + [ + request.method, + String(request.status || request.failure), + pathOf(request.url), + ].join('\t'), + ), + ), + (key) => key, + ); + lines.push('', '## Failed requests', ''); + if (failed.size === 0) { + lines.push('None.'); + } else { + lines.push('| Method | Answer | Path | Times |', '|---|---|---|---|'); + for (const [key, count] of byCount(failed)) { + const [method = '', answer = '', path = ''] = key.split('\t'); + lines.push(`| ${method} | ${answer} | ${cell(path, 200)} | ${count} |`); + } + } + + const consoleLines = tally( + reports.flatMap((report) => + unexpected(report.consoleErrors).map((entry) => hide(entry.text)), + ), + (text) => text, + ); + lines.push('', '## Console errors', ''); + if (consoleLines.size === 0) { + lines.push('None.'); + } else { + lines.push('| Text | Times |', '|---|---|'); + for (const [text, count] of byCount(consoleLines)) { + lines.push(`| ${cell(hide(text), 200)} | ${count} |`); + } + } + + const uncaught = tally( + reports.flatMap((report) => report.pageErrors.map(hide)), + (text) => text, + ); + lines.push('', '## Uncaught errors', ''); + if (uncaught.size === 0) { + lines.push('None.'); + } else { + lines.push('| Text | Times |', '|---|---|'); + for (const [text, count] of byCount(uncaught)) { + lines.push(`| ${cell(text, 200)} | ${count} |`); + } + } + + const repeated = tally( + reports.flatMap((report) => + report.duplicateRequests.map( + (request) => `${request.method} ${pathOf(request.url)}`, + ), + ), + (key) => key, + ); + lines.push('', '## Repeated requests', ''); + if (repeated.size === 0) { + lines.push('None.'); + } else { + lines.push('| Request | Screens |', '|---|---|'); + for (const [key, count] of byCount(repeated).slice(0, 40)) { + lines.push(`| ${cell(key, 200)} | ${count} |`); + } + } + + return `${lines.join('\n')}\n`; +} + +// What was left in a directory by crawls: the reports it holds, by file name. +function readReports(dir: string): Map { + const found = new Map(); + for (const file of readdirSync(dir)) { + if (!file.endsWith('.json') || !SCREEN_FILE.test(file)) { + continue; + } + try { + found.set(file, JSON.parse(readFileSync(join(dir, file), 'utf8'))); + } catch { + // A half-written file belongs to a run that did not finish. + } + } + return found; +} + +// Writes the summary of the run named `run` into dir, and its index when the +// run is complete. Reports and screenshots of other runs, and the index and +// summary of an earlier run, are removed first, so the directory always shows +// one run. Returns false when the run left no report (a run without crawl +// tests), in which case nothing is touched. +export function summarizeRun( + dir: string, + routes: CrawlRoute[], + run: string, +): boolean { + if (!run || !existsSync(dir)) { + return false; + } + const all = readReports(dir); + const mine = [...all].filter(([, report]) => report.run === run); + if (mine.length === 0) { + return false; + } + + for (const file of readdirSync(dir)) { + if (!SCREEN_FILE.test(file)) { + continue; + } + const report = all.get(file.replace(/\.png$/, '.json')); + if (report?.run !== run) { + rmSync(join(dir, file), { force: true }); + } + } + rmSync(join(dir, 'index.json'), { force: true }); + rmSync(join(dir, 'summary.md'), { force: true }); + + const reports = mine.map(([, report]) => report); + const slugs = new Set(routes.map((route) => route.slug)); + const present = new Set( + reports.map( + (report) => `${report.route}|${report.theme}|${report.viewport}`, + ), + ); + const complete = + routes.every((route) => + THEMES.every((theme) => + VIEWPORTS.every((viewport) => + present.has(`${route.pattern}|${theme}|${viewport}`), + ), + ), + ) && slugs.size === routes.length; + + writeFileSync( + join(dir, 'summary.md'), + renderSummary(reports, routes, complete), + ); + if (complete) { + writeFileSync( + join(dir, 'index.json'), + `${asciiJson(buildIndex(reports, routes, run, true))}\n`, + ); + } else { + console.log( + `crawl incomplete: ${present.size} of ${routes.length * THEMES.length * VIEWPORTS.length} screens captured, no index.json written`, + ); + } + return true; +} + +export default function globalTeardown(): void { + const run = process.env['PULP_UI_CRAWL_RUN'] ?? ''; + const dir = outDir(); + // Without reports of this run (smoke or remote runs) nothing is read from + // the source tree and nothing is written. + if ( + !run || + !existsSync(dir) || + !readdirSync(dir).some((f) => SCREEN_FILE.test(f)) + ) { + return; + } + summarizeRun(dir, crawlRoutes(), run); +} diff --git a/e2e/crawl/verify-report.mjs b/e2e/crawl/verify-report.mjs new file mode 100644 index 00000000..ae6945a3 --- /dev/null +++ b/e2e/crawl/verify-report.mjs @@ -0,0 +1,193 @@ +#!/usr/bin/env node +// Checks that a crawl output directory is complete: an index.json and a +// summary.md exist, every route pattern the index lists has a report in each of +// the four theme and width combinations, every report carries all of its fields +// and names a screenshot that exists. It checks completeness only; what the +// reports found (console errors, failed requests, accessibility findings, a +// page that crashed) never makes it fail. +// +// Usage: verify-report.mjs +// Prints "ok " or "FAIL : " per check and exits 1 on any FAIL. +import { existsSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export const THEMES = ['light', 'dark']; +export const VIEWPORTS = ['desktop', 'phone']; + +// The fields every per-screen report has (see e2e/crawl/report.ts). +export const REQUIRED_FIELDS = [ + 'route', + 'url', + 'params', + 'theme', + 'viewport', + 'account', + 'title', + 'errorBoundary', + 'loadMs', + 'screenshot', + 'consoleErrors', + 'pageErrors', + 'failedRequests', + 'duplicateRequests', + 'axe', +]; + +const ARRAY_FIELDS = [ + 'consoleErrors', + 'pageErrors', + 'failedRequests', + 'duplicateRequests', +]; + +function readJson(path) { + try { + return { value: JSON.parse(readFileSync(path, 'utf8')) }; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } +} + +const ok = (what) => ({ ok: true, what, why: '' }); +const fail = (what, why) => ({ ok: false, what, why }); + +// Returns one { ok, what, why } per check. +export function verifyReport(dir) { + const checks = []; + + if (!existsSync(dir) || !statSync(dir).isDirectory()) { + return [fail('output directory', `${dir} is not a directory`)]; + } + + const summary = join(dir, 'summary.md'); + checks.push( + existsSync(summary) + ? ok('summary.md') + : fail( + 'summary.md', + 'missing: the run did not finish writing its summary', + ), + ); + + const indexPath = join(dir, 'index.json'); + if (!existsSync(indexPath)) { + checks.push( + fail( + 'index.json', + 'missing: it is written only after every screen finished, so the crawl is incomplete or did not run', + ), + ); + return checks; + } + const index = readJson(indexPath); + if (index.error) { + checks.push(fail('index.json', `not valid JSON: ${index.error}`)); + return checks; + } + const routes = index.value?.routes; + if (!Array.isArray(routes) || routes.length === 0) { + checks.push(fail('index.json', 'lists no routes')); + return checks; + } + checks.push(ok(`index.json lists ${routes.length} routes`)); + if (index.value.complete === false) { + checks.push(fail('index.json', 'says the crawl is not complete')); + } + + for (const route of routes) { + for (const theme of THEMES) { + for (const viewport of VIEWPORTS) { + const what = `${route.pattern} ${theme} ${viewport}`; + const file = `${route.slug}.${theme}.${viewport}.json`; + const path = join(dir, file); + + if (!existsSync(path)) { + checks.push(fail(what, `${file} is missing`)); + continue; + } + const read = readJson(path); + if (read.error) { + checks.push(fail(what, `${file} is not valid JSON: ${read.error}`)); + continue; + } + + const report = read.value; + const problems = []; + for (const field of REQUIRED_FIELDS) { + if (!(field in report)) { + problems.push(`lacks the field ${field}`); + } + } + for (const field of ARRAY_FIELDS) { + if (field in report && !Array.isArray(report[field])) { + problems.push(`${field} is not a list`); + } + } + if ('axe' in report && !Array.isArray(report.axe?.violations)) { + problems.push('axe.violations is not a list'); + } + // A report left by another run next to this run's index is not part of it. + if ( + index.value.run !== undefined && + report.run !== undefined && + report.run !== index.value.run + ) { + problems.push('belongs to another crawl run than index.json'); + } + if (report.route !== undefined && report.route !== route.pattern) { + problems.push(`route is ${report.route}`); + } + if (report.theme !== undefined && report.theme !== theme) { + problems.push(`theme is ${report.theme}`); + } + if (report.viewport !== undefined && report.viewport !== viewport) { + problems.push(`viewport is ${report.viewport}`); + } + if ( + typeof report.screenshot === 'string' && + !existsSync(join(dir, report.screenshot)) + ) { + problems.push(`screenshot ${report.screenshot} is missing`); + } + + checks.push( + problems.length + ? fail(what, `${file} ${problems.join('; ')}`) + : ok(what), + ); + } + } + } + + return checks; +} + +export function main(argv) { + const dir = argv[0]; + if (!dir) { + console.error('usage: verify-report.mjs '); + return 2; + } + + const checks = verifyReport(dir); + for (const check of checks) { + console.log( + check.ok ? `ok ${check.what}` : `FAIL ${check.what}: ${check.why}`, + ); + } + const failed = checks.filter((check) => !check.ok).length; + console.log( + failed + ? `${failed} of ${checks.length} checks failed` + : `all ${checks.length} checks passed`, + ); + return failed ? 1 : 0; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + process.exitCode = main(process.argv.slice(2)); +} diff --git a/e2e/crawl/verify-report.test.mjs b/e2e/crawl/verify-report.test.mjs new file mode 100644 index 00000000..66bd829d --- /dev/null +++ b/e2e/crawl/verify-report.test.mjs @@ -0,0 +1,209 @@ +// @vitest-environment node +// Tests of the completeness check of a crawl output directory. The fixtures are +// written to a temporary directory and name only reserved example routes. +import { spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { REQUIRED_FIELDS, verifyReport } from './verify-report.mjs'; + +const SCRIPT = join( + dirname(fileURLToPath(import.meta.url)), + 'verify-report.mjs', +); +const ROUTES = [ + { pattern: '/status', slug: 'status' }, + { + pattern: '/deb/repositories/detail/:name', + slug: 'deb-repositories-detail-by-name', + }, +]; +const COMBINATIONS = [ + ['light', 'desktop'], + ['dark', 'desktop'], + ['light', 'phone'], + ['dark', 'phone'], +]; + +function screen(route, theme, viewport, slug) { + return { + route, + url: 'https://pulp.example.org/ui/x', + params: 'none', + theme, + viewport, + account: 'admin', + title: 'Status', + errorBoundary: false, + loadMs: 10, + screenshot: `${slug}.${theme}.${viewport}.png`, + consoleErrors: [], + pageErrors: [], + failedRequests: [], + duplicateRequests: [], + axe: { violations: [] }, + }; +} + +// A complete crawl of ROUTES: every combination, a screenshot for each, an +// index and a summary. +function writeFixture(dir) { + for (const { pattern, slug } of ROUTES) { + for (const [theme, viewport] of COMBINATIONS) { + const base = `${slug}.${theme}.${viewport}`; + writeFileSync( + join(dir, `${base}.json`), + JSON.stringify(screen(pattern, theme, viewport, slug)), + ); + writeFileSync(join(dir, `${base}.png`), 'png'); + } + } + writeFileSync(join(dir, 'index.json'), JSON.stringify({ routes: ROUTES })); + writeFileSync(join(dir, 'summary.md'), '# Crawl\n'); +} + +const failures = (dir) => + verifyReport(dir) + .filter((check) => !check.ok) + .map((check) => `${check.what}: ${check.why}`); + +let dir; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'verify-report-')); + writeFixture(dir); +}); +afterEach(() => rmSync(dir, { recursive: true, force: true })); + +describe('verifyReport', () => { + it('passes a directory with every route in all four combinations', () => { + expect(failures(dir)).toEqual([]); + }); + + it('fails when one combination of one route is missing', () => { + rmSync(join(dir, 'status.dark.phone.json')); + const found = failures(dir); + expect(found).toHaveLength(1); + expect(found[0]).toContain('/status dark phone'); + }); + + it('fails when a screenshot file is missing', () => { + rmSync(join(dir, 'deb-repositories-detail-by-name.light.desktop.png')); + const found = failures(dir); + expect(found).toHaveLength(1); + expect(found[0]).toContain('screenshot'); + }); + + it.each(REQUIRED_FIELDS)( + 'fails when a report lacks the field %s', + (field) => { + const file = join(dir, 'status.light.phone.json'); + const report = JSON.parse(readFileSync(file, 'utf8')); + delete report[field]; + writeFileSync(file, JSON.stringify(report)); + expect(failures(dir).join('\n')).toContain(field); + }, + ); + + it('fails when index.json is absent, as an unfinished crawl leaves it', () => { + rmSync(join(dir, 'index.json')); + expect(failures(dir).join('\n')).toContain('index.json'); + }); + + it('fails when summary.md is absent', () => { + rmSync(join(dir, 'summary.md')); + expect(failures(dir).join('\n')).toContain('summary.md'); + }); + + it('fails on an empty directory and on one that does not exist', () => { + const empty = mkdtempSync(join(tmpdir(), 'verify-report-empty-')); + try { + expect(failures(empty).length).toBeGreaterThan(0); + expect(failures(join(empty, 'missing')).length).toBeGreaterThan(0); + } finally { + rmSync(empty, { recursive: true, force: true }); + } + }); + + it('does not fail on findings in the reports', () => { + const file = join(dir, 'status.dark.desktop.json'); + const report = JSON.parse(readFileSync(file, 'utf8')); + report.errorBoundary = true; + report.consoleErrors = [{ text: 'boom', location: '', expected: false }]; + report.pageErrors = ['boom']; + report.failedRequests = [ + { method: 'GET', url: 'x', status: 500, expected: false }, + ]; + report.axe = { + violations: [ + { + id: 'color-contrast', + impact: 'serious', + help: 'h', + nodes: 3, + targets: [], + }, + ], + }; + writeFileSync(file, JSON.stringify(report)); + expect(failures(dir)).toEqual([]); + }); + + it('fails on a report whose route or theme disagrees with its file name', () => { + const file = join(dir, 'status.dark.phone.json'); + const report = JSON.parse(readFileSync(file, 'utf8')); + report.theme = 'light'; + writeFileSync(file, JSON.stringify(report)); + expect(failures(dir).join('\n')).toContain('theme'); + }); + + it('fails on a report left by another crawl run', () => { + writeFileSync( + join(dir, 'index.json'), + JSON.stringify({ routes: ROUTES, run: 'new' }), + ); + const file = join(dir, 'status.dark.phone.json'); + const report = JSON.parse(readFileSync(file, 'utf8')); + report.run = 'old'; + writeFileSync(file, JSON.stringify(report)); + const found = failures(dir); + expect(found.length).toBeGreaterThan(0); + expect(found.join('\n')).toContain('another crawl run'); + }); + + it('fails on a report that is not valid JSON', () => { + writeFileSync(join(dir, 'status.dark.phone.json'), '{ not json'); + expect(failures(dir).join('\n')).toContain('status.dark.phone.json'); + }); +}); + +describe('command line', () => { + const run = (...args) => + spawnSync(process.execPath, [SCRIPT, ...args], { encoding: 'utf8' }); + + it('prints ok lines and exits 0 on a complete directory', () => { + const result = run(dir); + expect(result.status).toBe(0); + expect(result.stdout).toMatch(/^ok /m); + expect(result.stdout).not.toMatch(/^FAIL /m); + }); + + it('prints FAIL lines and exits 1 on an incomplete directory', () => { + mkdirSync(join(dir, 'sub')); + rmSync(join(dir, 'index.json')); + const result = run(dir); + expect(result.status).toBe(1); + expect(result.stdout).toMatch(/^FAIL index\.json/m); + }); + + it('exits non-zero without a directory argument', () => { + expect(run().status).not.toBe(0); + }); +}); diff --git a/eslint.config.mjs b/eslint.config.mjs index 9d96efba..b1e01078 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -88,7 +88,7 @@ export default defineConfig([ }, }, { - files: ['scripts/**/*.mjs'], + files: ['scripts/**/*.mjs', 'e2e/**/*.mjs'], languageOptions: { globals: { ...globals.node }, }, diff --git a/package-lock.json b/package-lock.json index 5a105c0e..716347de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "react-router": "^7.18.2" }, "devDependencies": { + "@axe-core/playwright": "4.13.0", "@babel/core": "^7.29.6", "@babel/plugin-transform-runtime": "^7.28.5", "@babel/preset-env": "^7.28.5", @@ -96,6 +97,19 @@ "dev": true, "license": "MIT" }, + "node_modules/@axe-core/playwright": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", + "integrity": "sha512-6YLx+kxXu5GJceG4ozFg+33a2EMTdjYwWGloJ3sb9Kta5pp+ZNS53uxGVog5JetIY8s++P5UrtX+cri+u0VAVg==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.13.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -7039,6 +7053,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axe-core": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", + "integrity": "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/axios": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", diff --git a/package.json b/package.json index 375ad699..ffa23ada 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "react-router": "^7.18.2" }, "devDependencies": { + "@axe-core/playwright": "4.13.0", "@babel/core": "^7.29.6", "@babel/plugin-transform-runtime": "^7.28.5", "@babel/preset-env": "^7.28.5", diff --git a/playwright.config.ts b/playwright.config.ts index 6e517e7d..a18e59f5 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,4 +1,6 @@ import { defineConfig } from '@playwright/test'; +import { SCREEN_SIZES, crawlAccount } from './e2e/crawl/report'; +import { storageStatePath } from './e2e/support/accounts'; // The target and the account come from the environment only (see // e2e/support/env.ts); nothing here names a host or a credential. The base URL @@ -10,8 +12,34 @@ import { defineConfig } from '@playwright/test'; // sign in once through the API (remote-setup), reuse that session per role, and // end it again (remote-teardown). They are meant for a person at a terminal, not // for a hosted workflow: their output holds session data. +// The screen crawler (e2e/crawl) visits every route of the application, signed +// in as the account named by PULP_UI_CRAWL_ACCOUNT (admin unless set to +// viewer), and writes its report to PULP_UI_CRAWL_OUT (crawl-report/ unless +// set). One project is one theme at one width; the theme is the browser's +// colour scheme, which the application follows when the user has made no +// choice. Reports show a deployment's data and URLs and are meant to stay on +// the machine that ran the crawl. +function crawlProject(theme: 'light' | 'dark', viewport: 'desktop' | 'phone') { + return { + name: `crawl-${theme}-${viewport}`, + testMatch: 'crawl/**/*.spec.ts', + metadata: { theme, viewport }, + use: { + colorScheme: theme, + viewport: SCREEN_SIZES[viewport], + ...(viewport === 'phone' ? { isMobile: true, hasTouch: true } : {}), + storageState: storageStatePath(crawlAccount()), + }, + dependencies: ['remote-setup'], + }; +} + export default defineConfig({ testDir: 'e2e', + // The crawl run is named before the workers start and summarised after every + // project has finished; both do nothing in a run without crawl projects. + globalSetup: './e2e/crawl/global-setup.ts', + globalTeardown: './e2e/crawl/summarize.ts', // One worker in CI: the tests share one Pulp and sign in as the same account. workers: process.env['CI'] ? 1 : undefined, forbidOnly: !!process.env['CI'], @@ -50,5 +78,9 @@ export default defineConfig({ use: { storageState: 'playwright/.auth/viewer.json' }, dependencies: ['remote-setup'], }, + crawlProject('light', 'desktop'), + crawlProject('dark', 'desktop'), + crawlProject('light', 'phone'), + crawlProject('dark', 'phone'), ], }); diff --git a/vitest.config.ts b/vitest.config.ts index 825e1106..27067f8c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -5,13 +5,16 @@ import { defineConfig } from 'vitest/config'; // Two projects keep the browser-like setup away from Node tooling tests: // "app" runs the application tests under src/ in a DOM environment with the // HTTP mock server and the lingui macro transform, "node" runs the tests of the -// repository scripts under scripts/ and of the build configuration under -// config/ with no setup at all. The sdk workspace -// keeps its own configuration and is run through its own test script, so -// nothing under packages/ is collected here. There is deliberately no -// passWithNoTests: an include pattern that matches no file must fail the run, -// never pass silently. +// repository scripts under scripts/, of the build configuration under config/ +// and of the crawler's report tooling under e2e/crawl/ with no setup at all. +// The sdk workspace keeps its own configuration and is run through its own +// test script, so nothing under packages/ is collected here. There is +// deliberately no passWithNoTests: an include pattern that matches no file must +// fail the run, never pass silently. const exclude = ['node_modules/**', 'dist/**', 'e2e/**', 'packages/**']; +// The end-to-end directory belongs to Playwright except for the report tooling +// tests, which are plain Node tests. +const nodeExclude = exclude.filter((pattern) => pattern !== 'e2e/**'); const root = import.meta.dirname; @@ -53,8 +56,12 @@ export default defineConfig({ test: { name: 'node', environment: 'node', - include: ['scripts/**/*.test.{ts,mjs}', 'config/**/*.test.ts'], - exclude, + include: [ + 'scripts/**/*.test.{ts,mjs}', + 'config/**/*.test.ts', + 'e2e/**/*.test.mjs', + ], + exclude: nodeExclude, }, }, ],