test(mosaic): add screenshot testing POC

.changeset/mosaic-visual-tests.md

@@ -0,0 +1,2 @@
+---
+---

.github/workflows/ci.yml

@@ -440,6 +440,68 @@ jobs:
           retention-days: 1
           if-no-files-found: ignore
 
+  mosaic-visual-tests:
+    needs: [check-permissions, build-packages]
+    name: Mosaic Visual Tests
+    permissions:
+      contents: read
+    runs-on: 'blacksmith-8vcpu-ubuntu-2204'
+    defaults:
+      run:
+        shell: bash
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4.4.0
+        with:
+          persist-credentials: false
+          fetch-depth: 1
+          fetch-tags: false
+          filter: 'blob:none'
+          show-progress: false
+
+      - name: Setup
+        uses: $/.github/actions/init-blacksmith
+        with:
+          cache-enabled: true
+          turbo-signature: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
+          turbo-team: ${{ vars.TURBO_TEAM }}
+          turbo-token: ${{ secrets.TURBO_TOKEN }}
+
+      - name: Build Mosaic and its dependencies
+        run: pnpm turbo build $TURBO_ARGS --filter=@clerk/mosaic...
+
+      - name: Run visual tests
+        id: visual
+        run: |
+          status=0
+          bash packages/mosaic/scripts/test-visual.sh compare || status=$?
+          if [[ "$status" -ne 0 ]] && find packages/mosaic/.vitest/attachments -name '*.png' -print -quit 2>/dev/null | grep -q .; then
+            echo "differences=true" >> "$GITHUB_OUTPUT"
+          elif [[ "$status" -ne 0 ]]; then
+            exit "$status"
+          fi
+
+      - name: Report visual differences
+        if: ${{ steps.visual.outputs.differences == 'true' }}
+        run: |
+          echo "::warning title=Mosaic visual differences::Screenshots differ from baselines. Download mosaic-visual-failure-artifacts-${{ github.run_id }}-${{ github.run_attempt }} to review."
+          echo "### Mosaic screenshots differ from baselines" >> "$GITHUB_STEP_SUMMARY"
+          find packages/mosaic/.vitest/attachments -name '*.png' -printf '- %P\n' >> "$GITHUB_STEP_SUMMARY" || true
+
+      - name: Upload failure artifacts
+        if: ${{ steps.visual.outputs.differences == 'true' }}
+        uses: actions/upload-artifact@v4.6.2
+        with:
+          name: mosaic-visual-failure-artifacts-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            packages/mosaic/.vitest/attachments
+            packages/mosaic/src/**/__screenshots__/**/*-chromium-linux.png
+          include-hidden-files: true
+          retention-days: 7
+          if-no-files-found: warn
+
   integration-tests:
     needs: [check-permissions, build-packages]
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}

packages/mosaic/docs/testing.md

@@ -36,8 +36,10 @@ Rule of thumb: **test each behavior once, at the boundary that owns it.**
 | Feature | `*.feature.test.tsx`        | Chromium (`pnpm test:feature`)                | A feature end to end: real `Clerk`, real layers, FAPI faked with MSW    |
 | E2E     | `/integration` (Playwright) | Real apps against a real Clerk backend        | Framework wiring, redirects, real sessions. Not written per Mosaic flow |
 
-Visual states (loading, empty, every error, every variant) are swingset stories
-with plain props, reviewed by eye. Don't duplicate them as view tests.
+Most visual states (loading, empty, every error, every variant) are swingset
+stories with plain props, reviewed by eye. A small screenshot suite covers
+representative compositions, as described in [Screenshot tests](#screenshot-tests).
+Don't duplicate every story as a view or screenshot test.
 
 Each behavior lives in one tier. A feature test owns everything the feature
 does on screen, including edge cases, busy states, and errors. E2E only proves
@@ -99,8 +101,8 @@ pnpm --filter @clerk/mosaic test:browser              # layout tests (Chromium)
 pnpm --filter @clerk/mosaic exec vitest run           # every project
 ```
 
-CI runs feature tests in their own `Mosaic Feature Tests` job, so the repo-wide
-unit test job never needs a browser.
+CI runs feature and screenshot tests in separate `Mosaic Feature Tests` and
+`Mosaic Visual Tests` jobs, so the repo-wide unit test job never needs a browser.
 
 jsdom and happy-dom have no layout engine, so every rect is 0 and a test that
 asserts on position, size or scrolling always passes there. Name such a test
@@ -111,6 +113,56 @@ import the built `@clerk/clerk-js`, so run `pnpm turbo build --filter=@clerk/cle
 after changing it. Failure screenshots land in `packages/mosaic/.vitest/`.
 Console output from passing tests only shows with `--reporter=verbose`.
 
+## Screenshot tests
+
+The `visual` project runs `*.visual.test.tsx` in Chromium, using the feature
+project's Clerk and MSW setup. Tests and their fixtures live in Mosaic, beside
+the feature they cover. They do not import Swingset stories or fixtures.
+
+Screenshots are generated and compared only inside one digest-pinned Playwright
+image, locally and in CI, because fonts and Chromium draw slightly differently
+on each operating system.
+
+```bash
+pnpm --filter @clerk/mosaic test:visual
+pnpm --filter @clerk/mosaic test:visual:update
+```
+
+Both commands require Docker. Locally, they mount your checkout into the image
+and install Linux dependencies into Docker volumes, so your own `node_modules`
+is untouched and later runs reuse the install. The first command compares
+screenshots without changing baselines. The second deletes the Linux baselines,
+lets Vitest write every screenshot again, and restores the old ones if the run
+fails, so new screenshots are created and ones no test produces are removed.
+Review the changed images and commit them under each test's `__screenshots__/`
+directory. Failed comparisons write actual, expected, and diff images under
+`packages/mosaic/.vitest/attachments/`.
+
+Use these package commands for screenshots. Calling Vitest directly bypasses
+the pinned image.
+
+Add a screenshot only when it covers a distinct layout or visual state that the
+existing ones miss. Capture the element under test (a dialog, a popup, or the
+rendered container), not `page`, so page padding and unrelated content stay out
+of the baseline. Content taller than the viewport is cut off, so resize the
+viewport to the content's height first, as the user profile tests do.
+
+The suite fixes the viewport before each test, the bundled Inter font, color
+scheme, locale, time zone, and device scale, and compares with zero mismatched
+pixels allowed. The browser setup disables motion and hides carets. Use deterministic
+fixture data and avatar fallbacks, and wait for content before capturing the
+page so portaled menus and overlays are included.
+
+Baselines include the browser and operating system in their filenames. The
+`Mosaic Visual Tests` job builds the workspace on the runner, then compares in
+the same pinned image. Its comparison is advisory: screenshot differences
+appear as a warning and in the job summary without blocking the rest of CI. A
+failed build, or a comparison that crashes before writing images, still fails
+the job. On a difference, download
+`mosaic-visual-failure-artifacts-<run-id>-<run-attempt>` from the run and review
+the images in `.vitest/attachments/`, then run `test:visual:update` locally and
+commit the approved baselines.
+
 ## Feature tests
 
 A feature test runs the real `Clerk` from `@clerk/clerk-js` in Chromium. MSW

packages/mosaic/package.json

@@ -61,6 +61,8 @@
     "test:browser": "vitest run --project browser",
     "test:ci": "vitest run --project primitives --project mosaic --maxWorkers=70%",
     "test:feature": "vitest run --project feature",
+    "test:visual": "bash ./scripts/test-visual.sh compare",
+    "test:visual:update": "bash ./scripts/test-visual.sh update",
     "test:webkit": "vitest run --project webkit",
     "typecheck": "tsc --noEmit"
   },

packages/mosaic/scripts/test-visual.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
+image=mcr.microsoft.com/playwright:v1.62.1-noble@sha256:dcc5531e97840b9b5e794f2814476b21571c5124a3fca2267d73041f56e7580e
+vitest='node /repo/node_modules/vitest/vitest.mjs run --project visual'
+
+case "${1:-}" in
+  compare) update=none ;;
+  update) update=all ;;
+  *)
+    printf 'Usage: %s compare|update\n' "$0" >&2
+    exit 2
+    ;;
+esac
+
+run=(docker run --rm --init --ipc=host --platform linux/amd64 --volume "$root:/repo" --workdir /repo/packages/mosaic)
+
+if [[ "${CI:-}" == true ]]; then
+  run+=(--user "$(id -u):$(id -g)" --env HOME=/tmp "$image" sh -c "$vitest --update $update")
+else
+  run+=(--volume clerk-mosaic-visual-store:/pnpm-store --volume clerk-mosaic-visual-modules:/repo/node_modules)
+  for package in "$root"/packages/*/ "$root"/packages/electron-passkeys/npm/*/; do
+    relative=${package#"$root"/}
+    run+=(--volume "clerk-mosaic-visual-${relative//\//-}modules:/repo/${relative}node_modules")
+  done
+  run+=("$image" sh -c "corepack enable && pnpm install --frozen-lockfile --store-dir /pnpm-store && pnpm turbo build --filter=@clerk/mosaic... && $vitest --update $update")
+fi
+
+baselines=$(find "$root/packages/mosaic/src" -path '*/__screenshots__/*-chromium-linux.png')
+rm -rf "$root/packages/mosaic/.vitest/attachments"
+if [[ "$update" == all && -n "$baselines" ]]; then
+  printf '%s\n' "$baselines" | xargs rm
+fi
+
+status=0
+"${run[@]}" || status=$?
+
+if [[ "$update" == all && "$status" -ne 0 ]]; then
+  git -C "$root" checkout -- ':(glob)packages/mosaic/src/**/__screenshots__/**' 2>/dev/null || true
+fi
+exit "$status"

packages/mosaic/src/features/organization-switcher/__tests__/organization-switcher.visual.test.tsx

@@ -0,0 +1,15 @@
+import { describe, expect, it } from 'vitest';
+
+import { renderWithClerk } from '../../../__tests__/feature/render';
+import { openSwitcher, serveSwitcherFixture } from '../../switcher/__tests__/switcher.visual.fixtures';
+import { OrganizationSwitcher } from '../organization-switcher';
+
+describe('OrganizationSwitcher visual state', () => {
+  it('shows the organization menu open', async () => {
+    serveSwitcherFixture();
+    await renderWithClerk(<OrganizationSwitcher />);
+    const popup = await openSwitcher(/Open organization menu/);
+
+    await expect(popup).toMatchScreenshot('organization-switcher-open');
+  });
+});

packages/mosaic/src/features/switcher/__tests__/switcher.visual.fixtures.ts

@@ -0,0 +1,56 @@
+import { screen, waitFor } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import { page } from 'vitest/browser';
+
+import { serveFapi } from '../../../__tests__/feature/fake-fapi';
+import {
+  fapiClient,
+  fapiEmailAddress,
+  fapiInvitation,
+  fapiMembership,
+  fapiOrganization,
+  fapiSession,
+  fapiSuggestion,
+  fapiUser,
+} from '../../../__tests__/feature/fapi';
+
+type FapiSession = ReturnType<typeof fapiSession>;
+
+export function serveSwitcherFixture({ otherSessions = [] }: { otherSessions?: FapiSession[] } = {}) {
+  const acme = fapiOrganization({ id: 'org_acme', name: 'Acme Studio', members_count: 12 });
+  const orbit = fapiOrganization({ id: 'org_orbit', name: 'Orbit Labs', members_count: 8 });
+  const invitation = fapiOrganization({ id: 'org_north', name: 'Northstar Design' });
+  const suggestion = fapiOrganization({ id: 'org_field', name: 'Fieldwork' });
+  const memberships = [fapiMembership(acme, { permissions: ['org:sys_memberships:manage'] }), fapiMembership(orbit)];
+  const user = fapiUser({
+    id: 'user_alice',
+    first_name: 'Alice',
+    last_name: 'Rivera',
+    username: 'alice',
+    email_addresses: [fapiEmailAddress({ id: 'email_alice', email_address: 'alice@example.com' })],
+    organization_memberships: memberships,
+  });
+  serveFapi({
+    client: fapiClient([
+      fapiSession({ id: 'session_alice', user, last_active_organization_id: acme.id }),
+      ...otherSessions,
+    ]),
+    memberships,
+    invitations: [fapiInvitation('invitation_north', invitation)],
+    suggestions: [fapiSuggestion('suggestion_field', suggestion)],
+  });
+}
+
+export async function openSwitcher(trigger: RegExp) {
+  await userEvent.setup().click(screen.getByRole('button', { name: trigger }));
+  await screen.findByRole('button', { name: 'Orbit Labs' });
+  await screen.findByRole('button', { name: 'Accept' });
+  await screen.findByText('Fieldwork');
+  await waitFor(() => {
+    if (screen.queryByText('Loading organizations…')) {
+      throw new Error('Organizations are still loading');
+    }
+  });
+  await document.fonts.ready;
+  return page.elementLocator(screen.getByRole('dialog'));
+}

packages/mosaic/src/features/user-button/__tests__/user-button.visual.test.tsx

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest';
+
+import { fapiEmailAddress, fapiSession, fapiUser } from '../../../__tests__/feature/fapi';
+import { renderWithClerk } from '../../../__tests__/feature/render';
+import { openSwitcher, serveSwitcherFixture } from '../../switcher/__tests__/switcher.visual.fixtures';
+import { UserButton } from '../user-button';
+
+describe('UserButton visual state', () => {
+  it('shows the combined account and organization menu open', async () => {
+    const bob = fapiUser({
+      id: 'user_bob',
+      first_name: 'Bob',
+      last_name: 'Morgan',
+      email_addresses: [fapiEmailAddress({ id: 'email_bob', email_address: 'bob@example.com' })],
+    });
+    serveSwitcherFixture({ otherSessions: [fapiSession({ id: 'session_bob', user: bob })] });
+    await renderWithClerk(<UserButton mode='combined' />);
+    const popup = await openSwitcher(/Open account menu/);
+
+    await expect(popup).toMatchScreenshot('user-button-combined-open');
+  });
+});

packages/mosaic/src/features/user-profile/__tests__/user-profile.visual.fixtures.ts

@@ -0,0 +1,86 @@
+import { serveFapi } from '../../../__tests__/feature/fake-fapi';
+import {
+  fapiClient,
+  fapiEmailAddress,
+  fapiEnvironment,
+  fapiExternalAccount,
+  fapiPhoneNumber,
+  fapiSession,
+  fapiUser,
+  fapiVerification,
+  fapiWeb3Wallet,
+} from '../../../__tests__/feature/fapi';
+
+type VisualProfile = {
+  firstName: string;
+  lastName: string;
+  username: string;
+  emails: string[];
+  phones: string[];
+};
+
+export const shortProfile: VisualProfile = {
+  firstName: 'Alex',
+  lastName: 'Morgan',
+  username: 'alexmorgan',
+  emails: ['alex@example.com', 'alex.work@example.com'],
+  phones: ['+14155550123'],
+};
+
+export const longProfile: VisualProfile = {
+  firstName: 'Alexandria',
+  lastName: 'Montgomery-Wellington',
+  username: 'alexandria.montgomery',
+  emails: [
+    'alexandria.montgomery@longcompanydomain.example',
+    'alexandria.montgomery.personal@example.com',
+    'alexandria.montgomery.notifications@example.com',
+  ],
+  phones: ['+14155550123', '+14155550456'],
+};
+
+export function serveVisualProfile(profile: VisualProfile) {
+  const user = fapiUser({
+    id: 'user_alex',
+    first_name: profile.firstName,
+    last_name: profile.lastName,
+    username: profile.username,
+    email_addresses: profile.emails.map((email_address, index) =>
+      fapiEmailAddress({ id: `email_${index}`, email_address }),
+    ),
+    phone_numbers: profile.phones.map((phone_number, index) => fapiPhoneNumber({ id: `phone_${index}`, phone_number })),
+    primary_phone_number_id: 'phone_0',
+    web3_wallets: [
+      fapiWeb3Wallet({
+        id: 'wallet_primary',
+        web3_wallet: '0x1234567890abcdef1234567890abcdef12345678',
+        verification: fapiVerification('admin', { status: 'verified' }),
+      }),
+    ],
+    primary_web3_wallet_id: 'wallet_primary',
+    external_accounts: [
+      fapiExternalAccount({ id: 'idn_google', provider: 'google', email_address: profile.emails[0] }),
+    ],
+  });
+  serveFapi({
+    environment: fapiEnvironment({
+      attributes: {
+        phone_number: { enabled: true },
+        web3_wallet: { enabled: true },
+      },
+      user_settings: {
+        social: {
+          oauth_google: {
+            enabled: true,
+            required: false,
+            authenticatable: true,
+            strategy: 'oauth_google',
+            name: 'Google',
+            logo_url: null,
+          },
+        },
+      },
+    }),
+    client: fapiClient([fapiSession({ id: 'session_alex', user })]),
+  });
+}