diff --git a/app/lab/test-agent/page.tsx b/app/lab/test-agent/page.tsx new file mode 100644 index 00000000..f94261ba --- /dev/null +++ b/app/lab/test-agent/page.tsx @@ -0,0 +1,27 @@ +import type { Metadata } from 'next'; +import { notFound } from 'next/navigation'; +import { TestAgentFlow } from '@/components/test-agent-flow'; + +/** + * PREVIEW ONLY, like /lab/talk. The guided path for testing your own agent: a 404 on production, not linked, not + * indexed. It goes live only when the owner has tried it and the release it describes is published. + */ +export const metadata: Metadata = { + title: 'Preview: test your own agent', + robots: { index: false, follow: false }, +}; + +export default function LabTestAgentPage() { + if (process.env.VERCEL_ENV === 'production') notFound(); + return ( +
+
+

+ Preview. This page is not live on trustshell.dev. +

+

Test your own agent

+ +
+
+ ); +} diff --git a/components/test-agent-flow.tsx b/components/test-agent-flow.tsx new file mode 100644 index 00000000..57198a9e --- /dev/null +++ b/components/test-agent-flow.tsx @@ -0,0 +1,422 @@ +'use client'; + +import { useEffect, useRef, useState, type ChangeEvent, type ReactNode } from 'react'; +import { + MAX_REPORT_BYTES, + PATH_TWO_COMMANDS, + inspectCommand, + isRunUrl, + isShellSafe, + parseTrustReport, + type ReportView, +} from '@/lib/trust-report-view'; + +const STEPS = [ + { id: 'try', label: 'Try example' }, + { id: 'test', label: 'Test my agent' }, + { id: 'review', label: 'Review report' }, + { id: 'next', label: 'Next action' }, +] as const; +type StepId = (typeof STEPS)[number]['id']; + +const EXAMPLE_URL = '/examples/trust-report-example.json'; +const BUTTON = 'min-h-[44px] rounded-lg px-4 py-2 text-sm font-bold focus:outline-none focus-visible:ring-2 focus-visible:ring-amber-300'; +const PRIMARY = `${BUTTON} bg-amber-500 text-black hover:bg-amber-400 disabled:opacity-60`; +const SECONDARY = `${BUTTON} border border-slate-600 text-white hover:border-slate-400`; + +/** Every status carries its word as well as its colour. */ +function Status({ value }: { value: string }) { + const tone = + value === 'SUPPORTED' || value === 'ALLOWED' + ? 'border-emerald-500/60 text-emerald-300' + : value === 'CONTRADICTED' || value === 'DENIED' + ? 'border-red-500/60 text-red-300' + : 'border-amber-500/60 text-amber-300'; + return {value}; +} + +function Command({ text }: { text: string }) { + const [state, setState] = useState<'idle' | 'copied' | 'failed'>('idle'); + const copy = async () => { + try { + await navigator.clipboard.writeText(text); + setState('copied'); + } catch { + setState('failed'); + } + }; + return ( +
+ 
+        {text}
+
+
+ + + {state === 'copied' ? 'Copied.' : state === 'failed' ? 'Could not copy. Select the text above.' : ''} + +
+
+ ); +} + +function Section({ title, note, children }: { title: string; note?: string; children: ReactNode }) { + return ( +
+

{title}

+ {note ?

{note}

: null} + {children} +
+ ); +} + +function ReportBody({ report }: { report: ReportView }) { + return ( +
+
+

{report.title}

+

+ There is no overall score. Made {report.generated_at} by {report.generator}. +

+
+ + {report.inspect.map((i) => ( +
+ +
+ ))} + + {report.bound_claims.length ? ( +
+ +
+ ) : null} + +
+ +
+ +
+ +
+
+ ); +} + +function PathChoice({ path, onChange }: { path: 1 | 2; onChange: (p: 1 | 2) => void }) { + const options = [ + { p: 1 as const, title: 'Check what it said', body: 'Compare the agent’s claims with GitHub’s record of a public Actions run. No access to the agent or its keys.' }, + { p: 2 as const, title: 'Run a task through TrustShell', body: 'Opt in: start the agent’s commands with trustshell run. Only those commands are seen.' }, + ]; + return ( +
+ How do you want to test it? + {options.map((o) => ( + + ))} +
+ ); +} + +function PathOne() { + const [url, setUrl] = useState(''); + const [claim, setClaim] = useState(''); + const urlBad = url.trim() !== '' && !isRunUrl(url); + const claimBad = claim.trim() !== '' && !isShellSafe(claim); + const ready = isRunUrl(url) && isShellSafe(claim.trim()); + return ( +
+
+ + setUrl(e.target.value)} + inputMode="url" + autoComplete="off" + aria-invalid={urlBad} + aria-describedby="run-url-help" + placeholder="https://github.com/owner/repo/actions/runs/123" + className="min-h-[44px] w-full rounded-lg border border-slate-600 bg-slate-950 px-3 py-2 text-sm text-white" + /> +

+ {urlBad ? 'That is not a GitHub Actions run URL. It should end in /actions/runs/ and a number.' : 'A public repository’s run. Open its Actions tab and copy the address of one run.'} +

+
+
+ + setClaim(e.target.value)} + autoComplete="off" + aria-invalid={claimBad} + aria-describedby="claim-help" + placeholder="The check workflow succeeded, run 123." + className="min-h-[44px] w-full rounded-lg border border-slate-600 bg-slate-950 px-3 py-2 text-sm text-white" + /> +

+ {claimBad ? 'Double quotes, !, $, backticks and backslashes do not survive every shell. Put the text in a file and use --claims-file instead.' : 'One plain sentence that names the run. A vague claim comes back NOT_CHECKED, which is not a pass.'} +

+
+ {ready ? ( +
+

Run this. It works the same in bash, zsh and PowerShell. It contacts only api.github.com, and your text stays on your machine.

+ +
+ ) : ( +

Fill in both fields to get the command.

+ )} +
+ ); +} + +function PathTwo() { + return ( +
+

+ trust-report is not in a published release yet. Until it is, follow the steps in{' '} + + examples/own-agent + + , which run it from a clone. +

+

Use a fresh TRUSTSHELL_HOME for each test, then, in your project:

+
    + {PATH_TWO_COMMANDS.map((c, i) => ( +
  1. + Step {i + 1} + +
    2. + ))} +
+

Anything not in the policy is refused and never starts. Anything the agent does without trustshell run is not seen.

+
+ ); +} + +export function TestAgentFlow() { + const [step, setStep] = useState('try'); + const [path, setPath] = useState<1 | 2>(1); + const [report, setReport] = useState(null); + const [isExample, setIsExample] = useState(false); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const heading = useRef(null); + const first = useRef(true); + + // A step change moves focus to the new heading, so keyboard and screen-reader users land on it. + useEffect(() => { + if (first.current) { + first.current = false; + return; + } + heading.current?.focus(); + }, [step]); + + const go = (s: StepId) => { + setError(null); + setStep(s); + }; + + const loadExample = async () => { + setLoading(true); + setError(null); + try { + const res = await fetch(EXAMPLE_URL); + if (!res.ok) throw new Error(String(res.status)); + const parsed = parseTrustReport(await res.text()); + if (!parsed.ok) throw new Error(parsed.why); + setReport(parsed.report); + setIsExample(true); + setStep('review'); + } catch { + setError('The example could not be loaded. Check your connection and try again.'); + } finally { + setLoading(false); + } + }; + + const pickFile = async (e: ChangeEvent) => { + const file = e.target.files?.[0]; + e.target.value = ''; + if (!file) return; + setError(null); + if (file.size > MAX_REPORT_BYTES) { + setError('That file is larger than a Trust Report should be. Choose the report.json from your report folder.'); + return; + } + setLoading(true); + try { + const parsed = parseTrustReport(await file.text()); + if (!parsed.ok) { + setError(parsed.why); + return; + } + setReport(parsed.report); + setIsExample(false); + } catch { + setError('That file could not be read. Try choosing it again.'); + } finally { + setLoading(false); + } + }; + + const current = STEPS.findIndex((s) => s.id === step); + const title = STEPS[current]?.label ?? ''; + + return ( +
+ + +

+ {title} +

+ +
+ {loading ? 'Loading…' : ''} +
+ {error ? ( +
+ {error} +
+ ) : null} + + {step === 'try' ? ( +
+

See a real report before you install anything. No account, no key. It comes from a test task where the agent said it was finished and the test said otherwise.

+ +
+ ) : null} + + {step === 'test' ? ( +
+ + {path === 1 ? : } + +
+ ) : null} + + {step === 'review' ? ( +
+
+ + +

The file is read in this page and is not sent anywhere.

+
+ {report ? ( + <> + {isExample ?

This is the example report, not yours.

: null} + + + + ) : ( + + )} +
+ ) : null} + + {step === 'next' ? ( +
+ {report ? ( + <> +
    + {report.next_actions.map((a) => ( +
  • + {a} +
    • + ))} +
+
+ +
+ + + ) : ( + <> +

Open a report first, and the next steps it calls for will appear here.

+ + + )} +
+ ) : null} +
+ ); +} diff --git a/lib/trust-report-view.ts b/lib/trust-report-view.ts new file mode 100644 index 00000000..534e89c9 --- /dev/null +++ b/lib/trust-report-view.ts @@ -0,0 +1,90 @@ +/** + * What the guided page needs from a Trust Report (`trustshell trust-report`, report.json). PURE and browser-safe: + * the page reads a file the visitor picks, in the browser, and nothing here sends it anywhere. Reading a report + * here does NOT verify it: `trustshell trust-report verify` does that, and the page says so. + */ +export const TRUST_REPORT_SCHEMA = 'dev.trustshell.trust-report/0.1-draft'; +export const MAX_REPORT_BYTES = 8 * 1024 * 1024; + +export interface ClaimView { + status: string; + claim: string; + scope: string; +} +export interface InspectView { + run_url: string; + repo: string; + run_id: number; + verdict: string; + conclusion: string | null; + fetched_at: string; + claims: ClaimView[]; +} +export interface CommandView { + seq: number; + action: string; + decision: string; + reasons: string[]; + result: string; +} +export interface BoundView { + claim: string; + decided_by: string; + status: string; + why: string; +} +export interface ReportView { + schema: string; + title: string; + generated_at: string; + generator: string; + inspect: InspectView[]; + bound_claims: BoundView[]; + guard: { ledger: { verdict: string; count: number }; commands: CommandView[]; mediated: string[]; not_mediated: string[] }; + next_actions: string[]; +} + +export type ParsedReport = { ok: true; report: ReportView } | { ok: false; why: string }; + +const isList = (v: unknown): v is unknown[] => Array.isArray(v); + +/** Accepts only the shape this page can show in full, so a partial file is an error rather than a thin report. */ +export function parseTrustReport(text: string): ParsedReport { + let value: unknown; + try { + value = JSON.parse(text.charCodeAt(0) === 0xfeff ? text.slice(1) : text); + } catch { + return { ok: false, why: 'That file is not JSON. Choose the report.json from your report folder.' }; + } + const r = value as Partial | null; + if (!r || typeof r !== 'object' || r.schema !== TRUST_REPORT_SCHEMA) { + return { ok: false, why: 'That JSON is not a Trust Report. Choose the report.json that trustshell trust-report wrote.' }; + } + const guard = r.guard; + if (typeof r.title !== 'string' || !isList(r.inspect) || !isList(r.bound_claims) || !isList(r.next_actions) || !guard || !isList(guard.commands) || !isList(guard.not_mediated) || !isList(guard.mediated) || !guard.ledger) { + return { ok: false, why: 'That Trust Report is incomplete, so it cannot be shown. Make it again with trustshell trust-report.' }; + } + return { ok: true, report: r as ReportView }; +} + +const RUN_URL = /^https:\/\/github\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/actions\/runs\/[0-9]+(?:\/attempts\/[0-9]+)?\/?$/; + +export const isRunUrl = (s: string): boolean => RUN_URL.test(s.trim()); + +/** + * Text that means the same inside double quotes in bash, zsh and PowerShell. Anything else belongs in a file. + * `!` is history expansion in an interactive bash or zsh; PowerShell closes a double-quoted string on a curly double quote. + */ +export const isShellSafe = (s: string): boolean => s.length > 0 && !/["“”„!`$\\\r\n]/.test(s); + +export function inspectCommand(url: string, claim: string): string { + return `npx @hyperdag/trustshell@latest check ${url.trim()} --claims "${claim.trim()}" --out evidence`; +} + +/** The same commands as examples/own-agent/README.md, which an automated test holds this list to. */ +export const PATH_TWO_COMMANDS = [ + 'trustshell guard init', + 'trustshell guard allow tool:node --script test.js', + "trustshell run --tool node '--' node test.js", + 'trustshell trust-report --out report --bind "test.js=All tests pass"', +] as const; diff --git a/public/examples/trust-report-example.json b/public/examples/trust-report-example.json new file mode 100644 index 00000000..f2fc25b5 --- /dev/null +++ b/public/examples/trust-report-example.json @@ -0,0 +1,125 @@ +{ + "schema": "dev.trustshell.trust-report/0.1-draft", + "title": "Agent said it was finished", + "generated_at": "2026-11-08T23:32:31.789Z", + "generator": "trustshell 1.9.0-dev.0", + "inputs": { + "binds": [ + { + "claim": "All tests pass", + "decided_by": "test.js", + "tool": null, + "script_sha256": "ee55955c74d5ee8cbb7941e1265fbb863fa48e0350a5acb4b9777fcd1f2e907d" + }, + { + "claim": "Lint is clean", + "decided_by": "lint", + "tool": "lint", + "script_sha256": null + } + ], + "records": [ + "evidence/inspect-1/record.json" + ] + }, + "inspect": [ + { + "path": "evidence/inspect-1/record.json", + "run_url": "https://github.com/DealAppSeo/trustshell/actions/runs/42502305639", + "repo": "DealAppSeo/trustshell", + "run_id": 42502305639, + "head_sha": "f569e43ec8072f7530c2da8d25bacf28c30c510f", + "conclusion": "failure", + "verdict": "FAILED", + "fetched_at": "2026-11-08T23:32:16.234Z", + "digest": "aeb38b64eacef755727f2ed449116cff4d63d6a07bdd8108c513e6c63837b723", + "digest_matches": true, + "exit": 1, + "claims": [ + { + "status": "CONTRADICTED", + "claim": "The check workflow succeeded", + "scope": "Whether workflow run 42502305639 (attempt 1, commit f569e43ec807) concluded success with every job read. Not which tests it ran, not any other workflow or check on the commit, and not whether the change works." + }, + { + "status": "SUPPORTED", + "claim": "run 42502305639", + "scope": "Run identity only: whether the run this claim cites is the inspected run. Not what the run shows." + } + ] + } + ], + "guard": { + "policy": { + "state": "ok", + "sha256": "22996d766f99f6a7bcfa20c65491d14e6b558ee191a332f76ae27d961c0e54c7", + "nonce": "2", + "signed": false, + "actions": [ + "tool:bundle", + "tool:ground", + "tool:init", + "tool:inspect", + "tool:learn", + "tool:profile", + "tool:quarantine", + "tool:recall", + "tool:redact", + "tool:remember", + "tool:report", + "tool:traps", + "tool:trust-report", + "tool:node" + ] + }, + "ledger": { + "verdict": "INTACT", + "count": 5, + "breaks": [] + }, + "commands": [ + { + "seq": 3, + "ts": "2026-11-08T23:32:31.000Z", + "action": "tool:node", + "script_sha256": "ee55955c74d5ee8cbb7941e1265fbb863fa48e0350a5acb4b9777fcd1f2e907d", + "command_sha256": "fb82de7f5d562611e57524e1bd8e6af029e3f4163414556328a23a42ddcbae44", + "decision": "ALLOWED", + "reasons": [], + "result": "exited 1 after 67 ms", + "exit_code": 1 + } + ], + "mediated": [ + "the decision to start this command through `trustshell run`, checked against the owner policy before it started", + "the exit code of the process `trustshell run` started" + ], + "not_mediated": [ + "what the command did after it started: files, network, child processes", + "commands run without `trustshell run`", + "Anything the agent did without `trustshell run`: edits it made directly, its own network calls, commands in another terminal. This report cannot see them." + ] + }, + "bound_claims": [ + { + "claim": "All tests pass", + "decided_by": "test.js", + "status": "CONTRADICTED", + "why": "the last time it ran through trustshell run it exited 1 (record 3)", + "record_seq": 3 + }, + { + "claim": "Lint is clean", + "decided_by": "lint", + "status": "NOT_CHECKED", + "why": "that command was never asked for through TrustShell", + "record_seq": null + } + ], + "next_actions": [ + "Do not accept \"The check workflow succeeded\". GitHub's record disagrees: https://github.com/DealAppSeo/trustshell/actions/runs/42502305639", + "Do not accept \"All tests pass\". test.js says otherwise. Fix it or ask the agent why, then run it again through trustshell run.", + "1 claim was not checked, which is not a pass. To decide one: say it as one plain sentence that cites the run (\"The check workflow succeeded, run 123.\"), or pair it with a command using --bind." + ], + "exit": 1 +} diff --git a/tests/e2e/test-agent-walk.mjs b/tests/e2e/test-agent-walk.mjs new file mode 100644 index 00000000..660598ae --- /dev/null +++ b/tests/e2e/test-agent-walk.mjs @@ -0,0 +1,245 @@ +/** + * The guided page, /lab/test-agent, in a real browser, on a desktop and a phone: every step, the loading and error + * states, the keyboard path, accessibility (axe-core), and that a report the visitor opens never leaves the page. + * + * node tests/e2e/test-agent-walk.mjs [screenshot dir] + * + * Builds the app and serves it, like its siblings. Not in gating CI (needs a browser and a build). + */ +import { readFileSync, mkdtempSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { chromiumExecutablePath, LAUNCH_ARGS, loadPlaywrightOrExit, spawnNext } from './chromium-path.mjs'; + +const { chromium } = await loadPlaywrightOrExit(); +const PORT = 3108; +const base = `http://127.0.0.1:${PORT}`; +const PAGE = `${base}/lab/test-agent`; +const EXAMPLE = '**/examples/trust-report-example.json'; +const shots = process.argv[2]; +if (shots) mkdirSync(shots, { recursive: true }); + +const results = []; +const check = (name, pass, note = '') => { + results.push({ name, pass, note }); + console.log(`${pass ? 'PASS' : 'FAIL'} ${name}${note ? ` — ${note}` : ''}`); +}; + +try { + const probe = await fetch(`${base}/`, { signal: AbortSignal.timeout(2000) }); + if (probe) { + console.error(`NOT_CHECKED: port ${PORT} is already serving; this suite will not test a server it did not start.`); + process.exit(2); + } +} catch { + /* nothing listening: the state we want */ +} + +const env = { ...process.env }; +delete env.VERCEL_ENV; +const build = spawnNext(['build'], { env, stdio: 'ignore' }); +const code = await new Promise((r) => build.on('exit', r)); +if (code !== 0) { + console.error(`FATAL: next build exited ${code}`); + process.exit(1); +} +const app = spawnNext(['start', '--port', String(PORT)], { env, stdio: 'ignore' }); +const stop = () => { + try { + app.kill('SIGKILL'); + } catch { + /* gone */ + } +}; +process.on('exit', stop); +const deadline = Date.now() + 90_000; +for (;;) { + try { + if ((await fetch(PAGE)).ok) break; + } catch { + /* not up yet */ + } + if (Date.now() > deadline) { + console.error('FATAL: the app did not start'); + process.exit(1); + } + await new Promise((r) => setTimeout(r, 500)); +} + +const dir = mkdtempSync(join(tmpdir(), 'ts-test-agent-')); +const file = (name, text) => { + const p = join(dir, name); + writeFileSync(p, text); + return p; +}; +const goodReport = readFileSync(join(process.cwd(), 'public', 'examples', 'trust-report-example.json'), 'utf8'); +const axeSource = readFileSync(join(process.cwd(), 'node_modules', 'axe-core', 'axe.min.js'), 'utf8'); + +async function axe(page, label) { + await page.evaluate(axeSource); + const v = await page.evaluate(async () => { + // eslint-disable-next-line no-undef + const r = await axe.run('[data-testid="test-agent-flow"]', { resultTypes: ['violations'] }); + return r.violations.map((x) => `${x.id}(${x.impact}):${x.nodes.length}`); + }); + check(`axe ${label}`, v.length === 0, v.join(' ')); +} + +async function noSideScroll(page, label) { + const m = await page.evaluate(() => ({ scroll: document.documentElement.scrollWidth, width: window.innerWidth })); + check(`no sideways scroll ${label}`, m.scroll <= m.width, `${m.scroll} vs ${m.width}`); +} + +async function targets(page, label) { + const small = await page.evaluate(() => { + const root = document.querySelector('[data-testid="test-agent-flow"]'); + return [...root.querySelectorAll('button, input:not([type=radio]), label:has(input[type=radio])')] + .map((el) => ({ h: el.getBoundingClientRect().height, what: el.textContent?.slice(0, 30) || el.id })) + .filter((x) => x.h > 0 && x.h < 44); + }); + check(`tap targets at least 44px ${label}`, small.length === 0, JSON.stringify(small)); +} + +async function walk(label, viewport) { + const context = await browser.newContext({ viewport, permissions: ['clipboard-read', 'clipboard-write'] }); + const page = await context.newPage(); + const errors = []; + const external = []; + const sent = []; + page.on('pageerror', (e) => errors.push(String(e))); + page.on('request', (r) => { + if (!r.url().startsWith(base)) external.push(r.url()); + if (r.method() !== 'GET') sent.push(`${r.method()} ${r.url()}`); + }); + const step = () => page.getByTestId('test-agent-flow').getAttribute('data-step'); + const stepButton = (name) => page.getByRole('navigation', { name: 'Steps' }).getByRole('button', { name }); + + await page.goto(PAGE); + await page.waitForLoadState('networkidle'); + const shot = async (n) => { + if (shots) await page.screenshot({ path: join(shots, `${label}-${n}.png`), fullPage: true }); + }; + await shot('1-try'); + check(`${label}: starts on Try example`, (await step()) === 'try'); + // The control: axe must be able to report a violation here, or its silence below means nothing. + await page.evaluate(() => document.querySelector('[data-testid="test-agent-flow"]').insertAdjacentHTML('beforeend', '')); + await page.evaluate(axeSource); + const seen = await page.evaluate(async () => (await axe.run('[data-testid="test-agent-flow"]', { resultTypes: ['violations'] })).violations.map((x) => x.id)); + check(`${label}: axe can see a violation`, seen.includes('image-alt'), seen.join(' ')); + await page.evaluate(() => document.getElementById('axe-control').remove()); + check(`${label}: the current step is marked for assistive tech`, (await stepButton('Try example').getAttribute('aria-current')) === 'step'); + await axe(page, `${label} try`); + await noSideScroll(page, `${label} try`); + await targets(page, `${label} try`); + + // Loading state, then a failure, then a retry that works. + let release; + const held = new Promise((r) => (release = r)); + await page.route(EXAMPLE, async (route) => { + await held; + await route.abort(); + }); + await page.getByRole('button', { name: 'Show the example report' }).click(); + await page.waitForFunction(() => document.querySelector('[role=status]')?.textContent?.includes('Loading')); + check(`${label}: shows a loading state`, await page.getByRole('button', { name: 'Show the example report' }).isDisabled()); + release(); + await page.getByTestId('test-agent-flow').getByRole('alert').waitFor(); + check(`${label}: a failed load says so and offers a retry`, /could not be loaded/.test(await page.getByTestId('test-agent-flow').getByRole('alert').textContent()) && (await page.getByRole('button', { name: 'Try again' }).isVisible())); + await shot('1-error'); + await axe(page, `${label} error`); + await page.unroute(EXAMPLE); + await page.getByRole('button', { name: 'Try again' }).click(); + await page.getByTestId('report-view').waitFor(); + check(`${label}: the example opens on Review report`, (await step()) === 'review'); + check(`${label}: focus moves to the new step heading`, (await page.evaluate(() => document.activeElement?.tagName)) === 'H2'); + const body = await page.getByTestId('report-view').textContent(); + check(`${label}: the report shows a contradicted claim`, body.includes('CONTRADICTED')); + check(`${label}: it keeps the two paths apart`, body.includes('Path 1 ·') && body.includes('Path 2 ·')); + check(`${label}: it says what it cannot see`, body.includes('cannot see') && body.includes('This report cannot see them')); + check(`${label}: it is labelled as the example`, await page.getByText('This is the example report, not yours.').isVisible()); + await shot('3-review'); + await axe(page, `${label} review`); + await noSideScroll(page, `${label} review`); + await targets(page, `${label} review`); + + await page.getByRole('button', { name: 'What should I do next?' }).click(); + const next = await page.getByTestId('next-actions').textContent(); + check(`${label}: next action names what not to accept`, next.includes('Do not accept')); + check(`${label}: it says the page does not verify the file`, await page.getByText('does not prove the file is unchanged').isVisible()); + await shot('4-next'); + await axe(page, `${label} next`); + await noSideScroll(page, `${label} next`); + + await stepButton('Test my agent').click(); + check(`${label}: path 1 is first`, await page.getByTestId('path-one').isVisible()); + await axe(page, `${label} test`); + await page.getByLabel('GitHub Actions run URL').fill('https://example.com/nope'); + check(`${label}: a bad URL is explained`, /not a GitHub Actions run URL/.test(await page.locator('#run-url-help').textContent())); + check(`${label}: and marked invalid`, (await page.getByLabel('GitHub Actions run URL').getAttribute('aria-invalid')) === 'true'); + await page.getByLabel('GitHub Actions run URL').fill('https://github.com/o/r/actions/runs/123'); + await page.getByLabel('What your agent said').fill('It passed "fine" $(x)'); + check(`${label}: text a shell would reinterpret is refused`, /do not survive every shell/.test(await page.locator('#claim-help').textContent())); + check(`${label}: no command is offered for it`, (await page.locator('pre').count()) === 0); + await page.getByLabel('What your agent said').fill('The check workflow succeeded, run 123.'); + const command = await page.locator('pre').first().textContent(); + check(`${label}: a valid form yields the command`, command.includes('check https://github.com/o/r/actions/runs/123 --claims "The check workflow succeeded, run 123."')); + await page.getByRole('button', { name: /^Copy command/ }).first().click(); + await page.getByText('Copied.').waitFor(); + check(`${label}: copy works`, (await page.evaluate(() => navigator.clipboard.readText())) === command); + await shot('2-path1'); + await noSideScroll(page, `${label} path1`); + await targets(page, `${label} path1`); + + await page.getByRole('radio', { name: /Run a task through TrustShell/ }).check(); + check(`${label}: path 2 replaces path 1`, (await page.getByTestId('path-two').isVisible()) && !(await page.getByTestId('path-one').isVisible())); + check(`${label}: path 2 says it is not released`, await page.getByText('not in a published release yet').isVisible()); + check(`${label}: path 2 says what it does not see`, await page.getByText('without trustshell run is not seen').isVisible()); + await axe(page, `${label} path2`); + await shot('2-path2'); + await noSideScroll(page, `${label} path2`); + + await stepButton('Review report').click(); + for (const [name, content, want] of [ + ['not json', '{nope', 'not JSON'], + ['other json', '{"a":1}', 'not a Trust Report'], + ['partial', '{"schema":"dev.trustshell.trust-report/0.1-draft","title":"t"}', 'incomplete'], + ]) { + await page.locator('#report-file').setInputFiles(file(`${name}.json`, content)); + await page.getByTestId('test-agent-flow').getByRole('alert').waitFor(); + check(`${label}: ${name} is refused with a reason`, (await page.getByTestId('test-agent-flow').getByRole('alert').textContent()).includes(want)); + } + await axe(page, `${label} bad file`); + const before = external.length + sent.length; + await page.locator('#report-file').setInputFiles(file('report.json', goodReport)); + await page.getByText('Agent said it was finished').first().waitFor(); + check(`${label}: an opened report is not labelled as the example`, !(await page.getByText('This is the example report, not yours.').isVisible())); + check(`${label}: opening a report sends nothing`, external.length + sent.length === before, external.concat(sent).join(' ')); + + // Keyboard only: reach a step and open it. + await page.reload(); + await page.waitForLoadState('networkidle'); + await page.locator('nav[aria-label="Steps"] button').nth(1).focus(); + await page.keyboard.press('Enter'); + check(`${label}: a step opens from the keyboard`, (await step()) === 'test'); + check(`${label}: focus lands on its heading`, (await page.evaluate(() => document.activeElement?.tagName)) === 'H2'); + + check(`${label}: no script errors`, errors.length === 0, errors.join(' ')); + check(`${label}: nothing but this site was contacted`, external.length === 0, external.join(' ')); + check(`${label}: nothing was sent`, sent.length === 0, sent.join(' ')); + await context.close(); +} + +let browser; +try { + browser = await chromium.launch({ executablePath: chromiumExecutablePath(), args: LAUNCH_ARGS }); + await walk('desktop', { width: 1280, height: 800 }); + await walk('phone', { width: 375, height: 812 }); +} catch (e) { + check('the walk finished', false, e?.stack ?? String(e)); +} finally { + await browser?.close(); + stop(); +} +const failed = results.filter((r) => !r.pass); +console.log(`\n${results.length - failed.length} passed · ${failed.length} failed`); +process.exit(failed.length ? 1 : 0); diff --git a/tests/test-agent-flow.test.ts b/tests/test-agent-flow.test.ts new file mode 100644 index 00000000..fa9fbef8 --- /dev/null +++ b/tests/test-agent-flow.test.ts @@ -0,0 +1,56 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { PATH_TWO_COMMANDS, inspectCommand, isRunUrl, isShellSafe, parseTrustReport } from '../lib/trust-report-view'; + +const ROOT = join(__dirname, '..'); + +describe('the guided page reads a Trust Report', () => { + const example = readFileSync(join(ROOT, 'public', 'examples', 'trust-report-example.json'), 'utf8'); + + it('shows the shipped example, a real report with a contradiction and both paths', () => { + const parsed = parseTrustReport(example); + if (!parsed.ok) throw new Error(parsed.why); + expect(parsed.report.bound_claims.some((b) => b.status === 'CONTRADICTED')).toBe(true); + expect(parsed.report.inspect.length).toBeGreaterThan(0); + expect(parsed.report.guard.commands.length).toBeGreaterThan(0); + }); + + it('holds no local path of the machine that made it', () => { + expect(example).not.toMatch(/[A-Za-z]:\\\\|\/Users\/|\/home\//); + }); + + it.each([ + ['not JSON', '{nope', 'not JSON'], + ['another JSON file', '{"a":1}', 'not a Trust Report'], + ['a report with parts missing', '{"schema":"dev.trustshell.trust-report/0.1-draft","title":"t"}', 'incomplete'], + ])('refuses %s with a reason', (_name, text, why) => { + const parsed = parseTrustReport(text); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.why).toContain(why); + }); + + it('reads a report saved with a byte-order mark', () => { + expect(parseTrustReport(`${example}`).ok).toBe(true); + }); +}); + +describe('the commands the page builds', () => { + it('takes only GitHub Actions run URLs', () => { + expect(isRunUrl('https://github.com/o/r/actions/runs/123')).toBe(true); + expect(isRunUrl('https://github.com/o/r/actions/runs/123/attempts/2')).toBe(true); + for (const bad of ['http://github.com/o/r/actions/runs/1', 'https://evil.test/o/r/actions/runs/1', 'https://github.com/o/r/actions/runs/1; rm -rf /', 'https://github.com/o/r/pull/1']) { + expect(isRunUrl(bad)).toBe(false); + } + }); + + it('never puts text in a command that a shell would reinterpret', () => { + for (const bad of ['a"b', 'a`b', '$(x)', 'a\\b', 'a\nb', '', 'passed!!', 'a!b', 'a“b', 'a”b', 'a„b']) expect(isShellSafe(bad)).toBe(false); + expect(isShellSafe('The check workflow succeeded, run 123.')).toBe(true); + expect(inspectCommand('https://github.com/o/r/actions/runs/1', 'ok')).toBe('npx @hyperdag/trustshell@latest check https://github.com/o/r/actions/runs/1 --claims "ok" --out evidence'); + }); + + it('lists the commands the example guide documents', () => { + const guide = readFileSync(join(ROOT, 'examples', 'own-agent', 'README.md'), 'utf8'); + for (const c of PATH_TWO_COMMANDS.slice(0, 3)) expect(guide).toContain(c); + }); +});