feat(review): retain loaded runtime and full-stack workflow evidence

docs/ogc-1560-review-harness/execution-guide.md

@@ -42,3 +42,11 @@ The L3 reviewer maps requirements to appropriate test layers, tests, observable
 ## Dependencies and stack sources
 
 `--sources sources.json` attaches only explicitly selected repositories. Each source has a safe `id`, repository identity, absolute local `repo`, full pinned `revision`, role (`dependency`, `consumer`, `stack`) and `required` boolean. Optional `parent` validates stack ancestry; `pinPath` checks the primary candidate's gitlink. Missing required source or mismatched ancestry/pins is a coverage gap, not a code finding. Reviewers inspect these sources read-only and cite repository/revision with `sourceId`. `--history-depth N` explicitly includes needed ancestry; default is the selected revisions only. These checks do not merge or rewrite upstream branches.
+
+## Runtime and product evidence
+
+L4 uses `--scenarios` (behavior, category, applicable, reason, checkId) plus loaded runtime identity. OpenELIS collects image IDs and deployed Java class bytes itself. Other containers declare a `--runtime-check` whose `runtime` JSON artifact contains head, components (name, method, expected, observed), configuration and dependencies. Checkout HEAD or a login response alone cannot establish this identity.
+
+L5 additionally uses `--workflows` (id, requirement, checkId, designId or visualInapplicableReason) and `--designs` (id, file, sha256, source, revision, authority, approvalStatus). Designs may be draft for comparison, but only approved references support acceptance. A scenario declares artifacts by relative path and kind: http, readback, trace, screenshot, video, measurement, runtime or text. Fresh artifacts are copied out, bound to the candidate and checked for transport integrity. Screenshots and design images are attached to the reviewer. Missing media/readback, wrong loaded artifacts or visible deviations remain explicit in the report.
+
+The OpenELIS example `examples/openelis/label-preset-workflow.mjs` uses synthetic state in the disposable worker. Run the browser container as the worker UID/GID so subsequent checks can remove old evidence. The example records browser errors, screenshots, persisted readback and a trace, and restores the original preset. Setup exceptions are JUnit errors, not successful mutation detections. See `validation-v06.json` for the actual run and its limits. Artifacts are limited to 20 MiB each and 40 MiB per check; oversized evidence is reported as missing, never silently accepted.

docs/ogc-1560-review-harness/validation-v06.json

@@ -0,0 +1,11 @@
+{
+  "date": "2026-10-30",
+  "scope": "Runtime and product evidence implementation; not complete roadmap acceptance",
+  "source": "965d6d10559d9e11e8868cbc01a5157864056af9",
+  "workerRun": "4fc4fe91-9c62-425c-9a8b-f7fb8f5a8caa",
+  "runtime": {"loadedComponentsVerified": 5, "method": "Running image IDs and deployed Java class bytes compared with built artifacts"},
+  "workflow": {"name": "Label preset edit", "executed": 1, "status": "pass", "evidenceDirectory": "workflow-owned", "assertions": ["System name locked", "Height increment saved and read back", "Width, name and field selection preserved"], "artifacts": ["before screenshot", "after screenshot", "persisted readback", "browser trace", "browser errors"]},
+  "negativeControls": ["stale loaded component", "stale screenshot removed before execution", "altered artifact bytes", "missing saved-state evidence", "draft design cannot establish approved acceptance", "visible workflow deviation"],
+  "liveL3": {"run": "281d0189-0199-411d-bf6e-818ccf94ce80", "status": "complete", "assessment": "no_findings", "model": "gpt-6.1-sol", "reasoning": "low", "seconds": 120, "sessions": 1, "approvedRemovalRespected": true},
+  "limitations": ["Available OpenELIS design is draft; approved design acceptance remains unconfirmed", "Only selected synthetic workflow tested; not whole-product acceptance", "Probe executor mutates isolated source copies, not a second deployed full stack", "Artifacts limited to 20 MiB each and 40 MiB per check; larger media are explicit gaps"]
+}

examples/java-unit-image/JUnitEvidence.java

@@ -19,6 +19,14 @@ public static void main(String[] args) throws Exception {
       public void testStarted(Description d) {
         Element c=doc.createElement("testcase");c.setAttribute("classname",d.getClassName());c.setAttribute("name",d.getMethodName());suite.appendChild(c);cases.put(d,c);
       }
+      public void testIgnored(Description d) {
+        testStarted(d);cases.get(d).appendChild(doc.createElement("skipped"));
+      }
+      public void testAssumptionFailure(Failure f) {
+        Element c=cases.get(f.getDescription());
+        if(c==null){testStarted(f.getDescription());c=cases.get(f.getDescription());}
+        c.appendChild(doc.createElement("skipped"));
+      }
       public void testFailure(Failure f) {
         Element c=cases.get(f.getDescription());
         if(c==null){testStarted(f.getDescription());c=cases.get(f.getDescription());}

examples/openelis/label-preset-workflow.mjs

@@ -0,0 +1,48 @@
+// Runs only against the disposable review worker with synthetic administrator state.
+import {chromium,expect} from '@playwright/test';
+import {mkdir,writeFile} from 'node:fs/promises';
+const out=process.env.EVIDENCE_DIR ?? '/source/review-evidence';
+await mkdir(out,{recursive:true});
+const browser=await chromium.launch({headless:true});
+const context=await browser.newContext({ignoreHTTPSErrors:true,locale:'en',viewport:{width:1280,height:800},baseURL:process.env.BASE_URL});
+await context.tracing.start({screenshots:true,snapshots:true});
+const page=await context.newPage();
+const errors=[];page.on('pageerror',error=>errors.push(error.message));
+let original,token,failure;
+const api='/api/OpenELIS-Global/api/labelPresets';
+async function presets(){const response=await page.request.get(api);expect(response.status()).toBe(200);return response.json();}
+try {
+ await page.goto('/login');await page.locator('#loginName').fill('admin');await page.locator('#password').fill('adminADMIN!');await page.locator('[data-cy="loginButton"]').click();
+ await expect(page.getByRole('navigation',{name:'Side navigation'})).toBeVisible({timeout:60000});
+ token=await page.evaluate(()=>localStorage.getItem('CSRF'));
+ await page.goto('/MasterListsPage/labelPresets');
+ original=(await presets()).find(p=>p.name==='Order Label');expect(original).toBeTruthy();
+ const row=page.locator('tr',{hasText:'Order Label'}).first();await expect(row).toBeVisible();
+ await row.getByRole('button',{name:'Options'}).click();await page.getByRole('menuitem',{name:'Edit'}).click();
+ const dialog=page.getByRole('dialog');await expect(dialog.getByRole('heading',{name:'Edit Label Preset'})).toBeVisible();
+ await expect(dialog.locator('#preset-name')).toBeDisabled();
+ await page.screenshot({path:out+'/label-before.png'});
+ await dialog.getByRole('button',{name:'Increment number'}).first().click();
+ await expect(dialog.locator('#preset-heightMm')).toHaveValue(String(original.heightMm+1));
+ const saved=page.waitForResponse(r=>r.request().method()==='PUT' && /\/api\/labelPresets\/\d+$/.test(r.url()));
+ await dialog.getByRole('button',{name:'Save',exact:true}).click();expect((await saved).ok()).toBeTruthy();
+ await expect(page.locator('.cds--toast-notification--success')).toBeVisible();await expect(dialog).toBeHidden();
+ const after=(await presets()).find(p=>p.id===original.id);
+ expect(after.heightMm).toBe(original.heightMm+1);expect(after.widthMm).toBe(original.widthMm);expect(after.name).toBe(original.name);
+ expect(after.fields.map(f=>f.fieldKey)).toEqual(original.fields.map(f=>f.fieldKey));
+ await page.screenshot({path:out+'/label-after.png'});
+ await writeFile(out+'/label-readback.json',JSON.stringify({before:{name:original.name,heightMm:original.heightMm,widthMm:original.widthMm},after:{name:after.name,heightMm:after.heightMm,widthMm:after.widthMm},fieldsPreserved:true},null,2));
+} catch(error) {failure=error;await page.screenshot({path:out+'/label-failure.png'}).catch(()=>{});}
+finally {
+ if(original) {
+  const response=await page.request.put(api+'/'+original.id,{headers:{'X-CSRF-Token':token,'Content-Type':'application/json'},data:{...original}});
+  if(!response.ok() && !failure)failure=Error('Fixture restoration failed: '+response.status());
+ }
+ await writeFile(out+'/browser-errors.json',JSON.stringify(errors,null,2));
+ await context.tracing.stop({path:out+'/label-trace.zip'});
+ await context.close();await browser.close();
+ const escape=value=>String(value).replaceAll('&','&amp;').replaceAll('<','&lt;').replaceAll('"','&quot;');
+ const failureTag=failure?.matcherResult || failure?.name==='AssertionError' ? 'failure':'error';
+ await writeFile(out+'/label-workflow.xml','<testsuite><testcase name="label preset edit saves and reads back">'+(failure?'<'+failureTag+' message="'+escape(failure.message)+'"/>':'')+'</testcase></testsuite>');
+}
+if(failure){console.error(failure.message);process.exitCode=1;}

review/acceptance.mjs

@@ -0,0 +1,38 @@
+export function validateScenarios(rows, checks) {
+  if(!Array.isArray(rows))throw new Error('Scenarios must be an array');
+  for(const row of rows) {
+    if(!row.behavior?.trim() || !row.reason?.trim() || typeof row.applicable!=='boolean' || !['integration','persistence','negative','recovery','lifecycle','fresh-install','upgrade'].includes(row.category))throw new Error('Scenario needs behavior, category and applicability rationale');
+    if(row.applicable && !checks.some(c=>c.id===row.checkId && c.kind==='scenario'))throw new Error('Applicable scenario needs an executable scenario check');
+  }
+}
+export function validateWorkflows(rows, checks, designs) {
+  if(!Array.isArray(rows))throw new Error('Workflows must be an array');
+  const ids=new Set();
+  for(const row of rows) {
+    if(!/^[a-z][a-z0-9-]{0,63}$/.test(row.id) || ids.has(row.id) || !row.requirement?.trim())throw new Error('Workflow needs a unique ID and requirement');
+    ids.add(row.id);
+    if(!checks.some(c=>c.id===row.checkId && c.kind==='scenario'))throw new Error('Workflow needs a running scenario check');
+    if(!row.visualInapplicableReason?.trim() && !designs.some(d=>d.id===row.designId))throw new Error('Workflow needs an approved design or visual applicability rationale');
+  }
+}
+export function assessAcceptance(rows, report, checks, artifacts, designs) {
+  const gaps=[],deviations=[];
+  if(!rows.length)return {gaps:['No selected product workflows'],deviations};
+  for(const row of rows) {
+    const answer=report.acceptance?.find(a=>a.id===row.id);
+    const check=checks.find(c=>c.id===row.checkId);
+    if(!check || !['pass','fail'].includes(check.status) || check.executed<1)gaps.push('Workflow has no executed result: '+row.id);
+    if(!answer || !answer.observedOutcome?.trim() || !answer.persistedOutcome?.trim()) {gaps.push('Missing observed user/saved-state outcome: '+row.id);continue;}
+    if(answer.status==='unverified')gaps.push('Unverified acceptance: '+row.id+' — '+answer.reason);
+    if(answer.status==='fail' || check?.status==='fail')deviations.push(row.id+': '+answer.reason);
+    const evidence=answer.artifactIds.map(id=>artifacts.find(a=>a.id===id && a.checkId===row.checkId));
+    if(evidence.some(a=>!a))gaps.push('Unknown workflow artifact: '+row.id);
+    if(!evidence.some(a=>a?.kind==='readback' || a?.kind==='http'))gaps.push('Missing saved-state evidence: '+row.id);
+    if(!row.visualInapplicableReason) {
+      if(designs.find(d=>d.id===row.designId)?.approvalStatus!=='approved')gaps.push('Design approval unconfirmed: '+row.id);
+      if(answer.designId!==row.designId || !designs.some(d=>d.id===row.designId) || !answer.designComparison.trim())gaps.push('Missing design comparison: '+row.id);
+      if(!evidence.some(a=>a?.kind==='screenshot'))gaps.push('Missing rendered screenshot: '+row.id);
+    }
+  }
+  return {gaps,deviations};
+}

review/artifacts.mjs

@@ -0,0 +1,41 @@
+import {mkdir,writeFile,readFile,realpath} from 'node:fs/promises';
+import {createHash} from 'node:crypto';
+import {join,extname} from 'node:path';
+import {sourcePath} from './snapshot.mjs';
+export const artifactKinds=['http','readback','trace','screenshot','video','measurement','runtime','text'];
+export const digest=bytes=>createHash('sha256').update(bytes).digest('hex');
+export function validateArtifactSpecs(specs=[]) {
+  if(!Array.isArray(specs)) throw new Error('Artifacts must be an array');
+  for(const a of specs) {sourcePath(a.path);if(!artifactKinds.includes(a.kind))throw new Error('Unknown artifact kind');}
+}
+export async function persistArtifacts(check,directory,head) {
+  if(check.head!==head)throw new Error('Artifact candidate mismatch');
+  const out=join(directory,'artifacts');await mkdir(out,{recursive:true,mode:0o700});
+  const saved=[];
+  for(const [index,a] of (check.artifacts ?? []).entries()) {
+    sourcePath(a.path);
+    if(!artifactKinds.includes(a.kind) || typeof a.base64!=='string')throw new Error('Invalid artifact payload');
+    const bytes=Buffer.from(a.base64,'base64');
+    if(!bytes.length || bytes.length>20*1024*1024)throw new Error('Artifact empty or too large');
+    const sha256=digest(bytes);
+    if(a.sha256!==sha256)throw new Error('Artifact transport integrity mismatch');
+    const extension=extname(a.path).toLowerCase();
+    if(!/^\.[a-z0-9]{1,8}$/.test(extension))throw new Error('Artifact needs a safe extension');
+    if(a.kind==='screenshot' && !(bytes.subarray(0,8).equals(Buffer.from('89504e470d0a1a0a','hex')) || bytes.subarray(0,3).equals(Buffer.from('ffd8ff','hex'))))throw new Error('Screenshot is not PNG or JPEG');
+    const id=check.id+'-'+index,file=join(out,id+extension);
+    await writeFile(file,bytes,{flag:'wx',mode:0o600});
+    saved.push({id,kind:a.kind,sourcePath:a.path,path:file,sha256,bytes:bytes.length,head,checkId:check.id});
+  }
+  return saved;
+}
+export async function captureDesigns(designs,directory) {
+  const records=[];await mkdir(join(directory,'designs'),{recursive:true,mode:0o700});
+  for(const d of designs) {
+    if(!['approved','draft'].includes(d.approvalStatus) || !/^[a-z][a-z0-9-]{0,63}$/.test(d.id) || !d.authority?.trim() || !d.source?.trim() || !d.revision?.trim())throw new Error('Design needs identity and approval provenance');
+    const file=await realpath(d.file),bytes=await readFile(file);
+    if(digest(bytes)!==d.sha256)throw new Error('Design changed after planning');
+    const target=join(directory,'designs',d.id+extname(file));
+    await writeFile(target,bytes,{flag:'wx',mode:0o600});records.push({...d,file:target});
+  }
+  return records;
+}

review/checks.mjs

@@ -1,3 +1,4 @@
+import { validateArtifactSpecs } from './artifacts.mjs';
 import { sourcePath } from './snapshot.mjs';
 export function validateChecks(checks) {
   if (!Array.isArray(checks) || checks.length > 20) throw new Error('checks must be an array of at most 20 checks');
@@ -10,6 +11,7 @@ export function validateChecks(checks) {
     if (!Number.isInteger(check.timeoutSeconds) || check.timeoutSeconds < 1 || check.timeoutSeconds > 3600) throw new Error('Check timeout must be 1..3600 seconds');
     if (!Array.isArray(check.reports) || (check.kind !== 'command' && !check.reports.length)) throw new Error('Check needs JUnit XML report patterns');
     for (const path of check.reports) sourcePath(path);
+    validateArtifactSpecs(check.artifacts);
     if (check.cwd) sourcePath(check.cwd);
   }
   return checks;

review/cli.mjs

@@ -25,14 +25,14 @@ Options:
   --application pr|targeted --question TEXT --lens TEXT
   --path PATH (repeatable) --context FILE (repeatable)
   --requirements FILE --config FILE --environment none|openelis|container --image IMAGE
-  --decisions FILE --history FILE --sources FILE --history-depth N --probes FILE --experiments FILE --max-rounds 0..2
+  --decisions FILE --history FILE --scenarios FILE --workflows FILE --designs FILE --runtime-check ID --sources FILE --history-depth N --probes FILE --experiments FILE --max-rounds 0..2
   --profile development --adapter codex --model MODEL_ID --billing-mode subscription|api|unknown
   --review-seconds N --reasoning-effort low|medium|high
   --startup-seconds N --checks FILE
 Config/check files are explicit trusted operator inputs, not PR instructions.
 L1: no AI. L2: semantic review with explicit coverage and questions.
 L3: test-strength proof. L4: running integration. L5: product/design acceptance.
-L3 needs declared isolated probes and a behavior-to-test map. L4/L5 remain pending.
+L3 needs declared isolated probes and a behavior-to-test map. L4 requires runtime identity and scenarios; L5 adds workflows and rendered design evidence.
 L1/L2 can run declared checks in container/OpenELIS (kind: test|scenario|command, argv, reports,
 timeoutSeconds); those checks do not imply completion of a higher level.
 The development profile uses one inexpensive Sol session by default; --max-rounds explicitly reserves up to two reconsideration rounds within the same time allowance; --model overrides it.
@@ -48,7 +48,7 @@ try {
     out:{type:'string'}, plan:{type:'string'}, run:{type:'string'}, level:{type:'string'}, lens:{type:'string'},
     application:{type:'string'}, question:{type:'string'}, purpose:{type:'string'}, requirements:{type:'string'},
     path:{type:'string',multiple:true}, context:{type:'string',multiple:true},
-    sources:{type:'string'},'history-depth':{type:'string'},probes:{type:'string'},experiments:{type:'string'},'max-rounds':{type:'string'},image:{type:'string'},decisions:{type:'string'},history:{type:'string'},config:{type:'string'}, checks:{type:'string'}, environment:{type:'string'}, rigor:{type:'string'},
+    scenarios:{type:'string'},workflows:{type:'string'},designs:{type:'string'},'runtime-check':{type:'string'},sources:{type:'string'},'history-depth':{type:'string'},probes:{type:'string'},experiments:{type:'string'},'max-rounds':{type:'string'},image:{type:'string'},decisions:{type:'string'},history:{type:'string'},config:{type:'string'}, checks:{type:'string'}, environment:{type:'string'}, rigor:{type:'string'},
     profile:{type:'string'}, adapter:{type:'string'}, model:{type:'string'}, 'billing-mode':{type:'string'}, 'reasoning-effort':{type:'string'},
     'review-seconds':{type:'string'},'startup-seconds':{type:'string'},
   }});
@@ -69,6 +69,7 @@ try {
     const baseRequirements = await configurationRequirements(snap,config);
     const requirements = [baseRequirements, v.requirements ? await readFile(v.requirements,'utf8') : ''].filter(Boolean).join('\n\n');
     plan = await createPlan({...source,...neighbors,application:v.application,question:v.question,purpose:v.purpose,
+      scenarios:v.scenarios ? JSON.parse(await readFile(v.scenarios,'utf8')) : config.scenarios ?? [],workflows:v.workflows ? JSON.parse(await readFile(v.workflows,'utf8')) : config.workflows ?? [],designs:v.designs ? JSON.parse(await readFile(v.designs,'utf8')) : config.designs ?? [],runtimeCheckId:v['runtime-check'] ?? config.runtimeCheckId,
       sources:v.sources ? JSON.parse(await readFile(v.sources,'utf8')) : config.sources ?? [],historyDepth:Number(v['history-depth'] ?? config.historyDepth ?? 1),
       probes:v.probes ? JSON.parse(await readFile(v.probes,'utf8')) : config.probes ?? [],experiments:v.experiments ? JSON.parse(await readFile(v.experiments,'utf8')) : config.experiments ?? [],maxRounds:Number(v['max-rounds'] ?? config.maxRounds ?? 0),
       decisions:v.decisions ? JSON.parse(await readFile(v.decisions,'utf8')) : config.decisions ?? [], history:v.history ? JSON.parse(await readFile(v.history,'utf8')) : null,image:v.image ?? config.image,