|
| 1 | +"""Compares two bench-run output directories file by file and applies a strict regression gate. |
| 2 | +Usage: bench-compare.py <bench-dir> <before-out> <after-out> [report.md] |
| 3 | +""" |
| 4 | +import json, os, re, statistics, sys, unicodedata |
| 5 | +from collections import Counter, defaultdict |
| 6 | +from rapidfuzz import fuzz |
| 7 | + |
| 8 | +BENCH, BEFORE, AFTER = sys.argv[1:4] |
| 9 | +REPORT = sys.argv[4] if len(sys.argv) > 4 else os.path.join(BENCH, 'compare.md') |
| 10 | +REF = os.path.join(BENCH, 'reference') |
| 11 | +WORD = re.compile(r'\w+', re.UNICODE) |
| 12 | +NEEDLE_CAP, HAY_CAP, LINE_CAP = 160, 400_000, 300 |
| 13 | +PAGE_NUM = re.compile(r'^(page\s*)?\d{1,4}(\s*(of|/)\s*\d{1,4})?$', re.I) |
| 14 | +# Extensions whose ok->error flips are intended: legacy .ppt now refuses with a typed error. |
| 15 | +INTENDED_ERROR_EXTS = {'ppt'} |
| 16 | + |
| 17 | +def norm(t): |
| 18 | + t = unicodedata.normalize('NFKC', t or '').replace('', '').replace('‑', '-').lower() |
| 19 | + return re.sub(r'\s+', ' ', t).strip() |
| 20 | +def vocab(t): return {w for w in WORD.findall(norm(t)) if len(w) >= 2} |
| 21 | +def found(needle, hay): |
| 22 | + n = norm(needle)[:NEEDLE_CAP] |
| 23 | + if not n: return False |
| 24 | + if n in hay: return True |
| 25 | + return fuzz.partial_ratio(n, hay[:HAY_CAP], score_cutoff=90) >= 90 |
| 26 | +def blocks(t): return [l.strip() for l in (t or '').split('\n') if l.strip()] |
| 27 | +def junk_per_1k(t): |
| 28 | + bad = sum(1 for ch in t if (unicodedata.category(ch) in ('Cc', 'Co', 'Cn') and ch not in '\n\t\r') or ch in '�') |
| 29 | + return round(1000 * bad / max(1, len(t)), 2) |
| 30 | +def r(x): return None if x is None else round(x, 4) |
| 31 | + |
| 32 | +def load_ref(label): |
| 33 | + p = os.path.join(REF, label.split('__', 1)[1] + '.json') |
| 34 | + if not os.path.exists(p): return {} |
| 35 | + j = json.load(open(p)) |
| 36 | + return {k: v for k, v in j.items() if isinstance(v, str) and k != 'error' and v.strip()} |
| 37 | + |
| 38 | +def metrics(rec, refs): |
| 39 | + if not rec['ok']: |
| 40 | + return dict(ok=False, typed=bool(rec.get('typedError')), code=rec.get('errorCode') or rec.get('errorName'), error=rec.get('error', '')[:120], degraded=False) |
| 41 | + out = rec['content']; n_out = norm(out) |
| 42 | + m = dict(ok=True, typed=None, degraded=bool(rec['metadata'].get('degraded')), truncated=bool(rec['metadata'].get('truncated')), length=len(out), lines=len(blocks(out)), chunks=rec.get('chunkCount'), ms=rec['ms'], junk=junk_per_1k(out), detected=rec['metadata'].get('detectedType'), method=rec['metadata'].get('extractionMethod'), encoding=rec['metadata'].get('encoding')) |
| 43 | + line_counts = Counter(norm(l) for l in blocks(out) if 0 < len(l) <= 120) |
| 44 | + m['repeated_lines'] = sum(c - 1 for c in line_counts.values() if c >= 3) |
| 45 | + m['page_number_lines'] = sum(1 for l in blocks(out) if PAGE_NUM.match(l)) |
| 46 | + ref_vocab = set(); recalls = []; precisions = [] |
| 47 | + for name, text in refs.items(): |
| 48 | + n_ref = norm(text); ref_vocab |= vocab(text) |
| 49 | + ref_lines = [l for l in blocks(text) if len(l) >= 25][:LINE_CAP] |
| 50 | + out_lines = [l for l in blocks(out) if len(l) >= 25][:LINE_CAP] |
| 51 | + if ref_lines: recalls.append(sum(found(l, n_out) for l in ref_lines) / len(ref_lines)) |
| 52 | + if out_lines: precisions.append(sum(found(l, n_ref) for l in out_lines) / len(out_lines)) |
| 53 | + m['recall'] = r(max(recalls)) if recalls else None |
| 54 | + m['precision'] = r(max(precisions)) if precisions else None |
| 55 | + if ref_vocab: |
| 56 | + words = [w for w in WORD.findall(n_out) if len(w) >= 2] |
| 57 | + noise = [w for w in words if w not in ref_vocab] |
| 58 | + m['noise'] = r(len(noise) / max(1, len(words))) |
| 59 | + m['glued'] = len({w for w in set(noise) if len(w) >= 6 and any(w[:i] in ref_vocab and w[i:] in ref_vocab and i >= 2 and len(w) - i >= 2 for i in range(2, len(w) - 1))}) |
| 60 | + m['ref_vocab_recall'] = r(len(set(words) & ref_vocab) / max(1, len(ref_vocab))) |
| 61 | + return m |
| 62 | + |
| 63 | +rows = [] |
| 64 | +labels = sorted(set(os.listdir(BEFORE)) & set(os.listdir(AFTER))) |
| 65 | +for f in labels: |
| 66 | + if f.startswith('_'): continue |
| 67 | + b = json.load(open(os.path.join(BEFORE, f))); a = json.load(open(os.path.join(AFTER, f))) |
| 68 | + refs = load_ref(b['label']) |
| 69 | + mb, ma = metrics(b, refs), metrics(a, refs) |
| 70 | + flags = [] |
| 71 | + if mb['ok'] and not ma['ok']: |
| 72 | + if b['ext'] in INTENDED_ERROR_EXTS or (mb.get('degraded') and ma.get('typed')): flags.append('intended:ok->typed-error') |
| 73 | + elif ma.get('typed'): flags.append('REGRESSION:ok->error') |
| 74 | + else: flags.append('REGRESSION:ok->untyped-error') |
| 75 | + if not mb['ok'] and ma['ok']: flags.append('improved:error->ok') |
| 76 | + if not mb['ok'] and not ma['ok'] and not mb.get('typed') and ma.get('typed'): flags.append('improved:untyped->typed') |
| 77 | + if mb['ok'] and ma['ok']: |
| 78 | + if mb.get('recall') is not None and ma.get('recall') is not None and ma['recall'] < mb['recall'] - 0.02: flags.append(f"REGRESSION:recall {mb['recall']}->{ma['recall']}") |
| 79 | + if mb.get('ref_vocab_recall') is not None and ma.get('ref_vocab_recall') is not None and ma['ref_vocab_recall'] < mb['ref_vocab_recall'] - 0.02: flags.append(f"REGRESSION:vocab-recall {mb['ref_vocab_recall']}->{ma['ref_vocab_recall']}") |
| 80 | + if mb.get('noise') is not None and ma.get('noise') is not None and ma['noise'] > mb['noise'] + 0.02: flags.append(f"REGRESSION:noise {mb['noise']}->{ma['noise']}") |
| 81 | + if mb.get('glued') is not None and ma.get('glued', 0) > mb.get('glued', 0): flags.append(f"REGRESSION:glued {mb['glued']}->{ma['glued']}") |
| 82 | + if ma['junk'] > mb['junk'] + 0.5: flags.append(f"REGRESSION:junk {mb['junk']}->{ma['junk']}") |
| 83 | + if not mb['degraded'] and ma['degraded']: flags.append('REGRESSION:now-degraded') |
| 84 | + if ma['ms'] > 3 * mb['ms'] and ma['ms'] - mb['ms'] > 500: flags.append(f"SLOWER:{mb['ms']:.0f}->{ma['ms']:.0f}ms") |
| 85 | + if ma['length'] < 0.8 * mb['length'] and mb['length'] > 200 and (mb.get('recall') is None or ma.get('recall') is None): flags.append(f"CHECK:length {mb['length']}->{ma['length']} (no reference)") |
| 86 | + rows.append(dict(label=b['label'], ext=b['ext'], before=mb, after=ma, flags=flags)) |
| 87 | + |
| 88 | +json.dump(rows, open(os.path.join(BENCH, 'compare.json'), 'w'), indent=1) |
| 89 | + |
| 90 | +def mean(xs): |
| 91 | + xs = [x for x in xs if isinstance(x, (int, float)) and not isinstance(x, bool)] |
| 92 | + return f'{statistics.mean(xs):.3f}' if xs else '—' |
| 93 | +by_ext = defaultdict(list) |
| 94 | +for row in rows: by_ext[row['ext']].append(row) |
| 95 | +cols = ['recall', 'ref_vocab_recall', 'precision', 'noise', 'glued', 'lines', 'repeated_lines', 'page_number_lines', 'junk', 'chunks', 'ms'] |
| 96 | +L = ['# Before/after benchmark', '', f'{len(rows)} files compared. Gate: recall −0.02, vocab recall −0.02, noise +0.02, glued +1, junk +0.5, ok→error (except intended), now-degraded.', ''] |
| 97 | +regressions = [x for x in rows if any(f.startswith('REGRESSION') for f in x['flags'])] |
| 98 | +L += [f"**Regressions: {len(regressions)}**", ''] |
| 99 | +L += ['| ext | n | ok before→after | typed errors b→a | degraded b→a | ' + ' | '.join(f'{c} b→a' for c in cols) + ' |', '|' + '---|' * (len(cols) + 5)] |
| 100 | +for ext in sorted(by_ext): |
| 101 | + rs = by_ext[ext]; bs = [x['before'] for x in rs]; as_ = [x['after'] for x in rs] |
| 102 | + cells = [f"{mean([b.get(c) for b in bs])}→{mean([a.get(c) for a in as_])}" for c in cols] |
| 103 | + L.append(f"| {ext} | {len(rs)} | {sum(b['ok'] for b in bs)}→{sum(a['ok'] for a in as_)} | {sum(1 for b in bs if not b['ok'] and b.get('typed'))}→{sum(1 for a in as_ if not a['ok'] and a.get('typed'))} | {sum(1 for b in bs if b.get('degraded'))}→{sum(1 for a in as_ if a.get('degraded'))} | " + ' | '.join(cells) + ' |') |
| 104 | +L += ['', '## Flags per file', '', '| file | flags |', '|---|---|'] |
| 105 | +for row in rows: |
| 106 | + if row['flags']: L.append(f"| {row['label']} | {'; '.join(row['flags'])} |") |
| 107 | +open(REPORT, 'w').write('\n'.join(L) + '\n') |
| 108 | +print('\n'.join(L[:8 + len(by_ext)])) |
| 109 | +print(f"\nregressions={len(regressions)} flagged={sum(1 for x in rows if x['flags'])} report={REPORT}") |
0 commit comments