From 787f0c14ad447a0ce6344061b21e9fef1bc228f9 Mon Sep 17 00:00:00 2001 From: tactino <18781106300@163.com> Date: Sat, 3 Oct 2026 00:31:27 -0400 Subject: [PATCH] E56, E57: ruff format the five scripts that failed lint on main Layout only. Four files have an identical AST before and after; verdicts.py of E57 differs only in its docstring's indentation. The source hashes in results/*.sha256 refer to the versions that ran, which are in the E56 and E57 commits. --- experiments/e56-hopper-faults/verdicts.py | 113 +++++++++++++++---- experiments/e57-clean-eval/evaluate.py | 24 +++- experiments/e57-clean-eval/explore.py | 37 +++++-- experiments/e57-clean-eval/summarise.py | 129 +++++++++++++++++----- experiments/e57-clean-eval/verdicts.py | 83 +++++++++++--- 5 files changed, 306 insertions(+), 80 deletions(-) diff --git a/experiments/e56-hopper-faults/verdicts.py b/experiments/e56-hopper-faults/verdicts.py index 3325ac9..07292eb 100644 --- a/experiments/e56-hopper-faults/verdicts.py +++ b/experiments/e56-hopper-faults/verdicts.py @@ -31,7 +31,11 @@ def main() -> int: rows = read(root / "runs.csv") ledger = read(root / "ledger.csv") reg = [r for r in rows if r["seed"] in SEEDS] - changed = [r for r in reg if r["fault"] not in CONTROLS + LOUD and r["cells"] not in ("", "0", "None")] + changed = [ + r + for r in reg + if r["fault"] not in CONTROLS + LOUD and r["cells"] not in ("", "0", "None") + ] out = [] def verdict(name: str, ok: bool, detail: str) -> None: @@ -41,20 +45,38 @@ def runs_of(rs) -> list: return [(r["fault"], r["dose"], r["seed"]) for r in rs] bad = [r for r in rows if r["complete"] != "1"] - verdict("V1", not bad, f"{len(rows) - len(bad)} of {len(rows)} complete" + (f"; not: {runs_of(bad)[:5]}" if bad else "")) + verdict( + "V1", + not bad, + f"{len(rows) - len(bad)} of {len(rows)} complete" + + (f"; not: {runs_of(bad)[:5]}" if bad else ""), + ) none = {r["seed"]: r["sha"] for r in reg if r["fault"] == "none"} ctrl = [r for r in reg if r["fault"] in ("delay-1ms", "wire-float64")] v2_bad = [r for r in ctrl if r["sha"] != none.get(r["seed"])] - verdict("V2", not v2_bad and len(ctrl) == 6, f"controls on none's weights: {len(ctrl) - len(v2_bad)} of {len(ctrl)}") + verdict( + "V2", + not v2_bad and len(ctrl) == 6, + f"controls on none's weights: {len(ctrl) - len(v2_bad)} of {len(ctrl)}", + ) client = [r for r in changed if r["fault"] in SILENT] p1 = [r for r in client if r["fault"] not in FINAL_OBS] p1_bad = [r for r in p1 if r["weights"] != "1"] - verdict("P1", not p1_bad, f"weights differ in {len(p1) - len(p1_bad)} of {len(p1)}" + (f"; not: {runs_of(p1_bad)}" if p1_bad else "")) + verdict( + "P1", + not p1_bad, + f"weights differ in {len(p1) - len(p1_bad)} of {len(p1)}" + + (f"; not: {runs_of(p1_bad)}" if p1_bad else ""), + ) logs = [r for r in changed if r["fault"] in LOG] p2_bad = [r for r in logs if r["weights"] != "0"] - verdict("P2", not p2_bad, f"log faults on none's weights: {len(logs) - len(p2_bad)} of {len(logs)}") + verdict( + "P2", + not p2_bad, + f"log faults on none's weights: {len(logs) - len(p2_bad)} of {len(logs)}", + ) p3_ok, parts = True, [] for dose, cap in (("one", 0.10), ("0.01", 0.10), ("1.0", 0.60)): @@ -64,38 +86,85 @@ def runs_of(rs) -> list: parts.append(f"{dose}: {n}/{len(rs)} ({n / len(rs):.1%}, cap {cap:.0%})") verdict("P3", p3_ok, "; ".join(parts)) - must = ("reward:zero", "reward:stale", "reward:swap", "terminated:spurious", "terminated:drop", "truncated:drop") - never = tuple(f for f in SILENT if f.split(":")[0] in ("obs", "action", "final-obs")) + ( + must = ( + "reward:zero", + "reward:stale", + "reward:swap", + "terminated:spurious", + "terminated:drop", + "truncated:drop", + ) + never = tuple( + f for f in SILENT if f.split(":")[0] in ("obs", "action", "final-obs") + ) + ( "truncated:as-term", "terminated:as-trunc", ) - miss = [r for r in changed if r["fault"] in must and r["dose"] in ("0.01", "1.0") and r["record"] != "1"] - miss += [r for r in changed if r["fault"] == "step:fill" and r["dose"] == "0.01" and r["record"] != "1"] + miss = [ + r + for r in changed + if r["fault"] in must and r["dose"] in ("0.01", "1.0") and r["record"] != "1" + ] + miss += [ + r + for r in changed + if r["fault"] == "step:fill" and r["dose"] == "0.01" and r["record"] != "1" + ] miss += [r for r in logs if r["record"] != "1"] false = [r for r in changed if r["fault"] in never and r["record"] == "1"] - verdict("P4", not miss and not false, f"missed {len(miss)}, caught {len(false)} it should not" - + (f"; missed: {runs_of(miss)[:8]}" if miss else "") + (f"; caught: {runs_of(false)[:8]}" if false else "")) + verdict( + "P4", + not miss and not false, + f"missed {len(miss)}, caught {len(false)} it should not" + + (f"; missed: {runs_of(miss)[:8]}" if miss else "") + + (f"; caught: {runs_of(false)[:8]}" if false else ""), + ) - p5_ok = all(r["bridge"] == "1" for r in logs) and all(r["bridge"] == "0" for r in client) - verdict("P5", p5_ok, f"bridge caught {sum(r['bridge'] == '1' for r in logs)} of {len(logs)} log-fault runs, " - f"{sum(r['bridge'] == '1' for r in client)} of {len(client)} client-fault runs") + p5_ok = all(r["bridge"] == "1" for r in logs) and all( + r["bridge"] == "0" for r in client + ) + verdict( + "P5", + p5_ok, + f"bridge caught {sum(r['bridge'] == '1' for r in logs)} of {len(logs)} log-fault runs, " + f"{sum(r['bridge'] == '1' for r in client)} of {len(client)} client-fault runs", + ) led = [r for r in ledger if r["seed"] in SEEDS] - led_client = [r for r in led if r["fault"] in SILENT and r["cells"] not in ("", "0")] + led_client = [ + r for r in led if r["fault"] in SILENT and r["cells"] not in ("", "0") + ] led_quiet = [r for r in led if r["fault"] in CONTROLS + LOG] p6_bad = [r for r in led_client if r["caught"] != "1" or r["right"] != "1"] p6_false = [r for r in led_quiet if r["caught"] != "0"] - verdict("P6", not p6_bad and not p6_false, - f"caught and right in {len(led_client) - len(p6_bad)} of {len(led_client)} client-fault runs; " - f"no difference in {len(led_quiet) - len(p6_false)} of {len(led_quiet)} control and log-fault runs" - + (f"; not: {[(r['fault'], r['dose'], r['seed'], r['found']) for r in p6_bad][:6]}" if p6_bad else "")) + verdict( + "P6", + not p6_bad and not p6_false, + f"caught and right in {len(led_client) - len(p6_bad)} of {len(led_client)} client-fault runs; " + f"no difference in {len(led_quiet) - len(p6_false)} of {len(led_quiet)} control and log-fault runs" + + ( + f"; not: {[(r['fault'], r['dose'], r['seed'], r['found']) for r in p6_bad][:6]}" + if p6_bad + else "" + ), + ) - caught_ledger = {(r["fault"], r["dose"], r["seed"]) for r in led if r["caught"] == "1"} + caught_ledger = { + (r["fault"], r["dose"], r["seed"]) for r in led if r["caught"] == "1" + } p7_miss = [r for r in client + logs if r["record"] != "1" and (r["fault"], r["dose"], r["seed"]) not in caught_ledger] # fmt: skip - verdict("P7", not p7_miss, f"ledger or record caught {len(client + logs) - len(p7_miss)} of {len(client + logs)}") + verdict( + "P7", + not p7_miss, + f"ledger or record caught {len(client + logs) - len(p7_miss)} of {len(client + logs)}", + ) - fo = [(r["fault"], r["dose"], r["seed"], r["weights"]) for r in client if r["fault"] in FINAL_OBS] + fo = [ + (r["fault"], r["dose"], r["seed"], r["weights"]) + for r in client + if r["fault"] in FINAL_OBS + ] out.append(f"reported: final-obs runs (fault, dose, seed, weights caught): {fo}") text = "\n".join(out) print(text) diff --git a/experiments/e57-clean-eval/evaluate.py b/experiments/e57-clean-eval/evaluate.py index 6d5a9d9..9ee62d9 100644 --- a/experiments/e57-clean-eval/evaluate.py +++ b/experiments/e57-clean-eval/evaluate.py @@ -18,7 +18,11 @@ import torch from stable_baselines3 import PPO -IDS = {"pendulum": "Pendulum-v1", "halfcheetah": "HalfCheetah-v5", "hopper": "Hopper-v5"} +IDS = { + "pendulum": "Pendulum-v1", + "halfcheetah": "HalfCheetah-v5", + "hopper": "Hopper-v5", +} SEED0 = 10_000 @@ -49,14 +53,24 @@ def main() -> int: lengths.append(n) out = dict( - env=res["env"], fault=res["fault"], dose=res["dose"], seed=res["seed"], - weights_sha256=res["weights_sha256"], episodes=a.episodes, - returns=returns, lengths=lengths, mean=float(np.mean(returns)), + env=res["env"], + fault=res["fault"], + dose=res["dose"], + seed=res["seed"], + weights_sha256=res["weights_sha256"], + episodes=a.episodes, + returns=returns, + lengths=lengths, + mean=float(np.mean(returns)), eval_s=round(time.time() - t0, 3), ) a.out.mkdir(parents=True, exist_ok=True) (a.out / "clean_eval.json").write_text(json.dumps(out)) - print(json.dumps({k: out[k] for k in ("env", "fault", "dose", "seed", "mean", "eval_s")})) + print( + json.dumps( + {k: out[k] for k in ("env", "fault", "dose", "seed", "mean", "eval_s")} + ) + ) return 0 diff --git a/experiments/e57-clean-eval/explore.py b/experiments/e57-clean-eval/explore.py index 5372406..f984724 100644 --- a/experiments/e57-clean-eval/explore.py +++ b/experiments/e57-clean-eval/explore.py @@ -25,22 +25,43 @@ def main() -> int: rows = list(csv.DictReader(open(root / "clean.csv"))) for r in rows: r["seed"], r["clean"] = int(r["seed"]), float(r["clean"]) - out = ["exploratory: boundary-fault runs silent on the curve, against fault-free runs", - "env dose n mean z median z U-test p twin gap median (clean - none at same seed)"] + out = [ + "exploratory: boundary-fault runs silent on the curve, against fault-free runs", + "env dose n mean z median z U-test p twin gap median (clean - none at same seed)", + ] for env in sorted({r["env"] for r in rows}): - band = [r["clean"] for r in rows if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10] + band = [ + r["clean"] + for r in rows + if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10 + ] mu, sd = st.mean(band), st.stdev(band) free = [r["clean"] for r in rows if r["env"] == env and r["fault"] == "none"] - twin = {r["seed"]: r["clean"] for r in rows if r["env"] == env and r["fault"] == "none"} + twin = { + r["seed"]: r["clean"] + for r in rows + if r["env"] == env and r["fault"] == "none" + } for dose in ("one", "0.001", "0.01", "0.1", "1.0"): - s = [r for r in rows if r["env"] == env and r["dose"] == dose and r["kind"] == "boundary" - and r["seed"] < 10 and r["silent_curve"] == "1"] + s = [ + r + for r in rows + if r["env"] == env + and r["dose"] == dose + and r["kind"] == "boundary" + and r["seed"] < 10 + and r["silent_curve"] == "1" + ] if not s: continue z = [(r["clean"] - mu) / sd for r in s] - p = mannwhitneyu([r["clean"] for r in s], free, alternative="two-sided").pvalue + p = mannwhitneyu( + [r["clean"] for r in s], free, alternative="two-sided" + ).pvalue gap = st.median(r["clean"] - twin[r["seed"]] for r in s) - out.append(f"{env:12} {dose:6} {len(s):3} {st.mean(z):6.2f} {st.median(z):8.2f} {p:8.3f} {gap:9.1f}") + out.append( + f"{env:12} {dose:6} {len(s):3} {st.mean(z):6.2f} {st.median(z):8.2f} {p:8.3f} {gap:9.1f}" + ) text = "\n".join(out) (root / "explore.txt").write_text(text + "\n") print(text) diff --git a/experiments/e57-clean-eval/summarise.py b/experiments/e57-clean-eval/summarise.py index 8df63c3..8979d10 100644 --- a/experiments/e57-clean-eval/summarise.py +++ b/experiments/e57-clean-eval/summarise.py @@ -12,8 +12,10 @@ import sys HERE = pathlib.Path(__file__).resolve().parent -RUNS = [HERE.parent / "e54-boundary-faults/results/runs.csv", - HERE.parent / "e56-hopper-faults/results/runs.csv"] +RUNS = [ + HERE.parent / "e54-boundary-faults/results/runs.csv", + HERE.parent / "e56-hopper-faults/results/runs.csv", +] CONTROLS = {"none", "delay-1ms", "wire-float64"} LOUD = {"indices:reorder", "reward:short-array"} @@ -37,13 +39,22 @@ def load(root: pathlib.Path) -> list[dict]: for f in sorted(root.glob("*/*/*/seed*/clean_eval.json")): d = json.loads(f.read_text()) t = trained[(d["env"], d["fault"], d["dose"], int(d["seed"]))] - rows.append(dict( - env=d["env"], fault=d["fault"], dose=d["dose"], seed=int(d["seed"]), kind=kind(d["fault"]), - episodes=d["episodes"], clean=round(d["mean"], 2), sha=d["weights_sha256"][:8], - # The loud faults' runs have no `strict` (no return was logged); see AMENDMENT.md. - rule=int(t["rule"]) if t["rule"] else None, strict=int(t["strict"]) if t["strict"] else None, - last=float(t["last"]) if t["last"] else None, - )) + rows.append( + dict( + env=d["env"], + fault=d["fault"], + dose=d["dose"], + seed=int(d["seed"]), + kind=kind(d["fault"]), + episodes=d["episodes"], + clean=round(d["mean"], 2), + sha=d["weights_sha256"][:8], + # The loud faults' runs have no `strict` (no return was logged); see AMENDMENT.md. + rule=int(t["rule"]) if t["rule"] else None, + strict=int(t["strict"]) if t["strict"] else None, + last=float(t["last"]) if t["last"] else None, + ) + ) return rows @@ -51,7 +62,11 @@ def judge(rows: list[dict]) -> dict: """Band per env (none, seeds 10-19); harmed = more than 3 s.d. below its mean.""" bands = {} for env in sorted({r["env"] for r in rows}): - band = [r for r in rows if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10] + band = [ + r + for r in rows + if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10 + ] vals = [r["clean"] for r in band] bands[env] = dict(mean=st.mean(vals), sd=st.stdev(vals), n=len(vals)) # Each band seed judged against the other nine. @@ -71,8 +86,22 @@ def main() -> int: root = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else HERE / "results" rows = load(root) bands = judge(rows) - cols = ["env", "fault", "dose", "seed", "kind", "episodes", "clean", "harmed", "rule", "strict", - "silent_curve", "last", "log_error", "sha"] + cols = [ + "env", + "fault", + "dose", + "seed", + "kind", + "episodes", + "clean", + "harmed", + "rule", + "strict", + "silent_curve", + "last", + "log_error", + "sha", + ] with open(root / "clean.csv", "w", newline="") as fh: w = csv.DictWriter(fh, fieldnames=cols, lineterminator="\n") w.writeheader() @@ -81,35 +110,81 @@ def main() -> int: out = [] for env, b in bands.items(): - flagged = sum(r["harmed"] for r in rows if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10) - out.append(f"{env}: band (none, seeds 10-19) mean {b['mean']:.1f} sd {b['sd']:.1f} n {b['n']}; " - f"harm bar {b['mean'] - 3 * b['sd']:.1f}; band seeds flagged against the other nine: {flagged} of 10") + flagged = sum( + r["harmed"] + for r in rows + if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10 + ) + out.append( + f"{env}: band (none, seeds 10-19) mean {b['mean']:.1f} sd {b['sd']:.1f} n {b['n']}; " + f"harm bar {b['mean'] - 3 * b['sd']:.1f}; band seeds flagged against the other nine: {flagged} of 10" + ) out.append("") - out.append("boundary-fault runs, seeds 0-2: harmed / runs; harmed among those silent on the curve") + out.append( + "boundary-fault runs, seeds 0-2: harmed / runs; harmed among those silent on the curve" + ) for env in bands: for dose in ("one", "0.001", "0.01", "0.1", "1.0"): - d = [r for r in rows if r["env"] == env and r["dose"] == dose and r["kind"] == "boundary" and r["seed"] < 10] + d = [ + r + for r in rows + if r["env"] == env + and r["dose"] == dose + and r["kind"] == "boundary" + and r["seed"] < 10 + ] if not d: continue s = [r for r in d if r["silent_curve"]] - out.append(f" {env:12} {dose:6} harmed {sum(r['harmed'] for r in d):3}/{len(d):<3} " - f"silent on the curve {len(s):3}, of them harmed {sum(r['harmed'] for r in s):3}") + out.append( + f" {env:12} {dose:6} harmed {sum(r['harmed'] for r in d):3}/{len(d):<3} " + f"silent on the curve {len(s):3}, of them harmed {sum(r['harmed'] for r in s):3}" + ) out.append("") - out.append("faults silent on the curve at all three seeds and harmed at two or more (fault, dose: clean returns)") + out.append( + "faults silent on the curve at all three seeds and harmed at two or more (fault, dose: clean returns)" + ) for env in bands: b = bands[env] - for (fault, dose) in sorted({(r["fault"], r["dose"]) for r in rows if r["env"] == env and r["kind"] == "boundary"}): - d = [r for r in rows if r["env"] == env and r["fault"] == fault and r["dose"] == dose and r["seed"] < 10] - if len(d) == 3 and all(r["silent_curve"] for r in d) and sum(r["harmed"] for r in d) >= 2: + for fault, dose in sorted( + { + (r["fault"], r["dose"]) + for r in rows + if r["env"] == env and r["kind"] == "boundary" + } + ): + d = [ + r + for r in rows + if r["env"] == env + and r["fault"] == fault + and r["dose"] == dose + and r["seed"] < 10 + ] + if ( + len(d) == 3 + and all(r["silent_curve"] for r in d) + and sum(r["harmed"] for r in d) >= 2 + ): z = [round((r["clean"] - b["mean"]) / b["sd"], 1) for r in d] - out.append(f" {env:12} {fault:20} {dose:6} clean {[r['clean'] for r in d]} in band s.d. {z}") + out.append( + f" {env:12} {fault:20} {dose:6} clean {[r['clean'] for r in d]} in band s.d. {z}" + ) out.append("") - out.append("log error (logged final return - clean return): min / median / max by kind") + out.append( + "log error (logged final return - clean return): min / median / max by kind" + ) for env in bands: for k in ("control", "boundary", "log"): - e = [r["log_error"] for r in rows if r["env"] == env and r["kind"] == k and r["log_error"] is not None] + e = [ + r["log_error"] + for r in rows + if r["env"] == env and r["kind"] == k and r["log_error"] is not None + ] if e: - out.append(f" {env:12} {k:9} n {len(e):3} {min(e):9.1f} {st.median(e):9.1f} {max(e):9.1f}") + out.append( + f" {env:12} {k:9} n {len(e):3} {min(e):9.1f} {st.median(e):9.1f} {max(e):9.1f}" + ) text = "\n".join(out) (root / "summary.txt").write_text(text + "\n") print(text) diff --git a/experiments/e57-clean-eval/verdicts.py b/experiments/e57-clean-eval/verdicts.py index 12de40b..054f7ad 100644 --- a/experiments/e57-clean-eval/verdicts.py +++ b/experiments/e57-clean-eval/verdicts.py @@ -1,6 +1,6 @@ """E57: judge PROTOCOL.md's checks and predictions from results/clean.csv. - python verdicts.py [results] # prints, and writes results/verdicts.txt +python verdicts.py [results] # prints, and writes results/verdicts.txt """ import csv @@ -15,43 +15,90 @@ def main() -> int: root = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else HERE / "results" rows = list(csv.DictReader(open(root / "clean.csv"))) for r in rows: - r["seed"], r["harmed"], r["episodes"] = int(r["seed"]), int(r["harmed"]), int(r["episodes"]) + r["seed"], r["harmed"], r["episodes"] = ( + int(r["seed"]), + int(r["harmed"]), + int(r["episodes"]), + ) r["clean"] = float(r["clean"]) out = [] jobs = (root / "jobs.txt").read_text().split() full = sum(r["episodes"] == 50 for r in rows) - out.append(f"V1: {'holds' if full == len(jobs) == len(rows) else 'FAILS'} - " - f"{len(rows)} evaluations of {len(jobs)} runs, {full} with 50 episodes") + out.append( + f"V1: {'holds' if full == len(jobs) == len(rows) else 'FAILS'} - " + f"{len(rows)} evaluations of {len(jobs)} runs, {full} with 50 episodes" + ) by_sha = defaultdict(set) for r in rows: by_sha[(r["env"], r["sha"])].add(r["clean"]) - shared = [k for k in by_sha if sum(1 for r in rows if (r["env"], r["sha"]) == k) > 1] + shared = [ + k for k in by_sha if sum(1 for r in rows if (r["env"], r["sha"]) == k) > 1 + ] bad = [k for k in shared if len(by_sha[k]) > 1] - out.append(f"V2: {'holds' if not bad else 'FAILS'} - {len(shared)} weights shared by two or more runs; " - f"{len(bad)} with differing clean returns {bad[:5]}") + out.append( + f"V2: {'holds' if not bad else 'FAILS'} - {len(shared)} weights shared by two or more runs; " + f"{len(bad)} with differing clean returns {bad[:5]}" + ) p1 = {} for env in sorted({r["env"] for r in rows}): - p1[env] = sum(r["harmed"] for r in rows if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10) - out.append(f"P1: {'holds' if max(p1.values()) <= 1 else 'FALSIFIED'} - band seeds flagged: {p1}") + p1[env] = sum( + r["harmed"] + for r in rows + if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10 + ) + out.append( + f"P1: {'holds' if max(p1.values()) <= 1 else 'FALSIFIED'} - band seeds flagged: {p1}" + ) p2 = {} for env in sorted({r["env"] for r in rows}): - d = [r for r in rows if r["env"] == env and r["dose"] == "one" and r["kind"] == "boundary" and r["seed"] < 10] + d = [ + r + for r in rows + if r["env"] == env + and r["dose"] == "one" + and r["kind"] == "boundary" + and r["seed"] < 10 + ] p2[env] = (sum(r["harmed"] for r in d), len(d)) ok = all(h <= 0.10 * n for h, n in p2.values()) - out.append(f"P2: {'holds' if ok else 'FALSIFIED'} - harmed at one value (harmed, runs): {p2}") + out.append( + f"P2: {'holds' if ok else 'FALSIFIED'} - harmed at one value (harmed, runs): {p2}" + ) - d = [r for r in rows if r["env"] == "pendulum" and r["fault"] == "reward:zero" and r["dose"] == "1.0" and r["seed"] < 10] - ok = len(d) == 3 and all(r["harmed"] for r in d) and all(r["last"] and float(r["last"]) == 0.0 for r in d) - out.append(f"P3: {'holds' if ok else 'FALSIFIED'} - pendulum reward:zero 1.0: " - f"{[(r['seed'], r['clean'], r['last'], r['harmed']) for r in d]}") + d = [ + r + for r in rows + if r["env"] == "pendulum" + and r["fault"] == "reward:zero" + and r["dose"] == "1.0" + and r["seed"] < 10 + ] + ok = ( + len(d) == 3 + and all(r["harmed"] for r in d) + and all(r["last"] and float(r["last"]) == 0.0 for r in d) + ) + out.append( + f"P3: {'holds' if ok else 'FALSIFIED'} - pendulum reward:zero 1.0: " + f"{[(r['seed'], r['clean'], r['last'], r['harmed']) for r in d]}" + ) - h = [r for r in rows if r["env"] == "hopper" and r["fault"] == "obs:f16" and r["dose"] == "1.0" and r["seed"] < 10] - out.append(f"reported: hopper obs:f16 1.0 (seed, clean, harmed, silent on the curve): " - f"{[(r['seed'], r['clean'], r['harmed'], r['silent_curve']) for r in h]}") + h = [ + r + for r in rows + if r["env"] == "hopper" + and r["fault"] == "obs:f16" + and r["dose"] == "1.0" + and r["seed"] < 10 + ] + out.append( + f"reported: hopper obs:f16 1.0 (seed, clean, harmed, silent on the curve): " + f"{[(r['seed'], r['clean'], r['harmed'], r['silent_curve']) for r in h]}" + ) text = "\n".join(out) (root / "verdicts.txt").write_text(text + "\n") print(text)