E56, E57: ruff format the five scripts that failed lint on main

experiments/e56-hopper-faults/verdicts.py

@@ -31,7 +31,11 @@ def main() -> int:
     rows = read(root / "runs.csv")
     ledger = read(root / "ledger.csv")
     reg = [r for r in rows if r["seed"] in SEEDS]
-    changed = [r for r in reg if r["fault"] not in CONTROLS + LOUD and r["cells"] not in ("", "0", "None")]
+    changed = [
+        r
+        for r in reg
+        if r["fault"] not in CONTROLS + LOUD and r["cells"] not in ("", "0", "None")
+    ]
     out = []
 
     def verdict(name: str, ok: bool, detail: str) -> None:
@@ -41,20 +45,38 @@ def runs_of(rs) -> list:
         return [(r["fault"], r["dose"], r["seed"]) for r in rs]
 
     bad = [r for r in rows if r["complete"] != "1"]
-    verdict("V1", not bad, f"{len(rows) - len(bad)} of {len(rows)} complete" + (f"; not: {runs_of(bad)[:5]}" if bad else ""))
+    verdict(
+        "V1",
+        not bad,
+        f"{len(rows) - len(bad)} of {len(rows)} complete"
+        + (f"; not: {runs_of(bad)[:5]}" if bad else ""),
+    )
     none = {r["seed"]: r["sha"] for r in reg if r["fault"] == "none"}
     ctrl = [r for r in reg if r["fault"] in ("delay-1ms", "wire-float64")]
     v2_bad = [r for r in ctrl if r["sha"] != none.get(r["seed"])]
-    verdict("V2", not v2_bad and len(ctrl) == 6, f"controls on none's weights: {len(ctrl) - len(v2_bad)} of {len(ctrl)}")
+    verdict(
+        "V2",
+        not v2_bad and len(ctrl) == 6,
+        f"controls on none's weights: {len(ctrl) - len(v2_bad)} of {len(ctrl)}",
+    )
 
     client = [r for r in changed if r["fault"] in SILENT]
     p1 = [r for r in client if r["fault"] not in FINAL_OBS]
     p1_bad = [r for r in p1 if r["weights"] != "1"]
-    verdict("P1", not p1_bad, f"weights differ in {len(p1) - len(p1_bad)} of {len(p1)}" + (f"; not: {runs_of(p1_bad)}" if p1_bad else ""))
+    verdict(
+        "P1",
+        not p1_bad,
+        f"weights differ in {len(p1) - len(p1_bad)} of {len(p1)}"
+        + (f"; not: {runs_of(p1_bad)}" if p1_bad else ""),
+    )
 
     logs = [r for r in changed if r["fault"] in LOG]
     p2_bad = [r for r in logs if r["weights"] != "0"]
-    verdict("P2", not p2_bad, f"log faults on none's weights: {len(logs) - len(p2_bad)} of {len(logs)}")
+    verdict(
+        "P2",
+        not p2_bad,
+        f"log faults on none's weights: {len(logs) - len(p2_bad)} of {len(logs)}",
+    )
 
     p3_ok, parts = True, []
     for dose, cap in (("one", 0.10), ("0.01", 0.10), ("1.0", 0.60)):
@@ -64,38 +86,85 @@ def runs_of(rs) -> list:
         parts.append(f"{dose}: {n}/{len(rs)} ({n / len(rs):.1%}, cap {cap:.0%})")
     verdict("P3", p3_ok, "; ".join(parts))
 
-    must = ("reward:zero", "reward:stale", "reward:swap", "terminated:spurious", "terminated:drop", "truncated:drop")
-    never = tuple(f for f in SILENT if f.split(":")[0] in ("obs", "action", "final-obs")) + (
+    must = (
+        "reward:zero",
+        "reward:stale",
+        "reward:swap",
+        "terminated:spurious",
+        "terminated:drop",
+        "truncated:drop",
+    )
+    never = tuple(
+        f for f in SILENT if f.split(":")[0] in ("obs", "action", "final-obs")
+    ) + (
         "truncated:as-term",
         "terminated:as-trunc",
     )
-    miss = [r for r in changed if r["fault"] in must and r["dose"] in ("0.01", "1.0") and r["record"] != "1"]
-    miss += [r for r in changed if r["fault"] == "step:fill" and r["dose"] == "0.01" and r["record"] != "1"]
+    miss = [
+        r
+        for r in changed
+        if r["fault"] in must and r["dose"] in ("0.01", "1.0") and r["record"] != "1"
+    ]
+    miss += [
+        r
+        for r in changed
+        if r["fault"] == "step:fill" and r["dose"] == "0.01" and r["record"] != "1"
+    ]
     miss += [r for r in logs if r["record"] != "1"]
     false = [r for r in changed if r["fault"] in never and r["record"] == "1"]
-    verdict("P4", not miss and not false, f"missed {len(miss)}, caught {len(false)} it should not"
-            + (f"; missed: {runs_of(miss)[:8]}" if miss else "") + (f"; caught: {runs_of(false)[:8]}" if false else ""))
+    verdict(
+        "P4",
+        not miss and not false,
+        f"missed {len(miss)}, caught {len(false)} it should not"
+        + (f"; missed: {runs_of(miss)[:8]}" if miss else "")
+        + (f"; caught: {runs_of(false)[:8]}" if false else ""),
+    )
 
-    p5_ok = all(r["bridge"] == "1" for r in logs) and all(r["bridge"] == "0" for r in client)
-    verdict("P5", p5_ok, f"bridge caught {sum(r['bridge'] == '1' for r in logs)} of {len(logs)} log-fault runs, "
-            f"{sum(r['bridge'] == '1' for r in client)} of {len(client)} client-fault runs")
+    p5_ok = all(r["bridge"] == "1" for r in logs) and all(
+        r["bridge"] == "0" for r in client
+    )
+    verdict(
+        "P5",
+        p5_ok,
+        f"bridge caught {sum(r['bridge'] == '1' for r in logs)} of {len(logs)} log-fault runs, "
+        f"{sum(r['bridge'] == '1' for r in client)} of {len(client)} client-fault runs",
+    )
 
     led = [r for r in ledger if r["seed"] in SEEDS]
-    led_client = [r for r in led if r["fault"] in SILENT and r["cells"] not in ("", "0")]
+    led_client = [
+        r for r in led if r["fault"] in SILENT and r["cells"] not in ("", "0")
+    ]
     led_quiet = [r for r in led if r["fault"] in CONTROLS + LOG]
     p6_bad = [r for r in led_client if r["caught"] != "1" or r["right"] != "1"]
     p6_false = [r for r in led_quiet if r["caught"] != "0"]
-    verdict("P6", not p6_bad and not p6_false,
-            f"caught and right in {len(led_client) - len(p6_bad)} of {len(led_client)} client-fault runs; "
-            f"no difference in {len(led_quiet) - len(p6_false)} of {len(led_quiet)} control and log-fault runs"
-            + (f"; not: {[(r['fault'], r['dose'], r['seed'], r['found']) for r in p6_bad][:6]}" if p6_bad else ""))
+    verdict(
+        "P6",
+        not p6_bad and not p6_false,
+        f"caught and right in {len(led_client) - len(p6_bad)} of {len(led_client)} client-fault runs; "
+        f"no difference in {len(led_quiet) - len(p6_false)} of {len(led_quiet)} control and log-fault runs"
+        + (
+            f"; not: {[(r['fault'], r['dose'], r['seed'], r['found']) for r in p6_bad][:6]}"
+            if p6_bad
+            else ""
+        ),
+    )
 
-    caught_ledger = {(r["fault"], r["dose"], r["seed"]) for r in led if r["caught"] == "1"}
+    caught_ledger = {
+        (r["fault"], r["dose"], r["seed"]) for r in led if r["caught"] == "1"
+    }
     p7_miss = [r for r in client + logs
                if r["record"] != "1" and (r["fault"], r["dose"], r["seed"]) not in caught_ledger]  # fmt: skip
-    verdict("P7", not p7_miss, f"ledger or record caught {len(client + logs) - len(p7_miss)} of {len(client + logs)}")
+    verdict(
+        "P7",
+        not p7_miss,
+        f"ledger or record caught {len(client + logs) - len(p7_miss)} of {len(client + logs)}",
+    )
 
-    fo = [(r["fault"], r["dose"], r["seed"], r["weights"]) for r in client if r["fault"] in FINAL_OBS]
+    fo = [
+        (r["fault"], r["dose"], r["seed"], r["weights"])
+        for r in client
+        if r["fault"] in FINAL_OBS
+    ]
     out.append(f"reported: final-obs runs (fault, dose, seed, weights caught): {fo}")
     text = "\n".join(out)
     print(text)

experiments/e57-clean-eval/evaluate.py

@@ -18,7 +18,11 @@
 import torch
 from stable_baselines3 import PPO
 
-IDS = {"pendulum": "Pendulum-v1", "halfcheetah": "HalfCheetah-v5", "hopper": "Hopper-v5"}
+IDS = {
+    "pendulum": "Pendulum-v1",
+    "halfcheetah": "HalfCheetah-v5",
+    "hopper": "Hopper-v5",
+}
 SEED0 = 10_000
 
 
@@ -49,14 +53,24 @@ def main() -> int:
         lengths.append(n)
 
     out = dict(
-        env=res["env"], fault=res["fault"], dose=res["dose"], seed=res["seed"],
-        weights_sha256=res["weights_sha256"], episodes=a.episodes,
-        returns=returns, lengths=lengths, mean=float(np.mean(returns)),
+        env=res["env"],
+        fault=res["fault"],
+        dose=res["dose"],
+        seed=res["seed"],
+        weights_sha256=res["weights_sha256"],
+        episodes=a.episodes,
+        returns=returns,
+        lengths=lengths,
+        mean=float(np.mean(returns)),
         eval_s=round(time.time() - t0, 3),
     )
     a.out.mkdir(parents=True, exist_ok=True)
     (a.out / "clean_eval.json").write_text(json.dumps(out))
-    print(json.dumps({k: out[k] for k in ("env", "fault", "dose", "seed", "mean", "eval_s")}))
+    print(
+        json.dumps(
+            {k: out[k] for k in ("env", "fault", "dose", "seed", "mean", "eval_s")}
+        )
+    )
     return 0
 
 

experiments/e57-clean-eval/explore.py

@@ -25,22 +25,43 @@ def main() -> int:
     rows = list(csv.DictReader(open(root / "clean.csv")))
     for r in rows:
         r["seed"], r["clean"] = int(r["seed"]), float(r["clean"])
-    out = ["exploratory: boundary-fault runs silent on the curve, against fault-free runs",
-           "env          dose    n   mean z   median z   U-test p   twin gap median (clean - none at same seed)"]
+    out = [
+        "exploratory: boundary-fault runs silent on the curve, against fault-free runs",
+        "env          dose    n   mean z   median z   U-test p   twin gap median (clean - none at same seed)",
+    ]
     for env in sorted({r["env"] for r in rows}):
-        band = [r["clean"] for r in rows if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10]
+        band = [
+            r["clean"]
+            for r in rows
+            if r["env"] == env and r["fault"] == "none" and r["seed"] >= 10
+        ]
         mu, sd = st.mean(band), st.stdev(band)
         free = [r["clean"] for r in rows if r["env"] == env and r["fault"] == "none"]
-        twin = {r["seed"]: r["clean"] for r in rows if r["env"] == env and r["fault"] == "none"}
+        twin = {
+            r["seed"]: r["clean"]
+            for r in rows
+            if r["env"] == env and r["fault"] == "none"
+        }
         for dose in ("one", "0.001", "0.01", "0.1", "1.0"):
-            s = [r for r in rows if r["env"] == env and r["dose"] == dose and r["kind"] == "boundary"
-                 and r["seed"] < 10 and r["silent_curve"] == "1"]
+            s = [
+                r
+                for r in rows
+                if r["env"] == env
+                and r["dose"] == dose
+                and r["kind"] == "boundary"
+                and r["seed"] < 10
+                and r["silent_curve"] == "1"
+            ]
             if not s:
                 continue
             z = [(r["clean"] - mu) / sd for r in s]
-            p = mannwhitneyu([r["clean"] for r in s], free, alternative="two-sided").pvalue
+            p = mannwhitneyu(
+                [r["clean"] for r in s], free, alternative="two-sided"
+            ).pvalue
             gap = st.median(r["clean"] - twin[r["seed"]] for r in s)
-            out.append(f"{env:12} {dose:6} {len(s):3}   {st.mean(z):6.2f}   {st.median(z):8.2f}   {p:8.3f}   {gap:9.1f}")
+            out.append(
+                f"{env:12} {dose:6} {len(s):3}   {st.mean(z):6.2f}   {st.median(z):8.2f}   {p:8.3f}   {gap:9.1f}"
+            )
     text = "\n".join(out)
     (root / "explore.txt").write_text(text + "\n")
     print(text)