fix: harden evidence provenance

Author: Coding-Dev-Tools

eval/benchmark.py

@@ -256,7 +256,9 @@ def _redact_url(value: str) -> str:
     try:
         parsed = urlsplit(value)
     except ValueError:
-        return value
+        # An invalid authority can still contain credentials. Without a trustworthy parse,
+        # preserve neither the authority nor the rest of the URL in public evidence.
+        return "<redacted>"
     if not parsed.scheme or not parsed.netloc:
         return value
 

eval/longmemeval_v2_evidence.py

@@ -171,6 +171,10 @@ def build_evidence_report(
     per_question = Path(per_question_path)
     if re.fullmatch(r"[0-9a-f]{40}", reader_revision) is None:
         raise ValueError("reader_revision must be an immutable lowercase 40-character commit")
+    if bool(evaluator_model) != bool(evaluator_revision):
+        raise ValueError("evaluator_model and evaluator_revision must be used together")
+    if evaluator_revision and re.fullmatch(r"[0-9a-f]{40}", evaluator_revision) is None:
+        raise ValueError("evaluator_revision must be an immutable lowercase 40-character commit")
     source_paths = [
         per_question,
         Path(haystack_path),

tests/test_benchmark_evidence.py

@@ -442,6 +442,10 @@ def test_command_provenance_redacts_userinfo_when_a_url_port_is_malformed():
     ]
 
 
+def test_command_provenance_fails_closed_when_url_splitting_rejects_userinfo():
+    assert redact_command(["https://user:password@[invalid/path"]) == ["<redacted>"]
+
+
 def test_canonical_profile_validator_and_immutable_artifact_writer(tmp_path):
     dataset = tmp_path / "fixture.jsonl"
     dataset.write_text('{"id":"one"}\n', encoding="utf-8")

tests/test_longmemeval_v2_evidence.py

@@ -106,6 +106,28 @@ def test_official_v2_evidence_export_rejects_unpinned_reader_metadata(tmp_path):
         )
 
 
+@pytest.mark.parametrize(
+    ("evaluator_model", "evaluator_revision", "message"),
+    [
+        ("example/evaluator", None, "must be used together"),
+        ("example/evaluator", "main", "immutable lowercase 40-character commit"),
+    ],
+)
+def test_official_v2_evidence_export_requires_a_pinned_evaluator(
+    evaluator_model, evaluator_revision, message,
+):
+    with pytest.raises(ValueError, match=message):
+        build_evidence_report(
+            per_question_path="unused.jsonl",
+            questions_path="unused-questions.json",
+            haystack_path="unused-haystack.json",
+            trajectories_path="unused-trajectories.json",
+            memory_config_path="unused-memory.json",
+            evaluator_model=evaluator_model,
+            evaluator_revision=evaluator_revision,
+        )
+
+
 def test_official_v2_evidence_export_rejects_malformed_measured_fields(tmp_path):
     source_paths = [
         _write_json(tmp_path / name, {} if name != "questions.json" else [])