|
26 | 26 | from omm_agent_evals import ( |
27 | 27 | CANNED_EXPERIMENT_CODE, |
28 | 28 | CANNED_PAPER, |
| 29 | + CANNED_PROPOSALS_BY_VIEW, |
| 30 | + CANNED_REDUCE, |
29 | 31 | CANNED_VALIDATION_CODE, |
30 | 32 | FULL_CHAIN_CHAT_SEQUENCE, |
31 | 33 | FULL_CHAIN_GOLDEN_EVENT_TYPES, |
32 | 34 | FULL_CHAIN_METRICS, |
33 | 35 | FULL_CHAIN_PROMPT_SEQUENCE, |
34 | 36 | FULL_CHAIN_ROBUSTNESS_CHECKS, |
| 37 | + build_full_chain_llm, |
35 | 38 | build_full_chain_session, |
36 | 39 | robustness_success, |
37 | 40 | sandbox_failure, |
|
42 | 45 | G3_ACCEPT_OPTION_ID, |
43 | 46 | G4_CONFIRM_OPTION_ID, |
44 | 47 | PYTHON_TOOL_NAME, |
| 48 | + stub_response, |
45 | 49 | ) |
46 | 50 |
|
47 | 51 |
|
@@ -375,6 +379,95 @@ def test_review_rejection_then_retry_replans_and_completes(): |
375 | 379 | assert_replay_matches(session) |
376 | 380 |
|
377 | 381 |
|
| 382 | +# -- 4b. G1 三选（H3）：归约出的每一案都是选项，选 B 就按 B 往下做 ---------------- |
| 383 | + |
| 384 | + |
| 385 | +def g1_gate_event(session): |
| 386 | + gates = [ |
| 387 | + event for event in session.sink.events |
| 388 | + if event.event_type is EventType.REVIEW_REQUESTED |
| 389 | + and (event.payload.get("gate") or {}).get("gate") == "G1" |
| 390 | + ] |
| 391 | + return gates[-1] |
| 392 | + |
| 393 | + |
| 394 | +def test_g1_gate_offers_every_reduced_plan_and_adopting_b_flows_downstream(): |
| 395 | + session = build_full_chain_session() |
| 396 | + engine, snapshot = session.engine, session.snapshot |
| 397 | + |
| 398 | + outcome = engine.run_until_blocked(snapshot) |
| 399 | + assert outcome.status == AdvanceOutcome.REVIEW_REQUESTED |
| 400 | + |
| 401 | + gate = g1_gate_event(session).payload["gate"] |
| 402 | + assert gate["decision_type"] == "confirm_plan" |
| 403 | + assert [option["id"] for option in gate["options"]] == ["approve", "adopt:B", "reject"] |
| 404 | + assert [o["id"] for o in gate["options"] if o.get("recommended")] == ["approve"] |
| 405 | + assert gate["title"] == ( |
| 406 | + "请确认建模方案：推荐 A「线性回归 + 整数规划」；备选 B「时间序列 + 启发式」" |
| 407 | + ) |
| 408 | + assert gate["impact"]["proposers"] == { |
| 409 | + "succeeded": ["mechanism", "data_driven", "operations_research"], |
| 410 | + "failed": [], |
| 411 | + } |
| 412 | + assert gate["impact"]["dropped"] == CANNED_REDUCE["dropped"] |
| 413 | + planning = snapshot.outputs[TaskState.MODEL_PLANNING.value] |
| 414 | + assert [plan["role"] for plan in planning["plans"]] == ["primary", "baseline"] |
| 415 | + assert planning["quality_warnings"] == [] |
| 416 | + |
| 417 | + # 用户改选 B：决策进台账，实验任务卡 / 论文材料都按 B 走 |
| 418 | + engine.resolve_review(snapshot, approved=True, reason="adopt:B") |
| 419 | + outcome = confirm_delivery(session, engine.run_until_blocked(snapshot)) |
| 420 | + assert outcome.status == AdvanceOutcome.COMPLETED |
| 421 | + assert snapshot.review_decisions[TaskState.MODEL_PLANNING.value] == "adopt:B" |
| 422 | + experiment_system = next( |
| 423 | + call.messages[0]["content"] |
| 424 | + for call in session.llm.chat_calls |
| 425 | + if call.label == "experiment_code.sandbox" |
| 426 | + ) |
| 427 | + assert '"id": "B"' in experiment_system and "时间序列 + 启发式" in experiment_system |
| 428 | + assert '"id": "A"' not in experiment_system |
| 429 | + paper_calls = [call for call in session.llm.calls if call.prompt_id == "paper_outline.default"] |
| 430 | + assert "时间序列 + 启发式" in paper_calls[0].variables["chosen_plan"] |
| 431 | + |
| 432 | + assert_replay_matches(session) |
| 433 | + |
| 434 | + |
| 435 | +def test_planning_quorum_one_failed_proposer_still_reaches_the_gate_with_a_warning(): |
| 436 | + def flaky_proposer(variables): |
| 437 | + if variables["view_name"] == "数据驱动": |
| 438 | + return "（该视角的模型抽风了，两次都不是 JSON）" |
| 439 | + return stub_response(CANNED_PROPOSALS_BY_VIEW[variables["view_name"]]) |
| 440 | + |
| 441 | + session = build_full_chain_session( |
| 442 | + llm=build_full_chain_llm({"model_planning.proposer": flaky_proposer}) |
| 443 | + ) |
| 444 | + engine, snapshot = session.engine, session.snapshot |
| 445 | + |
| 446 | + outcome = engine.run_until_blocked(snapshot) |
| 447 | + assert outcome.status == AdvanceOutcome.REVIEW_REQUESTED |
| 448 | + |
| 449 | + # quorum：2/3 成功照常归约；缺席的那一路点名进警告、卡片与归约输入 |
| 450 | + planning = snapshot.outputs[TaskState.MODEL_PLANNING.value] |
| 451 | + [failure] = planning["proposer_failures"] |
| 452 | + assert failure.startswith("视角「数据驱动」未成功（failed") |
| 453 | + assert planning["quality_warnings"] == [failure] |
| 454 | + assert [proposal["view"] for proposal in planning["proposals"]] == [ |
| 455 | + "mechanism", "operations_research", |
| 456 | + ] |
| 457 | + gate = g1_gate_event(session).payload["gate"] |
| 458 | + assert gate["title"].endswith("；1 路视角提议未成功") |
| 459 | + assert gate["impact"]["proposers"]["failed"] == [failure] |
| 460 | + prompt_ids = [call.prompt_id for call in session.llm.calls] |
| 461 | + # 失败那一路用掉一次修复重试：3 + 1 次提议调用，归约仍只一次 |
| 462 | + assert prompt_ids.count("model_planning.proposer") == 4 |
| 463 | + assert prompt_ids.count("model_planning.reduce") == 1 |
| 464 | + |
| 465 | + engine.resolve_review(snapshot, approved=True, reason="approve") |
| 466 | + outcome = confirm_delivery(session, engine.run_until_blocked(snapshot)) |
| 467 | + assert outcome.status == AdvanceOutcome.COMPLETED |
| 468 | + assert_replay_matches(session) |
| 469 | + |
| 470 | + |
378 | 471 | # -- 5. experiment failure + retry recovery --------------------------------------- |
379 | 472 |
|
380 | 473 |
|
|
0 commit comments