diff --git a/skills/deeppapernote/references/user-configuration.md b/skills/deeppapernote/references/user-configuration.md index b0c7cc1..63f6aae 100644 --- a/skills/deeppapernote/references/user-configuration.md +++ b/skills/deeppapernote/references/user-configuration.md @@ -2,6 +2,8 @@ Use one device-local User Configuration at `~/.deeppapernote/config.json`: +For isolated validation only, `DEEPPAPERNOTE_CONFIG_PATH` may relocate this one file for the current process. It is not a preference, does not create a second configuration layer, and must not be persisted into the configuration itself. + - `output_language`: `zh-CN` or `en` - `save_mode`: `workspace` or `obsidian` - `obsidian_vault`: existing absolute directory, required only in Obsidian mode diff --git a/skills/deeppapernote/scripts/common.py b/skills/deeppapernote/scripts/common.py index 3130f88..024395c 100644 --- a/skills/deeppapernote/scripts/common.py +++ b/skills/deeppapernote/scripts/common.py @@ -1265,6 +1265,25 @@ def _is_unique_exact_zotero_title_observation( ) +def _is_unique_exact_arxiv_title_observation( + anchor: dict[str, Any], + item: dict[str, Any], + observation: dict[str, Any], +) -> bool: + relation = item.get("relation") + if not isinstance(relation, dict): + return False + return bool( + _string_field(item, "provider").lower() == "arxiv" + and _string_field(relation, "kind") == "arxiv_lookup" + and _string_field(relation, "match_kind") == "title" + and _string_field(relation, "match_resolution") == "unique_exact" + and _record_arxiv_id(observation) + and normalize_identity_title(_string_field(anchor, "title")) + == normalize_identity_title(_string_field(observation, "title")) + ) + + def adjudicate_identity_observations( anchor: dict[str, Any], observations: list[Any], @@ -1336,6 +1355,11 @@ def adjudicate_identity_observations( item, observation, ) + unique_exact_arxiv_title_match = _is_unique_exact_arxiv_title_observation( + anchor, + item, + observation, + ) if not title_author_year_match and not shared_identifiers: observation_provider = ( _string_field(item, "provider") @@ -1379,6 +1403,7 @@ def adjudicate_identity_observations( not shared_identifiers and not title_author_year_match and not unique_exact_zotero_title_match + and not unique_exact_arxiv_title_match ): rejected_observations.append( _identity_observation_summary( @@ -1393,6 +1418,8 @@ def adjudicate_identity_observations( acceptance_reason = "shared_identifier" elif unique_exact_zotero_title_match: acceptance_reason = "unique_exact_zotero_title" + elif unique_exact_arxiv_title_match: + acceptance_reason = "unique_exact_arxiv_title" else: acceptance_reason = "title_author_year" summary = _identity_observation_summary( @@ -2346,7 +2373,13 @@ def collect_metadata_observations(record: dict[str, Any]) -> list[dict[str, Any] title = normalize_whitespace(str(base.get("title", ""))) arxiv_id = normalize_whitespace(str(base.get("arxiv_id", ""))) - def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | None) -> None: + def append( + provider: str, + kind: str, + value: str, + candidate: dict[str, Any] | None, + relation: dict[str, str] | None = None, + ) -> None: if not candidate: return observation = { @@ -2354,6 +2387,8 @@ def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | Non "retrieved_by": {"kind": kind, "value": value}, "record": deepcopy(candidate), } + if relation: + observation["relation"] = relation if observation not in observations: observations.append(observation) @@ -2377,11 +2412,26 @@ def append(provider: str, kind: str, value: str, candidate: dict[str, Any] | Non append("openalex", "title", title, oa) cross = choose_best_title_match(title, search_crossref_by_title(title, limit=5)) append("crossref", "title", title, cross) - arxiv = choose_best_title_match( - title, - safe_fetch_arxiv_entries(search_query=f'ti:"{title}"', max_results=5), + arxiv_candidates = safe_fetch_arxiv_entries( + search_query=f'ti:"{title}"', max_results=5 + ) + arxiv = choose_best_title_match(title, arxiv_candidates) + exact_matches = [ + candidate + for candidate in arxiv_candidates + if normalize_identity_title(_string_field(candidate, "title")) + == normalize_identity_title(title) + ] + relation = ( + { + "kind": "arxiv_lookup", + "match_kind": "title", + "match_resolution": "unique_exact", + } + if arxiv is not None and len(exact_matches) == 1 and arxiv == exact_matches[0] + else None ) - append("arxiv", "title", title, arxiv) + append("arxiv", "title", title, arxiv, relation) return observations diff --git a/skills/deeppapernote/scripts/run_pipeline.py b/skills/deeppapernote/scripts/run_pipeline.py index b5b8424..cadc46b 100644 --- a/skills/deeppapernote/scripts/run_pipeline.py +++ b/skills/deeppapernote/scripts/run_pipeline.py @@ -82,11 +82,13 @@ def main() -> None: identity_json = workdir / f"{args.prefix}_identity.json" identity_trace_json = workdir / f"{args.prefix}_identity_repair_trace.json" fetch_json = workdir / f"{args.prefix}_fetch.json" + pdf_dir = workdir / f"{args.prefix}_pdfs" source_manifest_json = workdir / f"{args.prefix}_source_manifest.json" raw_sections_jsonl = workdir / f"{args.prefix}_raw_sections.jsonl" full_text_md = workdir / f"{args.prefix}_full_text.md" evidence_json = workdir / f"{args.prefix}_evidence.json" assets_json = workdir / f"{args.prefix}_assets.json" + assets_dir = workdir / f"{args.prefix}_assets" figures_json = workdir / f"{args.prefix}_figures.json" figure_decisions_json = workdir / f"{args.prefix}_figure_table_decisions.json" bundle_json = workdir / f"{args.prefix}_bundle.json" @@ -138,6 +140,8 @@ def main() -> None: str(metadata_json), "--identity", str(identity_json), + "--dest-dir", + str(pdf_dir), "--output", str(fetch_json), ], @@ -177,6 +181,8 @@ def main() -> None: str(scripts_dir / "extract_pdf_assets.py"), "--input", str(fetch_json), + "--assets-dir", + str(assets_dir), "--output", str(assets_json), ], diff --git a/skills/deeppapernote/scripts/user_configuration.py b/skills/deeppapernote/scripts/user_configuration.py index be44e1d..9811cf3 100644 --- a/skills/deeppapernote/scripts/user_configuration.py +++ b/skills/deeppapernote/scripts/user_configuration.py @@ -42,6 +42,9 @@ def __init__( def user_config_path() -> Path: + override = os.environ.get("DEEPPAPERNOTE_CONFIG_PATH", "").strip() + if override: + return Path(override).expanduser() return Path.home() / ".deeppapernote" / "config.json" diff --git a/skills/deeppapernote/scripts/write_obsidian_note.py b/skills/deeppapernote/scripts/write_obsidian_note.py index bd52c06..d659d8b 100644 --- a/skills/deeppapernote/scripts/write_obsidian_note.py +++ b/skills/deeppapernote/scripts/write_obsidian_note.py @@ -305,7 +305,7 @@ def main() -> None: if figure_decisions else [] ) - Path(target_path).write_text(note_text, encoding="utf-8") + Path(target_path).write_text(note_text, encoding="utf-8", newline="\n") require_reference_hygiene(Path(target_path).read_text(encoding="utf-8"), "after save") asset_dir.mkdir(parents=True, exist_ok=True) diff --git a/tests/conftest.py b/tests/conftest.py index 72addde..9430268 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,11 +15,24 @@ @pytest.fixture(autouse=True) def configured_user_home(tmp_path: Path, monkeypatch) -> Path: - monkeypatch.setenv("HOME", str(tmp_path)) config_path = tmp_path / ".deeppapernote" / "config.json" config_path.parent.mkdir(exist_ok=True) + for name in ( + "DEEPPAPERNOTE_OUTPUT_LANGUAGE", + "DEEPPAPERNOTE_SAVE_MODE", + "DEEPPAPERNOTE_OBSIDIAN_VAULT", + "DEEPPAPERNOTE_PAPERS_DIR", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(config_path)) config_path.write_text( - json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + json.dumps( + { + "output_language": "zh-CN", + "save_mode": "workspace", + "papers_dir": "Research/Papers", + } + ), encoding="utf-8", ) return config_path diff --git a/tests/test_acquisition_artifacts.py b/tests/test_acquisition_artifacts.py index bc9da64..aefebb8 100644 --- a/tests/test_acquisition_artifacts.py +++ b/tests/test_acquisition_artifacts.py @@ -1505,6 +1505,57 @@ def test_collect_metadata_preserves_provider_result_as_identity_observation( ] +def test_exact_swe_bench_title_admits_one_unique_arxiv_match( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + title = "SWE-bench: Can Language Models Resolve Real-world Github Issues?" + resolve_payload = { + "status": "ok", + "script": "resolve_paper.py", + "paper_id": "title:2156776fbf55", + "source_type": "title_query", + "title": title, + "metadata_sources": ["title_query"], + } + monkeypatch.setattr(common, "search_semantic_scholar", lambda *args, **kwargs: []) + monkeypatch.setattr(common, "search_openalex_by_title", lambda *args, **kwargs: []) + monkeypatch.setattr(common, "search_crossref_by_title", lambda *args, **kwargs: []) + monkeypatch.setattr( + common, + "safe_fetch_arxiv_entries", + lambda **kwargs: [ + { + "source": "arxiv", + "source_type": "arxiv", + "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", + "authors": ["Carlos E. Jimenez"], + "published": "2023-10-10T16:47:29Z", + "arxiv_id": "2310.06770", + "pdf_url": "https://arxiv.org/pdf/2310.06770v3", + } + ], + ) + + observations = common.collect_metadata_observations(resolve_payload) + identity, _ = build_identity_from_payloads( + tmp_path, + monkeypatch, + resolve_payload=resolve_payload, + metadata_payload={ + **resolve_payload, + "script": "collect_metadata.py", + "identity_observations": observations, + }, + ) + + assert identity["identity_verdict"] == "accepted" + assert identity["accepted_metadata"]["arxiv_id"] == "2310.06770" + assert identity["accepted_observations"][0]["reason"] == ( + "unique_exact_arxiv_title" + ) + + @pytest.mark.parametrize( "module", [extract_source_text, extract_evidence, extract_pdf_assets], diff --git a/tests/test_run_pipeline_manifest.py b/tests/test_run_pipeline_manifest.py index 8f6c7d2..e545227 100644 --- a/tests/test_run_pipeline_manifest.py +++ b/tests/test_run_pipeline_manifest.py @@ -253,12 +253,19 @@ def fake_run(cmd: list[str], check: bool = True, **kwargs) -> object: assert fetch_call[fetch_call.index("--identity") + 1] == str( (workdir / "paper_identity.json").resolve() ) + assert fetch_call[fetch_call.index("--dest-dir") + 1] == str( + (workdir / "paper_pdfs").resolve() + ) evidence_call = calls[5] assert "--source-manifest" in evidence_call assert evidence_call[evidence_call.index("--source-manifest") + 1] == str( (workdir / "paper_source_manifest.json").resolve() ) + assets_call = calls[6] + assert assets_call[assets_call.index("--assets-dir") + 1] == str( + (workdir / "paper_assets").resolve() + ) def test_run_pipeline_stops_at_configuration_before_identity( diff --git a/tests/test_user_configuration.py b/tests/test_user_configuration.py index 438774e..737cdd7 100644 --- a/tests/test_user_configuration.py +++ b/tests/test_user_configuration.py @@ -15,6 +15,7 @@ inspect_configuration, persist_preferences, resolve_preferences, + user_config_path, ) PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -22,6 +23,15 @@ ENVIRONMENT_SCRIPT = PROJECT_ROOT / "skills/deeppapernote/scripts/check_environment.py" +def test_user_config_path_honors_process_isolation_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + isolated_path = tmp_path / "isolated" / "config.json" + monkeypatch.setenv("DEEPPAPERNOTE_CONFIG_PATH", str(isolated_path)) + + assert user_config_path() == isolated_path + + def test_first_use_requests_one_workspace_prompt_batch(tmp_path: Path) -> None: result = inspect_configuration(config_path=tmp_path / "config.json", environ={}) @@ -500,6 +510,10 @@ def test_output_language_reference_matches_both_machine_schemas() -> None: def test_configuration_cli_keeps_semantic_failures_repairable( configured_user_home: Path, ) -> None: + configured_user_home.write_text( + json.dumps({"output_language": "zh-CN", "save_mode": "workspace"}), + encoding="utf-8", + ) result = subprocess.run( [sys.executable, str(CONFIG_SCRIPT), "--set-save-mode", "obsidian"], env=os.environ.copy(), @@ -520,7 +534,7 @@ def test_environment_report_survives_missing_user_configuration( home = tmp_path / "empty-home" home.mkdir() env = os.environ.copy() - env["HOME"] = str(home) + env["DEEPPAPERNOTE_CONFIG_PATH"] = str(home / ".deeppapernote" / "config.json") result = subprocess.run( [sys.executable, str(ENVIRONMENT_SCRIPT)], env=env,