diff --git a/.github/workflows/real-world-smoke.yml b/.github/workflows/real-world-smoke.yml new file mode 100644 index 0000000..8fdce60 --- /dev/null +++ b/.github/workflows/real-world-smoke.yml @@ -0,0 +1,40 @@ +name: real-world smoke + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + pinned-repositories: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Replay the maintained pinned corpus + env: + PYTHONPATH: src + GIT_TERMINAL_PROMPT: "0" + shell: bash + run: | + set -euo pipefail + mkdir -p .agent-config-score + python -u scripts/run_real_world_benchmark.py \ + --output .agent-config-score/real-world-smoke.json \ + 2>&1 | tee .agent-config-score/real-world-smoke.log + - name: Retain results or acquisition-failure logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: real-world-smoke-${{ github.sha }} + path: | + .agent-config-score/real-world-smoke.json + .agent-config-score/real-world-smoke.log + if-no-files-found: warn + retention-days: 7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e53c40..2e724cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +### Added + +- An explicitly manual, read-only GitHub Actions replay of the existing pinned real-repository smoke corpus. The same runner propagates acquisition/mismatch failures and retains logs/results for seven days; it does not execute external repository code, run on every PR or establish independent accuracy/adoption. + +### Fixed + +- Check simple full/collapsed Markdown reference links and images against document-relative local targets, reporting a used target at its definition line. Respect document-local, first-definition precedence, skip unused definitions and avoid treating supported definitions/link captions as extra prose paths. Code, ambiguous containers/layouts and unsupported Markdown forms retain explicit coverage limits. Stable rule IDs/penalties, runtime dependencies, published v0.24.0 artifacts and frozen research records are unchanged. + ## v0.24.0 A maintainer-focused Beta release: actionable findings, opt-in safeguard review, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 886b6c0..647055b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,15 @@ python scripts/run_real_world_benchmark.py The benchmark requires Git and network access only for cloning its public, pinned inputs. It does not execute code from those repositories. See [`benchmarks/README.md`](benchmarks/README.md) for its scope and limitations. +If local Git downloads are blocked, a maintainer can select **real-world smoke** +in GitHub Actions and use **Run workflow**, choosing the revision to check. The +optional workflow uses this same corpus/runner with read-only permissions, no +runtime dependencies and no external repository code execution. It is manual, +not a new network gate on every PR. A failed acquisition stays failed; the run +retains its log (and a result JSON only if produced) for seven days. Check the +run's exact commit, not just its branch name. Passing this maintained smoke suite +is not an independent accuracy result or evidence of user adoption. + Regenerate the README demo after changing its example output or visual design: ```bash diff --git a/docs/limitations.md b/docs/limitations.md index d128775..773a977 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -116,9 +116,30 @@ file's directory. For example, `[Guide](../CONTRIBUTING.md#setup)` checks that t file exists inside the repository; it does not validate the `setup` heading. URL-encoded spaces and angle-bracket destinations are supported. Remote links, anchor-only links, fenced examples, absolute paths, and paths escaping the -repository are skipped. Reference-style links and nested parentheses in link -destinations are not currently parsed. This is a limited link check, not a full -Markdown validator. +repository are skipped. Nested parentheses in inline link destinations are not +currently parsed. This is a limited link check, not a full Markdown validator. + +Unreleased source builds additionally check simple full `[Guide][setup]` and +collapsed `[setup][]` reference links, including images, when a matching +single-line definition such as `[setup]: GUIDE.md` exists in the same instruction +file. Paths stay document-relative; a missing used target is reported once at +its definition line. Labels use Unicode case folding and ASCII whitespace +normalization; the first matching definition wins, including external/anchor +targets. This follows the relevant [CommonMark reference-link conventions](https://spec.commonmark.org/0.31.2/#reference-link), +not full parser conformance. Unused supported definitions and resolved captions +are not separately treated as prose paths. These changes are not in v0.24.0. + +This subset accepts at most 999 label characters without escapes/nested brackets, +zero to three spaces before top-level definitions, bare/angle destinations and +optional single-line quoted/parenthesized titles. Definitions must start a block +or follow another supported definition; they cannot interrupt a paragraph. +Fenced/indented code, escaped links and matched inline code do not activate new +reference checks. Shortcut `[setup]`, multiline definitions/labels, container +definitions, escaped or parenthesized destinations and HTML-like layouts need +manual review. Unknown first definitions do not select later duplicates. Inline +code recognition is conservative, not a complete block parser; unsupported forms +may remain unchecked. Existing inline/prose checks and independent danger rules +still run; reference handling does not exempt commands or certify file contents. ## Plain prose path references diff --git a/docs/maintainer-workflow.md b/docs/maintainer-workflow.md index 1cac656..b4fa817 100644 --- a/docs/maintainer-workflow.md +++ b/docs/maintainer-workflow.md @@ -130,6 +130,12 @@ rollback is attempted. Alias layouts need manual integration, not `--force`. ## Coverage is deliberately bounded +Unreleased source builds also check simple `[Guide][setup]` / `[setup][]` links +with single-line definitions. Missing targets point to the definition line you +can review/edit; restoring the target or correcting that definition can then +pass recheck. This is not in v0.24.0. Shortcut, container and complex/HTML forms +still need manual review. See [Markdown link limits](limitations.md#local-markdown-links). + Path checks cover the [documented reference grammar](limitations.md#plain-prose-path-references), not every possible reference, generated artifact, or shell command argument. Prompt-injection flagging recognizes a narrow override grammar and certain diff --git a/docs/simplified/limitations.md b/docs/simplified/limitations.md index 2cd6b89..8834622 100644 --- a/docs/simplified/limitations.md +++ b/docs/simplified/limitations.md @@ -50,7 +50,11 @@ v0.24.0+ 提供[工具源码身份(英文)](../report-identity.md),用于 ## 本地 Markdown 链接 -简单的行内链接、图片目标相对于指令文件目录检查。URL 编码空格和尖括号目标受支持;只检查文件是否存在,不检查片段对应的标题。远程链接、纯锚点、代码块示例、绝对路径和逃出仓库的路径会跳过。引用式链接及目标中嵌套括号暂不解析。这不是完整 Markdown 校验器。 +简单的行内链接、图片目标相对于指令文件目录检查。URL 编码空格和尖括号目标受支持;只检查文件是否存在,不检查片段对应的标题。远程链接、纯锚点、代码块示例、绝对路径和逃出仓库的路径会跳过。行内目标中嵌套括号暂不解析。这不是完整 Markdown 校验器。 + +尚未发布的源码还检查简单的 `[指南][setup]`、`[setup][]` 引用式链接及图片,要求同一指令文件里有 `[setup]: GUIDE.md` 这样的单行定义。目标仍严格相对于文档目录;实际使用的缺失目标在定义行报告一次。标签忽略 Unicode 大小写并规范 ASCII 空白,首个匹配定义优先(包括远程目标和纯锚点);未使用的已支持定义、已解析的显示文本不会重复当成散文路径。这些变化不在 v0.24.0 中。 + +此子集只处理无转义/嵌套括号、最长 999 字符的标签,缩进零至三个空格的顶层单行定义,以及裸目标/尖括号目标和可选的单行标题;定义不能打断普通段落。代码块、缩进代码、已转义链接及匹配的行内代码不激活新的引用检查。简写 `[setup]`、跨行语法、容器内定义、含转义或括号的目标、HTML 类布局须人工检查;首个定义不明确时不会猜用后续副本。现有行内/散文检查和危险命令规则仍运行;它不是完整 Markdown 解析器,也不证明目标内容正确。[详细边界(英文)](../limitations.md#local-markdown-links)。 ## 普通散文中的路径 diff --git a/docs/traditional/limitations.md b/docs/traditional/limitations.md index b9a0b55..55bc44a 100644 --- a/docs/traditional/limitations.md +++ b/docs/traditional/limitations.md @@ -50,7 +50,11 @@ v0.24.0+ 提供[工具原始碼身分(英文)](../report-identity.md),用 ## 本機 Markdown 連結 -簡單的行內連結、圖片目標依指令檔目錄檢查。支援 URL 編碼空格與尖括號目標;只檢查檔案是否存在,不檢查片段對應的標題。遠端連結、純錨點、程式碼區塊範例、絕對路徑與離開儲存庫的路徑會略過。參照式連結及目標中巢狀括號暫不解析。這不是完整 Markdown 驗證工具。 +簡單的行內連結、圖片目標依指令檔目錄檢查。支援 URL 編碼空格與尖括號目標;只檢查檔案是否存在,不檢查片段對應的標題。遠端連結、純錨點、程式碼區塊範例、絕對路徑與離開儲存庫的路徑會略過。行內目標中巢狀括號暫不解析。這不是完整 Markdown 驗證工具。 + +尚未發布的原始碼也檢查簡單的 `[指南][setup]`、`[setup][]` 參照式連結及圖片,要求同一指令檔內有 `[setup]: GUIDE.md` 這類單行定義。目標仍嚴格依文件目錄解析;實際使用的缺失目標在定義行回報一次。標籤忽略 Unicode 大小寫並正規化 ASCII 空白,首個相符定義優先(包括遠端目標及純錨點);未使用的已支援定義、已解析的顯示文字不會重複視為敘述路徑。這些變更不在 v0.24.0 中。 + +此子集只處理無跳脫/巢狀括號、最長 999 字元的標籤,縮排零至三個空格的頂層單行定義,以及裸目標/尖括號目標和選用的單行標題;定義不能打斷一般段落。程式碼區塊、縮排程式碼、已跳脫連結及相符的行內程式碼不啟用新的參照檢查。簡寫 `[setup]`、跨行語法、容器內定義、含跳脫或括號的目標、HTML 類配置須人工檢查;首個定義不明確時不會猜用後續副本。既有行內/敘述檢查及危險命令規則仍執行;它不是完整 Markdown 解析器,也不證明目標內容正確。[詳細界線(英文)](../limitations.md#local-markdown-links)。 ## 一般敘述中的路徑 diff --git a/docs/translations.md b/docs/translations.md index 172651b..7f90cd3 100644 --- a/docs/translations.md +++ b/docs/translations.md @@ -103,6 +103,14 @@ does not bundle offline documentation or translated terminal messages. Earlier sync notes describe their historical state; the v0.23.0 artifacts and frozen research records remain unchanged. +After v0.24.0 on 2026-10-23, built on +`5d00ab1d5ce64f88fa392df1d0ba396bca98f8de`, both limitation summaries were +synchronized for the bounded reference-link check: document-local definitions, +first-definition precedence, definition-line reporting, unused-definition/caption +handling and unsupported Markdown/code/layout boundaries. This is an unreleased +source update, not a change to v0.24.0 packages, tags, rolling Action refs or +frozen research evidence. Executable guide examples are unchanged. + When an English source page changes, review both variants together and record the new source revision here. Preserve executable examples, rule IDs and JSON keys verbatim. Keep the score limitations and failure/exit semantics aligned; diff --git a/scripts/verify_installed_package.py b/scripts/verify_installed_package.py index d2c63e5..1730948 100644 --- a/scripts/verify_installed_package.py +++ b/scripts/verify_installed_package.py @@ -186,6 +186,25 @@ def verify(python: Path, expected_version: str) -> list[str]: "Fixed maintainer fixture did not pass recheck") checks.append("path/override finding, explanatory scan and Git regression, JSON compatibility, fix and recheck") + reference_target = repo / "REFERENCE.md" + reference_target.write_text("Synthetic guide\n", encoding="utf-8") + reference_text = "Read [build guide][setup].\n\n[setup]: REFERENCE.md\n" + instructions.write_text(reference_text, encoding="utf-8") + require(json.loads(run([*cli, ".", "--json"], repo))["findings"] == [], + "Existing reference-link target was reported missing") + reference_target.unlink() + reference_scan = json.loads(run([*cli, ".", "--json", "--explain"], repo)) + require([(f["code"], f["line"]) for f in reference_scan["findings"]] == [("dead-path", 3)], + "Missing reference-link target was not located at its definition") + reference_diff = json.loads(run([*cli, "diff", "--json"], repo, expected=1)) + require([f["code"] for f in reference_diff["new_findings"]] == ["dead-path"], + "Missing reference target did not fail the baseline score-drop gate") + reference_target.write_text("Restored guide\n", encoding="utf-8") + require(json.loads(run([*cli, ".", "--json"], repo))["findings"] == [], + "Restored reference target did not pass recheck") + instructions.write_text(clean_text, encoding="utf-8") + checks.append("reference-link target existence, definition location, regression blocking and fix/recheck") + preview = json.loads(run([*cli, "guardrails", ".", "--json"], repo)) require(preview["status"] == "review_needed" and preview["advisory_only"] is True, "Guardrail preview was not advisory") diff --git a/src/agent_config_score/scanner.py b/src/agent_config_score/scanner.py index 7828fd4..bc99bbb 100644 --- a/src/agent_config_score/scanner.py +++ b/src/agent_config_score/scanner.py @@ -881,14 +881,158 @@ def _looks_like_repository_path(value: str) -> bool: MARKDOWN_LINK = re.compile(r'!?\[[^\]\n]*\]\(\s*(?:<([^>\n]+)>|([^\s()]+))(?:\s+"[^"\n]*")?\s*\)') +MARKDOWN_REFERENCE = re.compile(r'!?\[([^\[\]\n]{0,999})\]\[([^\[\]\n]{0,999})\]') +REFERENCE_HEADER = re.compile(r' {0,3}\[([^\[\]\\\n]{1,999})\]:') +REFERENCE_DEFINITION = re.compile( + r' {0,3}\[([^\[\]\\\n]{1,999})\]:[ \t]*' + r'(?:<([^<>\\\n]*)>|([^\s<>\\()]+))' + r'''(?:[ \t]+(?:"[^"\\\n]*"|'[^'\\\n]*'|\([^()\\\n]*\)))?[ \t]*''' +) +REFERENCE_HTML = re.compile(r'", "
[guide][setup]
", + 'Example'): + with self.subTest(markup=markup): + self.assertEqual(self.dead(f"{markup}\n\n[setup]: ABSENT.md\n"), []) + + def test_reference_syntax_never_exempts_independent_danger_checks(self): + report = self.scan("Run rm -rf cache. Read [guide][setup].\n\n[setup]: GUIDE.md\n", + existing=("GUIDE.md",)) + self.assertIn("rm-rf", [f.code for f in report.findings]) + + def test_unsupported_first_definition_does_not_select_later_duplicate(self): + for first in ("[setup]: path(with).md", "> [setup]: GUIDE.md", "- [setup]: GUIDE.md"): + with self.subTest(first=first): + self.assertEqual(self.dead(f"Read [guide][setup].\n\n{first}\n\n" + "[setup]: ABSENT.md\n", existing=("GUIDE.md",)), []) + + def test_definition_scope_does_not_cross_instruction_files(self): + (self.root / "CLAUDE.md").write_text("[setup]: ABSENT.md\n", encoding="utf-8") + self.assertEqual(self.dead("Read [guide][setup].\n"), []) + + def test_encoded_entities_and_crlf_keep_definition_location(self): + self.assertEqual(self.dead("Read [guide][setup].\r\n\r\n[setup]: missing&guide.md\r\n"), + [(3, "Referenced path does not exist: missing&guide.md")]) + + def test_indented_code_and_unclosed_fences_do_not_activate_references(self): + self.assertEqual(self.dead(" [guide][setup]\n\n[setup]: ABSENT.md\n"), []) + self.assertEqual(self.dead("```md\n[guide][setup]\n\n[setup]: ABSENT.md\n"), []) + + def test_large_reference_list_is_deduplicated_at_definition_locations(self): + text = "\n".join(f"Read [guide][ref-{i}]." for i in range(600)) + "\n\n" + text += "\n".join(f"[ref-{i}]: EXISTING.md" for i in range(600)) + "\n" + self.assertEqual(self.dead(text, existing=("EXISTING.md",)), []) + + def test_removing_referenced_target_fails_the_baseline_gate_then_fix_passes(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + text = "Read [guide][setup].\n\n[setup]: GUIDE.md\n" + (base / "AGENTS.md").write_text(text, encoding="utf-8") + (base / "GUIDE.md").write_text("Guide\n", encoding="utf-8") + self.scan(text) + changed = compare(base, self.root) + self.assertLess(changed.delta, 0) + self.assertEqual([f.code for f in changed.new_findings], ["dead-path"]) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(main(["compare", str(base), str(self.root), "--max-drop", "0"]), 1) + self.scan(text, existing=("GUIDE.md",)) + self.assertEqual(compare(base, self.root).new_findings, []) + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(main(["compare", str(base), str(self.root), "--max-drop", "0"]), 0) + + def test_cli_json_explanations_and_sarif_use_the_definition_location(self): + self.scan("Read [guide][setup].\n\n[setup]: ABSENT.md\n") + sarif = self.root / "report.sarif" + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = main([str(self.root), "--json", "--explain", "--sarif", str(sarif)]) + self.assertEqual((code, stderr.getvalue()), (0, "")) + findings = json.loads(stdout.getvalue())["findings"] + self.assertEqual([(f["code"], f["line"]) for f in findings], [("dead-path", 3)]) + result = json.loads(sarif.read_text(encoding="utf-8"))["runs"][0]["results"][0] + self.assertEqual(result["locations"][0]["physicalLocation"]["region"]["startLine"], 3) + + +if __name__ == "__main__": + unittest.main()