diff --git a/.changeset/llms-full-and-sampling-fixes.md b/.changeset/llms-full-and-sampling-fixes.md new file mode 100644 index 0000000..241a11a --- /dev/null +++ b/.changeset/llms-full-and-sampling-fixes.md @@ -0,0 +1,14 @@ +--- +"agentimization": minor +"@agentimization/core": minor +--- + +expand audit coverage and fix page sampling: + +- add four llms-full.txt checks (exists, valid structure, size range, links resolve) +- add a dedicated mcp-tool-count check, split out of the mcp server card check +- scope page sampling to the audited path so auditing a sub-path (e.g. /docs) no longer samples unrelated site pages +- expand sitemap indexes to their nested sitemaps so real page URLs are sampled +- make page sampling deterministic so re-runs produce the same score +- fetch sampled pages with an html-only accept header so content-negotiating sites return rendered html instead of the agent markdown variant, fixing false negatives on structured data, open graph, meta description, and link checks +- parallelize the per-page markdown follow-up fetch diff --git a/packages/core/src/checks/agent-protocols.ts b/packages/core/src/checks/agent-protocols.ts index aad7cb3..d022ad4 100644 --- a/packages/core/src/checks/agent-protocols.ts +++ b/packages/core/src/checks/agent-protocols.ts @@ -64,6 +64,65 @@ const mcpServerCard: CheckDefinition = { }, } +// ─── MCP Tool Count ───────────────────────────────────── + +const mcpToolCount: CheckDefinition = { + id: "mcp-tool-count", + name: "MCP Tool Count", + category: "agent-protocols", + description: "Checks that the MCP server card exposes at least one tool", + weight: 0.4, + run: async (ctx) => { + if (!ctx.mcpServerCard) { + return { + id: "mcp-tool-count", + name: "MCP Tool Count", + category: "agent-protocols", + status: "skip", + message: "Skipped: no MCP server card found", + } + } + + let card: { tools?: unknown[]; capabilities?: { tools?: unknown[] } } + try { + card = JSON.parse(ctx.mcpServerCard) + } catch { + return { + id: "mcp-tool-count", + name: "MCP Tool Count", + category: "agent-protocols", + status: "skip", + message: "Skipped: MCP server card is invalid JSON", + } + } + + const toolCount = Array.isArray(card.tools) ? card.tools.length + : Array.isArray(card.capabilities?.tools) ? card.capabilities.tools.length + : 0 + + if (toolCount > 0) { + return { + id: "mcp-tool-count", + name: "MCP Tool Count", + category: "agent-protocols", + status: "pass", + message: `MCP server exposes ${toolCount} tool${toolCount === 1 ? "" : "s"}`, + metadata: { toolCount }, + } + } + + return { + id: "mcp-tool-count", + name: "MCP Tool Count", + category: "agent-protocols", + status: "warn", + message: "MCP server card found but exposes no tools", + suggestion: "List your MCP server's tools in the server card so agents know what actions are available before connecting.", + metadata: { toolCount }, + } + }, +} + // ─── API Catalog (RFC 9727) ───────────────────────────── const apiCatalog: CheckDefinition = { @@ -148,7 +207,7 @@ const contentSignals: CheckDefinition = { name: "Content Signals (AI Usage Declarations)", category: "agent-protocols", status: "info", - message: "No robots.txt found — cannot check for content signals", + message: "No robots.txt found, cannot check for content signals", suggestion: "Add a robots.txt with Content Signals directives to declare how AI agents may use your content (ai-train, ai-input, search).", } } @@ -363,7 +422,7 @@ const agentsMd: CheckDefinition = { category: "agent-protocols", status: "fail", message: "No AGENTS.md or AGENT.md found", - suggestion: "Add an AGENTS.md at the project root. This is the universal agent configuration file — a README for AI coding agents. Include build/test commands, architecture overview, conventions, and any gotchas. Used by 60k+ open-source projects.", + suggestion: "Add an AGENTS.md at the project root. This is the universal agent configuration file, a README for AI coding agents. Include build/test commands, architecture overview, conventions, and any gotchas. Used by 60k+ open-source projects.", } } @@ -422,6 +481,7 @@ const agentsMd: CheckDefinition = { export const agentProtocolChecks: CheckDefinition[] = [ mcpServerCard, + mcpToolCount, apiCatalog, contentSignals, linkHeaders, diff --git a/packages/core/src/checks/content-discoverability.ts b/packages/core/src/checks/content-discoverability.ts index 58a9ae5..2c7298e 100644 --- a/packages/core/src/checks/content-discoverability.ts +++ b/packages/core/src/checks/content-discoverability.ts @@ -1,4 +1,5 @@ -import type { CheckDefinition, CheckResult, AuditContext } from "@agentimization/shared" +import type { CheckDefinition } from "@agentimization/shared" +import { extractMarkdownLinks } from "../utils/html.js" /** Check if llms.txt exists */ const llmsTxtExists: CheckDefinition = { @@ -44,7 +45,7 @@ const llmsTxtValid: CheckDefinition = { name: "llms.txt Valid Structure", category: "content-discoverability", status: "skip", - message: "Skipped — no llms.txt found", + message: "Skipped: no llms.txt found", } } @@ -98,7 +99,7 @@ const llmsTxtSize: CheckDefinition = { name: "llms.txt Size", category: "content-discoverability", status: "skip", - message: "Skipped — no llms.txt found", + message: "Skipped: no llms.txt found", } } @@ -141,7 +142,7 @@ const llmsTxtFreshness: CheckDefinition = { name: "llms.txt Coverage", category: "content-discoverability", status: "skip", - message: "Skipped — no llms.txt found", + message: "Skipped: no llms.txt found", } } @@ -178,11 +179,9 @@ const llmsTxtFreshness: CheckDefinition = { } } - const linkRegex = /\[.+?\]\(([^)]+)\)/g const llmsKeys = new Set() - let match - while ((match = linkRegex.exec(ctx.llmsTxt)) !== null) { - const k = keyFor(match[1]!) + for (const link of extractMarkdownLinks(ctx.llmsTxt)) { + const k = keyFor(link) if (k) llmsKeys.add(k) } @@ -234,7 +233,7 @@ const llmsTxtFreshness: CheckDefinition = { message: `${message}${missingFromLlms > 0 ? ` · ${missingFromLlms} sitemap pages not in llms.txt` : ""}${staleInLlms > 0 ? ` · ${staleInLlms} llms.txt links not in sitemap` : ""}`, suggestion: coveragePct < freshnessPct ? "Add missing sitemap pages to llms.txt to improve AI agent discoverability." - : "Some llms.txt links aren't in the sitemap — they may be stale or your sitemap may be incomplete.", + : "Some llms.txt links aren't in the sitemap. They may be stale or your sitemap may be incomplete.", metadata: { coveragePct, freshnessPct, llmsCount: llmsKeys.size, sitemapCount: sitemapKeys.size, @@ -259,16 +258,14 @@ const llmsTxtLinksResolve: CheckDefinition = { name: "llms.txt Links Resolve", category: "content-discoverability", status: "skip", - message: "Skipped — no llms.txt found", + message: "Skipped: no llms.txt found", } } - const linkRegex = /\[.+?\]\(([^)]+)\)/g const urls: string[] = [] - let match - while ((match = linkRegex.exec(ctx.llmsTxt)) !== null) { + for (const link of extractMarkdownLinks(ctx.llmsTxt)) { try { - const resolved = new URL(match[1]!, ctx.baseUrl.origin) + const resolved = new URL(link, ctx.baseUrl.origin) if (resolved.origin === ctx.baseUrl.origin) { urls.push(resolved.href) } @@ -315,7 +312,7 @@ const llmsTxtLinksResolve: CheckDefinition = { name: "llms.txt Links Resolve", category: "content-discoverability", status: "fail", - message: `${resolved}/${sampled.length} sampled links resolve — ${sampled.length - resolved} broken`, + message: `${resolved}/${sampled.length} sampled links resolve, ${sampled.length - resolved} broken`, suggestion: "Fix broken links in llms.txt. AI agents will fail to fetch these pages.", metadata: { resolved, sampled: sampled.length, total: urls.length }, } @@ -336,16 +333,11 @@ const llmsTxtLinksMarkdown: CheckDefinition = { name: "llms.txt Links Markdown", category: "content-discoverability", status: "skip", - message: "Skipped — no llms.txt found", + message: "Skipped: no llms.txt found", } } - const linkRegex = /\[.+?\]\(([^)]+)\)/g - const urls: string[] = [] - let m - while ((m = linkRegex.exec(ctx.llmsTxt)) !== null) { - urls.push(m[1]!) - } + const urls = extractMarkdownLinks(ctx.llmsTxt) if (urls.length === 0) { return { @@ -399,7 +391,7 @@ const llmsTxtLinksMarkdown: CheckDefinition = { category: "content-discoverability", status: "fail", message: `Only ${mdLinks}/${urls.length} llms.txt links point to .md URLs (${pct}%)`, - suggestion: "Most llms.txt links are HTML-only. Serve a markdown version at .md URLs and link to those — agents get cleaner content and fewer parse failures.", + suggestion: "Most llms.txt links are HTML-only. Serve a markdown version at .md URLs and link to those, so agents get cleaner content and fewer parse failures.", metadata: { mdLinks, total: urls.length, pct }, } }, @@ -507,6 +499,205 @@ const robotsTxtAgentRules: CheckDefinition = { }, } +/** Check if llms-full.txt exists */ +const llmsFullExists: CheckDefinition = { + id: "llms-full-exists", + name: "llms-full.txt Exists", + category: "content-discoverability", + description: "Checks if llms-full.txt (the complete-content variant) is present at the site root", + weight: 0.4, + run: async (ctx) => { + if (ctx.llmsFullTxt) { + return { + id: "llms-full-exists", + name: "llms-full.txt Exists", + category: "content-discoverability", + status: "pass", + message: ctx.mode === "local" + ? "llms-full.txt found in project root" + : `llms-full.txt found at ${ctx.baseUrl.origin}/llms-full.txt`, + } + } + return { + id: "llms-full-exists", + name: "llms-full.txt Exists", + category: "content-discoverability", + status: "info", + message: "No llms-full.txt found (optional)", + suggestion: "If your llms.txt is large or you want agents to get full content in one fetch, add a /llms-full.txt containing the concatenated markdown of your docs.", + } + }, +} + +/** Check if llms-full.txt has recognizable markdown structure */ +const llmsFullValid: CheckDefinition = { + id: "llms-full-valid", + name: "llms-full.txt Valid Structure", + category: "content-discoverability", + description: "Checks if llms-full.txt has recognizable markdown structure (headings, content)", + weight: 0.4, + run: async (ctx) => { + if (!ctx.llmsFullTxt) { + return { + id: "llms-full-valid", + name: "llms-full.txt Valid Structure", + category: "content-discoverability", + status: "skip", + message: "Skipped: no llms-full.txt found", + } + } + + const hasHeadings = /^#{1,3}\s+/m.test(ctx.llmsFullTxt) + const hasProse = ctx.llmsFullTxt.length > 600 + + if (hasHeadings && hasProse) { + return { + id: "llms-full-valid", + name: "llms-full.txt Valid Structure", + category: "content-discoverability", + status: "pass", + message: "llms-full.txt has recognizable markdown structure", + } + } + + return { + id: "llms-full-valid", + name: "llms-full.txt Valid Structure", + category: "content-discoverability", + status: "warn", + message: `llms-full.txt found but ${!hasHeadings ? "has no markdown headings" : "has little content"}`, + suggestion: "llms-full.txt should contain the full markdown content of your docs, with headings, so agents can parse it.", + } + }, +} + +/** Check llms-full.txt size is within the expected range */ +const llmsFullSize: CheckDefinition = { + id: "llms-full-size", + name: "llms-full.txt Size", + category: "content-discoverability", + description: "Checks if llms-full.txt size is within the expected range (substantial but not excessive)", + weight: 0.3, + run: async (ctx) => { + if (!ctx.llmsFullTxt) { + return { + id: "llms-full-size", + name: "llms-full.txt Size", + category: "content-discoverability", + status: "skip", + message: "Skipped: no llms-full.txt found", + } + } + + const size = ctx.llmsFullTxt.length + // llms-full.txt is meant to hold full content: too small means it's not doing its job, + // too large risks blowing past agent context windows. + const MIN = 10_000 + const MAX = 5_000_000 + + if (size >= MIN && size <= MAX) { + return { + id: "llms-full-size", + name: "llms-full.txt Size", + category: "content-discoverability", + status: "pass", + message: `llms-full.txt is ${size.toLocaleString()} characters (within expected range)`, + metadata: { size }, + } + } + + return { + id: "llms-full-size", + name: "llms-full.txt Size", + category: "content-discoverability", + status: "warn", + message: size < MIN + ? `llms-full.txt is only ${size.toLocaleString()} characters, smaller than expected for a full-content file` + : `llms-full.txt is ${size.toLocaleString()} characters, large enough to overflow agent context windows`, + suggestion: size < MIN + ? "llms-full.txt should contain your complete documentation. If it's this small, llms.txt alone may be enough." + : "Consider trimming llms-full.txt or splitting content so agents can fetch what fits their context window.", + metadata: { size }, + } + }, +} + +/** Check if llms-full.txt links resolve */ +const llmsFullLinksResolve: CheckDefinition = { + id: "llms-full-links-resolve", + name: "llms-full.txt Links Resolve", + category: "content-discoverability", + description: "Checks if links in llms-full.txt return 200 OK", + weight: 0.4, + requiresNetwork: true, + run: async (ctx) => { + if (!ctx.llmsFullTxt) { + return { + id: "llms-full-links-resolve", + name: "llms-full.txt Links Resolve", + category: "content-discoverability", + status: "skip", + message: "Skipped: no llms-full.txt found", + } + } + + const urls: string[] = [] + for (const link of extractMarkdownLinks(ctx.llmsFullTxt)) { + try { + const resolved = new URL(link, ctx.baseUrl.origin) + if (resolved.origin === ctx.baseUrl.origin) { + urls.push(resolved.href) + } + } catch { + // skip + } + } + + if (urls.length === 0) { + return { + id: "llms-full-links-resolve", + name: "llms-full.txt Links Resolve", + category: "content-discoverability", + status: "info", + message: "No same-origin links found in llms-full.txt", + } + } + + const sampled = urls.slice(0, 10) + const results = await Promise.allSettled( + sampled.map(async (url) => { + const resp = await fetch(url, { method: "HEAD", redirect: "follow" }) + return { url, status: resp.status } + }), + ) + + const resolved = results.filter( + (r) => r.status === "fulfilled" && r.value.status >= 200 && r.value.status < 400, + ).length + + if (resolved === sampled.length) { + return { + id: "llms-full-links-resolve", + name: "llms-full.txt Links Resolve", + category: "content-discoverability", + status: "pass", + message: `All ${resolved} sampled same-origin links resolve (${urls.length} total links)`, + metadata: { resolved, sampled: sampled.length, total: urls.length }, + } + } + + return { + id: "llms-full-links-resolve", + name: "llms-full.txt Links Resolve", + category: "content-discoverability", + status: "fail", + message: `${resolved}/${sampled.length} sampled links resolve, ${sampled.length - resolved} broken`, + suggestion: "Fix broken links in llms-full.txt. AI agents will fail to fetch these pages.", + metadata: { resolved, sampled: sampled.length, total: urls.length }, + } + }, +} + export const contentDiscoverabilityChecks: CheckDefinition[] = [ llmsTxtExists, llmsTxtValid, @@ -514,6 +705,10 @@ export const contentDiscoverabilityChecks: CheckDefinition[] = [ llmsTxtFreshness, llmsTxtLinksResolve, llmsTxtLinksMarkdown, + llmsFullExists, + llmsFullValid, + llmsFullSize, + llmsFullLinksResolve, sitemapExists, robotsTxtAgentRules, ] diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 4175bc6..0d95ff5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -78,7 +78,7 @@ const buildRemoteContext = async ( const agentsMd = undefined // remote mode: not applicable, the check skips itself - const sitemapUrls = sitemapXml ? parseSitemapUrls(sitemapXml) : [] + let sitemapUrls = sitemapXml ? parseSitemapUrls(sitemapXml) : [] if (!sitemapXml && robotsTxt) { const sitemapMatch = robotsTxt.match(/Sitemap:\s*(.+)/i) @@ -90,11 +90,36 @@ const buildRemoteContext = async ( } } + // sitemap index lists nested sitemaps + const isSitemapIndex = (sitemapXml ?? "").includes(" 0) { + const nested = await Promise.allSettled( + sitemapUrls.slice(0, 20).map((u) => fetchText(u, config)), + ) + sitemapUrls = nested.flatMap((r) => + r.status === "fulfilled" && r.value?.statusCode === 200 + ? parseSitemapUrls(r.value.text) + : [], + ) + } + let pagesToSample: string[] = [] if (sitemapUrls.length > 0) { - const shuffled = [...sitemapUrls].sort(() => Math.random() - 0.5) - pagesToSample = shuffled.slice(0, config.sampleSize) + const pathPrefix = baseUrl.pathname.replace(/\/+$/, "") + const scoped = pathPrefix.length > 1 + ? sitemapUrls.filter((u) => { + try { + return new URL(u).pathname.startsWith(pathPrefix) + } catch { + return false + } + }) + : sitemapUrls + const pool = scoped.length > 0 ? scoped : sitemapUrls + // deterministic, evenly-spread sample so re-runs produce the same score + const step = Math.max(1, Math.floor(pool.length / config.sampleSize)) + pagesToSample = pool.filter((_, i) => i % step === 0).slice(0, config.sampleSize) } else { const mainPage = await fetchPage(targetUrl, config) const linkRegex = /]+href=["']([^"'#]+)["']/gi @@ -117,20 +142,22 @@ const buildRemoteContext = async ( pagesToSample.unshift(targetUrl) } - const sampledPages = await fetchMany(pagesToSample, config) + const sampledPages = await fetchMany(pagesToSample, config, true) emit({ type: "context-ready", pageCount: sampledPages.length }) - for (const page of sampledPages) { - const mdResult = await fetchWithContentNegotiation(page.url, "text/markdown", config) - if ( - mdResult && - mdResult.statusCode === 200 && - (mdResult.contentType.includes("text/markdown") || mdResult.contentType.includes("text/plain")) - ) { - page.markdown = mdResult.text - } - } + await Promise.allSettled( + sampledPages.map(async (page) => { + const mdResult = await fetchWithContentNegotiation(page.url, "text/markdown", config) + if ( + mdResult && + mdResult.statusCode === 200 && + (mdResult.contentType.includes("text/markdown") || mdResult.contentType.includes("text/plain")) + ) { + page.markdown = mdResult.text + } + }), + ) return { mode: "remote", @@ -161,7 +188,7 @@ const stripUndefined = (obj: AgentimizationConfig): AgentimizationConfig => { return result } -/** Shared scoring logic — runs checks against a context and computes results */ +/** Shared scoring logic: runs checks against a context and computes results */ const runAudit = async ( ctx: AuditContext, config: Required, diff --git a/packages/core/src/utils/fetch.ts b/packages/core/src/utils/fetch.ts index eae4146..f36777c 100644 --- a/packages/core/src/utils/fetch.ts +++ b/packages/core/src/utils/fetch.ts @@ -1,14 +1,26 @@ import type { PageSample, AgentimizationConfig } from "@agentimization/shared" import { DEFAULT_CONFIG } from "@agentimization/shared" -const makeHeaders = (config: AgentimizationConfig): Record => ({ - "User-Agent": config.userAgent ?? DEFAULT_CONFIG.userAgent, - Accept: "text/html,application/xhtml+xml,text/markdown,text/plain,*/*", -}) +// html-only Accept (no text/markdown) + browser UA so content-negotiating sites +// serve the real rendered HTML instead of the agent markdown variant +const BROWSER_UA = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + +const makeHeaders = (config: AgentimizationConfig, asBrowser = false): Record => + asBrowser + ? { + "User-Agent": BROWSER_UA, + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + } + : { + "User-Agent": config.userAgent ?? DEFAULT_CONFIG.userAgent, + Accept: "text/html,application/xhtml+xml,text/markdown,text/plain,*/*", + } export const fetchPage = async ( url: string, config: AgentimizationConfig = {}, + asBrowser = false, ): Promise => { const timeout = config.timeout ?? DEFAULT_CONFIG.timeout const start = Date.now() @@ -18,7 +30,7 @@ export const fetchPage = async ( try { const response = await fetch(url, { - headers: makeHeaders(config), + headers: makeHeaders(config, asBrowser), signal: controller.signal, redirect: "follow", }) @@ -90,6 +102,7 @@ export const fetchWithContentNegotiation = async ( export const fetchMany = async ( urls: string[], config: AgentimizationConfig = {}, + asBrowser = false, ): Promise => { const concurrency = config.concurrency ?? DEFAULT_CONFIG.concurrency const results: PageSample[] = [] @@ -97,7 +110,7 @@ export const fetchMany = async ( for (let i = 0; i < urls.length; i += concurrency) { const chunk = urls.slice(i, i + concurrency) const chunkResults = await Promise.allSettled( - chunk.map((url) => fetchPage(url, config)), + chunk.map((url) => fetchPage(url, config, asBrowser)), ) for (const result of chunkResults) { diff --git a/packages/core/src/utils/html.ts b/packages/core/src/utils/html.ts index 586cc96..4df2c3b 100644 --- a/packages/core/src/utils/html.ts +++ b/packages/core/src/utils/html.ts @@ -1,4 +1,4 @@ -/** Lightweight HTML parsing utilities — no heavy DOM dependencies */ +/** Lightweight HTML parsing utilities (no heavy DOM dependencies) */ /** Extract text content from HTML, stripping all tags */ export const stripHtml = (html: string): string => @@ -9,6 +9,19 @@ export const stripHtml = (html: string): string => .replace(/\s+/g, " ") .trim() +/** Extract the raw target URLs from markdown links `[text](url)` */ +export const extractMarkdownLinks = (markdown: string): string[] => { + const links: string[] = [] + const linkRegex = /\[.+?\]\(([^)]+)\)/g + let match + + while ((match = linkRegex.exec(markdown)) !== null) { + links.push(match[1]!) + } + + return links +} + /** Extract all links from HTML */ export const extractLinks = (html: string, baseUrl: string): string[] => { const links: string[] = []