From 8e99f8c555a851653e267038def87eb0624b6390 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:19:12 +0200 Subject: [PATCH 1/5] docs: promote HTTP Archive agent plugin on home page and docs --- astro.config.mjs | 1 + src/content/docs/docs/guides/ai-agents.mdx | 92 ++++++++++++++++++++++ src/content/docs/docs/index.mdx | 3 + src/pages/index.astro | 33 ++++++++ src/styles/styles.css | 35 ++++++++ 5 files changed, 164 insertions(+) create mode 100644 src/content/docs/docs/guides/ai-agents.mdx diff --git a/astro.config.mjs b/astro.config.mjs index cc7569a7..729d7dbc 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -43,6 +43,7 @@ export default defineConfig({ { label: 'Minimizing query costs', link: 'docs/guides/minimizing-costs' }, { label: 'Guided tour', link: 'docs/guides/guided-tour' }, { label: 'Release cycle', link: 'docs/guides/release-cycle' }, + { label: 'AI agents', link: 'docs/guides/ai-agents' }, ], }, { diff --git a/src/content/docs/docs/guides/ai-agents.mdx b/src/content/docs/docs/guides/ai-agents.mdx new file mode 100644 index 00000000..2ebce505 --- /dev/null +++ b/src/content/docs/docs/guides/ai-agents.mdx @@ -0,0 +1,92 @@ +--- +title: AI agents +description: Use HTTP Archive data from Claude Code, Codex, and other AI agents with the HTTP Archive agent plugin and MCP server +--- + +import { Tabs, TabItem } from '@astrojs/starlight/components'; + +The [HTTP Archive agent plugin](https://github.com/HTTPArchive/httparchive-plugin) lets AI coding agents such as Claude Code and Codex answer questions about how the web is built. It gives an agent two ways to reach HTTP Archive data: + +- **Tech Report tools** for technology adoption, Core Web Vitals, Lighthouse and page weight trends. These come from the [Tech Report](/reports/techreport/landing) through a remote MCP server and need no Google Cloud account. +- **BigQuery skills** for anything else. They teach the agent to inspect live table schemas, read these docs, preview rows for free, and write partition- and cluster-pruned SQL against the [public dataset](/docs/guides/getting-started/). + +## Install the plugin + + + + ```bash + claude plugin marketplace add httparchive/httparchive-plugin + claude plugin install httparchive@httparchive + ``` + + Or, inside a session, run `/plugin marketplace add httparchive/httparchive-plugin`, then `/plugin install httparchive@httparchive`. + + + ```bash + codex plugin marketplace add httparchive/httparchive-plugin + codex plugin add httparchive@httparchive + ``` + + + Copy the folders under [`skills/`](https://github.com/HTTPArchive/httparchive-plugin/tree/main/skills) into your agent's skills directory, for example `~/.agents/skills/`, and add the MCP server below. + + + +The plugin connects the Tech Report MCP server automatically. To use the server on its own, without the skills, add it to any MCP client that supports Streamable HTTP: + + + + ```bash + claude mcp add --transport http httparchive https://cdn.httparchive.org/mcp + ``` + + + ```bash + codex mcp add httparchive --url https://cdn.httparchive.org/mcp + ``` + + + Add a remote MCP server with the URL `https://cdn.httparchive.org/mcp`. It needs no authentication. + + + +## What's included + +### Tech Report tools + +| Tool | Returns | +|---|---| +| `search_technologies`, `list_categories`, `list_versions` | Technologies, categories and versions tracked by the Tech Report | +| `list_ranks`, `list_geos` | Rank segments and geographies available as filters | +| `get_adoption_metrics` | Share of sites using a technology over time | +| `get_cwv_metrics`, `get_cwv_distribution`, `get_geo_breakdown` | Core Web Vitals pass rates, histograms and per-country breakdowns | +| `get_lighthouse_metrics`, `get_audits_metrics` | Lighthouse category scores and audit results | +| `get_page_weight_metrics` | Page, JavaScript and CSS weight | + +### Skills + +| Skill | Use | +|---|---| +| `httparchive-docs-and-metrics` | Looks up these docs and discovers which `custom_metrics` keys exist in `crawl.pages` | +| `httparchive-query-optimizer` | Inspects schemas, previews rows with `bq head`, dry-runs queries, and applies the rules in [Minimizing query costs](/docs/guides/minimizing-costs/) | + +## Requirements + +The Tech Report tools work out of the box. + +The BigQuery skills run the `bq`, `curl` and `gh` command-line tools, so install the [Google Cloud CLI](https://cloud.google.com/sdk/docs/install) and authenticate against a project with BigQuery enabled, as described in [Getting started](/docs/guides/getting-started/). Queries are billed to that project. + +:::caution +An agent can write queries that scan terabytes. The skills tell it to dry-run before every query and to prototype on `rank <= 1000` first, but review the estimated bytes before letting it run anything large. Consider setting a [custom query quota](https://cloud.google.com/bigquery/docs/custom-quotas) on the project. +::: + +## Example prompts + +- "How has React adoption changed over the last two years, and how do React sites compare on Core Web Vitals?" +- "Which countries have the best LCP pass rate for WordPress sites?" +- "Write a query that counts how many of the top 10,000 mobile pages load a third-party font, and check its cost first." +- "Which `custom_metrics` keys describe structured data on a page?" + +## Contribute + +The plugin is open source on [GitHub](https://github.com/HTTPArchive/httparchive-plugin). Report issues or suggest new skills there. diff --git a/src/content/docs/docs/index.mdx b/src/content/docs/docs/index.mdx index 4ce5bd8d..59ca28d8 100644 --- a/src/content/docs/docs/index.mdx +++ b/src/content/docs/docs/index.mdx @@ -38,6 +38,9 @@ import { Card, CardGrid } from '@astrojs/starlight/components'; Learn how millions of URLs are crawled each month, how pipelines process data, and when new datasets are published in the [Release Cycle guide](/docs/guides/release-cycle/). + + Let Claude Code, Codex explore HTTP Archive dataset with the [HTTP Archive agent plugin](/docs/guides/ai-agents/), or ask any MCP client about web technologies and Core Web Vitals. + ## Dataset Reference diff --git a/src/pages/index.astro b/src/pages/index.astro index 705fa9b7..5b3e4405 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -148,6 +148,39 @@ const description = +
+
+
+ + +
+

+ AI Agents + Claude Code & Codex Plugin +

+ +

+ Ask your AI agent how the web is built. The HTTP Archive agent plugin + writes cost-safe BigQuery queries against the public dataset, and + answers technology adoption and Core Web Vitals questions straight + from the Tech Report. +

+ Set Up the Agent Plugin + View on GitHub +
+
+
+
+