diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c62c6f..8deb994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- **First use finds the docs sites too.** The automatic first-use fetch (`docs`/`api` before any `lockdocs fetch`) skipped the separate docs-site repositories, so React 19, Express, tokio and Tailwind answers came from package files only. It now reads them as well: the site archive within the same 64 MB cap, else one REST tree request plus raw files (tailwindcss.com is 190 MB and prisma/docs over 800 MB as archives, but a few hundred files). The REST call follows a redirect to the same API host once, which a renamed repository such as prisma/docs answers. A docs-site failure keeps the package docs and says what was skipped in the manifest note; `lockdocs fetch` retries it. On the benchmark's first-use run (empty isolated cache, no credentials, 97 of 105 questions installable locally) 76 become 88 correct and held-out 13/16 becomes 14/16; the first-use floor in `bench/run.py` is 88/105. `upstream::FORMAT` is 6. - **Free upgrade preview.** Without a Pro licence, `lockdocs upgrade ` (and `docs --upgrade-to`, and the `docs` tool's `upgrade_to`) now answers with a short local preview instead of only "Pro required": how many of the APIs your project calls were removed, renamed, re-signed or deprecated, across how many call sites, plus one sample, then the Pro price and link. The full list stays Pro. The MCP answer still carries `structuredContent.pro_required`, the CLI still exits 3, and the preview reads the same installed, cached or opted-in files as the report (no new downloads). - **`lockdocs licence buy`.** Built on mcp-kit 0.6.0. It opens the Pro page; in-terminal purchase turns on when checkout is live. diff --git a/bench/run.py b/bench/run.py index ddc7374..9d6ff90 100755 --- a/bench/run.py +++ b/bench/run.py @@ -252,7 +252,7 @@ def main(): def check_floors(summary, total): if total != 105: return - for variant, floor in [("fetched", 96), ("hybrid", 62)]: + for variant, floor in [("fetched", 96), ("hybrid", 62), ("first-use-default", 88)]: if variant in summary and summary[variant]["passed"] < floor: raise RuntimeError(f"{variant} regressed: {summary[variant]['passed']}/105, required >= {floor}/105") # Hybrid retrieval exists to beat keyword search; if it scores lower on the diff --git a/crates/lockdocs-core/src/upstream.rs b/crates/lockdocs-core/src/upstream.rs index 7010060..7cf3b81 100644 --- a/crates/lockdocs-core/src/upstream.rs +++ b/crates/lockdocs-core/src/upstream.rs @@ -29,7 +29,7 @@ pub struct Repo { /// Bump when what `fetch` downloads changes, so `lockdocs fetch` refreshes /// older copies (a stale copy is still used until then). -pub const FORMAT: u32 = 5; +pub const FORMAT: u32 = 6; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Manifest { @@ -290,7 +290,8 @@ fn site_docs(agent: &Client, dep: &Dep, pkg_repo: Option<&Repo>, out: &Path) -> } } Err(e) => { - // Over the cap or codeload failed: read this docs site file by file. + // Over the cap (tailwindcss.com is 190 MB, prisma/docs 860 MB) or codeload + // failed: read this docs site file by file (one tree request, then raw files). let _ = std::fs::remove_dir_all(&dst); let Some(items) = tree(agent, &repo, &reference, true).with_context(|| format!("codeload path failed ({e:#}); per-file fallback failed"))? else { return Ok(None); @@ -563,11 +564,26 @@ impl Client { /// GitHub API GET; Ok(None) on 404. fn api(agent: &Client, path: &str) -> Result> { agent.check_deadline()?; - let mut req = agent.get(&format!("{}{path}", agent.hosts.api)).header("Accept", "application/vnd.github+json"); - if let Some(t) = agent.authenticated.then(token).flatten() { - req = req.header("Authorization", &format!("Bearer {t}")); - } - let mut res = req.call()?; + let mut url = format!("{}{path}", agent.hosts.api); + let mut moved = false; + let mut res = loop { + let mut req = agent.get(&url).header("Accept", "application/vnd.github+json"); + if let Some(t) = agent.authenticated.then(token).flatten() { + req = req.header("Authorization", &format!("Bearer {t}")); + } + let res = req.call()?; + // A renamed repository (prisma/docs) answers the REST API with a redirect to + // its id-based URL. The automatic client follows no redirects, so follow this + // one, once, and only to the same API host. + let next = res.headers().get("location").and_then(|l| l.to_str().ok()).map(String::from); + match (res.status().as_u16(), next) { + (301 | 302 | 307 | 308, Some(l)) if !moved && l.starts_with(&format!("{}/", agent.hosts.api)) => { + moved = true; + url = l; + } + _ => break res, + } + }; let status = res.status().as_u16(); if status == 404 || status == 422 || status == 409 { return Ok(None); @@ -737,7 +753,8 @@ pub fn downloaded_bytes() -> u64 { /// requests (see the measurements in the PR for the choice). const MAX_ARCHIVE: u64 = 150 << 20; -/// The cap for the automatic (anonymous, 45 s budget) mode. +/// The cap for a package's own archive in the automatic (anonymous, 45 s +/// budget) mode: a bigger one falls back to per-file requests. const MAX_ARCHIVE_AUTOMATIC: u64 = 64 << 20; /// Counts compressed bytes and stops the stream (with an error) past `cap`. @@ -1328,6 +1345,22 @@ fn fetch_attempt(dep: &Dep, src: &Source, automatic: bool) -> Result { Ok(m) } +/// The docs site of `dep`, if it has one. An explicit fetch fails on an error; +/// an automatic one keeps the package docs it has and says what was skipped +/// (the cache stays unchecked, so `lockdocs fetch` retries it). +fn site_step(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result<(Option, Option)> { + match site_docs(agent, dep, Some(repo), out) { + Ok(s) => Ok((s, None)), + Err(e) if automatic => { + if let Some(site) = DOCS_SITES.iter().find(|s| s.eco == dep.eco && s.names.contains(&dep.name.as_str())) { + let _ = std::fs::remove_dir_all(out.join(site.repo.1)); + } + Ok((None, Some(format!("docs site skipped: {e:#}")))) + } + Err(e) => Err(e.context("docs-site selection failed")), + } +} + /// Download the docs of `dep`'s release (and docs site) into `out`; the caller publishes. fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> Result { let label = format!("github.com/{}/{}", repo.owner, repo.name); @@ -1349,18 +1382,16 @@ fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> pages: Vec::new(), via: None, }; - match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } { - Ok(Some(site)) => { - if site.files > 0 { - m.files = site.files; - m.bytes = site.bytes; - m.site = Some(site.label); - m.pages = site.pages; - m.note = None; - } - } - Ok(None) => {} - Err(e) => return Err(e.context("docs-site selection failed")), + let (found, skipped) = site_step(agent, dep, repo, automatic, out)?; + if let Some(site) = found.filter(|s| s.files > 0) { + m.files = site.files; + m.bytes = site.bytes; + m.site = Some(site.label); + m.pages = site.pages; + m.note = None; + } + if let Some(n) = skipped { + m.note = Some(m.note.take().map_or(n.clone(), |old| format!("{old}; {n}"))); } // The repository is gone, renamed or private. A docs site may still serve the // docs; fail clearly only when nothing was found anywhere. @@ -1386,15 +1417,12 @@ fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> let mut site_n = 0usize; let mut site_bytes = 0u64; let mut pages = Vec::new(); - match if automatic { Ok(None) } else { site_docs(agent, dep, Some(repo), out) } { - Ok(Some(s)) => { - (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages); - if site_n > 0 { - site = Some(s.label); - } + let (found, skipped) = site_step(agent, dep, repo, automatic, out)?; + if let Some(s) = found { + (site_n, site_bytes, pages) = (s.files, s.bytes, s.pages); + if site_n > 0 { + site = Some(s.label); } - Ok(None) => {} - Err(e) => return Err(e.context("docs-site selection failed")), } let m = Manifest { format: FORMAT, @@ -1404,10 +1432,10 @@ fn build(agent: &Client, dep: &Dep, repo: &Repo, automatic: bool, out: &Path) -> docs_sites_checked: !automatic, files: docs_n + site_n, bytes: docs_bytes + site_bytes, - note: if docs_n == 0 && site_n == 0 { - Some("no docs folder at that tag".into()) - } else { - None + note: match (docs_n == 0 && site_n == 0, skipped) { + (true, None) => Some("no docs folder at that tag".into()), + (true, Some(n)) => Some(format!("no docs folder at that tag; {n}")), + (false, n) => n, }, site, pages, @@ -1898,6 +1926,27 @@ mod tests { assert_eq!(std::fs::read_to_string(out.path().join("react.dev/src/content/learn/a.md")).unwrap(), "# learn"); } + #[test] + fn automatic_build_includes_the_docs_site_but_stays_unchecked() { + let f = fake(Codeload::PrivateWithSite, false, Some("v1.2.3")); + let out = tempfile::tempdir().unwrap(); + let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "19.0.0"), &repo(None), true, out.path()).unwrap(); + assert!(m.files >= 1 && m.site.is_some(), "{m:?}"); + // Explicit `lockdocs fetch` still completes what the automatic budget cannot. + assert!(!m.docs_sites_checked); + assert!(out.path().join("react.dev/src/content/learn/a.md").exists()); + } + + #[test] + fn automatic_docs_site_failure_keeps_the_package_docs() { + let f = fake(Codeload::Ok, false, Some("v1.2.3")); + let out = tempfile::tempdir().unwrap(); + // The fake serves no site repository, so the lookup fails; the package docs stay. + let m = build(&client(&f, MAX_ARCHIVE), &dep("react", "1.2.3"), &repo(None), true, out.path()).unwrap(); + assert_eq!(m.files, 2, "{m:?}"); + assert!(m.site.is_none()); + } + #[test] fn unavailable_repository_without_docs_site_is_a_clear_error() { let f = fake(Codeload::Private, false, Some("v1.2.3"));