From 0976443de8861c0a62914b6a2abc1410ac885243 Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Fri, 4 Sep 2026 09:59:39 +0000 Subject: [PATCH] fix(frontend): stop probing the docs API for non-latest versions (#1552) ## Problem Since #1473 the API only serves docs for the latest unyanked version and rejects every other pinned version with `docsOnlyForLatestVersion` (HTTP 404). The frontend still issued that docs request for every versioned doc URL (`/@scope/pkg@1.2.3/doc/...`) and only then redirected to the versionless page. Crawlers walk huge numbers of old versioned symbol URLs, so this guaranteed-to-fail probe is now the dominant API traffic. In a 50-minute production sample: | | count | |---|---| | all API requests | 22,457 | | docs-route requests | 17,961 | | docs-route 404s | ~7,450 | | of which pinned non-latest version | ~6,000 | About 70% of the 404s come from two header-rotating scraper pools and 18% from declared bots. Each hit cost the API two DB queries, an uncached 302 from the frontend, and usually a full docs render once the redirect was followed. ## Changes - **Skip the docs request for non-latest versions.** `Package.latestVersion` is computed by the same query the docs endpoint uses to pick "latest", so when it is set and differs from the pinned version the frontend redirects (doc pages) or renders the overview without docs, exactly as it did after the API 404, but without the round trip. When `latestVersion` is null (only prereleases published) the API stays the authority and the previous fallback path is unchanged. - **Cache the redirect at the edge** with the versioned doc pages' existing policy (`s-maxage=3600`). The target depends only on the URL. An old version only becomes latest again if every newer version is yanked, and then the versionless URL serves it anyway. - **`robots.txt`: `Disallow: /@*/*@*/doc`** so well-behaved crawlers stop requesting versioned doc pages that can only redirect. Versionless docs stay crawlable. ## Behaviour - Versioned doc page, version is not latest: 302 to the versionless URL, now cacheable, no docs API call. - Versioned doc page, version is latest: unchanged (docs API is called and serves it). - Versioned package overview, version is not latest: rendered without docs, unchanged, one fewer API call. - Unversioned URLs: unchanged. `deno lint` passes and the touched files type-check; there is no test coverage for the frontend data loaders. --- frontend/static/robots.txt | 7 ++- frontend/utils/data.ts | 126 ++++++++++++++++++++++++++----------- 2 files changed, 95 insertions(+), 38 deletions(-) diff --git a/frontend/static/robots.txt b/frontend/static/robots.txt index c999d55ab..e0defabb9 100644 --- a/frontend/static/robots.txt +++ b/frontend/static/robots.txt @@ -1 +1,6 @@ -Sitemap: https://jsr.io/sitemap.xml \ No newline at end of file +User-agent: * +# Versioned doc pages (/@scope/pkg@1.2.3/doc/...) only redirect to the +# versionless docs of the latest version, so crawling them is pure waste. +Disallow: /@*/*@*/doc + +Sitemap: https://jsr.io/sitemap.xml diff --git a/frontend/utils/data.ts b/frontend/utils/data.ts index 4ac66d74e..5906b38cf 100644 --- a/frontend/utils/data.ts +++ b/frontend/utils/data.ts @@ -105,16 +105,42 @@ export async function packageDataWithDocs( version: string | undefined, docs: DocsRequest, ): Promise { - let [data, pkgDocsResp] = await Promise.all([ - packageData(state, scope, pkg), - state.api.get( - path`/scopes/${scope}/packages/${pkg}/versions/${ - version || "latest" - }/docs`, + let data: PackageData | null; + let pkgDocsResp: APIResponse | null; + if (version) { + // The API only serves docs for the latest unyanked version and rejects + // every other pinned version with `docsOnlyForLatestVersion`. Crawlers + // walk huge numbers of old versioned symbol URLs, so decide locally from + // `latestVersion` (computed by the same query the docs endpoint uses) + // instead of paying for a docs request that is guaranteed to fail. + // `latestVersion` is null when only prereleases exist; the API is the + // authority in that case. + data = await packageData(state, scope, pkg); + if (data === null) return null; + if (data.pkg.latestVersion !== null && data.pkg.latestVersion !== version) { + return await nonLatestVersionDocs( + state, + scope, + pkg, + version, + docs, + data, + ); + } + pkgDocsResp = await state.api.get( + path`/scopes/${scope}/packages/${pkg}/versions/${version}/docs`, docs, - ) as Promise | null>, - ]); - if (data === null) return null; + ); + } else { + [data, pkgDocsResp] = await Promise.all([ + packageData(state, scope, pkg), + state.api.get( + path`/scopes/${scope}/packages/${pkg}/versions/latest/docs`, + docs, + ) as Promise | null>, + ]); + if (data === null) return null; + } if (pkgDocsResp && !pkgDocsResp.ok) { if (pkgDocsResp.code === "packageVersionNotFound") { @@ -127,35 +153,14 @@ export async function packageDataWithDocs( if (pkgDocsResp.code === "scopeNotFound") return null; if (pkgDocsResp.code === "packageNotFound") return null; if (pkgDocsResp.code === "docsOnlyForLatestVersion") { - if ("entrypoint" in docs || "all_symbols" in docs) { - // Docs are only served for the latest version; redirect to the - // canonical (versionless) docs URL for the latest version. - return new Response(null, { - status: 302, - headers: { - Location: `/@${scope}/${pkg}/doc${compileDocsRequestPath(docs)}`, - }, - }); - } - - // The package overview page of a non-latest version: render it - // without docs instead of redirecting away from the version. - const pkgVersionResp = await state.api.get( - path`/scopes/${scope}/packages/${pkg}/versions/${version!}`, + return await nonLatestVersionDocs( + state, + scope, + pkg, + version!, + docs, + data, ); - if (!pkgVersionResp.ok) { - if (pkgVersionResp.code === "packageVersionNotFound") return null; - if (pkgVersionResp.code === "scopeNotFound") return null; - if (pkgVersionResp.code === "packageNotFound") return null; - assertOk(pkgVersionResp); - } - return { - ...data, - kind: "content", - selectedVersion: pkgVersionResp.data, - selectedVersionIsLatestUnyanked: false, - docs: null, - }; } if (pkgDocsResp.code === "entrypointOrSymbolNotFound") { // redirect to all symbols page if there is no default entrypoint @@ -203,6 +208,53 @@ export async function packageDataWithDocs( } } +/** + * Docs for a pinned version that is not the latest unyanked version. Doc pages + * redirect to the canonical (versionless) URL, which serves the latest + * version; the package overview page is rendered without docs so the version + * stays visible. + */ +async function nonLatestVersionDocs( + state: State, + scope: string, + pkg: string, + version: string, + docs: DocsRequest, + data: PackageData, +): Promise { + if ("entrypoint" in docs || "all_symbols" in docs) { + return new Response(null, { + status: 302, + headers: { + Location: `/@${scope}/${pkg}/doc${compileDocsRequestPath(docs)}`, + // The target is derived from the URL alone, and an old version only + // becomes the latest again if every newer version is yanked (in which + // case the versionless URL serves it anyway), so the edge can absorb + // repeat hits. Mirrors the versioned doc pages' policy. + "Cache-Control": + "public, max-age=30, s-maxage=3600, stale-while-revalidate=10800", + }, + }); + } + + const pkgVersionResp = await state.api.get( + path`/scopes/${scope}/packages/${pkg}/versions/${version}`, + ); + if (!pkgVersionResp.ok) { + if (pkgVersionResp.code === "packageVersionNotFound") return null; + if (pkgVersionResp.code === "scopeNotFound") return null; + if (pkgVersionResp.code === "packageNotFound") return null; + assertOk(pkgVersionResp); + } + return { + ...data, + kind: "content", + selectedVersion: pkgVersionResp.data, + selectedVersionIsLatestUnyanked: false, + docs: null, + }; +} + export interface DocsData extends PackageData { kind: "content"; selectedVersion: PackageVersionWithUser | null;