diff --git a/api/.sqlx/query-fa5333f441b4533c8f1a2a68ffd68eca5a65f0bf664f22ef844219bf8d058631.json b/api/.sqlx/query-fa5333f441b4533c8f1a2a68ffd68eca5a65f0bf664f22ef844219bf8d058631.json new file mode 100644 index 000000000..c50196630 --- /dev/null +++ b/api/.sqlx/query-fa5333f441b4533c8f1a2a68ffd68eca5a65f0bf664f22ef844219bf8d058631.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE package_versions SET meta = $1\n WHERE scope = $2 AND name = $3 AND version = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa5333f441b4533c8f1a2a68ffd68eca5a65f0bf664f22ef844219bf8d058631" +} diff --git a/api/src/api/admin.rs b/api/src/api/admin.rs index 7aa57ae0e..afa4fc397 100644 --- a/api/src/api/admin.rs +++ b/api/src/api/admin.rs @@ -14,8 +14,13 @@ use tracing::Span; use tracing::field; use tracing::instrument; +use std::collections::HashMap; + +use crate::analysis::PackageAnalysisData; +use crate::analysis::analyze_package; use crate::db::*; use crate::iam::ReqIamExt; +use crate::ids::PackagePath; use crate::ids::ScopeDescription; use crate::publish::publish_task; use crate::util; @@ -40,6 +45,10 @@ pub fn admin_router() -> Router { .post("/scopes", util::auth(util::json(assign_scope))) .patch("/scopes/:scope", util::auth(util::json(patch_scopes))) .get("/packages", util::auth(util::json(list_packages))) + .post( + "/packages/:scope/:package/:version/recompute_meta", + util::auth(util::json(recompute_package_version_meta)), + ) .get( "/publishing_tasks", util::auth(util::json(list_publishing_tasks)), @@ -234,6 +243,108 @@ pub async fn list_packages( }) } +/// Re-runs the package analysis for a published version and stores the +/// freshly computed score meta, which is otherwise only computed at publish +/// time. Docs, the npm tarball, and provenance are left untouched. +#[instrument( + name = "POST /api/admin/packages/:scope/:package/:version/recompute_meta", + skip(req), + fields(scope, package, version) +)] +pub async fn recompute_package_version_meta( + req: Request, +) -> ApiResult { + let iam = req.iam(); + let staff = iam.check_admin_access()?; + let staff_id = staff.id; + + let scope = req.param_scope()?; + let package = req.param_package()?; + let version = req.param_version()?; + Span::current().record("scope", field::display(&scope)); + Span::current().record("package", field::display(&package)); + Span::current().record("version", field::display(&version)); + + let db = req.data::().unwrap(); + let buckets = req.data::().unwrap(); + let registry_url = req.data::().unwrap().0.clone(); + + let (pkg, _, _) = db + .get_package(&scope, &package) + .await? + .ok_or(ApiError::PackageNotFound)?; + let package_version = db + .get_package_version(&scope, &package, &version) + .await? + .ok_or(ApiError::PackageVersionNotFound)?; + + let mut files = HashMap::new(); + for file in db.list_package_files(&scope, &package, &version).await? { + let s3_path = + crate::s3_paths::file_path(&scope, &package, &version, &file.path); + let bytes = buckets + .modules_bucket + .download(s3_path.into()) + .await? + .ok_or_else(|| { + tracing::error!( + "module file '{}' of @{}/{}@{} is missing from the modules bucket", + file.path, + scope, + package, + version + ); + ApiError::InternalServerError + })?; + files.insert(file.path, bytes.to_vec()); + } + + // the config file path is only used in analysis error messages; the + // exports driving the analysis come from the database + let config_file = ["/jsr.json", "/jsr.jsonc", "/deno.json", "/deno.jsonc"] + .into_iter() + .map(|path| PackagePath::new(path.to_string()).unwrap()) + .find(|path| files.contains_key(path)) + .unwrap_or_else(|| PackagePath::new("/jsr.json".to_string()).unwrap()); + + let span = Span::current(); + let analysis_scope = scope.clone(); + let analysis_package = package.clone(); + let analysis_version = version.clone(); + let data = PackageAnalysisData { + exports: package_version.exports.clone(), + files, + }; + let output = tokio::task::spawn_blocking(move || { + analyze_package( + span, + registry_url, + analysis_scope, + analysis_package, + analysis_version, + config_file, + data, + ) + }) + .await + .map_err(|join_error| { + tracing::error!("analysis task panicked: {join_error:?}"); + ApiError::InternalServerError + })? + .map_err(|analysis_error| ApiError::PackageAnalysisFailed { + msg: analysis_error.to_string(), + })?; + + let mut meta = output.meta; + // provenance is recorded post-publish and not derivable from the files + meta.has_provenance = package_version.meta.has_provenance; + + db.update_package_version_meta(&staff_id, &scope, &package, &version, &meta) + .await?; + + Ok(ApiPackageScore::from((&meta, &pkg))) +} + #[instrument(name = "GET /api/admin/publishing_tasks", skip(req))] pub async fn list_publishing_tasks( req: Request, @@ -537,4 +648,127 @@ mod tests { .expect_err_code(StatusCode::CONFLICT, "scopeAlreadyExists") .await; } + + #[tokio::test] + async fn recompute_package_version_meta() { + use std::io::Write as _; + + use bytes::Bytes; + use flate2::Compression; + use flate2::write::GzEncoder; + + use crate::api::ApiPackageScore; + use crate::db::PublishingTaskStatus; + use crate::ids::PackageName; + use crate::ids::ScopeName; + use crate::ids::Version; + + let mut t = TestSetup::new().await; + + // a JS entrypoint typed via `/// `, the layout from + // jsr-io/jsr#698 + let mut tar_bytes = Vec::new(); + let mut tar = tar::Builder::new(&mut tar_bytes); + let mut append = |path: &str, content: &[u8]| { + let mut header = tar::Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + tar.append_data(&mut header, path, content).unwrap(); + }; + append( + "jsr.json", + br#"{ "name": "@scope/foo", "version": "1.2.3", "license": "MIT", "exports": "./mod.mjs" }"#, + ); + append("README.md", b"# foo\n\n```ts\nadd(1, 2);\n```\n"); + append( + "mod.mjs", + b"/// \n/** Adds two numbers. */\nexport function add(a, b) {\n return a + b;\n}\n", + ); + append( + "mod.d.ts", + b"/** A module. @module */\n\n/** Adds two numbers. */\nexport declare function add(a: number, b: number): number;\n", + ); + tar.finish().unwrap(); + drop(tar); + let mut gz_bytes = Vec::new(); + let mut encoder = GzEncoder::new(&mut gz_bytes, Compression::default()); + encoder.write_all(&tar_bytes).unwrap(); + encoder.finish().unwrap(); + + let task = + crate::publish::tests::process_tarball_setup(&t, Bytes::from(gz_bytes)) + .await; + assert_eq!( + task.status, + PublishingTaskStatus::Success, + "{:?}", + task.error + ); + + let scope = ScopeName::try_from("scope").unwrap(); + let name = PackageName::try_from("foo").unwrap(); + let version = Version::try_from("1.2.3").unwrap(); + + let fresh = t + .db() + .get_package_version(&scope, &name, &version) + .await + .unwrap() + .unwrap(); + assert!(fresh.meta.all_fast_check); + assert!(fresh.meta.has_readme); + + // simulate stale meta from an older analysis pipeline + let mut stale = fresh.meta.clone(); + stale.all_fast_check = false; + stale.has_readme = false; + stale.has_provenance = true; + t.db() + .update_package_version_meta( + &t.staff_user.user.id, + &scope, + &name, + &version, + &stale, + ) + .await + .unwrap(); + + let path = "/api/admin/packages/scope/foo/1.2.3/recompute_meta"; + + let user_token = t.user1.token.clone(); + t.http() + .post(path) + .token(Some(&user_token)) + .call() + .await + .unwrap() + .expect_err_code(StatusCode::FORBIDDEN, "actorNotAuthorized") + .await; + + let staff_token = t.staff_user.token.clone(); + let score = t + .http() + .post(path) + .token(Some(&staff_token)) + .call() + .await + .unwrap() + .expect_ok::() + .await; + assert!(score.all_fast_check); + assert!(score.has_readme); + assert!(score.has_provenance); + + let healed = t + .db() + .get_package_version(&scope, &name, &version) + .await + .unwrap() + .unwrap(); + assert!(healed.meta.all_fast_check); + assert!(healed.meta.has_readme); + assert!(healed.meta.has_provenance); + } } diff --git a/api/src/api/errors.rs b/api/src/api/errors.rs index ad1c7051b..369a4b155 100644 --- a/api/src/api/errors.rs +++ b/api/src/api/errors.rs @@ -43,6 +43,11 @@ errors!( status: NOT_FOUND, "The requested package version was not found.", }, + PackageAnalysisFailed { + status: BAD_REQUEST, + fields: { msg: String }, + ({ msg }) => "Failed to re-analyze the package version: {msg}.", + }, DiffNoIndex { status: NOT_FOUND, "Diffs do not have an index.", diff --git a/api/src/db/database.rs b/api/src/db/database.rs index 223a32ace..99bbe6bb4 100644 --- a/api/src/db/database.rs +++ b/api/src/db/database.rs @@ -577,6 +577,51 @@ impl Database { Ok(()) } + /// Replaces the stored score metadata of a published version. + #[instrument( + name = "Database::update_package_version_meta", + skip(self, meta), + err + )] + pub async fn update_package_version_meta( + &self, + staff_id: &Uuid, + scope: &ScopeName, + name: &PackageName, + version: &Version, + meta: &PackageVersionMeta, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + + audit_log( + &mut tx, + staff_id, + true, + "update_package_version_meta", + json!({ + "scope": scope, + "name": name, + "version": version, + }), + ) + .await?; + + sqlx::query!( + r#"UPDATE package_versions SET meta = $1 + WHERE scope = $2 AND name = $3 AND version = $4"#, + meta as _, + scope as _, + name as _, + version as _ + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(()) + } + #[instrument(name = "Database::update_package_description", skip(self), err)] pub async fn update_package_description( &self, diff --git a/api/src/npm/specifiers.rs b/api/src/npm/specifiers.rs index 2b75df417..3f95895c5 100644 --- a/api/src/npm/specifiers.rs +++ b/api/src/npm/specifiers.rs @@ -22,6 +22,9 @@ pub enum RewriteKind { #[derive(Clone, Copy)] pub struct SpecifierRewriter<'a> { pub base_specifier: &'a ModuleSpecifier, + /// The specifier of the module in the module graph whose specifiers are + /// being rewritten. Relative specifiers are resolved against this. + pub module_specifier: &'a ModuleSpecifier, pub source_rewrites: &'a HashMap<&'a ModuleSpecifier, ModuleSpecifier>, pub declaration_rewrites: &'a HashMap<&'a ModuleSpecifier, ModuleSpecifier>, pub dependencies: &'a IndexMap, @@ -30,12 +33,29 @@ pub struct SpecifierRewriter<'a> { impl SpecifierRewriter<'_> { pub fn rewrite(&self, specifier: &str, kind: RewriteKind) -> Option { let source_text_specifier = specifier; - let dep = self.dependencies.get(specifier)?; - let specifier = match kind { - RewriteKind::Source => dep.get_code(), - RewriteKind::Declaration => dep.get_type().or_else(|| dep.get_code()), - }?; + let resolved; + let specifier = if let Some(dep) = self.dependencies.get(specifier) { + match kind { + RewriteKind::Source => dep.get_code(), + RewriteKind::Declaration => dep.get_type().or_else(|| dep.get_code()), + }? + } else if matches!(kind, RewriteKind::Declaration) + && specifier.starts_with('.') + { + // The fast check transform in deno_graph rewrites relative import + // specifiers in the generated .d.ts to point at the resolved (types) + // dependency, so the specifier may not match a source text specifier + // in the dependency map anymore. For example, an import of `./a.js` + // where `./a.js` has an adjacent hand-written `./a.d.ts` is rewritten + // to `./a.d.ts`. Resolve such specifiers against the original module + // specifier so they can still be rewritten (the generated .d.ts is + // emitted to a different directory than the original module). + resolved = self.module_specifier.join(specifier).ok()?; + &resolved + } else { + return None; + }; let rewrites = match kind { RewriteKind::Source => self.source_rewrites, diff --git a/api/src/npm/tarball.rs b/api/src/npm/tarball.rs index 4722ce7bd..abfcffed0 100644 --- a/api/src/npm/tarball.rs +++ b/api/src/npm/tarball.rs @@ -188,6 +188,7 @@ pub async fn create_npm_tarball<'a>( .unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: &js.specifier, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, @@ -209,6 +210,7 @@ pub async fn create_npm_tarball<'a>( .unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: &js.specifier, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, @@ -227,6 +229,7 @@ pub async fn create_npm_tarball<'a>( let source_target = source_rewrites.get(&js.specifier).unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: source_target, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, @@ -246,6 +249,7 @@ pub async fn create_npm_tarball<'a>( .unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: &js.specifier, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, @@ -263,6 +267,7 @@ pub async fn create_npm_tarball<'a>( let source_target = source_rewrites.get(&js.specifier).unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: source_target, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, @@ -279,6 +284,7 @@ pub async fn create_npm_tarball<'a>( declaration_rewrites.get(&js.specifier).unwrap(); let specifier_rewriter = SpecifierRewriter { base_specifier: declaration_target, + module_specifier: &js.specifier, source_rewrites: &source_rewrites, declaration_rewrites: &declaration_rewrites, dependencies: &js.dependencies, diff --git a/api/testdata/specs/npm_tarballs/ts_importing_js_with_dts_types.txt b/api/testdata/specs/npm_tarballs/ts_importing_js_with_dts_types.txt new file mode 100644 index 000000000..e3ec6516f --- /dev/null +++ b/api/testdata/specs/npm_tarballs/ts_importing_js_with_dts_types.txt @@ -0,0 +1,88 @@ +# mod.ts +import * as wasm from "./wasm/lib.js"; + +export type Algorithm = wasm.Algorithm; + +export function defaultAlgorithm(): Algorithm { + return wasm.algorithm_default(); +} + +# wasm/lib.js +// @ts-self-types="./lib.d.ts" +export function algorithm_default() { + return "ed25519"; +} + +# wasm/lib.d.ts +export type Algorithm = "ed25519" | "secp256k1"; +export function algorithm_default(): Algorithm; + +# jsr.json +{ + "name": "@scope/foo", + "version": "0.0.1", + "exports": "./mod.ts" +} + +# output +== /_dist/mod.d.ts == +import * as wasm from "../wasm/lib.js"; +export type Algorithm = wasm.Algorithm; +export declare function defaultAlgorithm(): Algorithm; +//# sourceMappingURL=mod.d.ts.map + +== /_dist/mod.d.ts.map == +{"version":3,"file":"mod.d.ts","sources":["../mod.ts"],"names":[],"mappings":"AAAA,YAAY,2BAA0B;AAEtC,YAAY,YAAY,KAAK;AAE7B,OAAO,iBAAS,oBAAoB"} + +== /jsr.json == +{ + "name": "@scope/foo", + "version": "0.0.1", + "exports": "./mod.ts" +} + +== /mod.js == +import * as wasm from "./wasm/lib.js"; +export function defaultAlgorithm() { + return wasm.algorithm_default(); +} +//# sourceMappingURL=mod.js.map + +== /mod.js.map == +{"version":3,"file":"mod.js","sources":["./mod.ts"],"names":[],"mappings":"AAAA,YAAY,UAAU,gBAAgB;AAItC,OAAO,SAAS;EACd,OAAO,KAAK,iBAAiB;AAC/B"} + +== /mod.ts == +import * as wasm from "./wasm/lib.js"; + +export type Algorithm = wasm.Algorithm; + +export function defaultAlgorithm(): Algorithm { + return wasm.algorithm_default(); +} + +== /package.json == +{ + "name": "@jsr/scope__foo", + "version": "0.0.1", + "homepage": "http://jsr.test/@scope/foo", + "type": "module", + "dependencies": {}, + "exports": { + ".": { + "types": "./_dist/mod.d.ts", + "default": "./mod.js" + } + }, + "_jsr_revision": 0 +} + +== /wasm/lib.d.ts == +export type Algorithm = "ed25519" | "secp256k1"; +export function algorithm_default(): Algorithm; + +== /wasm/lib.js == +// @ts-self-types="./lib.d.ts" +export function algorithm_default() { + return "ed25519"; +} + diff --git a/api/testdata/specs/npm_tarballs/type_imports.txt b/api/testdata/specs/npm_tarballs/type_imports.txt index 5c59ae076..f90c171d0 100644 --- a/api/testdata/specs/npm_tarballs/type_imports.txt +++ b/api/testdata/specs/npm_tarballs/type_imports.txt @@ -65,16 +65,16 @@ export declare function createAdd(): Add; # output == /_dist/main.d.ts == -export { type Add as Add2 } from "./foo.d.ts"; -export type { Add as Add3 } from "./foo.d.ts"; -export type * from "./foo.d.ts"; -export type * as Foo from "./foo.d.ts"; -import { Add } from "./foo.d.ts"; +export { type Add as Add2 } from "../foo.js"; +export type { Add as Add3 } from "../foo.js"; +export type * from "../foo.js"; +export type * as Foo from "../foo.js"; +import { Add } from "../foo.js"; export declare function createAdd(): Add; //# sourceMappingURL=main.d.ts.map == /_dist/main.d.ts.map == -{"version":3,"file":"main.d.ts","sources":["../main.ts"],"names":[],"mappings":"AAAA,SAAS,KAAK,OAAO,IAAI,QAAQ,aAAW;AAC5C,cAAc,OAAO,IAAI,QAAQ,aAAW;AAC5C,mBAAmB,aAAW;AAC9B,YAAO,KAAU,GAAG,MAAM,aAAW;AACrC,SAAc,GAAG,QAAQ,aAAW;AAEpC,OAAO,iBAAS,aAAa"} +{"version":3,"file":"main.d.ts","sources":["../main.ts"],"names":[],"mappings":"AAAA,SAAS,KAAK,OAAO,IAAI,oBAAmB;AAC5C,cAAc,OAAO,IAAI,oBAAmB;AAC5C,+BAA8B;AAC9B,YAAO,KAAU,GAAG,kBAAiB;AACrC,SAAc,GAAG,oBAAmB;AAEpC,OAAO,iBAAS,aAAa"} == /adder.d.ts == export interface Adder { diff --git a/frontend/routes/admin/(_islands)/RecomputeVersionMeta.tsx b/frontend/routes/admin/(_islands)/RecomputeVersionMeta.tsx new file mode 100644 index 000000000..01d3e4faa --- /dev/null +++ b/frontend/routes/admin/(_islands)/RecomputeVersionMeta.tsx @@ -0,0 +1,102 @@ +// Copyright 2024 the JSR authors. All rights reserved. MIT license. +import { useEffect, useId, useRef } from "preact/hooks"; +import { useSignal } from "@preact/signals"; +import TbLoader2 from "tb-icons/TbLoader2"; +import { api, path } from "../../../utils/api.ts"; +import { PackageScore } from "../../../utils/api_types.ts"; + +export function RecomputeVersionMeta( + { scope, pkg }: { scope: string; pkg: string }, +) { + const open = useSignal(false); + const submitting = useSignal(false); + const version = useSignal(""); + const result = useSignal(null); + const buttonRef = useRef(null); + const ref = useRef(null); + + useEffect(() => { + function outsideClick(e: Event) { + if ( + (ref.current && !ref.current.contains(e.target as Element)) && + (buttonRef.current && !buttonRef.current.contains(e.target as Element)) + ) { + open.value = false; + } + } + document.addEventListener("click", outsideClick); + return () => document.removeEventListener("click", outsideClick); + }, []); + + const prefix = useId(); + + return ( +
+ +
+
{ + e.preventDefault(); + submitting.value = true; + result.value = null; + api.post( + path`/admin/packages/${scope}/${pkg}/${version.value}/recompute_meta`, + null, + ).then((res) => { + submitting.value = false; + result.value = res.ok + ? `done — recomputed score: ${res.data.total}%` + : `${res.code}: ${res.message}`; + }); + }} + > +

+ Recompute meta of @{scope}/{pkg} +

+

+ Re-runs the analysis on the stored module files of a version and + replaces its score meta. +

+ + {result.value &&

{result.value}

} + + +
+
+ ); +} diff --git a/frontend/routes/admin/packages.tsx b/frontend/routes/admin/packages.tsx index 335318df5..c5ace0eaf 100644 --- a/frontend/routes/admin/packages.tsx +++ b/frontend/routes/admin/packages.tsx @@ -7,6 +7,7 @@ import { URLQuerySearch } from "./(_components)/URLQuerySearch.tsx"; import { define } from "../../util.ts"; import twas from "twas"; import { AdminCopyButton } from "./(_islands)/AdminCopyButton.tsx"; +import { RecomputeVersionMeta } from "./(_islands)/RecomputeVersionMeta.tsx"; export default define.page(function Packages({ data, url }) { return ( @@ -33,6 +34,7 @@ export default define.page(function Packages({ data, url }) { fieldName: "created_at", align: "right", }, + { title: "", class: "w-0", align: "right" }, ]} pagination={data} sortBy={data.sortBy} @@ -96,6 +98,9 @@ export default define.page(function Packages({ data, url }) { > {twas(new Date(pkg.createdAt).getTime())} + + + ))}