diff --git a/Cargo.lock b/Cargo.lock index 1cc97df..9269672 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,7 @@ name = "ai-catalog-cli" version = "0.3.0" dependencies = [ "ai-catalog", + "ai-catalog-discover", "ai-catalog-oci", "ai-catalog-trust", "ai-catalog-validate", @@ -46,6 +47,19 @@ dependencies = [ "url", ] +[[package]] +name = "ai-catalog-discover" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5c98bbff69f42d4dee321372ebf2bfed177a706e1f9d7506f4697f0413b3ef6" +dependencies = [ + "ai-catalog", + "reqwest", + "scraper", + "thiserror", + "url", +] + [[package]] name = "ai-catalog-oci" version = "0.3.0" @@ -377,12 +391,56 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cssparser" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a" +dependencies = [ + "quote", + "syn 3.0.9", +] + [[package]] name = "deranged" version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.11.3" @@ -426,12 +484,33 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ego-tree" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd50d24b1b75aa817bb0339cb9f2e0e664bdfab1e160a69a0fda9145b77fab2" + [[package]] name = "errno" version = "0.3.14" @@ -557,6 +636,17 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "html5ever" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130" +dependencies = [ + "log", + "markup5ever", + "memchr", +] + [[package]] name = "http" version = "1.5.0" @@ -906,6 +996,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -918,6 +1017,17 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "markup5ever" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842" +dependencies = [ + "log", + "tendril", + "web_atoms", +] + [[package]] name = "memchr" version = "2.8.3" @@ -935,6 +1045,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "num-conv" version = "0.2.2" @@ -974,12 +1090,88 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros", + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1007,6 +1199,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -1114,6 +1312,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.5.3" @@ -1337,6 +1544,25 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scraper" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6844b5f49a5579f12ba4f89d9f7eec6d71889e4aa0bc7181821040ce66f08fff" +dependencies = [ + "cssparser", + "ego-tree", + "html5ever", + "precomputed-hash", + "selectors", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -1360,6 +1586,23 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8322aebea94a6cc288ff884e8a49bdbceef60206c400c6c46d22e62fc4fd91a6" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.28" @@ -1420,6 +1663,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "servo_arc" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eceb519ec34d8290aaf6e46f435102f2bfa63f22b05a144fb16ab266d3b198" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha2" version = "0.11.1" @@ -1453,6 +1705,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -1481,6 +1739,30 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "string_cache" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1548,6 +1830,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08" +dependencies = [ + "new_debug_unreachable", +] + [[package]] name = "thiserror" version = "2.0.21" @@ -1851,6 +2142,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada" +dependencies = [ + "phf", + "phf_codegen", + "string_cache", + "string_cache_codegen", +] + [[package]] name = "webpki-root-certs" version = "1.0.11" diff --git a/Cargo.toml b/Cargo.toml index 8fa3e37..55fb4ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] ai-catalog = "0.3.0" +ai-catalog-discover = "0.3.0" ai-catalog-oci = "0.3.0" ai-catalog-trust = "0.3.0" ai-catalog-validate = "0.3.0" diff --git a/README.md b/README.md index bc7bb00..279d6be 100644 --- a/README.md +++ b/README.md @@ -162,17 +162,27 @@ cat catalog.json | ai-catalog format - ### `catalog add` -Fetches a catalog from a URL (or `file://` path), stores all catalog blobs in -the local content-addressed cache, and registers the catalog in the local +Fetches a catalog from a URL, a domain or a local file, stores all catalog blobs +in the local content-addressed cache, and registers the catalog in the local registry. Nested catalogs are fetched recursively (up to depth 4). ```sh ai-catalog catalog add my-registry https://example.com/ai-catalog.json +ai-catalog catalog add example https://example.com # or just: example.com ai-catalog catalog add local-demo file:///path/to/catalog.json ``` Makes network requests. All other consumer commands operate on the local cache. +A domain or an `https://` URL does not have to be the catalog itself. If it +isn't one, the CLI follows the specification's +[discovery procedure](https://ai-catalog.io/spec/#discovery): a `Link` header +with `rel="ai-catalog"`, then an HTML `` element, then +`/.well-known/ai-catalog.json`. The catalog that is found is the one that gets +registered, under its own URL, which is what `catalog update` fetches again. If +nothing is found, the error lists every URL that was tried. Files and `http://` +URLs are used as they are, without discovery. + References inside a catalog must be `https` (or relative to the catalog). A local `file://` path is followed only from a local catalog, so a remote catalog can't read your disk. Cycles, `http://` references and nesting beyond depth 4 are diff --git a/src/cli.rs b/src/cli.rs index 57e1d0e..7d32a6d 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -60,11 +60,11 @@ pub enum Command { #[derive(Debug, Subcommand)] pub enum CatalogCommand { - /// Register a catalog from a URL or a file + /// Register a catalog from a URL, a domain or a file Add { /// Name to register the catalog under name: String, - /// URL or path of the catalog + /// URL or path of the catalog, or a website or domain to discover it from url: String, }, /// List the registered catalogs diff --git a/src/commands/catalog_add.rs b/src/commands/catalog_add.rs index 873b800..d99b747 100644 --- a/src/commands/catalog_add.rs +++ b/src/commands/catalog_add.rs @@ -5,51 +5,43 @@ use colored::Colorize; use ai_catalog::CatalogEntry; +use ai_catalog_discover::Fetcher; use crate::cache::CacheManager; use crate::error::{Error, Result}; -use crate::fetch::local_path_to_file_url; +use crate::fetch::{Located, build_fetcher, locate}; use crate::resolver::{Resolver, cli_extension, make_entry_extensions, registry_identifier}; -pub fn execute(name: &str, url: &str) -> Result<()> { - let url = local_path_to_file_url(url)?; - let url = url.as_str(); +pub fn execute(name: &str, input: &str) -> Result<()> { let cache = CacheManager::new()?; + add(&cache, &build_fetcher()?, name, input) +} + +fn add(cache: &CacheManager, fetcher: &dyn Fetcher, name: &str, input: &str) -> Result<()> { cache.ensure_dirs()?; - { - let registry = cache.read_registry()?; - for entry in ®istry.entries { - if entry - .display_name - .as_deref() - .map(|n| n.eq_ignore_ascii_case(name)) - .unwrap_or(false) - { - return Err(Error::Other(format!( - "a catalog named \"{name}\" is already registered. Use `ai-catalog catalog update {name}` to refresh it." - ))); - } - if let Some(source_url) = cli_extension(entry) - .and_then(|m| m.get("sourceUrl")) - .and_then(|v| v.as_str()) - && source_url.eq_ignore_ascii_case(url) - { - return Err(Error::Other(format!( - "URL is already registered as \"{}\". Use `ai-catalog catalog update {}` to refresh it.", - entry.display_name.as_deref().unwrap_or("?"), - entry.display_name.as_deref().unwrap_or("?") - ))); + ensure_name_is_free(cache, name)?; + println!("Fetching catalog from {}...", input.trim().cyan()); + let mut resolver = Resolver::online(cache)?; + let (url, root_hash) = match locate(input, fetcher)? { + Located::Url(url) => { + ensure_url_is_free(cache, &url)?; + let hash = resolver.fetch(&url)?; + (url, hash) + } + Located::Discovered { url, bytes, method } => { + if method != ai_catalog_discover::Method::Direct { + println!("Found the catalog at {} ({method})", url.cyan()); } + ensure_url_is_free(cache, &url)?; + let hash = resolver.fetch_bytes(&url, &bytes)?; + (url, hash) } - } - println!("Fetching catalog from {}...", url.cyan()); - let mut resolver = Resolver::online(&cache)?; - let root_hash = resolver.fetch(url)?; + }; let entry_count = resolver.leaves().count(); resolver.into_found(); let file_url = cache.object_file_url(&root_hash); let identifier = registry_identifier("local", name); - let extensions = make_entry_extensions(url, &root_hash, entry_count); + let extensions = make_entry_extensions(&url, &root_hash, entry_count); let new_entry = CatalogEntry { identifier, entry_type: "application/ai-catalog+json".to_string(), @@ -76,3 +68,173 @@ pub fn execute(name: &str, url: &str) -> Result<()> { ); Ok(()) } + +fn ensure_name_is_free(cache: &CacheManager, name: &str) -> Result<()> { + let registry = cache.read_registry()?; + let taken = registry.entries.iter().any(|entry| { + entry + .display_name + .as_deref() + .is_some_and(|n| n.eq_ignore_ascii_case(name)) + }); + if taken { + return Err(Error::Other(format!( + "a catalog named \"{name}\" is already registered. Use `ai-catalog catalog update {name}` to refresh it." + ))); + } + Ok(()) +} + +fn ensure_url_is_free(cache: &CacheManager, url: &str) -> Result<()> { + let registry = cache.read_registry()?; + for entry in ®istry.entries { + if let Some(source_url) = cli_extension(entry) + .and_then(|m| m.get("sourceUrl")) + .and_then(|v| v.as_str()) + && source_url.eq_ignore_ascii_case(url) + { + let name = entry.display_name.as_deref().unwrap_or("?"); + return Err(Error::Other(format!( + "URL is already registered as \"{name}\". Use `ai-catalog catalog update {name}` to refresh it." + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::cell::Cell; + + use ai_catalog_discover::Response; + use url::Url; + + use super::*; + + const CATALOG: &str = r#"{ + "specVersion": "1.0", + "entries": [{"identifier": "a", "type": "application/json", "url": "https://example.com/a.json"}] + }"#; + + /// A site with a home page that advertises its catalog in a `Link` header. + struct Site { + requests: Cell, + } + + impl Fetcher for Site { + fn get(&self, url: &Url) -> std::result::Result { + self.requests.set(self.requests.get() + 1); + let (status, content_type, links, body) = match url.as_str() { + "https://example.com/" => ( + 200, + "text/html", + vec![r#"; rel="ai-catalog""#.to_string()], + "", + ), + "https://example.com/catalog/ai.json" => { + (200, "application/ai-catalog+json", vec![], CATALOG) + } + _ => (404, "text/plain", vec![], ""), + }; + Ok(Response { + url: url.clone(), + status, + content_type: Some(content_type.to_string()), + link_headers: links, + body: body.as_bytes().to_vec(), + }) + } + } + + fn cache(dir: &tempfile::TempDir) -> CacheManager { + CacheManager::with_base_dir(dir.path().join("cache")) + } + + fn registered(cache: &CacheManager) -> Vec<(String, String)> { + cache + .read_registry() + .unwrap() + .entries + .iter() + .map(|e| { + let source = cli_extension(e).unwrap()["sourceUrl"].as_str().unwrap(); + (e.display_name.clone().unwrap(), source.to_string()) + }) + .collect() + } + + #[test] + fn a_bare_domain_registers_the_catalog_it_advertises() { + let dir = tempfile::TempDir::new().unwrap(); + let cache = cache(&dir); + let site = Site { + requests: Cell::new(0), + }; + + add(&cache, &site, "example", "example.com").unwrap(); + + assert_eq!( + registered(&cache), + [( + "example".to_string(), + "https://example.com/catalog/ai.json".to_string() + )] + ); + } + + #[test] + fn the_same_catalog_cannot_be_registered_under_two_names() { + let dir = tempfile::TempDir::new().unwrap(); + let cache = cache(&dir); + let site = Site { + requests: Cell::new(0), + }; + add(&cache, &site, "first", "example.com").unwrap(); + + let error = add( + &cache, + &site, + "second", + "https://example.com/catalog/ai.json", + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("already registered as \"first\""), "{error}"); + assert_eq!(registered(&cache).len(), 1); + } + + #[test] + fn a_name_cannot_be_used_twice() { + let dir = tempfile::TempDir::new().unwrap(); + let cache = cache(&dir); + let site = Site { + requests: Cell::new(0), + }; + add(&cache, &site, "example", "example.com").unwrap(); + let before = site.requests.get(); + + let error = add(&cache, &site, "EXAMPLE", "example.com") + .unwrap_err() + .to_string(); + + assert!(error.contains("already registered"), "{error}"); + assert_eq!(site.requests.get(), before, "nothing should be fetched"); + } + + #[test] + fn a_site_without_a_catalog_is_not_registered() { + let dir = tempfile::TempDir::new().unwrap(); + let cache = cache(&dir); + let site = Site { + requests: Cell::new(0), + }; + + let error = add(&cache, &site, "nothing", "https://example.com/missing") + .unwrap_err() + .to_string(); + + assert!(error.contains("no AI Catalog found"), "{error}"); + assert!(registered(&cache).is_empty()); + } +} diff --git a/src/fetch.rs b/src/fetch.rs index d073c51..e5cf5c5 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -3,9 +3,10 @@ // SPDX-License-Identifier: Apache-2.0 use std::io::Read; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; +use ai_catalog_discover::{Fetcher, HttpFetcher, Method, discover_with}; use reqwest::blocking::Client; use reqwest::redirect::Policy; use url::Url; @@ -70,6 +71,56 @@ pub fn join(base: &str, reference: &str) -> Result { }) } +/// Where the catalog that a user pointed the CLI at actually is. +#[derive(Debug)] +pub enum Located { + /// A catalog URL, to be fetched. + Url(String), + /// A catalog that discovery has fetched already. + Discovered { + url: String, + bytes: Vec, + method: Method, + }, +} + +/// Turn what the user typed into a catalog. Local files and `http://` URLs are used as +/// they are; an `https://` URL or a bare domain goes through discovery +/// (). +pub fn locate(input: &str, fetcher: &dyn Fetcher) -> Result { + let input = input.trim(); + if input.starts_with("file://") || input.starts_with("http://") || Path::new(input).exists() { + return Ok(Located::Url(local_path_to_file_url(input)?)); + } + let bare = !input.contains("://"); + if bare && !looks_like_host(input) { + return Ok(Located::Url(local_path_to_file_url(input)?)); + } + match discover_with(fetcher, input) { + Ok(found) => Ok(Located::Discovered { + url: found.url, + bytes: found.bytes, + method: found.method, + }), + Err(e) if bare => Err(Error::Other(format!( + "{e}\n (there is no local file \"{input}\" either)" + ))), + Err(e) => Err(Error::Other(e.to_string())), + } +} + +/// A bare argument is a domain, not a path, when its first segment has a dot or a port. +fn looks_like_host(input: &str) -> bool { + let host = input.split('/').next().unwrap_or(""); + host.starts_with(|c: char| c.is_ascii_alphanumeric()) + && (host.contains('.') || host.contains(':') || host == "localhost") +} + +/// The fetcher discovery uses, with the CLI's own client. +pub fn build_fetcher() -> Result { + Ok(HttpFetcher::with_client(build_client()?)) +} + /// Fetch a catalog document, refusing anything over the size limit. pub fn fetch_catalog(url: &str, client: &Client) -> Result> { let mut bytes = Vec::new(); @@ -119,6 +170,105 @@ mod tests { ); } + use std::cell::RefCell; + + use ai_catalog_discover::Response; + + const CATALOG: &str = r#"{"specVersion": "1.0", "entries": []}"#; + + /// Serves canned responses, and records the requests. + #[derive(Default)] + struct Mock { + pages: Vec<(&'static str, &'static str, Vec<&'static str>, &'static str)>, + requests: RefCell>, + } + + impl Mock { + fn page( + mut self, + url: &'static str, + content_type: &'static str, + links: &[&'static str], + body: &'static str, + ) -> Self { + self.pages.push((url, content_type, links.to_vec(), body)); + self + } + } + + impl Fetcher for Mock { + fn get(&self, url: &Url) -> std::result::Result { + self.requests.borrow_mut().push(url.to_string()); + let page = self.pages.iter().find(|p| p.0 == url.as_str()); + Ok(Response { + url: url.clone(), + status: if page.is_some() { 200 } else { 404 }, + content_type: page.map(|p| p.1.to_string()), + link_headers: page + .map(|p| p.2.iter().map(|l| l.to_string()).collect()) + .unwrap_or_default(), + body: page.map(|p| p.3.as_bytes().to_vec()).unwrap_or_default(), + }) + } + } + + #[test] + fn locate_leaves_files_and_http_urls_alone() { + let mock = Mock::default(); + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("catalog.json"); + std::fs::write(&path, CATALOG).unwrap(); + + for input in [ + path.to_str().unwrap(), + "file:///tmp/catalog.json", + "http://localhost:8080/catalog.json", + ] { + let Located::Url(url) = locate(input, &mock).unwrap() else { + panic!("{input} should not be discovered"); + }; + assert!(url.contains("://"), "{url}"); + } + assert!(mock.requests.borrow().is_empty()); + } + + #[test] + fn locate_discovers_the_catalog_of_a_bare_domain() { + let mock = Mock::default() + .page( + "https://example.com/", + "text/html", + &[], + r#""#, + ) + .page( + "https://example.com/ai.json", + "application/ai-catalog+json", + &[], + CATALOG, + ); + + let Located::Discovered { url, bytes, method } = locate("example.com", &mock).unwrap() + else { + panic!("expected discovery"); + }; + + assert_eq!(url, "https://example.com/ai.json"); + assert_eq!(bytes, CATALOG.as_bytes()); + assert_eq!(method, Method::HtmlLink); + } + + #[test] + fn locate_keeps_the_error_for_a_mistyped_path() { + let mock = Mock::default(); + + for input in ["catalogs/missing.json", "./missing.json", "~/missing.json"] { + let error = locate(input, &mock).unwrap_err().to_string(); + assert!(error.contains("cannot resolve path"), "{error}"); + } + assert!(mock.requests.borrow().is_empty()); + } + #[test] fn fetch_catalog_rejects_oversized_files() { let dir = tempfile::TempDir::new().unwrap(); diff --git a/src/resolver.rs b/src/resolver.rs index fb1063d..8321801 100644 --- a/src/resolver.rs +++ b/src/resolver.rs @@ -69,8 +69,13 @@ impl<'a> Resolver<'a> { .as_ref() .expect("fetch needs an online resolver"); let bytes = fetch_catalog(url, client)?; - let catalog = serde_json::from_slice(&bytes)?; - let hash = self.store(url, &bytes)?; + self.fetch_bytes(url, &bytes) + } + + /// Like [`Resolver::fetch`], for a root document the caller has fetched from `url`. + pub fn fetch_bytes(&mut self, url: &str, bytes: &[u8]) -> Result { + let catalog = serde_json::from_slice(bytes)?; + let hash = self.store(url, bytes)?; self.path.push(url.to_string()); self.walk(&catalog, url, 0); self.path.pop();