diff --git a/package-manager/Cargo.lock b/package-manager/Cargo.lock index 4a7fb13..d3c9d2c 100644 --- a/package-manager/Cargo.lock +++ b/package-manager/Cargo.lock @@ -2223,7 +2223,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "w3bs-cli" -version = "0.1.19" +version = "0.1.20" dependencies = [ "aws-lc-rs", "clap", @@ -2251,7 +2251,7 @@ dependencies = [ [[package]] name = "w3bs-core" -version = "0.1.19" +version = "0.1.20" dependencies = [ "async-trait", "aws-lc-rs", @@ -2281,7 +2281,7 @@ dependencies = [ [[package]] name = "w3bs-publish" -version = "0.1.19" +version = "0.1.20" dependencies = [ "anyhow", "aws-lc-rs", @@ -2300,7 +2300,7 @@ dependencies = [ [[package]] name = "w3bs-schema" -version = "0.1.19" +version = "0.1.20" dependencies = [ "hex", "semver", diff --git a/package-manager/Cargo.toml b/package-manager/Cargo.toml index e0fad23..d71608f 100644 --- a/package-manager/Cargo.toml +++ b/package-manager/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/w3bs-schema", "crates/w3bs-core", "crates/w3bs-cli", "crates/w3bs-publish"] [workspace.package] -version = "0.1.19" +version = "0.1.20" edition = "2024" license = "MIT OR Apache-2.0" repository = "https://github.com/profullstack/w3bs.org" diff --git a/package-manager/crates/w3bs-cli/src/ask.rs b/package-manager/crates/w3bs-cli/src/ask.rs index 18f2762..ec145f9 100644 --- a/package-manager/crates/w3bs-cli/src/ask.rs +++ b/package-manager/crates/w3bs-cli/src/ask.rs @@ -210,16 +210,55 @@ pub fn strip_echo(answer: &str, system: &str) -> String { } /// The chat messages: the agent's system prompt, then the question with numbered background. -pub fn messages(agent: &Agent, question: &str, sources: &[Source]) -> Value { - let user = if sources.is_empty() { - question.to_owned() +/// +/// Layout 0 puts the question first in the user turn (with the background first, a 0.5B model +/// tends to repeat its system prompt). Layout 1, the retry after such an echo, moves the +/// background into the system message and leaves the user turn as just the question. +pub fn messages(agent: &Agent, question: &str, sources: &[Source], layout: u8) -> Value { + if sources.is_empty() { + return json!([{ "role": "system", "content": agent.system }, { "role": "user", "content": question }]); + } + let ctx = sources.iter().enumerate().map(|(i, s)| format!("[{}] {} ({})\n{}", i + 1, s.title, s.url, s.text)).collect::>().join("\n\n"); + if layout == 0 { + let user = format!("Question: {question}\n\nNotes from the web:\n{ctx}\n\nAnswer the question in a few practical steps."); + json!([{ "role": "system", "content": agent.system }, { "role": "user", "content": user }]) } else { - let ctx = sources.iter().enumerate().map(|(i, s)| format!("[{}] {} ({})\n{}", i + 1, s.title, s.url, s.text)).collect::>().join("\n\n"); - // Question first: with the background first, a 0.5B model tends to repeat its system - // prompt instead of answering. - format!("Question: {question}\n\nNotes from the web:\n{ctx}\n\nAnswer the question in a few practical steps.") - }; - json!([{ "role": "system", "content": agent.system }, { "role": "user", "content": user }]) + let system = format!("{}\n\nNotes from the web you may use:\n{ctx}", agent.system); + json!([{ "role": "system", "content": system }, { "role": "user", "content": question }]) + } +} + +/// Holds streamed tokens back while they could still be a copy of the system prompt, so an +/// echo never reaches the screen; everything after the echo (or a real answer) flows through. +#[derive(Debug, Default)] +pub struct EchoGuard { + held: String, + released: bool, +} + +impl EchoGuard { + /// Text to show now, if any. + pub fn push(&mut self, t: &str, system: &str) -> Option { + if self.released { + return Some(t.to_owned()); + } + self.held.push_str(t); + let h = self.held.trim_start(); + let sys = system.trim(); + if sys.starts_with(h) { + return None; + } + if let Some(rest) = h.strip_prefix(sys) { + let rest = rest.trim_start(); + if rest.is_empty() { + return None; + } + self.released = true; + return Some(rest.to_owned()); + } + self.released = true; + Some(std::mem::take(&mut self.held)) + } } // ------------------------------------------------------------------ Infernet SSE @@ -289,115 +328,127 @@ pub async fn ask(req: Request<'_>, on_token: &mut dyn FnMut(&str), on_status: &m let origin_raw = req.infernet.map(str::to_owned).or_else(|| std::env::var("W3BS_INFERNET_API").ok().filter(|s| !s.is_empty())).unwrap_or_else(|| INFERNET.into()); let origin = checked_url(&origin_raw, "the Infernet API")?.origin().ascii_serialization(); let key = std::env::var("INFERNET_API_KEY").ok().filter(|s| !s.trim().is_empty()); - let body = json!({ - "modelName": agent.model, - "messages": messages(agent, question, &sources), - "maxTokens": req.max_tokens.unwrap_or(agent.max_tokens), - "temperature": agent.temperature, - // Only a live peer node may answer: never Infernet's hosted (paid) fallback. - "p2p_only": true, - }); - on_status(&format!("asking {} on the c0mpute network", agent.name)); - let client = http(Some(Duration::from_secs(30))); - let mut post = client.post(format!("{origin}/api/chat")).json(&body); - if let Some(k) = &key { - post = post.bearer_auth(k.trim()); - } - let resp = post.send().await.map_err(|e| Error::package("ASK_UNAVAILABLE", format!("{origin}/api/chat: {e}")).retryable())?; - let status = resp.status().as_u16(); - let v: Value = serde_json::from_str(&resp.text().await.unwrap_or_default()).unwrap_or(Value::Null); - let msg = v["error"].as_str().map(str::to_owned).unwrap_or_else(|| format!("the Infernet API answered {status}")); - match status { - 200..=299 => {} - 429 => { - let at = v["detail"]["resetAt"].as_str().map(|t| format!(" (resets {t})")).unwrap_or_default(); - let hint = if key.is_some() { "your INFERNET_API_KEY budget is spent".to_owned() } else { "the public limit is 20 questions an hour per IP; set INFERNET_API_KEY to a trusted client key for more".to_owned() }; - return Err(Error::package("ASK_RATE_LIMITED", format!("Infernet rate limit reached{at}: {hint}")).retryable()); + let (mut node, mut job, mut final_text) = (Value::Null, String::new(), String::new()); + let mut first_token = None::; + for attempt in 0..2u8 { + let body = json!({ + "modelName": agent.model, + "messages": messages(agent, question, &sources, attempt), + "maxTokens": req.max_tokens.unwrap_or(agent.max_tokens), + "temperature": agent.temperature, + // Only a live peer node may answer: never Infernet's hosted (paid) fallback. + "p2p_only": true, + }); + on_status(&format!("asking {} on the c0mpute network", agent.name)); + let client = http(Some(Duration::from_secs(30))); + let mut post = client.post(format!("{origin}/api/chat")).json(&body); + if let Some(k) = &key { + post = post.bearer_auth(k.trim()); } - 503 => { - let live = v["detail"]["availableModels"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::>().join(", ")).unwrap_or_default(); - return Err(Error::package("ASK_NO_NODE", format!("no c0mpute node is serving {} right now; try again in a few minutes{}", agent.model, if live.is_empty() { String::new() } else { format!(" (live models: {live})") })).retryable()); + let resp = post.send().await.map_err(|e| Error::package("ASK_UNAVAILABLE", format!("{origin}/api/chat: {e}")).retryable())?; + let status = resp.status().as_u16(); + let v: Value = serde_json::from_str(&resp.text().await.unwrap_or_default()).unwrap_or(Value::Null); + let msg = v["error"].as_str().map(str::to_owned).unwrap_or_else(|| format!("the Infernet API answered {status}")); + match status { + 200..=299 => {} + 429 => { + let at = v["detail"]["resetAt"].as_str().map(|t| format!(" (resets {t})")).unwrap_or_default(); + let hint = if key.is_some() { "your INFERNET_API_KEY budget is spent".to_owned() } else { "the public limit is 20 questions an hour per IP; set INFERNET_API_KEY to a trusted client key for more".to_owned() }; + return Err(Error::package("ASK_RATE_LIMITED", format!("Infernet rate limit reached{at}: {hint}")).retryable()); + } + 503 => { + let live = v["detail"]["availableModels"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::>().join(", ")).unwrap_or_default(); + return Err(Error::package("ASK_NO_NODE", format!("no c0mpute node is serving {} right now; try again in a few minutes{}", agent.model, if live.is_empty() { String::new() } else { format!(" (live models: {live})") })).retryable()); + } + 401 | 403 => return Err(Error::package("ASK_REFUSED", format!("Infernet refused the question: {msg}"))), + _ => return Err(Error::package("ASK_FAILED", format!("Infernet: {msg}")).retryable()), } - 401 | 403 => return Err(Error::package("ASK_REFUSED", format!("Infernet refused the question: {msg}"))), - _ => return Err(Error::package("ASK_FAILED", format!("Infernet: {msg}")).retryable()), - } - if v["source"].as_str() != Some("p2p") { - return Err(Error::package("ASK_NOT_P2P", format!("Infernet offered a {:?} route, not a c0mpute node; refused", v["source"].as_str().unwrap_or("unknown")))); - } - let job = v["jobId"].as_str().unwrap_or_default().to_owned(); - let stream_path = v["streamUrl"].as_str().unwrap_or_default(); - if job.is_empty() || !stream_path.starts_with("/api/chat/stream/") || stream_path.contains("..") { - return Err(Error::package("ASK_FAILED", "Infernet's answer had no job stream")); - } - let mut node = json!({ "id": v["provider"]["nodeId"], "name": v["provider"]["name"], "provider_id": v["provider"]["id"], "gpu": v["provider"]["gpuModel"] }); - on_status(&format!("{} is answering", s(&node["name"]))); + if v["source"].as_str() != Some("p2p") { + return Err(Error::package("ASK_NOT_P2P", format!("Infernet offered a {:?} route, not a c0mpute node; refused", v["source"].as_str().unwrap_or("unknown")))); + } + job = v["jobId"].as_str().unwrap_or_default().to_owned(); + let stream_path = v["streamUrl"].as_str().unwrap_or_default(); + if job.is_empty() || !stream_path.starts_with("/api/chat/stream/") || stream_path.contains("..") { + return Err(Error::package("ASK_FAILED", "Infernet's answer had no job stream")); + } + node = json!({ "id": v["provider"]["nodeId"], "name": v["provider"]["name"], "provider_id": v["provider"]["id"], "gpu": v["provider"]["gpuModel"] }); + on_status(&format!("{} is answering", s(&node["name"]))); - let mut get = http(None).get(format!("{origin}{stream_path}")).header("accept", "text/event-stream"); - if let Some(k) = &key { - get = get.bearer_auth(k.trim()); - } - let deadline = started + req.timeout; - let resp = tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), get.send()) - .await - .map_err(|_| Error::package("ASK_TIMEOUT", "the node did not start answering in time").retryable())? - .map_err(|e| Error::package("ASK_UNAVAILABLE", format!("{origin}{stream_path}: {e}")).retryable())?; - if !resp.status().is_success() { - return Err(Error::package("ASK_FAILED", format!("the answer stream returned HTTP {}", resp.status().as_u16())).retryable()); - } - let mut stream = resp.bytes_stream(); - let mut sse = Sse::default(); - let (mut answer, mut finished, mut first_token) = (String::new(), None::, None::); - 'read: loop { - let chunk = match tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), stream.next()).await { - Err(_) => return Err(Error::package("ASK_TIMEOUT", format!("no complete answer after {} s (job {job})", req.timeout.as_secs())).retryable()), - Ok(None) => break, - Ok(Some(Err(e))) => return Err(Error::package("ASK_UNAVAILABLE", format!("the answer stream broke: {e}")).retryable()), - Ok(Some(Ok(b))) => b, - }; - for (event, data) in sse.push(&String::from_utf8_lossy(&chunk)) { - match event.as_str() { - "meta" => { - for (k, from) in [("id", "provider_node_id"), ("name", "provider_name"), ("engine", "engine")] { - if let Some(x) = data[from].as_str() { - node[k] = json!(x); + let mut get = http(None).get(format!("{origin}{stream_path}")).header("accept", "text/event-stream"); + if let Some(k) = &key { + get = get.bearer_auth(k.trim()); + } + let deadline = started + req.timeout; + let resp = tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), get.send()) + .await + .map_err(|_| Error::package("ASK_TIMEOUT", "the node did not start answering in time").retryable())? + .map_err(|e| Error::package("ASK_UNAVAILABLE", format!("{origin}{stream_path}: {e}")).retryable())?; + if !resp.status().is_success() { + return Err(Error::package("ASK_FAILED", format!("the answer stream returned HTTP {}", resp.status().as_u16())).retryable()); + } + let mut stream = resp.bytes_stream(); + let mut sse = Sse::default(); + let (mut answer, mut finished, mut guard) = (String::new(), None::, EchoGuard::default()); + 'read: loop { + let chunk = match tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), stream.next()).await { + Err(_) => return Err(Error::package("ASK_TIMEOUT", format!("no complete answer after {} s (job {job})", req.timeout.as_secs())).retryable()), + Ok(None) => break, + Ok(Some(Err(e))) => return Err(Error::package("ASK_UNAVAILABLE", format!("the answer stream broke: {e}")).retryable()), + Ok(Some(Ok(b))) => b, + }; + for (event, data) in sse.push(&String::from_utf8_lossy(&chunk)) { + match event.as_str() { + "meta" => { + for (k, from) in [("id", "provider_node_id"), ("name", "provider_name"), ("engine", "engine")] { + if let Some(x) = data[from].as_str() { + node[k] = json!(x); + } } } - } - "token" => { - if let Some(t) = data["text"].as_str() { - first_token.get_or_insert(started.elapsed().as_millis()); - answer.push_str(t); - on_token(t); + "token" => { + if let Some(t) = data["text"].as_str() { + answer.push_str(t); + if let Some(show) = guard.push(t, &agent.system) { + first_token.get_or_insert(started.elapsed().as_millis()); + on_token(&show); + } + } } + "done" => { + finished = Some(data["text"].as_str().map(str::to_owned).unwrap_or_else(|| answer.clone())); + break 'read; + } + "error" => { + let m = data["message"].as_str().or(data["error"].as_str()).or(data.as_str()).unwrap_or("the node failed").to_owned(); + return Err(Error::package("ASK_FAILED", format!("{} could not answer: {m}", s(&node["name"]))).retryable()); + } + _ => {} } - "done" => { - finished = Some(data["text"].as_str().map(str::to_owned).unwrap_or_else(|| answer.clone())); - break 'read; - } - "error" => { - let m = data["message"].as_str().or(data["error"].as_str()).or(data.as_str()).unwrap_or("the node failed").to_owned(); - return Err(Error::package("ASK_FAILED", format!("{} could not answer: {m}", s(&node["name"]))).retryable()); - } - _ => {} } } - } - let Some(text) = finished else { - return Err(Error::package("ASK_FAILED", format!("the answer stream ended before the node finished (job {job})")).retryable()); - }; - let text = strip_echo(&text, &agent.system); - if text.is_empty() { - return Err(Error::package("ASK_NO_ANSWER", format!("{} repeated its instructions instead of answering (job {job}); ask again, perhaps in other words", agent.name)).retryable()); - } - if answer.is_empty() { - on_token(&text); + let Some(text) = finished else { + return Err(Error::package("ASK_FAILED", format!("the answer stream ended before the node finished (job {job})")).retryable()); + }; + let text = strip_echo(&text, &agent.system); + if text.is_empty() { + if attempt == 0 { + on_status(&format!("{} repeated its instructions; asking again", s(&node["name"]))); + continue; + } + return Err(Error::package("ASK_NO_ANSWER", format!("{} repeated its instructions instead of answering, twice (job {job}); ask again, perhaps in other words", agent.name)).retryable()); + } + if !guard.released { + on_token(&text); + } + final_text = text; + break; } Ok(json!({ "agent": agent.name, "title": agent.title, "model": agent.model, "question": question, - "answer": text, + "answer": final_text, "sources": sources.iter().enumerate().map(|(i, s)| json!({ "n": i + 1, "title": s.title, "url": s.url })).collect::>(), "node": node, "transport": "infernet", @@ -635,12 +686,28 @@ mod tests { let a = agents(); let b = find(&a, "business").unwrap(); let src = vec![Source { title: "T".into(), url: "https://t.example/".into(), text: "body".into() }]; - let m = messages(b, "Q?", &src); + let m = messages(b, "Q?", &src, 0); assert_eq!(m[0]["role"], "system"); assert_eq!(m[1]["content"], "Question: Q?\n\nNotes from the web:\n[1] T (https://t.example/)\nbody\n\nAnswer the question in a few practical steps."); assert_eq!(strip_echo(&format!("{} Do X.", b.system), &b.system), "Do X."); assert_eq!(strip_echo(&b.system, &b.system), ""); - assert_eq!(messages(b, "Q?", &[])[1]["content"], "Q?"); + assert_eq!(messages(b, "Q?", &[], 0)[1]["content"], "Q?"); + let m1 = messages(b, "Q?", &src, 1); + assert_eq!(m1[1]["content"], "Q?"); + assert!(m1[0]["content"].as_str().unwrap().ends_with("Notes from the web you may use:\n[1] T (https://t.example/)\nbody")); + } + + #[test] + fn echo_guard_hides_a_repeated_system_prompt() { + let sys = "You are x. Be brief."; + let mut g = EchoGuard::default(); + assert_eq!(g.push("You are", sys), None); + assert_eq!(g.push(" x. Be brief.", sys), None); + assert_eq!(g.push(" Do A.", sys).as_deref(), Some("Do A.")); + assert_eq!(g.push(" Then B.", sys).as_deref(), Some(" Then B.")); + let mut g = EchoGuard::default(); + assert_eq!(g.push("You ", sys), None); + assert_eq!(g.push("can sell.", sys).as_deref(), Some("You can sell.")); } #[test] diff --git a/package-manager/crates/w3bs-cli/tests/ask.rs b/package-manager/crates/w3bs-cli/tests/ask.rs index 7006824..b5e7e0f 100644 --- a/package-manager/crates/w3bs-cli/tests/ask.rs +++ b/package-manager/crates/w3bs-cli/tests/ask.rs @@ -18,6 +18,7 @@ fn fake(chat_status: u16) -> Fake { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let origin = format!("http://{}", listener.local_addr().unwrap()); let seen: Seen = Arc::new(Mutex::new(vec![])); + let posts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let log = seen.clone(); std::thread::spawn(move || { for stream in listener.incoming() { @@ -60,9 +61,19 @@ fn fake(chat_status: u16) -> Fake { { "title": "Three", "url": "https://three.example/", "description": "third" } ]}).to_string()), ("POST", "/api/chat") if chat_status == 429 => (429, "application/json", json!({ "error": "Rate limit exceeded", "detail": { "resetAt": "2026-11-02T13:00:00.000Z" } }).to_string()), - ("POST", "/api/chat") => (200, "application/json", json!({ "jobId": "j1", "status": "assigned", "source": "p2p", "model": "hf.co/x/y-GGUF:latest", - "provider": { "id": "p1", "name": "node-one", "nodeId": "provider-1" }, "streamUrl": "/api/chat/stream/j1" }).to_string()), - ("GET", "/api/chat/stream/j1") => (200, "text/event-stream", [ + ("POST", "/api/chat") => { + let n = posts.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + (200, "application/json", json!({ "jobId": format!("j{n}"), "status": "assigned", "source": "p2p", "model": "hf.co/x/y-GGUF:latest", + "provider": { "id": "p1", "name": "node-one", "nodeId": "provider-1" }, "streamUrl": format!("/api/chat/stream/j{n}") }).to_string()) + } + // Every odd job only repeats the business agent's system prompt; the CLI must + // hide it and ask again. + ("GET", r) if r.starts_with("/api/chat/stream/j") && r[18..].parse::().unwrap_or(0) % 2 == 1 => (200, "text/event-stream", [ + "event: token\ndata: {\"text\":\"You are \"}\n\n", + "event: token\ndata: {\"text\":\"y.\"}\n\n", + "event: done\ndata: {\"text\":\"You are y.\",\"reason\":\"stop\"}\n\n", + ].concat()), + ("GET", r) if r.starts_with("/api/chat/stream/j") => (200, "text/event-stream", [ "event: job\ndata: {\"id\":\"j1\"}\n\n", "event: meta\ndata: {\"provider_node_id\":\"provider-1\",\"provider_name\":\"node-one\",\"engine\":\"ollama\"}\n\n", "event: token\ndata: {\"text\":\"Find \"}\n\n", @@ -111,7 +122,12 @@ fn ask_grounds_routes_p2p_and_streams() { let seen = f.seen.lock().unwrap().clone(); let search = seen.iter().find(|(_, p, _, _)| p.starts_with("/search")).unwrap(); assert_eq!(search.1, "/search?q=How+do+I+make+money%3F+startup"); - let (_, _, chat, auth) = seen.iter().find(|(m, p, _, _)| m == "POST" && p == "/api/chat").unwrap(); + let posts: Vec<_> = seen.iter().filter(|(m, p, _, _)| m == "POST" && p == "/api/chat").collect(); + assert_eq!(posts.len(), 2, "the echo was retried once"); + assert_eq!(posts[1].2["messages"][1]["content"], "How do I make money??", "the retry moves the background into the system message"); + assert!(posts[1].2["messages"][0]["content"].as_str().unwrap().contains("Notes from the web you may use:")); + assert_eq!(d["job_id"], "j2"); + let (_, _, chat, auth) = posts[0]; assert_eq!(chat["p2p_only"], true); assert_eq!(chat["modelName"], "hf.co/x/y-GGUF:latest"); assert_eq!((chat["maxTokens"].as_u64(), chat["temperature"].as_f64()), (Some(99), Some(0.1))); @@ -123,7 +139,7 @@ fn ask_grounds_routes_p2p_and_streams() { // Human output: the streamed answer, then the sources; who served it goes to stderr. let (code, out, err) = w3bs(&f, &["ask", "business", "Help?"], &[("INFERNET_API_KEY", "ifc_test")]); assert_eq!(code, 0, "{out} {err}"); - assert!(out.starts_with("Find customers.\n"), "{out}"); + assert!(out.starts_with("Find customers.\n"), "the echo never reaches the screen: {out}"); assert!(out.contains("Sources:\n [1] One\n https://one.example/"), "{out}"); assert!(err.contains("on c0mpute node node-one (provider-1)"), "{err}"); let last = f.seen.lock().unwrap().iter().rev().find(|(m, p, _, _)| m == "POST" && p == "/api/chat").unwrap().3.clone(); diff --git a/src/ask.mjs b/src/ask.mjs index 12b7cb5..2391b6d 100644 --- a/src/ask.mjs +++ b/src/ask.mjs @@ -5,7 +5,9 @@ import { readFileSync } from 'node:fs'; import { W3bsError } from './core.mjs'; -const REGISTRY = JSON.parse(readFileSync(new URL('../fixtures/agents.json', import.meta.url), 'utf8')); +const REGISTRY = JSON.parse( + readFileSync(new URL('../fixtures/agents.json', import.meta.url), 'utf8'), +); const SNIPPET = 700; export function agents(registry = REGISTRY) { @@ -40,36 +42,59 @@ export const publicAgent = (a) => ({ }); export function findAgent(name, list = agents()) { - const n = String(name ?? '').trim().replace(/^@/, '').toLowerCase(); + const n = String(name ?? '') + .trim() + .replace(/^@/, '') + .toLowerCase(); return list.find((a) => a.name === n) ?? list.find((a) => a.aliases.includes(n)) ?? null; } export function searchQuery(question, suffix = '') { - const q = question.replace(/([?!.])\1+/g, '$1').split(/\s+/).filter(Boolean).join(' '); + const q = question + .replace(/([?!.])\1+/g, '$1') + .split(/\s+/) + .filter(Boolean) + .join(' '); return suffix ? `${q} ${suffix}` : q; } export function pickSources(body, k) { const all = (Array.isArray(body?.results) ? body.results : []) - .filter((r) => typeof r?.title === 'string' && r.title.trim() && /^https?:\/\//.test(r?.url ?? '')) + .filter( + (r) => typeof r?.title === 'string' && r.title.trim() && /^https?:\/\//.test(r?.url ?? ''), + ) .map((r) => ({ title: r.title.trim(), url: r.url.trim(), - text: String(r.description ?? '').split(/\s+/).filter(Boolean).join(' ').slice(0, SNIPPET), + text: String(r.description ?? '') + .split(/\s+/) + .filter(Boolean) + .join(' ') + .slice(0, SNIPPET), })); return [...all.filter((s) => s.text), ...all.filter((s) => !s.text)].slice(0, k); } // Question first: with the background first, the 0.5B strategist repeats its system prompt. -export function messages(agent, question, sources) { - const user = sources.length - ? `Question: ${question}\n\nNotes from the web:\n${sources - .map((s, i) => `[${i + 1}] ${s.title} (${s.url})\n${s.text}`) - .join('\n\n')}\n\nAnswer the question in a few practical steps.` - : question; +// Layout 1 is the retry after an echo: the background moves into the system message. +export function messages(agent, question, sources, layout = 0) { + if (!sources.length) + return [ + { role: 'system', content: agent.system }, + { role: 'user', content: question }, + ]; + const ctx = sources.map((s, i) => `[${i + 1}] ${s.title} (${s.url})\n${s.text}`).join('\n\n'); + if (layout === 1) + return [ + { role: 'system', content: `${agent.system}\n\nNotes from the web you may use:\n${ctx}` }, + { role: 'user', content: question }, + ]; return [ { role: 'system', content: agent.system }, - { role: 'user', content: user }, + { + role: 'user', + content: `Question: ${question}\n\nNotes from the web:\n${ctx}\n\nAnswer the question in a few practical steps.`, + }, ]; } @@ -145,76 +170,105 @@ export async function ask( warnings.push(`pit.world search failed (${e.message}); answering without sources`); } } - const auth = key ? { authorization: `Bearer ${key}` } : {}; - const origin = new URL(infernet).origin; - const posted = await fetchImpl(`${origin}/api/chat`, { - method: 'POST', - headers: { 'content-type': 'application/json', ...auth }, - body: JSON.stringify({ - modelName: agent.model, - messages: messages(agent, q, sources), - maxTokens: agent.maxTokens, - temperature: agent.temperature, - p2p_only: true, - }), - signal: AbortSignal.timeout(30_000), - }).catch((e) => { - throw fail('ASK_UNAVAILABLE', `Infernet is unreachable: ${e.message}`); - }); - const job = await posted.json().catch(() => ({})); - if (posted.status === 429) - throw fail('ASK_RATE_LIMITED', 'The c0mpute network is rate limiting questions right now; try again later.', 429); - if (posted.status === 503) - throw fail('ASK_NO_NODE', `No c0mpute node is serving ${agent.model} right now; try again in a few minutes.`, 503); - if (!posted.ok) throw fail('ASK_FAILED', `Infernet: ${job.error ?? `HTTP ${posted.status}`}`); - if (job.source !== 'p2p') - throw fail('ASK_NOT_P2P', `Infernet offered a ${job.source ?? 'unknown'} route, not a c0mpute node; refused.`); - if (!job.jobId || !/^\/api\/chat\/stream\/[\w-]+$/.test(job.streamUrl ?? '')) - throw fail('ASK_FAILED', "Infernet's answer had no job stream."); - const node = { - id: job.provider?.nodeId ?? null, - name: job.provider?.name ?? null, - provider_id: job.provider?.id ?? null, - engine: null, - }; - const stream = await fetchImpl(`${origin}${job.streamUrl}`, { - headers: { accept: 'text/event-stream', ...auth }, - signal: AbortSignal.timeout(timeoutMs), - }).catch((e) => { - throw fail('ASK_UNAVAILABLE', `The answer stream failed: ${e.message}`); - }); - if (!stream.ok || !stream.body) throw fail('ASK_FAILED', `The answer stream returned HTTP ${stream.status}.`); - const parse = sseParser(); - const decoder = new TextDecoder(); - let streamed = '', - done = null, - firstToken = null; - try { - outer: for await (const chunk of stream.body) { - for (const [event, data] of parse(decoder.decode(chunk, { stream: true }))) { - if (event === 'meta') { - node.id = data.provider_node_id ?? node.id; - node.name = data.provider_name ?? node.name; - node.engine = data.engine ?? null; - } else if (event === 'token' && typeof data.text === 'string') { - firstToken ??= Date.now() - started; - streamed += data.text; - } else if (event === 'done') { - done = typeof data.text === 'string' ? data.text : streamed; - break outer; - } else if (event === 'error') { - throw fail('ASK_FAILED', `${node.name ?? 'The node'} could not answer: ${data.message ?? data.error ?? 'error'}`); + let result; + for (let attempt = 0; attempt < 2 && !result; attempt++) { + const auth = key ? { authorization: `Bearer ${key}` } : {}; + const origin = new URL(infernet).origin; + const posted = await fetchImpl(`${origin}/api/chat`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...auth }, + body: JSON.stringify({ + modelName: agent.model, + messages: messages(agent, q, sources, attempt), + maxTokens: agent.maxTokens, + temperature: agent.temperature, + p2p_only: true, + }), + signal: AbortSignal.timeout(30_000), + }).catch((e) => { + throw fail('ASK_UNAVAILABLE', `Infernet is unreachable: ${e.message}`); + }); + const job = await posted.json().catch(() => ({})); + if (posted.status === 429) + throw fail( + 'ASK_RATE_LIMITED', + 'The c0mpute network is rate limiting questions right now; try again later.', + 429, + ); + if (posted.status === 503) + throw fail( + 'ASK_NO_NODE', + `No c0mpute node is serving ${agent.model} right now; try again in a few minutes.`, + 503, + ); + if (!posted.ok) throw fail('ASK_FAILED', `Infernet: ${job.error ?? `HTTP ${posted.status}`}`); + if (job.source !== 'p2p') + throw fail( + 'ASK_NOT_P2P', + `Infernet offered a ${job.source ?? 'unknown'} route, not a c0mpute node; refused.`, + ); + if (!job.jobId || !/^\/api\/chat\/stream\/[\w-]+$/.test(job.streamUrl ?? '')) + throw fail('ASK_FAILED', "Infernet's answer had no job stream."); + const node = { + id: job.provider?.nodeId ?? null, + name: job.provider?.name ?? null, + provider_id: job.provider?.id ?? null, + engine: null, + }; + const stream = await fetchImpl(`${origin}${job.streamUrl}`, { + headers: { accept: 'text/event-stream', ...auth }, + signal: AbortSignal.timeout(timeoutMs), + }).catch((e) => { + throw fail('ASK_UNAVAILABLE', `The answer stream failed: ${e.message}`); + }); + if (!stream.ok || !stream.body) + throw fail('ASK_FAILED', `The answer stream returned HTTP ${stream.status}.`); + const parse = sseParser(); + const decoder = new TextDecoder(); + let streamed = '', + done = null, + firstToken = null; + try { + outer: for await (const chunk of stream.body) { + for (const [event, data] of parse(decoder.decode(chunk, { stream: true }))) { + if (event === 'meta') { + node.id = data.provider_node_id ?? node.id; + node.name = data.provider_name ?? node.name; + node.engine = data.engine ?? null; + } else if (event === 'token' && typeof data.text === 'string') { + firstToken ??= Date.now() - started; + streamed += data.text; + } else if (event === 'done') { + done = typeof data.text === 'string' ? data.text : streamed; + break outer; + } else if (event === 'error') { + throw fail( + 'ASK_FAILED', + `${node.name ?? 'The node'} could not answer: ${data.message ?? data.error ?? 'error'}`, + ); + } } } + } catch (e) { + if (e instanceof W3bsError) throw e; + throw fail('ASK_TIMEOUT', `No complete answer (job ${job.jobId}): ${e.message}`, 504); + } + if (done === null) + throw fail( + 'ASK_FAILED', + `The answer stream ended before the node finished (job ${job.jobId}).`, + ); + const answer = stripEcho(done, agent.system); + if (!answer) { + if (attempt === 0) continue; + throw fail( + 'ASK_NO_ANSWER', + `${agent.name} repeated its instructions instead of answering, twice (job ${job.jobId}); ask again, perhaps in other words.`, + ); } - } catch (e) { - if (e instanceof W3bsError) throw e; - throw fail('ASK_TIMEOUT', `No complete answer (job ${job.jobId}): ${e.message}`, 504); + result = { answer, node, job, firstToken }; } - if (done === null) throw fail('ASK_FAILED', `The answer stream ended before the node finished (job ${job.jobId}).`); - const answer = stripEcho(done, agent.system); - if (!answer) - throw fail('ASK_NO_ANSWER', `${agent.name} repeated its instructions instead of answering (job ${job.jobId}); ask again, perhaps in other words.`); + const { answer, node, job, firstToken } = result; return { agent: agent.name, title: agent.title, @@ -239,6 +293,10 @@ export function limiter({ max, windowMs }) { let h = hits.get(key); if (!h || h.reset <= now) hits.set(key, (h = { count: 0, reset: now + windowMs })); h.count += 1; - return { ok: h.count <= max, remaining: Math.max(0, max - h.count), resetAt: new Date(h.reset).toISOString() }; + return { + ok: h.count <= max, + remaining: Math.max(0, max - h.count), + resetAt: new Date(h.reset).toISOString(), + }; }; } diff --git a/test/ask.test.mjs b/test/ask.test.mjs index 60fefc4..f7394fb 100644 --- a/test/ask.test.mjs +++ b/test/ask.test.mjs @@ -46,6 +46,7 @@ test('the registry, aliases, query, sources, prompt and SSE framing', () => { // A fake pit.world search + Infernet chat API on one port. async function fakeNetwork() { const seen = []; + let posts = 0; const server = createServer(async (req, res) => { let body = ''; for await (const c of req) body += c; @@ -56,9 +57,17 @@ async function fakeNetwork() { } if (req.method === 'POST' && req.url === '/api/chat') { res.setHeader('content-type', 'application/json'); - return res.end(JSON.stringify({ jobId: 'j1', source: 'p2p', provider: { id: 'p1', name: 'node-one', nodeId: 'provider-1' }, streamUrl: '/api/chat/stream/j1' })); + posts += 1; + return res.end(JSON.stringify({ jobId: `j${posts}`, source: 'p2p', provider: { id: 'p1', name: 'node-one', nodeId: 'provider-1' }, streamUrl: `/api/chat/stream/j${posts}` })); } - if (req.url === '/api/chat/stream/j1') { + // Odd jobs only repeat the system prompt: the server must ask again. + const job = /^\/api\/chat\/stream\/j(\d+)$/.exec(req.url); + if (job && Number(job[1]) % 2 === 1) { + res.setHeader('content-type', 'text/event-stream'); + const sys = 'You are saas-business-strategist-agent. You give concise, practical business strategy grounded in what independent writers on these subjects are publishing.'; + return res.end(`event: done\ndata: ${JSON.stringify({ text: sys })}\n\n`); + } + if (job) { res.setHeader('content-type', 'text/event-stream'); res.write('event: meta\ndata: {"provider_name":"node-one","provider_node_id":"provider-1","engine":"ollama"}\n\n'); res.write('event: token\ndata: {"text":"Find "}\n\n'); @@ -96,7 +105,12 @@ test('POST /api/v1/ask and the w3bs_ask MCP tool route p2p_only, ground and thro assert.deepEqual(d.sources, [{ n: 1, title: 'One', url: 'https://one.example/' }]); assert.equal(d.node.name, 'node-one'); assert.equal(d.node.engine, 'ollama'); - const chat = net.seen.find((x) => x.url === '/api/chat'); + const chats = net.seen.filter((x) => x.url === '/api/chat'); + assert.equal(chats.length, 2, 'the echo was retried once'); + assert.equal(chats[1].body.messages[1].content, 'How do I make money??'); + assert.match(chats[1].body.messages[0].content, /Notes from the web you may use:/); + assert.equal(d.job_id, 'j2'); + const chat = chats[0]; assert.equal(chat.body.p2p_only, true); assert.equal(chat.body.modelName, 'hf.co/profullstack/saas-business-strategist-agent-GGUF:latest'); assert.equal(chat.auth, 'Bearer ifc_test');