diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 1f08263dc..70377ae34 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**,不是最大的编号(守卫 `schema_version_policy_compares_against_current` // 按个数比):编号有空缺时两者不同——0071 由一个开放 PR 占着,0072 先落,个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 79; +const CURRENT_SCHEMA_VERSION: u32 = 80; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs index c87ebb43a..1f44119a1 100644 --- a/crates/utopia-server/src/api/review_routes.rs +++ b/crates/utopia-server/src/api/review_routes.rs @@ -1361,7 +1361,8 @@ pub struct DecideAlignmentKindWordReq { } /// 人定一个类别词绑到哪个类(#725 对齐队列)。绑上的类写到它名下的实体上;两端的类变了, -/// 短语签名跟着变,所以再排一次短语对齐。 +/// 短语签名跟着变,所以再排一次短语对齐——与判定同一事务排下(`decide_and_apply_human`), +/// 提交之后再排的话,进程在两步之间退出就只剩判定。 pub async fn decide_alignment_kind_word( State(state): State, AuthUser(user): AuthUser, @@ -1399,12 +1400,6 @@ pub async fn decide_alignment_kind_word( if !written { return Err(utopia_core::AppError::NotFound.into()); } - utopia_store::jobs::enqueue_unless_queued( - &state.pool, - "align_phrases", - json!({ "kb_id": kb_id }), - ) - .await?; let _ = utopia_store::audit::record( &state.pool, Some(kb_id), diff --git a/crates/utopia-server/src/type_alignment.rs b/crates/utopia-server/src/type_alignment.rs index 50f591494..48f79a1b2 100644 --- a/crates/utopia-server/src/type_alignment.rs +++ b/crates/utopia-server/src/type_alignment.rs @@ -4,10 +4,15 @@ //! 不选类。类是本体的事,绑定是对齐的事:一个库里 distinct 的类别词就几十上百个,每个 //! 只判一次——例句、它参与的关系短语、候选类的定义一起给模型,**两票一致**才绑,不一致 //! 记成 undecided 留给审核(#725 队列 2),没有类对得上的记成 none 并按老流程提成 -//! 「建议加类」(`proposed_type` → 本体页采纳)。绑定存在 `type_bindings`,按类的 -//! `updated_at` 与库里最新的类判过期,本体一改只重判过期的(决策记录里说的「版本」就是 -//! 这两个时间戳)。绑上的类写到该类别词下每个实体的 `type_id`(`type_source = 'aligned'`, -//! 人定过类的不动),身份消解的按类圈范围随之恢复。 +//! 「建议加类」(`proposed_type` → 本体页采纳)。绑定存在 `type_bindings`,连同判定时给 +//! 模型看的候选类的指纹(`basis`:各自的 `updated_at` 与祖先闭包,0053 的类别词那一半)。 +//! 过期 = 按现在的类重算的指纹对不上,本体一改只重判对不上的。从前按时间戳判,看不见 +//! 两件事:模型答题期间改的定义(判定写在答完之后,时间戳说判定更新,#795),和不碰 +//! `updated_at` 的父边增删。绑上的类写到该类别词下每个实体的 `type_id`(`type_source = +//! 'aligned'`,人定过类的不动),身份消解的按类圈范围随之恢复。 +//! +//! 输入在调模型之前从一个快照里读齐;写判定时在同一事务里按当前的类再算一遍指纹,对不上 +//! 就不收这份回复、再排一轮(`type_bindings::decide_and_apply_if_current`)。 //! //! 候选类怎么来:库配了嵌入模型就按类别词加例名检索最近的类;没配就在类不多时整表给; //! 类太多又没嵌入时不判——瞎判比不判糟。 @@ -21,7 +26,7 @@ use utopia_core::models::EntityType; use utopia_extract::align::{ build_kind_word_messages, parse_kind_word_response, ClassCandidate, KindWordItem, }; -use utopia_store::type_bindings::{self, KindWordSignature}; +use utopia_store::type_bindings::{self, Acceptance, Binding, KindWordSignature}; use uuid::Uuid; /// 一次问多少个类别词。 @@ -35,6 +40,9 @@ const WHOLE_LIST_LIMIT: usize = 60; type Vote = Option; /// 给一批类别词找候选类:有嵌入就检索,没有就整表(类少时)。返回每个签名的候选 id 列表。 +/// +/// 检索报错也退回整表,同短语那边:候选算在判定的指纹里,报错这一轮判的词恢复以后 +/// 指纹对不上、再问一轮。不退回的话,嵌入端点一直坏着时新来的词就永远没有类 async fn candidates_for( state: &AppState, kb_id: Uuid, @@ -45,10 +53,21 @@ async fn candidates_for( .iter() .map(|s| format!("{}: {}", s.kind_word, s.examples.join(", "))) .collect(); - let nearest = - ontology_index::nearest_for_each(state, kb_id, &queries, CANDIDATES, Target::ClassLabel) - .await - .unwrap_or_default(); + let nearest = match ontology_index::nearest_for_each( + state, + kb_id, + &queries, + CANDIDATES, + Target::ClassLabel, + ) + .await + { + Ok(nearest) => nearest, + Err(e) => { + tracing::warn!(%kb_id, error = %e, "类别词对齐检索候选类失败,退回整表"); + Vec::new() + } + }; let mut out = Vec::with_capacity(sigs.len()); for (i, _) in sigs.iter().enumerate() { let found: Vec = nearest @@ -128,34 +147,58 @@ async fn align_types_locked( client: &utopia_llm::LlmClient, ) -> anyhow::Result<()> { let pool = &state.pool; - let classes = utopia_store::graph::entity_types(pool, kb_id).await?; - let by_id: HashMap = classes.iter().map(|c| (c.id, c)).collect(); - let by_key: HashMap<&str, &EntityType> = classes.iter().map(|c| (c.key.as_str(), c)).collect(); - let sigs = type_bindings::signatures(pool, kb_id).await?; - let existing: HashMap = - type_bindings::bindings(pool, kb_id) + // 调模型之前在一个快照里读齐输入(#795):类的定义、类的版本与父边、类别词、已有的 + // 判定。分几次读的话,两次读之间提交的编辑会让提示词里的定义和指纹里的版本对不上 + let (classes, snapshot, sigs, existing) = { + let mut tx = pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *tx) + .await?; + let classes = utopia_store::graph::entity_types(&mut *tx, kb_id).await?; + let snapshot = type_bindings::class_snapshot(&mut *tx, kb_id).await?; + let sigs = type_bindings::signatures(&mut *tx, kb_id).await?; + let existing: HashMap = type_bindings::bindings(&mut *tx, kb_id) .await? .into_iter() .map(|b| (b.kind_word.clone(), b)) .collect(); - let stale: HashSet = type_bindings::stale(pool, kb_id) - .await? - .into_iter() + tx.commit().await?; + (classes, snapshot, sigs, existing) + }; + let by_id: HashMap = classes.iter().map(|c| (c.id, c)).collect(); + let by_key: HashMap<&str, &EntityType> = classes.iter().map(|c| (c.key.as_str(), c)).collect(); + // 每个活着的类别词此刻给模型看的候选与指纹。检索一轮只做一次:开跑时拿它决定判谁, + // 收尾时拿同一份候选、按重新读的类再算指纹,看跑着的时候有没有东西过期 + let all: Vec<&KindWordSignature> = sigs.iter().collect(); + let considered: HashMap, String)> = sigs + .iter() + .zip(candidates_for(state, kb_id, &all, &classes).await?) + .map(|(s, c)| { + let basis = snapshot.basis(&c); + (s.kind_word.clone(), (c, basis)) + }) .collect(); + // 过期 = 存下的指纹和此刻的不一样(没有指纹的是这一列出现前判的,各重判一次)。人的判定不重判 let todo: Vec<&KindWordSignature> = sigs .iter() .filter(|s| match existing.get(&s.kind_word) { None => true, - Some(b) => b.decided_by != "person" && stale.contains(&s.kind_word), + Some(b) => { + b.decided_by != "person" + && b.basis.as_deref() != Some(considered[&s.kind_word].1.as_str()) + } }) .collect(); let attempted: HashSet = todo.iter().map(|s| s.kind_word.clone()).collect(); + // 本轮写下的,或人已判过、代理不覆盖的。问了却没落下结论的不算,见收尾 + let mut settled: HashSet = HashSet::new(); tracing::info!(%kb_id, kind_words = sigs.len(), to_decide = todo.len(), classes = classes.len(), "类别词对齐开始"); - // 没有类可绑:每个词都是「没有」,并提成建议;类出现后 `stale` 会把它们再交回来 + // 没有类可绑:每个词都是「没有」,并提成建议;类出现后候选变了,指纹对不上,它们会再交回来 if classes.is_empty() { for s in &todo { - type_bindings::decide_and_apply( + let (candidates, basis) = &considered[&s.kind_word]; + let accepted = type_bindings::decide_and_apply_if_current( pool, kb_id, &s.kind_word, @@ -163,16 +206,19 @@ async fn align_types_locked( None, "none", &serde_json::json!({ "reason": "no classes" }), - "agent", - ) - .await?; - type_bindings::propose( - pool, - kb_id, - &s.kind_word, - s.words.first().unwrap_or(&s.kind_word), + basis, + candidates, ) .await?; + if accepted == Acceptance::Written { + type_bindings::propose( + pool, + kb_id, + &s.kind_word, + s.words.first().unwrap_or(&s.kind_word), + ) + .await?; + } } return Ok(()); } @@ -182,8 +228,13 @@ async fn align_types_locked( let mut failed = 0usize; // 问了、模型也答了、却没答到的词:两票缺一票就不下结论 let mut unanswered = 0usize; + // 模型答题期间候选类变了、回复没收下的词:它们答的是旧输入,收尾时再排一轮拿新的问 + let mut moved = 0usize; for batch in todo.chunks(BATCH) { - let cands = candidates_for(state, kb_id, batch, &classes).await?; + let cands: Vec<&[Uuid]> = batch + .iter() + .map(|s| considered[&s.kind_word].0.as_slice()) + .collect(); // 两票:第二票把候选倒过来给,防止「选第一个」这种顺序偏好冒充一致 let mut votes: Vec<(Vote, Vote)> = vec![(None, None); batch.len()]; let mut answered = vec![(false, false); batch.len()]; @@ -193,7 +244,7 @@ async fn align_types_locked( .enumerate() .filter(|(i, _)| !cands[*i].is_empty()) .map(|(i, s)| { - let mut ids: Vec = cands[i].clone(); + let mut ids: Vec = cands[i].to_vec(); if pass == 1 { ids.reverse(); } @@ -284,67 +335,56 @@ async fn align_types_locked( continue; } let record = serde_json::json!({ "first": a, "second": b }); - if !agree(a, b) { - if type_bindings::decide_and_apply( - pool, - kb_id, - &s.kind_word, - &s.words, - None, - "undecided", - &record, - "agent", - ) - .await? - { - undecided += 1; + let (type_id, status) = if !agree(a, b) { + (None, "undecided") + } else { + match a.as_deref().and_then(|k| by_key.get(k)) { + Some(class) => (Some(class.id), "bound"), + None => (None, "none"), } - continue; - } - match a.as_deref().and_then(|k| by_key.get(k)) { - Some(class) => { - if type_bindings::decide_and_apply( - pool, - kb_id, - &s.kind_word, - &s.words, - Some(class.id), - "bound", - &record, - "agent", - ) - .await? - { - bound += 1; + }; + let (candidates, basis) = &considered[&s.kind_word]; + match type_bindings::decide_and_apply_if_current( + pool, + kb_id, + &s.kind_word, + &s.words, + type_id, + status, + &record, + basis, + candidates, + ) + .await? + { + Acceptance::Written => { + settled.insert(s.kind_word.clone()); + match status { + "bound" => bound += 1, + "undecided" => undecided += 1, + _ => { + type_bindings::propose( + pool, + kb_id, + &s.kind_word, + s.words.first().unwrap_or(&s.kind_word), + ) + .await?; + none += 1; + } } } - None => { - if type_bindings::decide_and_apply( - pool, - kb_id, - &s.kind_word, - &s.words, - None, - "none", - &record, - "agent", - ) - .await? - { - type_bindings::propose( - pool, - kb_id, - &s.kind_word, - s.words.first().unwrap_or(&s.kind_word), - ) - .await?; - none += 1; - } + Acceptance::KeptPerson => { + settled.insert(s.kind_word.clone()); + } + Acceptance::Moved => { + tracing::info!(%kb_id, kind_word = %s.kind_word, "类别词对齐:模型答题期间候选类变了,这份回复不收"); + moved += 1; } } } } - tracing::info!(%kb_id, bound, none, undecided, skipped, failed, unanswered, "类别词对齐完成"); + tracing::info!(%kb_id, bound, none, undecided, skipped, failed, unanswered, moved, "类别词对齐完成"); if unanswered > 0 { tracing::warn!(%kb_id, unanswered, "类别词对齐有词模型没答到,这些词这轮没有结论"); } @@ -352,21 +392,45 @@ async fn align_types_locked( state.emit_graph(kb_id); } // 同短语对齐:来了没试过的新词、本轮判完的又过期了,就再排一次(从头算一份, - // 新词换了提示词)。「过期」不限本轮判的,跑着时建的类也要让老绑定再判一次 - let changed = { - let stale_now: HashSet = type_bindings::stale(pool, kb_id) + // 新词换了提示词)。「过期」不限本轮判的,跑着时改的类也要让老绑定再判一次——比的 + // 是按**重新读**的类算出来的指纹(候选沿用开跑时检索的那份)。本轮问了却没落下结论 + // 的词不在这里算:调用失败、读不出、漏答、没有候选的走下面有上限的再问;算进来的话, + // 一个永久报错的端点会让任务一轮接一轮立刻重排 + let changed = moved > 0 || { + let now = type_bindings::class_snapshot(pool, kb_id).await?; + let decided: HashMap = type_bindings::bindings(pool, kb_id) .await? .into_iter() + .map(|b| (b.kind_word.clone(), b)) .collect(); type_bindings::signatures(pool, kb_id) .await? .iter() - .any(|s| !attempted.contains(&s.kind_word) && !existing.contains_key(&s.kind_word)) - || type_bindings::bindings(pool, kb_id) - .await? - .iter() - .any(|b| b.decided_by != "person" && stale_now.contains(&b.kind_word)) + .any(|s| match decided.get(&s.kind_word) { + None => !attempted.contains(&s.kind_word), + Some(b) => { + b.decided_by != "person" + && (settled.contains(&s.kind_word) || !attempted.contains(&s.kind_word)) + && considered.get(&s.kind_word).is_some_and(|(candidates, _)| { + b.basis.as_deref() != Some(now.basis(candidates).as_str()) + }) + } + }) }; + finish(state, kb_id, reask, changed, failed, unanswered).await +} + +/// 一轮的收尾:有新活就立刻再排一轮;没判完的延时再问,次数有上限;不再排自己时才排 +/// 短语对齐。 +async fn finish( + state: &AppState, + kb_id: Uuid, + reask: u32, + changed: bool, + failed: usize, + unanswered: usize, +) -> anyhow::Result<()> { + let pool = &state.pool; // 本轮没判完的(调用失败、回复读不出、模型漏答了几个 id)自己再排,最多 MAX_REASK 次, // 每次多等一会。从前只有失败的批次会再排,漏答的词就只能等下一篇文档来排——最后 // 一篇之后没有下一篇,它们就永远没有结论;而漏答不写任何行,本体页也看不见 diff --git a/crates/utopia-server/src/type_alignment_tests.rs b/crates/utopia-server/src/type_alignment_tests.rs index e75e9ecd2..8d9d82918 100644 --- a/crates/utopia-server/src/type_alignment_tests.rs +++ b/crates/utopia-server/src/type_alignment_tests.rs @@ -33,6 +33,13 @@ async fn reply(State(m): State, Json(body): Json) -> impl IntoResp format!("data: {frame}\n\ndata: [DONE]\n\n"), ) } +/// 嵌入端点一直坏着:只有配了嵌入模型的测试会走到这里 +async fn embeddings_down() -> impl IntoResponse { + ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "embedding backend down", + ) +} struct Fx { pool: sqlx::PgPool, state: AppState, @@ -87,6 +94,7 @@ impl Fx { let endpoint = format!("http://{}", listener.local_addr()?); let router = Router::new() .route("/chat/completions", post(reply)) + .route("/embeddings", post(embeddings_down)) .with_state(model.clone()); let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap(); @@ -250,11 +258,16 @@ async fn human_decision_during_disagreement_survives() -> anyhow::Result<()> { .bind(f.entity) .fetch_one(&f.pool) .await?; + let requeued = align_types_jobs(&f).await?; let class = f.class; f.cleanup().await?; assert_eq!(binding.decided_by, "person"); assert_eq!(binding.type_id, Some(class)); assert_eq!(projected, Some(class)); + assert!( + requeued.is_empty(), + "a person's decision carries no basis and is never asked about again" + ); Ok(()) } @@ -732,3 +745,247 @@ async fn an_unreadable_reply_is_asked_again_a_bounded_number_of_times() -> anyho assert_eq!(phrases_queued_late, 1, "最后一轮排短语对齐"); Ok(()) } + +/// #795:模型还在答的时候类的定义改了。两票读的都是旧定义,它们的答案不能当成对新定义 +/// 的判定留下来;下一轮要拿新定义再问,问完之后输入没变就不再问 +#[tokio::test] +async fn an_edit_during_the_model_request_is_asked_again() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![ + vote(None), + vote(None), + vote(Some("organization")), + vote(Some("organization")), + ]) + .await? + else { + return Ok(()); + }; + let run = async { + f.model + .hold + .store(true, std::sync::atomic::Ordering::SeqCst); + let state = f.state.clone(); + let kb = f.kb; + let worker = tokio::spawn(async move { align_types_reasking(&state, kb, 0).await }); + tokio::time::timeout( + std::time::Duration::from_secs(10), + f.model.entered.notified(), + ) + .await?; + sqlx::query( + "UPDATE entity_types SET description='NEW definition', updated_at=clock_timestamp() + WHERE id=$1", + ) + .bind(f.class) + .execute(&f.pool) + .await?; + f.model.release.notify_one(); + worker.await??; + let first = f.requests(); + anyhow::ensure!(first.len() == 2, "two votes, got {}", first.len()); + anyhow::ensure!( + first + .iter() + .all(|r| r.to_string().contains("OLD definition")), + "both votes were built before the edit" + ); + sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id'=$1") + .bind(f.kb.to_string()) + .execute(&f.pool) + .await?; + f.run().await?; + let second = f.requests(); + anyhow::ensure!( + second.len() == 4, + "the edit made during the request was taken as seen: the next run asked {} more", + second.len() - 2 + ); + anyhow::ensure!( + second[2..] + .iter() + .all(|r| r.to_string().contains("NEW definition")), + "the next run asks with the new definition" + ); + let binding = type_bindings::bindings(&f.pool, f.kb).await?.remove(0); + anyhow::ensure!(binding.status == "bound" && binding.type_id == Some(f.class)); + f.run().await?; + anyhow::ensure!(f.requests().len() == 4, "unchanged inputs ask nothing"); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} + +async fn align_types_jobs(f: &Fx) -> anyhow::Result>> { + Ok(sqlx::query_scalar( + "SELECT (payload->>'reask')::bigint FROM jobs + WHERE kind='align_types' AND payload->>'kb_id'=$1 ORDER BY id", + ) + .bind(f.kb.to_string()) + .fetch_all(&f.pool) + .await?) +} + +async fn clear_jobs(f: &Fx) -> anyhow::Result<()> { + sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id'=$1") + .bind(f.kb.to_string()) + .execute(&f.pool) + .await?; + Ok(()) +} + +/// 父边的增删不碰 `updated_at`,时间戳看不见它;指纹里有祖先闭包,看得见 +#[tokio::test] +async fn a_parent_edge_makes_an_agent_binding_stale() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![ + vote(Some("organization")), + vote(Some("organization")), + vote(Some("organization")), + vote(Some("organization")), + ]) + .await? + else { + return Ok(()); + }; + let run = async { + let legal = Uuid::now_v7(); + sqlx::query("INSERT INTO entity_types(id,kb_id,key,label,description) VALUES($1,$2,'legal_entity','Legal entity','A body the law treats as a person')") + .bind(legal).bind(f.kb).execute(&f.pool).await?; + f.run().await?; + anyhow::ensure!(f.requests().len() == 2); + clear_jobs(&f).await?; + let versions = "SELECT id, updated_at FROM entity_types WHERE kb_id=$1 ORDER BY id"; + let before: Vec<(Uuid, chrono::DateTime)> = sqlx::query_as(versions) + .bind(f.kb) + .fetch_all(&f.pool) + .await?; + sqlx::query( + "INSERT INTO entity_type_parents(child_id,parent_id,is_primary) VALUES($1,$2,true)", + ) + .bind(f.class) + .bind(legal) + .execute(&f.pool) + .await?; + let after: Vec<(Uuid, chrono::DateTime)> = sqlx::query_as(versions) + .bind(f.kb) + .fetch_all(&f.pool) + .await?; + anyhow::ensure!(before == after, "a parent edge leaves updated_at alone"); + f.run().await?; + anyhow::ensure!( + f.requests().len() == 4, + "the ancestor closure is part of the basis, so the word is asked again" + ); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} + +/// 这一列出现之前代理判的行没有指纹:各重判一次,之后不再问 +#[tokio::test] +async fn rows_without_a_basis_are_decided_again_once() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![vote(Some("organization")), vote(Some("organization"))]).await? + else { + return Ok(()); + }; + let run = async { + f.seed_bound().await?; + f.run().await?; + anyhow::ensure!(f.requests().len() == 2); + let binding = type_bindings::bindings(&f.pool, f.kb).await?.remove(0); + anyhow::ensure!(binding.basis.is_some() && binding.decided_by == "agent"); + f.run().await?; + anyhow::ensure!( + f.requests().len() == 2, + "a recorded basis that still matches asks nothing" + ); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} + +/// 过期的词这一轮问了却没问出结论(端点一直报错、回复读不出):走有上限的延时再问, +/// 不能因为「还过期」立刻再排——那样一个永久报错的端点会让任务一轮接一轮地跑 +#[tokio::test] +async fn a_stale_word_whose_batch_fails_takes_the_bounded_reask() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![ + json!({"answer": "organization"}), + json!({"answer": "organization"}), + ]) + .await? + else { + return Ok(()); + }; + let run = async { + f.seed_bound().await?; + f.run().await?; + anyhow::ensure!(f.requests().len() == 2); + let jobs = align_types_jobs(&f).await?; + anyhow::ensure!( + jobs == vec![Some(1)], + "only the delayed re-ask is queued, got {jobs:?}" + ); + let binding = type_bindings::bindings(&f.pool, f.kb).await?.remove(0); + anyhow::ensure!(binding.basis.is_none(), "nothing was decided this round"); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} + +/// 嵌入端点一直坏着:检索退回整表,小库照样绑得上(#894 修过的「整库没有类型」不能回来); +/// 一直坏下去候选不变、指纹不变,也不会每轮再问一遍 +#[tokio::test] +async fn a_retrieval_error_falls_back_to_the_whole_class_list() -> anyhow::Result<()> { + let Some(f) = Fx::new(vec![vote(Some("organization")), vote(Some("organization"))]).await? + else { + return Ok(()); + }; + let run = async { + let ws: Uuid = sqlx::query_scalar("SELECT workspace_id FROM knowledge_bases WHERE id=$1") + .bind(f.kb) + .fetch_one(&f.pool) + .await?; + let chat = utopia_store::settings::get(&f.pool, ws) + .await? + .and_then(|s| s.chat_base_url) + .expect("fixture endpoint"); + // upsert 整行覆盖(只有密钥传 None 才保留旧值),对话端点得原样再给一次 + utopia_store::settings::upsert( + &f.pool, + ws, + Some(&chat), + None, + Some("scripted"), + Some(&chat), + None, + Some("scripted-embedding"), + Some(4), + ) + .await?; + f.run().await?; + anyhow::ensure!(f.requests().len() == 2, "the fallback still asks"); + let binding = type_bindings::bindings(&f.pool, f.kb).await?.remove(0); + anyhow::ensure!(binding.status == "bound" && binding.type_id == Some(f.class)); + let jobs = align_types_jobs(&f).await?; + anyhow::ensure!( + jobs.is_empty(), + "a settled word queues nothing, got {jobs:?}" + ); + f.run().await?; + anyhow::ensure!( + f.requests().len() == 2, + "while retrieval keeps failing the candidates, and so the basis, stay the same" + ); + anyhow::Ok(()) + } + .await; + f.cleanup().await?; + run +} diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index e31c0f6f4..22218e901 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -39,7 +39,11 @@ const ADOPT_MERGED: &str = "merged"; // 剩下的那个函数于是只是在遍历一张空表。**本体从建库第一天起就只有 // 用户自己导入的词表**——与 0009 删掉内置实体类是同一件事的下半段。 -pub async fn entity_types(pool: &PgPool, kb_id: Uuid) -> AppResult> { +/// 库里全部类。执行器取泛型:类别词对齐要在调模型之前的同一个快照事务里读它(#795)。 +pub async fn entity_types<'e>( + pool: impl sqlx::Executor<'e, Database = sqlx::Postgres>, + kb_id: Uuid, +) -> AppResult> { Ok( // 又一次 SELECT *:parents 在关联表里,`*` 取不到。 // 这是同一个陷阱的第三次——SQL 在字符串里,cargo check 全绿, diff --git a/crates/utopia-store/src/jobs.rs b/crates/utopia-store/src/jobs.rs index e7ae331aa..8206f8a7d 100644 --- a/crates/utopia-store/src/jobs.rs +++ b/crates/utopia-store/src/jobs.rs @@ -69,6 +69,19 @@ pub async fn enqueue_unless_queued( payload: serde_json::Value, ) -> AppResult> { let mut tx = pool.begin().await?; + let id = enqueue_unless_queued_tx(&mut tx, kind, payload).await?; + tx.commit().await?; + Ok(id) +} + +/// 同 [`enqueue_unless_queued`],但排在调用方的事务里:任务与事务里别的写一起提交、 +/// 一起回滚。人定一个类别词时,判定和它的短语对齐任务要么都在、要么都不在——先提交 +/// 判定再排,进程在两步之间退出就只剩判定,它改了的签名没人去重判 +pub async fn enqueue_unless_queued_tx( + tx: &mut Transaction<'_, Postgres>, + kind: &str, + payload: serde_json::Value, +) -> AppResult> { let row: Option<(i64,)> = sqlx::query_as( "INSERT INTO jobs (kind, payload) SELECT $1, $2 @@ -77,12 +90,11 @@ pub async fn enqueue_unless_queued( ) .bind(kind) .bind(payload) - .fetch_optional(&mut *tx) + .fetch_optional(&mut **tx) .await?; if row.is_some() { - notify_worker_tx(&mut tx).await?; + notify_worker_tx(tx).await?; } - tx.commit().await?; Ok(row.map(|(id,)| id)) } diff --git a/crates/utopia-store/src/type_bindings.rs b/crates/utopia-store/src/type_bindings.rs index dd0ae1bbf..5c6859772 100644 --- a/crates/utopia-store/src/type_bindings.rs +++ b/crates/utopia-store/src/type_bindings.rs @@ -9,6 +9,12 @@ //! 类别词在 Rust 与 SQL 两侧用同一种归一:空白折成一个空格、去两端、小写。 //! [`normalize`] 与 [`KIND_WORD_SQL`] 必须说同一件事,否则 `signatures` 数出来的词 //! `apply` 找不着。 +//! +//! 代理的判定记下它看到的输入(`basis`,0053 的类别词那一半,#795):给模型看的每个 +//! 候选类的 id、`updated_at` 与祖先闭包([`ClassSnapshot::basis`])。过期 = 按现在的类 +//! 重算出来的指纹对不上,不再比时刻——判定写在模型答完之后,答题期间改的定义时间戳 +//! 看不见,父边的增删也不碰 `updated_at`。写判定时在同一事务里再算一遍,对不上的回复 +//! 不收([`decide_and_apply_if_current`])。 use chrono::{DateTime, Utc}; use sqlx::{Executor, PgPool, Postgres}; @@ -48,7 +54,10 @@ pub struct KindWordSignature { } /// 库里每个 distinct 的类别词:活着的(`merged_into` 空)、带类别词的实体。 -pub async fn signatures(pool: &PgPool, kb_id: Uuid) -> AppResult> { +pub async fn signatures<'e>( + pool: impl Executor<'e, Database = Postgres>, + kb_id: Uuid, +) -> AppResult> { let sql = format!( "WITH live AS ( SELECT e.id, e.canonical_name, e.description, e.created_at, @@ -97,12 +106,18 @@ pub struct Binding { pub decided_at: DateTime, /// agent / person pub decided_by: String, + /// 代理判定时输入的指纹([`ClassSnapshot::basis`])。人的判定不带——人不按指纹重判; + /// 这一列出现之前的代理判定也为空,各重判一次 + pub basis: Option, } /// 库里全部绑定,按词序。 -pub async fn bindings(pool: &PgPool, kb_id: Uuid) -> AppResult> { +pub async fn bindings<'e>( + pool: impl Executor<'e, Database = Postgres>, + kb_id: Uuid, +) -> AppResult> { Ok(sqlx::query_as( - "SELECT kind_word, type_id, status, decided_at, decided_by + "SELECT kind_word, type_id, status, decided_at, decided_by, basis FROM type_bindings WHERE kb_id = $1 ORDER BY kind_word", ) .bind(kb_id) @@ -110,9 +125,92 @@ pub async fn bindings(pool: &PgPool, kb_id: Uuid) -> AppResult> { .await?) } +/// 一个库此刻的类:每个类的 `updated_at` 与直接父类。判定的指纹从它算——开跑时在调模型 +/// 之前的快照事务里读一份,写判定时在写的事务里再读一份,两份算出来不一样就是答题 +/// 期间候选类变了 +#[derive(Debug, Clone, Default)] +pub struct ClassSnapshot { + versions: HashMap>, + parents: HashMap>, +} + +/// 读一个库的 [`ClassSnapshot`]。 +pub async fn class_snapshot<'e>( + pool: impl Executor<'e, Database = Postgres>, + kb_id: Uuid, +) -> AppResult { + let rows: Vec<(Uuid, DateTime, Vec)> = sqlx::query_as( + "SELECT t.id, t.updated_at, + ARRAY(SELECT p.parent_id FROM entity_type_parents p WHERE p.child_id = t.id) + FROM entity_types t WHERE t.kb_id = $1", + ) + .bind(kb_id) + .fetch_all(pool) + .await?; + let mut snapshot = ClassSnapshot::default(); + for (id, at, parents) in rows { + snapshot.versions.insert(id, at); + snapshot.parents.insert(id, parents); + } + Ok(snapshot) +} + +impl ClassSnapshot { + /// 一个类别词判定的指纹:给模型看的每个候选类的 id、`updated_at` 与祖先闭包。 + /// 候选的先后不算(第二票本来就倒着给);候选已经不在了记成 gone,同样算变了。 + /// 父边的增删不碰 `updated_at`,所以闭包要单独算进来。 + pub fn basis(&self, candidates: &[Uuid]) -> String { + let mut parts: Vec = candidates + .iter() + .map(|id| match self.versions.get(id) { + Some(at) => { + let up: Vec = self.ancestors(*id).iter().map(Uuid::to_string).collect(); + format!("{id}@{}^{}", at.to_rfc3339(), up.join(",")) + } + None => format!("{id}@gone"), + }) + .collect(); + parts.sort(); + parts.dedup(); + fingerprint(&parts.join(";")) + } + + /// 一个类的全部祖先(不含自己),排好序。多继承与菱形按并集走,同一个祖先只进一次; + /// 编辑器不允许环,见过就不再走,万一有环也走得完 + fn ancestors(&self, id: Uuid) -> Vec { + let mut seen: Vec = Vec::new(); + let mut stack: Vec = self.parents.get(&id).cloned().unwrap_or_default(); + while let Some(p) = stack.pop() { + if p == id || seen.contains(&p) { + continue; + } + seen.push(p); + if let Some(up) = self.parents.get(&p) { + stack.extend_from_slice(up); + } + } + seen.sort(); + seen + } +} + +/// 与 `phrase_bindings::basis_of` 同一个哈希:FNV-1a 64 位。只是缓存失效的键,不是安全 +/// 用途,不值得为它拉一个哈希依赖;也不去动那边——那边的值一变,全部短语判定都得重判 +fn fingerprint(text: &str) -> String { + let mut h: u64 = 0xcbf29ce484222325; + for b in text.as_bytes() { + h ^= u64::from(*b); + h = h.wrapping_mul(0x100000001b3); + } + format!("{h:016x}") +} + /// 不再成立的绑定:绑到的类在判定之后改过;或判成 none / undecided 之后库里有类 /// 新建或修改。负向判定没有选中的类,已有类的新定义也可能让它对得上。 /// 绑到的类被删了的,行已随级联消失,这里不会出现。 +/// +/// 对齐的 worker 已不读它:时间戳看不见模型答题期间的编辑(#795),过期改按指纹判 +/// ([`ClassSnapshot::basis`])。留着给只要粗信号的调用方。 pub async fn stale(pool: &PgPool, kb_id: Uuid) -> AppResult> { Ok(sqlx::query_scalar( "SELECT b.kind_word @@ -134,7 +232,7 @@ pub async fn stale(pool: &PgPool, kb_id: Uuid) -> AppResult> { /// /// **人的判定不被代理覆盖**:已有行是人判的而这次是代理,原样留着、返回 false。 /// 反过来人可以改代理的。`words` 传空时保留已有的写法——人在界面上拍板时手里 -/// 未必有签名。 +/// 未必有签名。这里写下的判定不带指纹:代理的判定走 [`decide_and_apply_if_current`]。 #[allow(clippy::too_many_arguments)] pub async fn decide<'e>( pool: impl Executor<'e, Database = Postgres>, @@ -145,6 +243,24 @@ pub async fn decide<'e>( status: &str, votes: &serde_json::Value, decided_by: &str, +) -> AppResult { + decide_with_basis( + pool, kb_id, kind_word, words, type_id, status, votes, decided_by, None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn decide_with_basis<'e>( + pool: impl Executor<'e, Database = Postgres>, + kb_id: Uuid, + kind_word: &str, + words: &[String], + type_id: Option, + status: &str, + votes: &serde_json::Value, + decided_by: &str, + basis: Option<&str>, ) -> AppResult { if !matches!(status, "bound" | "none" | "undecided") { return Err(AppError::Validation(format!( @@ -169,8 +285,8 @@ pub async fn decide<'e>( } let res = sqlx::query( "INSERT INTO type_bindings - (id, kb_id, kind_word, words, type_id, status, votes, decided_at, decided_by) - VALUES ($1, $2, $3, $4, $5, $6, $7, now(), $8) + (id, kb_id, kind_word, words, type_id, status, votes, decided_at, decided_by, basis) + VALUES ($1, $2, $3, $4, $5, $6, $7, now(), $8, $9) ON CONFLICT (kb_id, kind_word) DO UPDATE SET words = CASE WHEN cardinality(EXCLUDED.words) = 0 THEN type_bindings.words ELSE EXCLUDED.words END, @@ -178,7 +294,8 @@ pub async fn decide<'e>( status = EXCLUDED.status, votes = EXCLUDED.votes, decided_at = now(), - decided_by = EXCLUDED.decided_by + decided_by = EXCLUDED.decided_by, + basis = EXCLUDED.basis WHERE NOT (type_bindings.decided_by = 'person' AND EXCLUDED.decided_by = 'agent')", ) .bind(Uuid::now_v7()) @@ -189,6 +306,7 @@ pub async fn decide<'e>( .bind(status) .bind(votes) .bind(decided_by) + .bind(basis) .execute(pool) .await?; Ok(res.rows_affected() > 0) @@ -211,16 +329,89 @@ pub async fn decide_and_apply( ) -> AppResult { let mut tx = pool.begin().await?; let written = write_decision_and_projection( - &mut tx, kb_id, kind_word, words, type_id, status, votes, decided_by, + &mut tx, kb_id, kind_word, words, type_id, status, votes, decided_by, None, ) .await?; tx.commit().await?; Ok(written) } +/// What became of an agent decision offered to [`decide_and_apply_if_current`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Acceptance { + /// The decision and its projection are written, with the basis. + Written, + /// A person decided this kind word; the agent's answer does not replace it. + KeptPerson, + /// The candidate classes changed while the model was answering. The answer + /// is about inputs that no longer exist, so nothing is written. + Moved, +} + +/// Accept an agent decision only for the inputs it actually read (#795). +/// +/// `basis` is what the run computed from its snapshot before calling the model. +/// The candidate classes are locked `FOR SHARE` before the binding row is written: +/// a class delete takes the class row first and then cascades to the binding, so +/// taking them in the same order cannot deadlock with it. The basis is then +/// recomputed from the rows as they are now; a mismatch means the model answered +/// about a definition or hierarchy that has since changed, and nothing is written. +/// +/// A parent edge or a new class committed after this check is not blocked. The +/// stored basis then no longer matches the current inputs, so the next run finds +/// the decision stale and asks again. +#[allow(clippy::too_many_arguments)] +pub async fn decide_and_apply_if_current( + pool: &PgPool, + kb_id: Uuid, + kind_word: &str, + words: &[String], + type_id: Option, + status: &str, + votes: &serde_json::Value, + basis: &str, + candidates: &[Uuid], +) -> AppResult { + let mut tx = pool.begin().await?; + sqlx::query( + "SELECT id FROM entity_types WHERE kb_id = $1 AND id = ANY($2) ORDER BY id FOR SHARE", + ) + .bind(kb_id) + .bind(candidates) + .fetch_all(&mut *tx) + .await?; + if class_snapshot(&mut *tx, kb_id).await?.basis(candidates) != basis { + tx.rollback().await?; + return Ok(Acceptance::Moved); + } + let written = write_decision_and_projection( + &mut tx, + kb_id, + kind_word, + words, + type_id, + status, + votes, + "agent", + Some(basis), + ) + .await?; + tx.commit().await?; + Ok(if written { + Acceptance::Written + } else { + Acceptance::KeptPerson + }) +} + /// The review request may wait briefly for a concurrent writer, but must not /// pin a connection indefinitely. This is per lock acquisition, not a request /// deadline, and does not change the background aligner's waiting policy. +/// +/// The decision, its projection and the phrase alignment it makes necessary +/// commit together: a changed class moves the phrase signatures of every entity +/// under this kind word, and a job queued after the commit could be lost with +/// the process in between. pub async fn decide_and_apply_human( pool: &PgPool, kb_id: Uuid, @@ -233,7 +424,7 @@ pub async fn decide_and_apply_human( sqlx::query("SET LOCAL lock_timeout = '2s'") .execute(&mut *tx) .await?; - write_decision_and_projection( + let written = write_decision_and_projection( &mut tx, kb_id, kind_word, @@ -242,8 +433,18 @@ pub async fn decide_and_apply_human( if type_id.is_some() { "bound" } else { "none" }, votes, "person", + None, ) - .await + .await?; + if written { + crate::jobs::enqueue_unless_queued_tx( + &mut tx, + "align_phrases", + serde_json::json!({ "kb_id": kb_id }), + ) + .await?; + } + Ok::(written) } .await; match result { @@ -283,8 +484,9 @@ async fn write_decision_and_projection( status: &str, votes: &serde_json::Value, decided_by: &str, + basis: Option<&str>, ) -> AppResult { - let written = decide( + let written = decide_with_basis( &mut *connection, kb_id, kind_word, @@ -293,6 +495,7 @@ async fn write_decision_and_projection( status, votes, decided_by, + basis, ) .await?; if written { @@ -405,7 +608,9 @@ pub async fn propose( #[cfg(test)] mod tests { - use super::normalize; + use super::{normalize, ClassSnapshot}; + use chrono::{DateTime, Utc}; + use uuid::Uuid; #[test] fn a_kind_word_is_one_word_however_spaced_or_cased() { @@ -417,4 +622,74 @@ mod tests { assert_eq!(normalize("指标"), "指标"); assert_eq!(normalize(" "), ""); } + + fn at(s: &str) -> DateTime { + s.parse().expect("timestamp") + } + + /// organization ⊂ legal_entity;agent 暂时没有父类 + fn snapshot() -> (ClassSnapshot, [Uuid; 3]) { + let ids = [Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()]; + let [org, legal, _] = ids; + let mut s = ClassSnapshot::default(); + for id in ids { + s.versions.insert(id, at("2026-09-26T00:00:00Z")); + s.parents.insert(id, Vec::new()); + } + s.parents.insert(org, vec![legal]); + (s, ids) + } + + #[test] + fn a_basis_names_what_was_shown_not_the_order_it_was_shown_in() { + let (s, [org, legal, _]) = snapshot(); + assert_eq!(s.basis(&[org, legal]), s.basis(&[legal, org])); + assert_eq!(s.basis(&[org, legal]), s.basis(&[org, legal, org])); + assert_ne!(s.basis(&[org, legal]), s.basis(&[org])); + } + + #[test] + fn an_edit_a_grandparent_edge_or_a_deleted_candidate_changes_the_basis() { + let (s, [org, legal, agent]) = snapshot(); + let before = s.basis(&[org]); + + let mut edited = s.clone(); + edited.versions.insert(org, at("2026-09-26T00:00:01Z")); + assert_ne!(before, edited.basis(&[org]), "an edit moves updated_at"); + + // 父边的增删不碰 updated_at:org 的祖父变了,org 的指纹也得变 + let mut rooted = s.clone(); + rooted.parents.insert(legal, vec![agent]); + assert_ne!( + before, + rooted.basis(&[org]), + "a grandparent is in the closure" + ); + + let mut gone = s.clone(); + gone.versions.remove(&org); + gone.parents.remove(&org); + assert_ne!( + before, + gone.basis(&[org]), + "a deleted candidate is a change" + ); + } + + #[test] + fn a_diamond_or_a_cycle_still_gives_one_closure() { + let (mut s, [org, legal, agent]) = snapshot(); + // 菱形:org → legal、org → agent,legal → agent + s.parents.insert(org, vec![legal, agent]); + s.parents.insert(legal, vec![agent]); + assert_eq!(s.ancestors(org), { + let mut v = vec![legal, agent]; + v.sort(); + v + }); + // 编辑器不许环;万一有,走得完且不把自己算进祖先 + s.parents.insert(agent, vec![org]); + assert!(!s.ancestors(org).contains(&org)); + let _ = s.basis(&[org, legal, agent]); + } } diff --git a/crates/utopia-store/tests/store/a_kind_word_binds_to_a_class.rs b/crates/utopia-store/tests/store/a_kind_word_binds_to_a_class.rs index 2a2d69ad6..930bf7e72 100644 --- a/crates/utopia-store/tests/store/a_kind_word_binds_to_a_class.rs +++ b/crates/utopia-store/tests/store/a_kind_word_binds_to_a_class.rs @@ -486,3 +486,399 @@ async fn a_kind_word_is_counted_once_bound_once_and_applied_to_its_entities() -> .await?; run } + +/// 自己的组织名:上面那个测试开跑时按名字清场,共用名字会互相删掉对方的数据 +const BASIS_ORG: &str = "kind-word-basis-test"; + +/// 代理的判定只在它读到的输入还成立时才收(#795);人的判定不带指纹,代理不覆盖; +/// 人定的判定与它的短语对齐任务同一事务提交 +#[tokio::test] +async fn an_agent_decision_is_accepted_only_for_the_inputs_it_read() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(BASIS_ORG) + .execute(&pool) + .await?; + let (org, ws, kb, class, acme) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)") + .bind(org) + .bind(BASIS_ORG) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)") + .bind(ws) + .bind(org) + .bind(BASIS_ORG) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)") + .bind(kb) + .bind(ws) + .bind(BASIS_ORG) + .execute(&pool) + .await?; + + let run = async { + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label, description) + VALUES ($1, $2, 'organization', 'Organization', 'OLD definition')", + ) + .bind(class) + .bind(kb) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO entities (id, kb_id, canonical_name, specific_type) + VALUES ($1, $2, 'Acme', 'company')", + ) + .bind(acme) + .bind(kb) + .execute(&pool) + .await?; + let shown = [class]; + let before = type_bindings::class_snapshot(&pool, kb) + .await? + .basis(&shown); + + // 模型答题期间定义改了:这份回复答的是旧定义,不收,什么都不写 + sqlx::query( + "UPDATE entity_types SET description = 'NEW definition', updated_at = clock_timestamp() + WHERE id = $1", + ) + .bind(class) + .execute(&pool) + .await?; + let votes = serde_json::json!({ "first": "organization", "second": "organization" }); + assert_eq!( + type_bindings::decide_and_apply_if_current( + &pool, + kb, + "company", + &[], + Some(class), + "bound", + &votes, + &before, + &shown, + ) + .await?, + type_bindings::Acceptance::Moved + ); + assert!(type_bindings::bindings(&pool, kb).await?.is_empty()); + assert_eq!(typed(&pool, acme).await?.type_id, None); + + // 按新输入问出来的:收下,指纹一起存 + let current = type_bindings::class_snapshot(&pool, kb) + .await? + .basis(&shown); + assert_ne!(before, current); + assert_eq!( + type_bindings::decide_and_apply_if_current( + &pool, + kb, + "company", + &[], + Some(class), + "bound", + &votes, + ¤t, + &shown, + ) + .await?, + type_bindings::Acceptance::Written + ); + let b = type_bindings::bindings(&pool, kb).await?.remove(0); + assert_eq!(b.basis.as_deref(), Some(current.as_str())); + assert_eq!(typed(&pool, acme).await?.type_id, Some(class)); + + // 人判的:不带指纹;它的短语对齐任务与判定一起提交 + assert!(type_bindings::decide_and_apply_human(&pool, kb, "company", None, &votes).await?); + let b = type_bindings::bindings(&pool, kb).await?.remove(0); + assert_eq!((b.decided_by.as_str(), b.basis), ("person", None)); + let queued: i64 = sqlx::query_scalar( + "SELECT count(*) FROM jobs + WHERE kind = 'align_phrases' AND payload->>'kb_id' = $1 AND status = 'queued'", + ) + .bind(kb.to_string()) + .fetch_one(&pool) + .await?; + assert_eq!(queued, 1); + + // 代理晚到的回复不覆盖人 + assert_eq!( + type_bindings::decide_and_apply_if_current( + &pool, + kb, + "company", + &[], + Some(class), + "bound", + &votes, + ¤t, + &shown, + ) + .await?, + type_bindings::Acceptance::KeptPerson + ); + assert_eq!( + type_bindings::bindings(&pool, kb) + .await? + .remove(0) + .decided_by, + "person" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + + sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id' = $1") + .bind(kb.to_string()) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(BASIS_ORG) + .execute(&pool) + .await?; + run +} + +/// 锁顺序的两个测试用:一个类、一个带类别词的实体、一行绑在这个类上的代理判定。 +/// 判定不投影——`entities.type_id` 删类时是 RESTRICT,实体一指着这个类,删类本身就会失败, +/// 测的就不是锁了。返回 (库, 类) +async fn seed_lock_order(pool: &PgPool, name: &str) -> anyhow::Result<(Uuid, Uuid)> { + let (org, ws, kb, class) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)") + .bind(org) + .bind(name) + .execute(pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)") + .bind(ws) + .bind(org) + .bind(name) + .execute(pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)") + .bind(kb) + .bind(ws) + .bind(name) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, 'organization', 'Organization')", + ) + .bind(class) + .bind(kb) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO entities (id, kb_id, canonical_name, specific_type) + VALUES ($1, $2, 'Acme', 'company')", + ) + .bind(Uuid::now_v7()) + .bind(kb) + .execute(pool) + .await?; + type_bindings::decide( + pool, + kb, + "company", + &[], + Some(class), + "bound", + &serde_json::json!({}), + "agent", + ) + .await?; + Ok((kb, class)) +} + +/// 等到有人排在 `pid` 后面等锁。`chain` 时等的是更长的一串:有人排在一个正排在 `pid` +/// 后面的人后面 +async fn wait_behind(pool: &PgPool, pid: i32, chain: bool) -> anyhow::Result<()> { + let sql = if chain { + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity p + WHERE EXISTS (SELECT 1 FROM unnest(pg_blocking_pids(p.pid)) b(pid) + WHERE $1 = ANY(pg_blocking_pids(b.pid))))" + } else { + "SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE $1 = ANY(pg_blocking_pids(pid)))" + }; + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let waiting: bool = sqlx::query_scalar(sql).bind(pid).fetch_one(pool).await?; + if waiting { + return anyhow::Ok(()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await? +} + +/// 代理收判定时先锁候选类、再写判定行,与删类的顺序一致(删类先拿类的行,再级联到判定行)。 +/// 反过来就是死锁:代理拿着判定行等类,删类拿着类等判定行。这里让代理停在两步之间, +/// 删类排到它后面,再放行:两边都得走完,谁也不报 40P01 +#[tokio::test] +async fn an_acceptance_and_a_class_delete_wait_for_each_other_instead_of_deadlocking( +) -> anyhow::Result<()> { + const NAME: &str = "kind-word-lock-order-test"; + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(NAME) + .execute(&pool) + .await?; + let (kb, class) = seed_lock_order(&pool, NAME).await?; + + let run = async { + let shown = [class]; + let basis = type_bindings::class_snapshot(&pool, kb) + .await? + .basis(&shown); + // 闸门拿着判定行:代理锁完候选类、算完指纹,停在写判定这一步 + let mut gate = pool.begin().await?; + let gate_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *gate) + .await?; + sqlx::query("SELECT id FROM type_bindings WHERE kb_id = $1 FOR UPDATE") + .bind(kb) + .fetch_one(&mut *gate) + .await?; + let agent = { + let pool = pool.clone(); + tokio::spawn(async move { + type_bindings::decide_and_apply_if_current( + &pool, + kb, + "company", + &[], + None, + "undecided", + &serde_json::json!({}), + &basis, + &shown, + ) + .await + }) + }; + wait_behind(&pool, gate_pid, false).await?; + let deleter = { + let pool = pool.clone(); + tokio::spawn(async move { + sqlx::query("DELETE FROM entity_types WHERE id = $1") + .bind(class) + .execute(&pool) + .await + }) + }; + // 删类排在代理后面(代理拿着类的共享锁),代理排在闸门后面 + wait_behind(&pool, gate_pid, true).await?; + gate.rollback().await?; + let accepted = tokio::time::timeout(Duration::from_secs(10), agent).await???; + tokio::time::timeout(Duration::from_secs(10), deleter).await???; + assert_eq!(accepted, type_bindings::Acceptance::Written); + let b = type_bindings::bindings(&pool, kb).await?; + assert_eq!( + b.len(), + 1, + "the undecided row no longer points at the class" + ); + assert_eq!((b[0].status.as_str(), b[0].type_id), ("undecided", None)); + let classes: i64 = sqlx::query_scalar("SELECT count(*) FROM entity_types WHERE kb_id = $1") + .bind(kb) + .fetch_one(&pool) + .await?; + assert_eq!( + classes, 0, + "the delete went through after the agent committed" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(NAME) + .execute(&pool) + .await?; + run +} + +/// 候选类在收判定之前被删了:代理等删类提交,读到的类已经不在,指纹对不上,这份回复 +/// 不收;判定行随级联走了,代理也不把它写回来 +#[tokio::test] +async fn a_candidate_deleted_while_the_reply_waits_moves_it() -> anyhow::Result<()> { + const NAME: &str = "kind-word-deleted-candidate-test"; + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(NAME) + .execute(&pool) + .await?; + let (kb, class) = seed_lock_order(&pool, NAME).await?; + + let run = async { + let shown = [class]; + let basis = type_bindings::class_snapshot(&pool, kb) + .await? + .basis(&shown); + let mut deleter = pool.begin().await?; + let deleter_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *deleter) + .await?; + sqlx::query("DELETE FROM entity_types WHERE id = $1") + .bind(class) + .execute(&mut *deleter) + .await?; + let agent = { + let pool = pool.clone(); + tokio::spawn(async move { + type_bindings::decide_and_apply_if_current( + &pool, + kb, + "company", + &[], + None, + "none", + &serde_json::json!({}), + &basis, + &shown, + ) + .await + }) + }; + wait_behind(&pool, deleter_pid, false).await?; + deleter.commit().await?; + let accepted = tokio::time::timeout(Duration::from_secs(10), agent).await???; + assert_eq!(accepted, type_bindings::Acceptance::Moved); + assert!( + type_bindings::bindings(&pool, kb).await?.is_empty(), + "the cascade took the row and the moved reply did not write it back" + ); + Ok::<(), anyhow::Error>(()) + } + .await; + + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(NAME) + .execute(&pool) + .await?; + run +} diff --git a/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md b/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md index 48c432c5f..b77056555 100644 --- a/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md +++ b/docs/decisions/0053-a-phrase-decision-records-the-inputs-it-considered.md @@ -1,6 +1,6 @@ # 0053 · A phrase decision records the inputs it considered -- **Status**: implemented 2026-09-23 in PR #878 · `phrase_bindings.basis` (migration 0072), candidates admitted through the class hierarchy and shown to the model as such, structural outcomes recorded instead of skipped, the requeue condition reads live signatures only · closes the lifecycle half of #807 and the phrase half of #795; the kind-word aligner keeps its timestamp staleness for now +- **Status**: implemented 2026-09-23 in PR #878 · `phrase_bindings.basis` (migration 0072), candidates admitted through the class hierarchy and shown to the model as such, structural outcomes recorded instead of skipped, the requeue condition reads live signatures only · closes the lifecycle half of #807 and the phrase half of #795 · kind words since 2026-09-26: `type_bindings.basis` (migration 0092), read from one snapshot and compared again when a reply is accepted, see [the revision below](#revision-2026-09-26-kind-words) - **Written**: 2026-09-23 - **Related**: [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) decision 3; [0051](0051-a-human-phrase-decision-carries-its-materialization-work.md); #807, #795, #801 (withdrawn), #773, #754 @@ -73,7 +73,8 @@ property stops fitting keeps its projection: the person said so. - Fingerprinting the kind-word aligner. #795's reproduction is on `align_types`; the same design applies and is the obvious next cut, but its inputs (kind words, class definitions, the - hierarchy) are a different set and this record does not claim them. + hierarchy) are a different set and this record does not claim them. Done since, as the + [revision below](#revision-2026-09-26-kind-words). - A revision table of decisions. The old decision is overwritten in place as before; the audit ledger keeps the person's decisions and the projection changes. #807 asked what records are retained: the answer here is the current decision plus its basis, nothing historical. @@ -92,3 +93,53 @@ for the next run; a person's decision made during a request is not overwritten. The cost is one fingerprint per live signature per run, computed from data the run already loads, plus one query for property versions. Rows decided before this record have no basis and are re-decided once. + +## Revision 2026-09-26 (kind words) + +The kind-word aligner records a basis too (`type_bindings.basis`, migration 0092, #795). Its inputs +are not a phrase's: the model is shown a kind word's candidate classes with their labels and +definitions, so the fingerprint covers each candidate class's `updated_at` and ancestor closure. +Candidates come from embedding retrieval, or the whole class list when there is no embedding model +and few classes. They are retrieved once per run for every live kind word, and the same lists are +reused when the run checks for staleness at its end. + +Two things go further than the phrase half, which records its basis and catches an edit during +the request at the next run: + +- **One snapshot.** The classes, their versions and parent edges, the signatures and the existing + decisions are read in one `REPEATABLE READ, READ ONLY` transaction that closes before the first + model call, so the definitions in the prompt and the versions in the fingerprint are one state. +- **Acceptance compares.** Writing an agent decision first locks the candidate class rows + `FOR SHARE`, the order a class delete takes before it cascades to the binding, then recomputes + the fingerprint from the rows as they are now. A reply whose fingerprint moved is discarded and + the run queues another. Updates and deletes of the candidates wait for that transaction; a parent + edge or a new class committed in the same window is not blocked, and leaves the decision + detectably stale for the next run. + +A failed retrieval falls back to the whole class list, as the phrase shortlist does: the words +decided during the failure carry that list in their fingerprints and are asked once more when +retrieval recovers. Deciding nothing instead would leave a small base untyped for as long as its +embedding endpoint is broken. + +A stale word the run asked about but could not settle (a failed call, an unreadable reply, a +missing vote, no candidates) takes the bounded re-ask rather than the immediate requeue, so an +endpoint that fails every time cannot requeue the job round after round. A person's kind-word +decision commits together with its `align_phrases` job, the way 0051 pairs a phrase decision with +its materialisation. Every statement that changes a class's label or definition moves `updated_at`, +parent edges are in the closure, and the vector refresh (`set_type_embeddings`) changes neither, so +no semantic change escapes the fingerprint and re-embedding does not make decisions stale. What +imports and packs still lack is a shared writers' guard, which would only narrow the window above +from "stale next run" to "rejected now". + +The cost is one retrieval per live kind word per run, one embedding request per 64 words and one +nearest-class query per word, where before only the words being decided were retrieved; the phrase +shortlist likewise embeds its wide signatures on every run. Rows decided before this revision have +no basis and are decided again once. + +Regression coverage, with a scripted model and a real PostgreSQL: an edit during the model request +is not accepted and the next run asks with the new definition (red before this revision); a parent +edge alone makes an agent binding stale; rows without a basis are decided again once; a stale word +whose batch fails takes the bounded re-ask; a failed retrieval falls back and is not asked again +while it keeps failing; a person's decision carries no basis, is not overwritten, and commits with +its job; an acceptance and a class delete wait for each other instead of deadlocking, and a +candidate deleted before the check turns the reply away. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index c792e92be..8a8fb0e11 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -78,7 +78,7 @@ The test for writing one: if someone (including us) looks at a piece of code in | 0050 | [An action attempt keeps its identity and uncertain outcome](0050-an-action-attempt-keeps-its-identity-and-uncertain-outcome.md) | Proposed · durable execution identity and uncertain outcomes; no sender | | 0051 | [A human phrase decision carries its materialization work](0051-a-human-phrase-decision-carries-its-materialization-work.md) | Proposed · decision and materialization delivery; shared refactors and real regressions only | | 0052 | [Document content is a read contract over the retained ledger](0052-document-content-is-a-read-contract.md) | Proposed 2026-09-21 · implemented in #860 · two Viewer-level reads serve the retained originals the export already names by digest: `/documents/{id}/content[?version=N]` and `/documents/{id}/versions`; the handler locks the document and its ledger row through the blob read, purge answers 410, a ledger-referenced missing blob is a 500 invariant failure, a session or a scoped PAT may read, ingest tokens may not -| 0053 | [A phrase decision records the inputs it considered](0053-a-phrase-decision-records-the-inputs-it-considered.md) | Implemented 2026-09-23 · a decision stores a fingerprint of the ancestor closures and admitted candidates it saw; stale means the fingerprint of the current inputs differs, which is what timestamps could not see (#807, #795): inheritance, parent edges, edits during the request; no-candidate and overflow become recorded outcomes; requeue reads live signatures only, so orphaned rows stop looping +| 0053 | [A phrase decision records the inputs it considered](0053-a-phrase-decision-records-the-inputs-it-considered.md) | Implemented 2026-09-23 · a decision stores a fingerprint of the ancestor closures and admitted candidates it saw; stale means the fingerprint of the current inputs differs, which is what timestamps could not see (#807, #795): inheritance, parent edges, edits during the request; no-candidate and overflow become recorded outcomes; requeue reads live signatures only, so orphaned rows stop looping · revised 2026-09-26: kind words fingerprinted too, read from one snapshot and compared again when a reply is accepted | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Proposed 2026-09-23 · cut 1 in its PR · a `statements` source accepts the open extraction contract (`e`/`s`/`n`) verbatim on `POST /sources/{id}/statements` with the `api` push's identity, versions and tombstones; the payload is stored as one chunk and extraction parses it instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement and reaches the typed graph only through alignment; there is no slot for a property or class; an update marks earlier statements stale, it does not close them; tables stay on the mount (0036) | 0060 | [A rule's definition has a history](0060-a-rule-definition-has-a-history.md) | Implemented 2026-09-25 (#912, migration 0076) · A business rule was edited in place and a derivation pointed at the row, so an invalidated conclusion pointed at a rule that now said something else. Every edit that changes what a rule says opens a **version**, a full snapshot with a record time; a derivation names the version it was drawn under, a kept conclusion moves to the new one, and the proof, the rules panel and a versions endpoint read the history. Name, description and the switch open nothing. Exporting rule bodies per version is now honest and stays #902's second cut | diff --git a/docs/design/ontology.md b/docs/design/ontology.md index b7a07b159..829ace553 100644 --- a/docs/design/ontology.md +++ b/docs/design/ontology.md @@ -79,7 +79,11 @@ when the fingerprint of the current inputs differs, which is what timestamps cou parent edge added or removed, an edit committed while the model was answering [0053, #807, #795]. A signature with no admissible property is recorded as `none` (its projection retires); one with more candidates than the limit is `undecided` for the queue, not silently skipped. Kind-word -bindings still use `updated_at`, so cosmetic edits can also trigger their reevaluation. +bindings carry a basis as well: each candidate class the model was shown, with its `updated_at` +and ancestor closure, read from one snapshot before the model is called. A reply is accepted only +if that basis still matches the rows when it is written; otherwise it is discarded and the kind +word asked again [0053 revision, #795]. Cosmetic edits to a candidate class still count as a +change. **A shape of statement can imply a fact of another property** [0044 decision 3, migration 0073]. An implication rule is keyed like a binding (a signature) or by a kind word, names the property it diff --git a/migrations/0092_a_kind_word_decision_records_the_inputs_it_considered.sql b/migrations/0092_a_kind_word_decision_records_the_inputs_it_considered.sql new file mode 100644 index 000000000..2f6eef7ae --- /dev/null +++ b/migrations/0092_a_kind_word_decision_records_the_inputs_it_considered.sql @@ -0,0 +1,12 @@ +-- 一个类别词的判定记下它当时看到的输入(0053 的类别词那一半,#795)。 +-- +-- 过期从前按时间戳判:绑到的类在判定之后改过,或判成 none / undecided 之后库里有类新建或 +-- 修改。判定写在模型答完之后(`decided_at = now()`),模型答题期间改了类的定义,时间戳说 +-- 判定更新,可两票看到的都是旧定义——这次改动从此不会再被问到(#795 复现过)。父边的增删 +-- 不碰 `updated_at`,时间戳同样看不见。 +-- +-- `basis` 是给模型看的候选类的指纹:每个候选的 id、`updated_at` 与祖先闭包,在调模型之前 +-- 从同一个快照里读。写判定时在同一事务里按当前的类重算,对不上就不收这份回复;worker 每轮 +-- 按当前的输入重算,对不上就是过期。NULL = 这一列出现之前的代理判定,各重判一次;人的 +-- 判定不带指纹,也不按指纹重判。 +ALTER TABLE type_bindings ADD COLUMN basis TEXT;