From e7e36463ec8231da12bea7c491b929ef7a8ee8f5 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 18:29:20 +0530 Subject: [PATCH 1/5] Deleted memory stays refused and digests require current source facts Persist value refusals before explicit removal and fence memory, extraction, restoration and projection writes through publication. Rebuild session digests from complete eligible source facts and verify that proof before serving them. Settle native delivery only for acknowledged rows so withheld projections remain owed. Signed-off-by: Lareb Zafar --- bench/conflict_resolver_long_session.py | 21 +- bench/conflict_resolver_scenario.py | 12 +- docs/architecture/clickhouse.md | 17 +- docs/architecture/memory-architecture.md | 21 +- docs/architecture/memory-consent.md | 108 +++ docs/architecture/storage-layout.md | 4 +- docs/concepts/memory.md | 20 +- .../features/memory/history-and-provenance.md | 4 +- docs/features/memory/retention-and-erasure.md | 28 +- docs/features/memory/writing.md | 32 +- docs/features/tasks-and-decisions.md | 2 +- docs/glossary.md | 5 +- docs/operations/backup-and-recovery.md | 24 +- docs/operations/maintenance.md | 2 +- docs/operations/okf-sync.md | 10 +- docs/reference/configuration.md | 4 +- docs/reference/operations.md | 2 +- docs/reference/operations/memory.md | 8 +- docs/reference/schema.md | 3 +- llms.txt | 1 + loom.example.yaml | 12 +- schemas/loom-settings.schema.json | 4 +- schemas/migrations/115_memory_refusals.sql | 24 + schemas/pg_migrations/014_memory_refusals.sql | 16 + schemas/schema.sql | 28 +- scripts/codegen/push_langfuse_prompts.py | 2 - scripts/gates/function_length_baseline.json | 1 - sdk/python/loom/memory.py | 18 +- sdk/python/loom/ops.py | 4 +- sdk/python/loom/profile.py | 6 +- server/cli/import_clickgap.py | 6 +- server/core/ch.py | 14 +- server/core/repo_base.py | 2 + server/erasure/app/saga_steps.py | 3 + server/erasure/store/census.py | 6 + server/erasure/store/consent.py | 32 + server/erasure/store/refusals.py | 251 +++++++ server/erasure/store/steps_pg.py | 54 ++ server/memory/api/models_crud.py | 2 +- server/memory/api/models_evidence_transfer.py | 12 +- server/memory/app/chunk_projection.py | 9 +- server/memory/app/curation.py | 14 +- server/memory/app/deep_evidence.py | 163 +--- server/memory/app/evidence.py | 8 +- server/memory/app/evidence_transfer.py | 17 +- server/memory/app/extract/handler.py | 71 +- server/memory/app/extraction_queue.py | 125 +++- server/memory/app/handlers.py | 143 +++- server/memory/app/move.py | 23 +- server/memory/app/ops.py | 19 +- server/memory/app/period_digest.py | 125 +++- server/memory/app/pg_delivery.py | 35 +- server/memory/app/pipeline.py | 26 +- server/memory/app/profile_search.py | 4 +- server/memory/app/promote.py | 55 +- server/memory/app/reembed.py | 123 +++- server/memory/app/restore.py | 22 +- server/memory/app/retrieval/summaries.py | 2 +- server/memory/app/runs_manifest.py | 7 +- server/memory/app/sensitivity_sweep.py | 5 +- server/memory/app/session_outcomes.py | 70 +- server/memory/app/session_summary.py | 311 ++------ server/memory/app/set_chunks.py | 17 +- server/memory/domain/promotion_source.py | 74 ++ server/memory/domain/provenance.py | 2 + server/memory/domain/refusals.py | 178 +++++ server/memory/domain/summary_provenance.py | 57 ++ server/memory/store/delete_checks.py | 15 +- server/memory/store/embeddings.py | 77 +- server/memory/store/entities.py | 12 +- server/memory/store/inserts.py | 48 +- server/memory/store/integrity.py | 6 +- server/memory/store/lifecycle.py | 89 ++- server/memory/store/pg_batch.py | 16 +- server/memory/store/pg_memory.py | 63 +- server/memory/store/pg_publish.py | 5 +- server/memory/store/reads.py | 17 +- server/memory/store/refusals.py | 697 ++++++++++++++++++ server/memory/store/shared.py | 102 +++ server/memory/store/source_review.py | 52 -- server/memory/store/summary_sources.py | 219 ++++-- server/memory/store/write_fence.py | 12 +- server/memory/workers/embedding.py | 72 +- server/memory/workers/usage_fold.py | 6 +- .../observability/langfuse_prompts.lock.json | 4 - server/settings/catalog.py | 8 +- server/wiring.py | 13 +- tests/analyzer/cases/erasure.py | 9 + tests/analyzer/cases/legs.py | 11 +- tests/analyzer/cases/workers.py | 50 +- tests/analyzer/corpus.py | 6 +- tests/analyzer/shapes.txt | 56 +- tests/integration/_memory.py | 49 ++ tests/integration/same_key/_rows.py | 6 +- tests/integration/test_blacklist.py | 25 +- .../integration/test_conflict_resolver_ch.py | 49 +- .../test_conflict_resolver_invariants_ch.py | 46 +- .../test_conflict_resolver_long_session_ch.py | 67 +- tests/integration/test_embedding_worker.py | 31 +- tests/integration/test_erasure.py | 9 + .../integration/test_erasure_replicated_ch.py | 2 + .../test_inference_identity_workers.py | 8 +- .../integration/test_mcp_session_run_scope.py | 15 + tests/integration/test_memory_code_links.py | 6 +- tests/integration/test_memory_promote.py | 3 +- .../integration/test_memory_read_lifecycle.py | 22 +- .../test_memory_refusal_capture_live_ch.py | 124 ++++ .../integration/test_memory_refusal_policy.py | 191 +++++ tests/integration/test_memory_webhooks.py | 14 +- tests/integration/test_okf_sync.py | 16 +- tests/integration/test_pg_authority.py | 245 +++++- tests/integration/test_profile.py | 24 +- tests/integration/test_ratelimit.py | 7 +- .../test_raw_evidence_sensitivity.py | 31 +- .../integration/test_read_scope_isolation.py | 18 +- .../integration/test_repositories_live_ch.py | 76 +- .../test_restore_reopens_facts_ch.py | 4 +- tests/integration/test_scim_provider_flows.py | 4 +- .../test_sensitivity_high_canary.py | 17 +- tests/integration/test_vector_retire_race.py | 45 +- tests/unit/bulk_stubs.py | 6 +- tests/unit/ch_fake.py | 9 +- tests/unit/clickhouse_local.py | 7 + tests/unit/conftest.py | 9 +- tests/unit/memory_store.py | 34 + tests/unit/session_content_senders.json | 2 +- tests/unit/test_bm25_late_rows.py | 15 +- tests/unit/test_cell_scan_tables.py | 1 + tests/unit/test_ch_client_settings.py | 22 + tests/unit/test_chunk_parent_identity.py | 2 +- .../test_clickhouse_mutation_inventory.py | 4 +- tests/unit/test_code_member_writes.py | 15 +- tests/unit/test_dedup_ops.py | 34 +- tests/unit/test_delete_by_key.py | 97 ++- .../unit/test_delete_version_consent_races.py | 249 +++++++ tests/unit/test_delete_version_sql.py | 156 ++-- tests/unit/test_digest_chain.py | 47 +- tests/unit/test_embedding_reemit.py | 304 ++++++-- tests/unit/test_embedding_worker_fence.py | 1 + tests/unit/test_entity_mirror_heal.py | 18 + tests/unit/test_erasure_no_content_columns.py | 3 + .../test_erasure_publication_tombstones.py | 404 ++++++++++ tests/unit/test_erasure_step_order.py | 4 + tests/unit/test_export_evidence_holds_high.py | 45 +- .../test_extract_concurrent_projections.py | 89 +-- tests/unit/test_extract_defer_allowlist.py | 1 + tests/unit/test_extract_write_dates.py | 1 + tests/unit/test_extraction_policy.py | 64 +- tests/unit/test_extraction_queue.py | 22 +- tests/unit/test_extraction_retry_mine.py | 19 +- tests/unit/test_file_rules_shared.py | 15 +- tests/unit/test_high_rows_sql.py | 47 +- ...est_import_evidence_refuses_blacklisted.py | 47 +- tests/unit/test_in_use_coverage.py | 2 + tests/unit/test_key_axis_text.py | 17 + tests/unit/test_legacy_vector_retirement.py | 59 ++ tests/unit/test_lock_guard_consistent_read.py | 11 +- tests/unit/test_memory_apply.py | 11 +- tests/unit/test_memory_ask_deep_evidence.py | 146 +--- tests/unit/test_memory_clear.py | 17 +- tests/unit/test_memory_consolidate_handler.py | 1 + ...test_memory_delete_tombstones_all_types.py | 16 +- tests/unit/test_memory_digests_op.py | 32 +- tests/unit/test_memory_drops.py | 1 + tests/unit/test_memory_expiry.py | 6 +- tests/unit/test_memory_fence_publication.py | 20 + tests/unit/test_memory_metadata_contention.py | 23 +- tests/unit/test_memory_move.py | 57 +- .../unit/test_memory_outline_backlinks_op.py | 23 +- tests/unit/test_memory_pipeline.py | 2 + tests/unit/test_memory_refusals.py | 462 ++++++++++++ tests/unit/test_memory_reindex_mirror.py | 58 +- tests/unit/test_memory_set_chunks.py | 25 +- tests/unit/test_memory_set_verdict.py | 8 +- tests/unit/test_memory_update_conflict.py | 2 + tests/unit/test_memory_verify.py | 13 +- tests/unit/test_model_call_customer.py | 22 +- tests/unit/test_model_call_holds.py | 3 +- tests/unit/test_model_call_recovery.py | 19 + tests/unit/test_model_call_writers.py | 4 + tests/unit/test_model_calls_held_mid_pass.py | 8 +- tests/unit/test_model_calls_switch.py | 35 +- tests/unit/test_model_calls_switch_workers.py | 9 +- .../test_namespace_rollback_unembedded.py | 8 +- tests/unit/test_negative_escalation.py | 55 +- tests/unit/test_observation_date_default.py | 8 +- tests/unit/test_passage_sensitivity.py | 112 ++- tests/unit/test_period_digest.py | 92 ++- tests/unit/test_pg_authority.py | 149 +++- tests/unit/test_pg_batch.py | 46 +- tests/unit/test_pg_durable_reads.py | 7 +- .../test_pipeline_actor_sensitivity_gauges.py | 6 +- tests/unit/test_profile_leg.py | 3 +- tests/unit/test_promote_definition.py | 22 +- tests/unit/test_promote_refuses_high.py | 6 + tests/unit/test_promote_source_identity.py | 8 + tests/unit/test_promotion_source.py | 58 ++ tests/unit/test_proposal_commit_review.py | 2 + tests/unit/test_proposals.py | 3 + tests/unit/test_read_scope_admission.py | 11 - tests/unit/test_read_your_write_inserts.py | 2 +- tests/unit/test_reembed_in_database.py | 57 +- tests/unit/test_reembed_paging.py | 40 + tests/unit/test_restore_mirrors_all_types.py | 10 + tests/unit/test_schema_baseline.py | 1 + tests/unit/test_session_digest_prompt.py | 91 +-- tests/unit/test_session_outcomes.py | 34 +- tests/unit/test_session_summaries.py | 363 ++++----- tests/unit/test_session_summary_retrieval.py | 9 + .../test_set_from_messages_fail_closed.py | 34 +- tests/unit/test_set_from_messages_refusals.py | 7 + .../test_shared_and_documents_list_repo.py | 131 +++- tests/unit/test_status_mirror.py | 59 +- tests/unit/test_summary_leg_links.py | 11 + tests/unit/test_summary_sources.py | 203 ++++- tests/unit/test_value_updates.py | 14 + tests/unit/test_version_row_value_bytes.py | 15 +- tests/unit/test_workers_poll_the_work.py | 139 +++- tests/unit/tunable_constants.txt | 1 - 219 files changed, 7819 insertions(+), 2252 deletions(-) create mode 100644 docs/architecture/memory-consent.md create mode 100644 schemas/migrations/115_memory_refusals.sql create mode 100644 schemas/pg_migrations/014_memory_refusals.sql create mode 100644 server/erasure/store/consent.py create mode 100644 server/erasure/store/refusals.py create mode 100644 server/memory/domain/refusals.py create mode 100644 server/memory/domain/summary_provenance.py create mode 100644 server/memory/store/refusals.py delete mode 100644 server/memory/store/source_review.py create mode 100644 tests/integration/_memory.py create mode 100644 tests/integration/test_memory_refusal_capture_live_ch.py create mode 100644 tests/integration/test_memory_refusal_policy.py create mode 100644 tests/unit/test_delete_version_consent_races.py create mode 100644 tests/unit/test_erasure_publication_tombstones.py create mode 100644 tests/unit/test_legacy_vector_retirement.py create mode 100644 tests/unit/test_memory_refusals.py diff --git a/bench/conflict_resolver_long_session.py b/bench/conflict_resolver_long_session.py index 0795a84991..9f1b93b42e 100644 --- a/bench/conflict_resolver_long_session.py +++ b/bench/conflict_resolver_long_session.py @@ -5,7 +5,7 @@ namespace: six attributes (home city, employer, manager, standup time, CI provider, diet) change once or twice, each change under a new key and in new words, among restated, elaborated and neighbouring facts, a pinned fact that is -contradicted later, a fact deleted and brought back with Undo, and eighteen +contradicted later, a permanently deleted fact, and eighteen unrelated ones (`bench/conflict_resolver_scenario.py`, the scenario and its ground truth; `tests/integration/test_conflict_resolver_long_session_ch.py` runs it with a scripted judge). The harness then waits for the server's @@ -91,12 +91,13 @@ from bench.conflict_resolver_scenario import ( ATTRIBUTES, BY_KEY, + CHECK_DELETED_IN, CURRENT, DELETE_IN, + DELETED, FACTS, KEPT, OLDER, - RESTORE_IN, SESSIONS, dialogue, relation_of, @@ -150,7 +151,7 @@ def _entry(start: datetime, session: int, i: int, key: str) -> dict[str, Any]: def write_set_session(post: Post, namespace: str, session: int, start: datetime) -> None: - """The session's facts under the scenario's keys, then its delete or Undo.""" + """The session's facts under the scenario's keys, then its deletion or absence check.""" keys = [f.key for f in FACTS if f.session == session] entries = [_entry(start, session, i, k) for i, k in enumerate(keys)] ident = _identity(namespace, session) @@ -162,9 +163,12 @@ def write_set_session(post: Post, namespace: str, session: int, start: datetime) if session in DELETE_IN: key = DELETE_IN[session] post("memory.delete", {**_identity(namespace, BY_KEY[key].session), "memory_key": key}) - if session in RESTORE_IN: - key = RESTORE_IN[session] - post("memory.restore", {**_identity(namespace, BY_KEY[key].session), "memory_key": key}) + if session in CHECK_DELETED_IN: + key = CHECK_DELETED_IN[session] + absent = post( + "memory.get", {**_identity(namespace, BY_KEY[key].session), "memory_key": key} + ) + assert absent["found"] is False, absent def write_messages_session(post: Post, namespace: str, session: int, start: datetime) -> None: @@ -320,7 +324,8 @@ def score_by_key(rows: dict[str, dict[str, Any]], top: dict[str, str]) -> dict[s "flagged_current": sum( 1 for k in CURRENT if k in rows and rows[k]["status"] == "conflicted" ), - "missing_rows": sorted({f.key for f in FACTS} - set(rows)), + "missing_rows": sorted({f.key for f in FACTS} - DELETED - set(rows)), + "deleted_returned": sorted(DELETED & set(rows)), "older_live": sorted(k for k in OLDER if k in rows and _live(rows[k])), } @@ -480,7 +485,7 @@ def print_plan(servers: list[tuple[str, str]], variants: list[str]) -> None: for name, a in ATTRIBUTES.items(): values = " -> ".join(f"{BY_KEY[k].value} (s{BY_KEY[k].session})" for k in a.keys) print(f" {name}: {values}") - print(f"kept live: {len(KEPT)} facts; deleted+restored: {sorted(set(DELETE_IN.values()))}") + print(f"kept live: {len(KEPT)} facts; permanently deleted: {sorted(set(DELETE_IN.values()))}") for label, url in servers or [("on", ""), ("off", "")]: for v in variants: print(f"plan: {label} {url} variant={v}") diff --git a/bench/conflict_resolver_scenario.py b/bench/conflict_resolver_scenario.py index bd5c4130d9..34bc04adc2 100644 --- a/bench/conflict_resolver_scenario.py +++ b/bench/conflict_resolver_scenario.py @@ -10,8 +10,8 @@ key and in new words. The last value is current (`CURRENT`); every earlier one should end superseded behind the next (`OLDER`). - Four facts are restated, four elaborated, four sit near a different fact, one - pinned fact is contradicted later, one fact is deleted and brought back with - Undo, and eighteen stand alone. All of them should stay live and unflagged + pinned fact is contradicted later, one fact is permanently deleted, and eighteen + stand alone. The other facts stay live and unflagged (`KEPT`). - `JUDGE` holds the right verdict for every pair a cluster should hold, except for the scripted judge's deliberate mistakes: two wrong answers at low @@ -170,7 +170,7 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: # Pinned, and contradicted later: a pinned row is never retired. F(3, "code_freeze", "Code freeze starts every Friday at noon", alone("freeze"), pinned=True), F(9, "freeze_change", "Code freeze now starts on Thursday evening", near("freeze", 0.75)), - # Deleted in session 4, brought back with Undo in session 6, elaborated in 9. + # Deleted in session 4; session 6 checks its key stays absent. F(3, "badge", "The user's badge number is 4471", alone("badge")), F(9, "badge_lab", "Badge 4471 also opens the third-floor lab", near("badge", 0.75)), # Unrelated, each in its own direction: no peer, never judged. @@ -198,7 +198,8 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: # The key deleted in a session, and the session whose Undo brings it back (after # each session's writes). -DELETE_IN, RESTORE_IN = {4: "badge"}, {6: "badge"} +DELETE_IN, CHECK_DELETED_IN = {4: "badge"}, {6: "badge"} +DELETED = set(DELETE_IN.values()) # The relations are not single-valued, so no write closes a fact on its own. ATTRIBUTES: dict[str, Attribute] = { @@ -240,7 +241,6 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: frozenset({"marathon", "marathon_plan"}): ("elaboration", 0.96, True, True), frozenset({"rust", "rust_plan"}): ("elaboration", 0.95, True, False), frozenset({"guitar", "guitar_band"}): ("elaboration", 0.96, True, True), - frozenset({"badge", "badge_lab"}): ("elaboration", 0.96, True, False), frozenset({"sister_city", "home_city"}): ("unrelated", 0.97, False, False), frozenset({"skip_level", "manager"}): ("related", 0.95, False, False), frozenset({"car", "partner_car"}): ("unrelated", 0.97, False, True), @@ -262,7 +262,7 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: CURRENT = {a.keys[-1] for a in ATTRIBUTES.values()} OLDER = {k for a in ATTRIBUTES.values() for k in a.keys[:-1]} -KEPT = {f.key for f in FACTS} - CURRENT - OLDER +KEPT = {f.key for f in FACTS} - CURRENT - OLDER - DELETED BY_KEY = {f.key: f for f in FACTS} diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 148fb11ca8..bdc988ab57 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -652,8 +652,9 @@ on `valid_to`, `invalid_at`, `version`; `set` on `status` and `memory_type`. Loom deletes memory by writing a tombstone version (`tombstoned_at` set, every other column carried). It is visible to the next `FINAL` read, keeps prior versions in -`memory_versions` and leaves `memory.restore` a window. Physical removal is for erasure -with `purge=true`, runner state, and the supersession rows `knowledge.restore` removes. +`memory_versions`. Only automatic retirement without permanent refusal has a restore window. Physical removal is for erasure +with `purge=true`, stale vector caches, runner state, and the supersession rows +`knowledge.restore` removes. - **`knowledge.restore`** runs a lightweight `DELETE FROM loom.chunk_supersessions` for the rows one delete wrote: the document's `source_uri`, the ingestions of its revision and @@ -705,6 +706,18 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r candidate review lease. An invalidated model attempt instead removes only its own candidate versions through a lightweight `DELETE FROM`, bounded by the claim version, so it cannot remove a later review decision. +- **Orphaned vector retirement** deletes cache versions below the caller's cutoff by + the complete vector sort key, across all creation partitions. A missing authoritative + source or matching erasure policy cannot be represented by a newly inserted marker: + even an empty marker would recreate residue after a purge. Admission fences stay held + until the synchronous lightweight delete is acknowledged; equal and newer versions stay. +- **Shared canonical retirement** uses the same bounded lightweight deletion when its + shared head is absent, source identities are unproven or erasure policy matches. The + cutoff preserves a later promotion and avoids writing a tombstone after a purge. +- **Refused extraction jobs** end through a control-only `ALTER TABLE … UPDATE` on the + exact identity, queue id, source version and payload hash. It writes a fixed reason and + terminal status without copying transcript bytes or creating a row. Unrelated jobs in + the same claim batch can still run. - **Extraction retries** (`memory.extraction_retry_mine`) update failed queue rows in place, asynchronously. The update targets the exact identity and version observed, with the caller's namespace, principal and read scope. Copying the transcript into a new diff --git a/docs/architecture/memory-architecture.md b/docs/architecture/memory-architecture.md index bac74089fe..2b7bae5b05 100644 --- a/docs/architecture/memory-architecture.md +++ b/docs/architecture/memory-architecture.md @@ -724,10 +724,10 @@ is left. Its triples close as for a row that holds none. A revision (`memory.set`, `memory.bulk_set`, `memory.update_by_id`) of a row that existed before the write closes the facts its row held that no row holds any longer, and -an Undo of a retirement or of a delete opens again those the restored row's current value +an Undo of reversible retirement opens again those the restored row's current value holds by the same rule (another row's claim keeps a fact from closing, but never reopens one); either, when it fails, is recorded for `projection_repair` as `triples_reconcile`. -A delete closes none of its row's facts, so the only facts a delete's Undo opens again +A tombstone closes none of its row's facts, so the only facts a reversible restore opens again are ones another row's revision or retirement closed while the row was gone. A dropped fact still valid later ("until Dec 1") ends now, and so does one a retired row @@ -1004,14 +1004,15 @@ session, agent or run identity. A read bounded on one of those axes omits the ag graph signals, including hub pruning and graph paths. Its fact legs retain their scoped cosine and query-entity ranking. Namespace-wide reads use the graph aggregates. -Session digests carry the keys they compressed in `derived_from`. Before deduplication, -the summary leg checks those known sources under the digest's full identity. A source -that is missing, unserved, deleted, expired, outside its validity, high-sensitivity or at a newer -version withholds the digest. An unreadable source check withholds all unchecked digests. -Without source revision hashes, a later source version conservatively withholds a digest -during replay too. Lineage covers extracted facts, not every sentence of the transcript: -a digest without a known source remains session evidence, and does not prove that its -statements are current facts. +Session digests contain whole committed source values joined by newlines. Their ordered +`derived_from` and `key_facts` reconstruct every byte of `summary_text`; raw transcript +text and model-generated additions cannot contribute. Every digest reader verifies that +proof against the authoritative heads under the digest's exact agent, user and session. +Missing, changed, unserved, expired, truncated, sensitive or unreviewed sources withhold +the digest. A pin or touch that leaves the same eligible content does not invalidate it. +Legacy digests without complete proof are withheld. Source-read failures fail closed. +A digest hit cannot open unrelated raw turns from its session. Model-generated period +lessons and contrast reflections enter proposed state for independent review. `memory.list` excludes rejected rows from both sort orders and counts. A key lookup can inspect a rejected row. `memory.for_file` also checks `valid_to` before adding a fact to diff --git a/docs/architecture/memory-consent.md b/docs/architecture/memory-consent.md new file mode 100644 index 0000000000..e9879658d3 --- /dev/null +++ b/docs/architecture/memory-consent.md @@ -0,0 +1,108 @@ +# Memory consent and derived text + +A deletion contracts memory and records content refusal before its tombstone commits. +`memory.delete`, a successful `memory.delete_version`, natural-language forgetting and +`memory.clear` keep canonical JSON hashes and normalized literal fingerprints in +`memory_refusals`. Refusal records contain no deleted plaintext and have no TTL. Ordinary +revision, supersession, expiry and an internal move do not express permanent refusal. + +A point deletion applies to the same user in its namespace, across agents and sessions. +A user erasure applies across the organization, including namespaces created later. +A namespace erasure applies to later intake by every user in that namespace. The policy +scope is separate from the original row's identity tuple, so a derived row keeps its +writer identity without narrowing the erased subject's refusal. Captured derived content +also records an indexed erasure refusal under that writer's namespace and user identity. + +Canonical values are compared after JSON decoding. Literal comparisons normalize Unicode, +letter case and whitespace, and check passages within later strings. Wrapping a deleted +value in another JSON object does not bypass the check. These fingerprints compare +content; they do not recognize arbitrary semantic paraphrases. Verification work is +bounded and an incomplete check refuses the write. + +ClickHouse writers share admission leases with deletion and erasure. A successful delete +records refusal under the exact head's write fence. Erasure blocks new intake while it +captures fingerprints and drains admitted sends before deleting content. Postgres stores +policy beside canonical memory. Every publisher takes an organization admission stripe; +erasure drains every stripe before capture. Native writers take the organization policy +lock before namespace, user, generation and head locks. Erasure takes that organization +lock exclusively, including when it spans several namespaces. +A failed policy read refuses intake. Raw evidence, deferred extraction, canonical writes, +embeddings and shared promotion use the same policy. A delayed projection rechecks its +source under admission before publishing. + +Erasure captures ClickHouse revision history under either storage authority. A native +revision whose payload was cleared still supplies its typed identity for raw dependent +capture; ClickHouse history supplies its content before either copy is removed. + +Canonical tombstones check erasure policy even though their content is no longer served. +Point deletion can retain its canonical history. Native authority commits check point +refusal before entering the journal. Delivery can publish those committed revisions after +point deletion, while rechecking erasure policy under the policy locks. Raw evidence, +chunks, extraction payloads and new vector publication still check point refusal. + +A vector retirement keeps cache bytes +only while its authoritative source and policy permit them; point refusal produces an +empty marker. With no source or an erasure match, cleanup deletes older vector versions +across every partition, bounded by the caller's retirement version. It inserts no new row +and preserves equal or newer versions. Native publication proves source absence with a +durable read before borrowing a connection, then rechecks on that connection under policy +locks. Present heads remain locked until publication finishes. + +Native delivery clears search-document debt only for acknowledged publications or a +source that no longer needs a document. A vector withheld after source selection stays +owed for another pass. Receipts match the complete identity and published version; +a newer metadata copy settles its original revision's delivery. Copy follow-ups confirm +the exact document in ClickHouse before settling debt. + +Shared canonical retirement holds admission through its final send. It checks the +promoted source identities against erasure policy and requires the shared head to remain +present. Missing heads, erased content and unproven legacy sources are removed below the +retirement version without inserting plaintext or a marker. Definition promotion and +demotion acquire the namespace claim before borrowing a source connection. + +Deferred extraction checks caller data in current and prior messages, names, instructions, +schema enums and pipeline arguments. Standard protocol fields do not become refusal +patterns. Enqueue, claim and every payload-carrying transition share this admission. +A transition also requires its exact source revision and payload hash to remain present. +A refused job ends through a control-only update bound to that same source; its transcript +is never copied into the terminal update. A missing source cannot be recreated. + +An owed `memory.delete_version` check carries a hash of the exact checked state. +Recovery compares that hash before writing a missing tombstone. A numeric version +collision cannot turn a different fact into the target of permanent refusal. + +Vector maintenance reports only acknowledged replacements. A legacy vector without a +source hash stays held when its complete source cannot be proved; `--delete-old` keeps +that organization's old vectors. Run `memory.reindex` with `force=true` to rebuild such +vectors from the complete authoritative values. + +Permanent refusal also applies to `memory.restore` and snapshot restoration. Automatic +retirement can still be undone when the content has not been refused. Keeping the +historical row for inspection does not grant permission to publish it again. + +## Session digests + +A session digest contains complete committed source facts joined by newlines. Each fact +has a corresponding key in `derived_from`, under the same complete identity. A model does +not draft session digest text. Search and other digest consumers reconstruct the text +from the authoritative current heads and withhold any digest whose source is missing, +changed, retired, expired, sensitive, incomplete or awaiting review. A metadata touch +keeps an unchanged fact valid. A legacy digest without a complete proof is withheld. + +Period and contrast synthesis remain proposals. A model-generated restatement therefore +waits for independent review before ordinary fact retrieval serves it. + +## Rollout + +Stop and drain older APIs, workers and command-line writers before applying the additive +ClickHouse and Postgres migrations. Finish pending versioned deletion checks before +the drain. Without a checked-state hash, recovery closes a check that would need +a missing tombstone without deleting the fact; retry the deletion to check its current state. +Keep writes paused until every publisher enforces the +consent rules. If an older writer ran alongside a migrated store, keep traffic paused +until the affected deletion and erasure scopes have been verified again; a migration does +not certify writes admitted by the older build. Existing rows removed before policy recording was +introduced have no reconstructed consent record. Operators needing a historical guarantee +must establish the intended refusals from their deletion records before resuming intake. +Rolling back restores the earlier admission behavior. Audit records use their separate +retention policy and do not inherit namespace deletion windows. diff --git a/docs/architecture/storage-layout.md b/docs/architecture/storage-layout.md index 278d9712cc..bf249d2301 100644 --- a/docs/architecture/storage-layout.md +++ b/docs/architecture/storage-layout.md @@ -950,6 +950,7 @@ copies of tables listed here, owned and removed by that run, and erasure reaches | `memory` | RMT(version) `(org, namespace, agent, user_id, session_id, memory_key)`, by `toYYYYMM(created_at)` | stays | Derived | census, `memory.tombstone`, user `memory.retire`; step `consolidated.tombstone` | — | the current rows in ClickHouse: the source of `memory_versions_mv`, granted to `insights.query`, read by ClickHouse-side joins; published from `memory_current` and rebuilt by republishing it; named in `server/memory/store/history.py`, `server/memory/store/inserts.py` | | `memory_embeddings` | RMT(version) `(org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key)`, by `toYYYYMM(created_at)` | stays | Derived | census, `memory_embeddings.tombstone`, user scope too; step `shared_embeddings.tombstone`; step `consolidated.tombstone` | — | the vectors of the memory heads (`memory_current`) and the shared heads (`shared`, and from wave 3 `shared_current`); Postgres stores none (re-embedding costs provider calls); wave 4's vector levers change its columns and codec; named in `server/memory/store/inserts.py`, `server/memory/store/shared.py` | | `memory_versions` | RMT `(org, namespace, agent, user_id, session_id, memory_key, version)`, by `toYYYYMM(created_at)` | stays | SoR | census, `memory_versions.delete`, user scope too | — | version history, fed by `memory_versions_mv`: the system of record for history written before a deployment's journal existed, derived from `memory_revisions` for later rows; for a version superseded with its journal payload cleared, the only copy of its content; named in `server/memory/store/conflict_resolver.py`, `server/memory/store/history.py` | +| `memory_refusals` | MT `(org, scope_kind, scope_value, user_id, kind, units, digest)` | stays | SoR | no content | — | Durable consent fingerprints recorded before deletion; policy scopes preserve the original identity and prevent later canonical or literal intake. The native authority stores the same policy in Postgres; named in `server/memory/store/refusals.py`. | | `shared` | RMT(version) `(org, namespace, memory_key)`, by `(org, namespace)` | Postgres `shared_current` + ClickHouse `shared` | SoR (head) | census, `shared.tombstone` | 3 | promoted namespace-wide memory: a head per `(org, namespace, memory_key)` on the memory journal (its revisions in `memory_revisions`, its deliveries in `memory_deliveries`); the ClickHouse table stays its search projection; a namespace erasure tombstones it and a user erasure matches it by provenance; named in `server/memory/store/shared.py`, `server/erasure/store/steps_memory.py` | | `working_memory` | RMT(version) `(org, namespace, agent, user_id, session_id, run_id, block_key)`, by `(org, namespace)` | Postgres `working_memory` | SoR | census, `working_memory.tombstone`, user scope too | 3 | keyed blocks read and replaced by key, never ranked; named in `server/memory/app/working_memory.py`, `server/erasure/store/steps_memory.py` | | `memory_reinforcement` | RMT(version) `(org, namespace, agent, user_id, memory_key)` | stays | Derived | census, `memory_reinforcement.delete`, user scope too | — | folded usage counters, rebuildable from `memory_usage_events`; named in `server/memory/app/reinforcement.py`, `server/erasure/store/steps_memory.py` | @@ -963,7 +964,7 @@ copies of tables listed here, owned and removed by that run, and erasure reaches | `memory_chunks` | RMT(created_at) `(org, namespace, session_id, chunk_index, content_sha256)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for chunks `memory.import_evidence` writes) | census, `memory_chunks.tombstone`, user scope too | — | chunked evidence and chunks of long `memory.set` values, with vectors (`server/memory/app/set_chunks.py`); rebuilt from `memory_evidence` and the heads at embedding cost, except the chunks `memory.import_evidence` writes with no `memory_evidence` row (`server/memory/app/evidence_transfer.py`), held only here; named in `server/memory/app/set_chunks.py`, `server/memory/app/chunk_projection.py` | | `memory_chunk_sources` | RMT(created_at) `(org, namespace, session_id, chunk_index, memory_key, evidence_id)`, by `toYYYYMM(created_at)` | stays | SoR | census, `source_links.tombstone`, user scope too | — | chunk-to-memory links written by the extraction handler after a run and by `memory.import_evidence` (`server/memory/app/evidence_transfer.py`), which takes links a caller states; held only here, so lost without a backup; named in `server/memory/app/chunk_links.py`, `server/erasure/store/steps_graph.py` | | `memory_belief_sources` | RMT(created_at) `(org, namespace, memory_key, evidence_id)` | stays | SoR | census, `source_links.tombstone`, user scope too | — | evidence links of a belief, written by the extraction handler after a run (`server/memory/app/extract/handler.py`), by consolidation linking merged rows to their summary (`server/memory/app/consolidate.py`) and by period digests (`server/memory/app/period_digest.py`); held only here, so lost without a backup; named in `server/memory/app/evidence.py`, `server/erasure/store/steps_graph.py` | -| `session_summaries` | RMT(version) `(org, namespace, agent, user_id, session_id, embedding_model)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for digests `memory.import_evidence` writes) | census, `summaries.tombstone`, user scope too | — | session digests with vectors, written by a model from the session's turns (`server/memory/app/session_summary.py`) and rebuildable at model cost from those turns in `memory_evidence`; the digests `memory.import_evidence` writes are held only here; wave 3 retires a digest whose `derived_from` names a redacted key ([deliveries](#deliveries)); named in `server/memory/app/session_summary.py`, `server/memory/app/evidence_transfer.py` | +| `session_summaries` | RMT(version) `(org, namespace, agent, user_id, session_id, embedding_model)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for digests `memory.import_evidence` writes) | census, `summaries.tombstone`, user scope too | — | Complete committed source facts joined into an extractive digest, with one key per fact. Consumers reconstruct the text from current authoritative heads and withhold unproven or stale digests; named in `server/memory/store/summary_sources.py`. | | `derived_facts` | MT `(org, namespace, subject, predicate, category)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for operands a `memory.bulk_set` entry states) | census, `derived_facts.delete`, user scope too | — | per-occurrence operands for multi-hop answers, append-only, written by `memory.bulk_set` from each entry's `derived` list, which a caller states or extraction from messages fills; a stated operand is held only here, so a rebuild, by re-running extraction over the messages, restores only the extracted ones; wave 3 adds the fact's memory `version` and retires the facts of a redacted version ([deliveries](#deliveries)); named in `server/memory/app/triple_projections.py`, `server/erasure/store/steps_memory.py` | | `memory_records` | RMT(version) `(org, namespace, schema_name, memory_key, record_id)` | Postgres `records` + ClickHouse projection | SoR (head) | census, `records.tombstone`, user scope too | 3 | a record's current fields, read by key and compared on write; Postgres `records` keeps each field's state and evidence id with its value, which only `memory_record_fields` holds today; the projection stays for the record side leg and record query, which join it with `memory_record_fields`, and feeds `memory_record_versions_mv`; named in `server/memory/store/records.py`, `server/memory/app/records.py` | | `memory_record_versions` | RMT `(org, namespace, schema_name, memory_key, record_id, version)`, by `toYYYYMM(created_at)` | stays | SoR | census, `records.tombstone`, user scope too | — | record history, fed by `memory_record_versions_mv` from `memory_records`; from wave 3, one row per delivered revision, reconciled after a restore by the journal replay; named in `server/erasure/store/steps_graph.py`, `server/memory/app/records.py` | @@ -1181,6 +1182,7 @@ Postgres step the flip splits off it, which joins `PG_ONLY_STEPS` | `cache_generations` | `(org, kind)` | Coord | no content | | `memory_current` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.memory_current` | | `memory_revisions` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.memory_revisions` | +| `memory_refusals` | `(org, scope_kind, scope_value, user_id, kind, units, digest)` | SoR, hash-only consent policy committed beside memory; no TTL | no content; retained across erasure | | `memory_deliveries` | [storage authority](storage-authority.md#the-tables) | SoR | cascades with its revision | | `idempotency`, `idempotency_fences` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.idempotency`; `idempotency_fences` holds no content (the census's bound) | | `erased_memories`, `erased_documents` | [storage authority](storage-authority.md#the-tables) | SoR | evidence, no content | diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 46b7e25492..29a089801c 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -164,13 +164,13 @@ answer confidently with a fact it has reason to doubt. Five operations cover the memory lifecycle: -| Lens | Operation | Effect | -| --------- | --------------------- | -------------------------------------------------------------------------- | -| Expand | `memory.set` | Learn a fact under a key. | -| Revise | `memory.update_by_id` | Correct one row by id (`memory.update` does the same by key). | -| Contract | `memory.delete` | Tombstone a key. `memory.restore` revives it within 14 days of the delete. | -| Reinforce | `memory.pin` | Store and pin: no later write that does not itself pin replaces it. | -| Refuse | `blacklist.add` | Refuse every future write matching a substring or regex. | +| Lens | Operation | Effect | +| --------- | --------------------- | ------------------------------------------------------------------- | +| Expand | `memory.set` | Learn a fact under a key. | +| Revise | `memory.update_by_id` | Correct one row by id (`memory.update` does the same by key). | +| Contract | `memory.delete` | Tombstone a key and refuse later intake of its removed value. | +| Reinforce | `memory.pin` | Store and pin: no later write that does not itself pin replaces it. | +| Refuse | `blacklist.add` | Refuse every future write matching a substring or regex. | A searchable `memory.set` runs these checks in order: @@ -264,7 +264,7 @@ stateDiagram-v2 active --> superseded: newer write (newest_wins, refines, conflict resolver) superseded --> active: memory.restore active --> tombstoned: memory.delete, blacklist sweep, expiry - tombstoned --> active: memory.restore (within 14 days) + tombstoned --> active: memory.restore (automatic retirement, no refusal) ``` Every arrow is a new version; the old one stays in `loom.memory_versions`. @@ -570,8 +570,8 @@ ahead of the durable sections, so an agent sees its own state first. Loom deletes only what its writer asked for. No memory table has a default TTL and no sweep retires rows by age. A row leaves the served set when its writer deletes or supersedes it, when its writer gave it an expiry (`ttl_seconds`, `valid_to`), or when the -namespace owner or operator configured a retention window. `memory.restore` revives a -tombstone within 14 days. +namespace owner or operator configured a retention window. `memory.restore` reverses an automatic +tombstone within 14 days when no permanent refusal exists. [Retention and erasure](../features/memory/retention-and-erasure.md) covers the settings and the erasure saga. diff --git a/docs/features/memory/history-and-provenance.md b/docs/features/memory/history-and-provenance.md index 331db28f47..2e078d6a94 100644 --- a/docs/features/memory/history-and-provenance.md +++ b/docs/features/memory/history-and-provenance.md @@ -48,8 +48,8 @@ async with Agent(org="acme", namespace="support", agent="auditor") as agent: print(v["written_at"], v["actor"], v["source_class"], v["value"]) ``` -A tombstone is a version too, so a delete shows in history and `memory.restore` can -revive the key within 14 days. For point-in-time reads of current state, use `as_of` on +A tombstone is a version too, so a delete shows in history. `memory.restore` can +reverse an automatic tombstone within 14 days when no permanent refusal exists. For point-in-time reads of current state, use `as_of` on `memory.get`, `memory.search` and `memory.ask` ([validity time](./writing.md#validity-time-valid_at-and-valid_to)). diff --git a/docs/features/memory/retention-and-erasure.md b/docs/features/memory/retention-and-erasure.md index 830898de8f..3472bbe228 100644 --- a/docs/features/memory/retention-and-erasure.md +++ b/docs/features/memory/retention-and-erasure.md @@ -37,14 +37,16 @@ other kinds. See [namespaces and sharing](../namespaces-and-sharing.md). A tombstone is a new version with `tombstoned_at` set. Reads stop serving it at once, but the row and its prior versions in `loom.memory_versions` stay on disk: -- `memory.restore` revives a tombstoned key within 14 days (after that it answers - `expired=true`). It also reverses an automatic supersession. +- Explicit deletion records permanent content refusal. `memory.restore` and snapshot + restoration cannot publish a refused value again. `memory.restore` still reverses an + automatic supersession or expiry when no refusal applies. See + [memory consent](../../architecture/memory-consent.md). - `memory.clear` (admin) tombstones every memory for an agent or user in a namespace; `confirm_subject` must repeat the concrete `user_id` or `agent` (neither `'*'` nor `'-'` confirms). `agent="*"` with a `user_id` clears that user's rows under every agent, and `user_id="*"` with an `agent` that agent's rows for every user. A scope that names nobody but `'-'` is refused, as a namespace-wide clear is. Locked rows stay and are counted in - `locked_skipped`. Cleared keys revive the same way. + `locked_skipped`. Cleared values receive the same permanent refusal as a point deletion. To remove the bytes, use an [erasure](#erasure). To strip one value but keep the row and audit trail, use [`memory.redact`](./history-and-provenance.md#scrubbing-history-memoryredact). @@ -172,17 +174,16 @@ flowchart TD hosts with Keeper, within one process without it. It waits up to 15 minutes for a pass to finish; past that the step fails with `LeaseBusy` and `erasure_recovery` retries it. - **Tombstone or purge.** By default the soft-delete tables (memory, embeddings, chunks, - summaries, links) are tombstoned, and `memory.restore` can still revive them inside - the window; `erasure.list` reports such revivals as `reversals_count`. `purge=true` - deletes the rows instead (a lightweight `DELETE FROM` on the wide tables). A scope + summaries, links) are tombstoned. Permanent refusal prevents publishing the captured + values again, including through `memory.restore`. `purge=true` deletes the rows instead + (a lightweight `DELETE FROM` on the wide tables). A scope that holds a namespace that has ever had a mirror is purged whole, every namespace in it, so none of them has such a window ([document sources](../../architecture/document-sources.md#the-mirror)). A user erasure finds what the user's memories fed through their keys, so in a purge its `loom.memory` rows, and the consolidated summaries built from them, are tombstoned first and deleted in - `memory.retire`, after every step that reads those keys; until then `memory.restore` - can revive them (the user's own rows count as a reversal, the summaries do not), and - `memory.retire` deletes what `memory.restore` revived. + `memory.retire`, after every step that reads those keys. Refusal fingerprints are + recorded before this content is removed and survive the erasure. - **Uploaded originals** of knowledge documents: the namespace's rows of `loom.document_blobs` are deleted first, and `blob_erased_at` is stamped. A user erasure deletes the originals that user uploaded and no other writer's live document shares, @@ -336,13 +337,14 @@ What the receipt does not cover: - backups, point-in-time recovery, WAL and standbys of either service; - copies made outside Loom, such as a table built in an org's own Postgres from the [SQL surface](../../reference/sql-surface.md); -- rows a worker writes after the census from a read it made before it - ([storage authority](../../architecture/storage-authority.md#erasure)); -- a `namespace.restore` from a snapshot taken before the erasure, which brings rows back: - run the erasure again, and its census is the new receipt; +- an unrelated row written after the census, outside its erasure scope; - for a user scope, document originals in the bucket, which the census counts for a namespace scope only. +Consent checks fence delayed memory publishers and refuse removed values in later +snapshot restoration. See [memory consent](../../architecture/memory-consent.md) for +the matching rules and the rollout required to enforce them. + The erased id stays where the proof of erasure needs it: in the ledger row (until the table's TTL drops it, 90 days after the request), in audit, in `loom.erasure_reversals`, and in the deployment's own logs of the statements that named it (the ClickHouse mutation log, diff --git a/docs/features/memory/writing.md b/docs/features/memory/writing.md index 0c99c4db9d..6e7d47eb52 100644 --- a/docs/features/memory/writing.md +++ b/docs/features/memory/writing.md @@ -18,7 +18,7 @@ merge, duplicate, conflict or rejected. The verdicts and conflict policies are i | Correct a fact you found through search or ask | `memory.update` | Resolves the key namespace-wide; keeps a pin. | | Correct one exact row by id | `memory.update_by_id` | Addressed by `(org, memory_id)`. | | Change only the expiry | `memory.touch` | One key or up to 1,000 `memory_keys`; value untouched. | -| Retire a fact | `memory.delete` | Tombstone; `memory.restore` revives it within 14 days. | +| Retire a fact | `memory.delete` | Tombstone and permanent refusal of the removed value. | | Rename a key and keep the old one readable | `memory.move` | `keep_redirect` (default true) leaves a redirect. | | Guard a row against edits or deletion | `memory.lock` / `memory.unlock` | Unlock is admin. | | Apply one change to many existing rows | `memory.apply` | Dry run by default. | @@ -178,14 +178,14 @@ key then answers `conflict_verdict: "conflict"`, `peers[0].memory_key: ## Changing and retiring facts -| Op | Behaviour | -| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `memory.update` | Resolves `memory_key` across the namespace when no identity is given (`"-"` counts as none, and `"*"` for `agent`). One live row: updated. None: `updated=false, reason="not_found"`. Several: 409 `AMBIGUOUS_KEY` with `candidates`. Runs the conflict check and keeps a pin. An expired row answers 409 `EXPIRED` unless `revive=true`. | -| `memory.update_by_id` | One row by `(org, memory_id)`. The namespace is resolved and checked before the read; a refusal answers `updated=false`, not 403. | -| `memory.delete` | Writes a tombstone, looking past the caller's identity for the key (`found=false` when no live row exists). `memory.delete_version` instead reads only the caller's exact identity (a concrete agent) and, under the key's write lock, deletes only the version it names, else `tombstoned=false, reason="version_changed"`. Consolidated summaries lose the row: retired when no source survives, else `conflicted`. `memory.restore` revives the key within 14 days. | -| `memory.move` | Renames a key, carrying value, type, tags, entities, validity and provenance. The source becomes a redirect (`canonical_key`, `follow=true`) or, with `keep_redirect=false`, a tombstone. See [optimize a namespace](../../guides/optimize-namespace.md). | -| `memory.lock` | `read_only` refuses value, expiry and deletion changes; `no_delete` refuses tombstone, redact, clear and consolidation. Both refuse `memory.reject` (423 `LOCKED`); a negative-outcome quarantine leaves a locked row as it is, while its negatives keep counting. A writer may lock or tighten (`until`, `ttl_seconds`); `memory.unlock` is admin. `locked=false, reason="not_found"` is not a lock. | -| `memory.apply` | A `target` (keys, a filter or a search; filter and search need `limit` ≤ 1,000), an `action` (`delete`, `expire`, `lock`, `label`, `demote`), a `reason`, `dry_run` on by default. Each row goes through the single-row path; refusals come back as `refused` rows. Refuses `agent="*"`. | +| Op | Behaviour | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `memory.update` | Resolves `memory_key` across the namespace when no identity is given (`"-"` counts as none, and `"*"` for `agent`). One live row: updated. None: `updated=false, reason="not_found"`. Several: 409 `AMBIGUOUS_KEY` with `candidates`. Runs the conflict check and keeps a pin. An expired row answers 409 `EXPIRED` unless `revive=true`. | +| `memory.update_by_id` | One row by `(org, memory_id)`. The namespace is resolved and checked before the read; a refusal answers `updated=false`, not 403. | +| `memory.delete` | Writes a tombstone, looking past the caller's identity for the key (`found=false` when no live row exists). `memory.delete_version` instead reads only the caller's exact identity (a concrete agent) and, under the key's write lock, deletes only the version it names, else `tombstoned=false, reason="version_changed"`. Consolidated summaries lose the row: retired when no source survives, else `conflicted`. Permanent refusal blocks restoration and later intake of the removed content. | +| `memory.move` | Renames a key, carrying value, type, tags, entities, validity and provenance. The source becomes a redirect (`canonical_key`, `follow=true`) or, with `keep_redirect=false`, a tombstone. See [optimize a namespace](../../guides/optimize-namespace.md). | +| `memory.lock` | `read_only` refuses value, expiry and deletion changes; `no_delete` refuses tombstone, redact, clear and consolidation. Both refuse `memory.reject` (423 `LOCKED`); a negative-outcome quarantine leaves a locked row as it is, while its negatives keep counting. A writer may lock or tighten (`until`, `ttl_seconds`); `memory.unlock` is admin. `locked=false, reason="not_found"` is not a lock. | +| `memory.apply` | A `target` (keys, a filter or a search; filter and search need `limit` ≤ 1,000), an `action` (`delete`, `expire`, `lock`, `label`, `demote`), a `reason`, `dry_run` on by default. Each row goes through the single-row path; refusals come back as `refused` rows. Refuses `agent="*"`. | ## Provenance fields @@ -278,14 +278,12 @@ extraction path (`LOOM_MEMORY_EXTRACT_VALUE_UPDATES`) never runs for an import. (`on_expire=anonymize`) and comes back `skipped` (`revised`), before the bulk write's own pin and lock guards. A status the import does not recognise leaves the key as it is too, as `skipped` (`unknown`). The read and the write are not atomic: a delete that lands - between them is written over. Text deleted, retired or rejected under another key is - imported again under its `import:` key: the bulk write's dedup sees live rows only. -- **A tombstone has no way back through the import:** `memory.delete`, `memory.clear` and - an expiry with `on_expire=tombstone` all write one, so clearing an agent and importing - from the source again answers `skipped` (`deleted`) for every such item. Bring the keys - back with `memory.restore` within 14 days of the delete (a re-import then answers - `unchanged`), or remove the scope's rows with `erasure.request` and `purge=true`: a key - with no rows left is imported as new. + between them must still pass the committed write's policy checks. Refused content is + blocked under another key too; ordinary retirement does not record permanent refusal. +- **Import does not reverse a tombstone:** explicit deletion and clearing also refuse + the removed content. `memory.restore` can reverse an automatic tombstone within + 14 days when no refusal exists. Erasure and purge preserve consent fingerprints, + so removing stored rows does not grant permission to import their content again. - **Provenance:** every row is `source_class=imported`, `valid_at` is the source's time and `created_at` the import's. An entry with no `source_trace` is traced by its key. - **Categories:** a `category` the namespace registered (`namespace.register_category`) is diff --git a/docs/features/tasks-and-decisions.md b/docs/features/tasks-and-decisions.md index 6e870a7584..4861d83156 100644 --- a/docs/features/tasks-and-decisions.md +++ b/docs/features/tasks-and-decisions.md @@ -341,7 +341,7 @@ superseded in the record plane and does not keep the record. with `sweep=true` and `blacklist.sweep`. `decision.chain`, `decision.precedents` and `decision.outcome` stop serving it, and `memory.record.history` shows the last version with `status="tombstoned"`. Two retirements that race can each write a tombstoned version, so history may list the tombstone twice. -- **After a retirement**: `memory.restore`, or `namespace.rollback` past the delete, revives the row but not the +- **After a retirement**: a reversible automatic retirement can be restored, but this revives the row without the record; call `decision.record` again. A row moved with `keep_redirect=false` is no longer a decision: its new key has no record, and `decision.record` with the same category and scenario records it again under its derived key. - **Leaves the record active**: a delete of another identity's row; `memory.clear` and `namespace.rollback`, which diff --git a/docs/glossary.md b/docs/glossary.md index 8dd4bdacf4..954ac759ff 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -199,8 +199,9 @@ the rows agents write (`memory.promote` / `memory.demote`, table `loom.shared`). **Tier** — which extractor produced a code-index fact (parser or compiler); answers say which tier backs them. -**Tombstone** — a higher-version row marking a memory deleted. `memory.restore` can undo -it within the grace window; nothing is physically deleted by `memory.delete`. +**Tombstone** — a higher-version row marking a memory deleted. Automatic tombstones can +be reversed within the grace window when no refusal exists. Explicit deletion also records +permanent refusal; its historical rows remain until physical erasure. **Toolset** — the MCP listing a client asks for (`core`, `coding`, `atomic`, `tasks`, `tracker`, `readonly`, `all`); it changes what is listed, not what the token may call. diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 0a47670b4f..79128e4004 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -196,7 +196,7 @@ Repairs tab, where a namespace admin downloads a snapshot, restores one and roll | Operation | Role | Does | | -------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `memory.restore` | `WRITER` | brings one tombstoned key back | +| `memory.restore` | `WRITER` | reverses one automatic tombstone when no refusal exists | | `namespace.snapshot` | `ADMIN` | freezes a namespace's memories, types, grants and an audit tail (`include_audit_tail_n`, default 1000) as a JSON payload; `include_embeddings=false` (the default) lets the target re-embed under its own model | | `namespace.restore` | `VIEWER` on a fresh target, `ADMIN` on an existing one | writes a snapshot payload into `target_namespace` (optionally `rebind_agent`); idempotent by `snapshot_id`. Rows keep `version`, `content_sha256` and the attestation columns, so `verified_by` stays; a payload value its column cannot hold (a memory id that is not a UUID) is refused with 422. A collection is refused: its snapshot restores into a plain namespace | | `namespace.rollback` | `ADMIN` | returns every memory head (or `memory_keys`) to the version it had at `as_of`: reinstates the older version as the new head, tombstones keys created after it. Dry run by default (`dry_run=false` applies); at most `max_keys` per call (`truncated` says when to call again) | @@ -382,17 +382,17 @@ again whole before the deployment starts; deliveries it held are listed as parke ## What is and is not recoverable -| Situation | Recoverable? | How | -| ---------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A memory tombstoned by `memory.delete` or `memory.clear` | yes | `memory.restore`, or `namespace.rollback` to before the delete | -| A memory overwritten by a later version | yes | `memory.history` shows the versions; `namespace.rollback` reinstates one | -| An erasure with `purge: true`, or a purged bench namespace | no | purge hard-deletes; there is no restore window. Only a backup taken before it | -| An erasure that tombstoned (no `purge`) | treat as final | the saga's census verified the scope is gone from every read path; bringing it back would undo a deletion request, so restore from a backup only when the erasure itself was the mistake | -| A newer build's schema, after rolling back the image | not needed | migrations are additive; the previous build reads the newer schema ([upgrades](./upgrades.md#rollback)) | -| An earlier schema | from a backup | there are no down-migrations; restore a backup taken before the upgrade | -| A single-node ClickHouse OSS server's disk | from a backup | no replication and no point-in-time recovery: the node is durable only to its own disk | -| Code index tables | rebuildable | `loom code backfill` re-derives them from the git mirror ([code index operations](./code-index-operations.md#first-index)) | -| Rows removed by an opt-in retention TTL | no | TTL deletes are final; only a backup taken before them | +| Situation | Recoverable? | How | +| ---------------------------------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------- | +| A memory tombstoned by `memory.delete` or `memory.clear` | no | refusal blocks restoration and reimport | +| A memory overwritten by a later version | yes | `memory.history` shows the versions; `namespace.rollback` reinstates one | +| An erasure with `purge: true`, or a purged bench namespace | no | purge removes stored content; permanent refusal also blocks logical restoration | +| An erasure that tombstoned (no `purge`) | treat as final | the saga's census verified the scope is gone from every read path; permanent refusal remains in force during restoration | +| A newer build's schema, after rolling back the image | not needed | migrations are additive; the previous build reads the newer schema ([upgrades](./upgrades.md#rollback)) | +| An earlier schema | from a backup | there are no down-migrations; restore a backup taken before the upgrade | +| A single-node ClickHouse OSS server's disk | from a backup | no replication and no point-in-time recovery: the node is durable only to its own disk | +| Code index tables | rebuildable | `loom code backfill` re-derives them from the git mirror ([code index operations](./code-index-operations.md#first-index)) | +| Rows removed by an opt-in retention TTL | no | TTL deletes are final; only a backup taken before them | On a single self-managed OSS node, schedule `BACKUP DATABASE` (or `clickhouse-backup`) to storage off the host. ClickHouse Cloud replicates and backs up the service itself, so diff --git a/docs/operations/maintenance.md b/docs/operations/maintenance.md index 1decb06de6..aa3536b822 100644 --- a/docs/operations/maintenance.md +++ b/docs/operations/maintenance.md @@ -126,7 +126,7 @@ A namespace whose graph outlived its memories is listed by | Call | Effect | Use for | | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | -| `memory.clear` (`ADMIN`) | tombstones the memories of one `user_id` or `agent` in a namespace; `confirm_subject` must repeat the scope value | cleanup, test resets; reversible with `memory.restore` | +| `memory.clear` (`ADMIN`) | tombstones the memories of one `user_id` or `agent` in a namespace; `confirm_subject` must repeat the scope value | explicit removal; refused content cannot be restored | | `erasure.request` with `scope_type`: `user_id`, `namespace` or `namespace_list` | a resumable saga across every Loom table that can hold the subject's content, then the configured external backends (Langfuse, ClickStack) | erasing a user's data (the audit trail is kept); retiring namespaces | The saga checkpoints each step in `loom.erasure_requests` (`ReplacingMergeTree`, rows read diff --git a/docs/operations/okf-sync.md b/docs/operations/okf-sync.md index d6e5ec3b4a..500c330d8c 100644 --- a/docs/operations/okf-sync.md +++ b/docs/operations/okf-sync.md @@ -37,8 +37,7 @@ LOOM_API_KEY="$PUBLIC_WRITER" loom import-okf /tmp/bundle --server-url "$PUBLIC_ import's `--agent`, written by its token, whose `source_trace` starts with `okf:` and whose key is no longer in the bundle. Rows anyone else wrote in the namespace stay. Give each bundle imported into the namespace its own `--agent` (`--prune` refuses without - one). A delete is a tombstone: `memory.restore` brings the row back within its 14-day - window. + one). Pruning tombstones the row and permanently refuses its removed content. The last line reports the counts: `500 rows from the bundle into acme/shared-lessons: 0 written, 500 unchanged, 0 failed …, @@ -92,8 +91,8 @@ Suspend the CronJob (`kubectl patch cronjob okf-sync -p '{"spec":{"suspend":true delete the copies on the public Loom with `memory.delete`, or through erasure when the content must go. -A key pruned by mistake is written again by the next run whose bundle holds it, as a new -row: its `memory_id` changes. `memory.restore` brings back the original row instead. +Pruning is permanent content refusal. A later bundle or `memory.restore` cannot republish +the removed value. Review the prune preview before applying it. ## What a run costs @@ -162,8 +161,7 @@ knows a `high` row by its `sensitivity`, which it reads through `memory.bulk_get the `sensitivity:high` tag `memory.set` and `memory.update` add. Against a private Loom whose `memory.bulk_get` does not return `sensitivity`, every row stays home (the export counts them under "no known sensitivity"), so the bundle has no concepts and `--prune` -refuses it. A prune is a tombstone: the value stays restorable for 14 days and in the -public Loom's version history. Content that must leave the public Loom goes through +refuses it. Pruning retains version history and refuses later publication of the removed value. Content that must leave the public Loom goes through [erasure](../features/memory/retention-and-erasure.md). ## Network diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index a80be64230..cafd4edc51 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1104,7 +1104,7 @@ The memory plane outside search/ask/extraction: writes, conflicts, lifecycle, wo | `LOOM_MEMORY_REINFORCEMENT_CLOSER_WINDOW_DAYS` | `int` | `14` | | Days a consumer's closer keeps its run-less reads credited. | | `LOOM_MEMORY_REINFORCEMENT_CLOSER_CACHE_SECONDS` | `int` | `300` | | Seconds one (org, consumer) closer answer is cached per worker. | | `LOOM_MEMORY_LIST_VIEWER_LIMIT` | `int` | `100` | | memory.list rows per call for a token below writer on the namespace; it pages with `offset` (wire cap stays 1000). 0 = no viewer cap. | -| `LOOM_MEMORY_SESSION_DIGEST_MAX_INPUT_CHARS` | `int` | `3000` | | Transcript characters the session-digest call reads (~4 chars a token); 0 = the built-in 3000. | +| `LOOM_MEMORY_SESSION_DIGEST_MAX_INPUT_CHARS` | `int` | `3000` | | Characters of complete source facts included in an extractive session digest; 0 = the built-in 3000. Facts are never clipped to fit. | | `LOOM_MEMORY_STRUCTURED_ANSWER_EXCERPT_LIMIT` | `int` | `4096` | | content_excerpt ceiling for answer-bearing facts (question_topic set) when memory_capture_structured_answers is on, so an enumerated answer is not clipped mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). | | `LOOM_MEMORY_VALUE_ADVISORY_CHARS` | `int` | `30000` | | A searchable memory.set value serializing longer than this many chars gets one advisory line (the write still lands): only the first EXCERPT_LIMIT chars are embedded, and past the embedder's 8,191-token cap nothing is. 30,000 chars ≈ 8k tokens. 0 disables. | @@ -1698,7 +1698,7 @@ Telemetry, audit, tracing and the ops surface. | `LOOM_LANGFUSE_CIRCUIT_FAILURE_THRESHOLD` | `int` | `5` | | Score-POST circuit breaker: consecutive failures that open it. | | `LOOM_LANGFUSE_CIRCUIT_COOLDOWN_S` | `float` | `30.0` | | Seconds the open breaker waits before a trial call (floor 1). | | `LOOM_LANGFUSE_TIMEOUT_S` | `float` | `15.0` | live | Seconds one Langfuse API call may take: posting a score, and the traces and scores the prompt evaluation reads. | -| `LOOM_LANGFUSE_CAPTURE_IO_LEVEL` | `'off' \| 'query' \| 'all'` | `'off'` | | How much LLM prompt and completion text is recorded, by what the prompt holds: "off" records none (the default; consent is given, not withdrawn); "query" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); "all" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated. | +| `LOOM_LANGFUSE_CAPTURE_IO_LEVEL` | `'off' \| 'query' \| 'all'` | `'off'` | | How much LLM prompt and completion text is recorded, by what the prompt holds: "off" records none (the default; consent is given, not withdrawn); "query" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); "all" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated. | | `LOOM_LANGFUSE_GENERATION_IO_MAX_CHARS` | `int` | `2000` | | Per-field cap on recorded prompt/completion text; truncation adds an explicit marker. | | `LOOM_RETRIEVAL_JUDGE_SAMPLE_RATE` | `float` | `0.0` | | Fraction of searches sent to an independent LLM judge, written back as a `retrieval_relevance` score (unlike loom's own confidence scores, it shares no state with retrieval). 0.0 = off: it costs money per search; a few percent (e.g. 0.05) suits production. | | `LOOM_ANALYTICS_EXPORT_NAME_KEY` | `str` | *(unset)* | | Secret that `loom export-analytics` hashes org, namespace and agent names under: a random string of 32 or more characters; a new key changes every hash. | diff --git a/docs/reference/operations.md b/docs/reference/operations.md index ba1127c137..9211aa7f43 100644 --- a/docs/reference/operations.md +++ b/docs/reference/operations.md @@ -538,7 +538,7 @@ Fields for every `memory` op: [operations/memory.md](operations/memory.md). | [`memory.reject`](operations/memory.md#memoryreject) | Mark a memory as explicitly rejected (extractor / user review decided the candidate fact is wrong), so it isn't re-extracted. | writer | idempotent | no | yes | | [`memory.related`](operations/memory.md#memoryrelated) | Find memories sharing entities with the source memory. | viewer | read | no | | | [`memory.resolve_orphan_duplicates`](operations/memory.md#memoryresolve_orphan_duplicates) | Scan (org, namespace) for orphaned duplicate rows and re-activate them. | writer | idempotent | yes | yes | -| [`memory.restore`](operations/memory.md#memoryrestore) | Bring back a deleted memory (within 14 days) or one Loom retired on its own. | writer | idempotent | yes | yes | +| [`memory.restore`](operations/memory.md#memoryrestore) | Reverse automatic retirement when no deletion or erasure refusal blocks the value. | writer | idempotent | yes | yes | | [`memory.search`](operations/memory.md#memorysearch) | Semantic search over the agent's own memory. | viewer | read | no | | | [`memory.search_trace`](operations/memory.md#memorysearch_trace) | Same shape as memory.search; returns the vector leg's per-candidate scoring breakdown (no fusion, assembly or reranker). | viewer | read | no | | | [`memory.search_trace_log`](operations/memory.md#memorysearch_trace_log) | Read loom.search_trace_log for a namespace (requires the telemetry flag). | viewer | read | yes | | diff --git a/docs/reference/operations/memory.md b/docs/reference/operations/memory.md index 5b0e8c789d..97e81689b2 100644 --- a/docs/reference/operations/memory.md +++ b/docs/reference/operations/memory.md @@ -757,7 +757,7 @@ The raw chunks and session digests behind a namespace's memories, as portable ro | `session_id` | `str` | `''` | optional: one session only; "" or "-" = every session | | `limit` | `int` | `500` | Rows per page (1–2000); `has_more` says whether another page follows. | | `cursor` | `str` | `''` | opaque keyset cursor from a previous page | -| `include_high` | `bool` | `False` | Also export the chunks and digests behind a 'high' row or whose text holds a high shape (a credential, card number, SSN or stated PIN), and the chunks stored without a vector (one may hold part of a key). | +| `include_high` | `bool` | `False` | Include high chunks and history-held digests. | | Output field | Type | Meaning | | --- | --- | --- | @@ -1105,7 +1105,7 @@ Write exported evidence chunks and digests into a namespace, re-embedded here an | `agent` | `str` | required | The writing or reading agent; `'-'` or `'*'` means any agent where the op allows it. | | `user_id` | `str` | `'-'` | The end user the row belongs to; `'-'` for none. | | `chunks` | `list[EvidenceChunk]` | *(computed)* | Chunks to re-embed and insert (≤256); empty-content or session-less ones are skipped, one that reads as high is inserted without a vector, and a key cut between two calls' chunks is not found. | -| `digests` | `list[EvidenceDigest]` | *(computed)* | Session digests to re-embed and insert (≤256); empty summaries are skipped, and one whose text reads as high is inserted without a vector. | +| `digests` | `list[EvidenceDigest]` | *(computed)* | Session digests to re-embed and insert (≤256); only exact extractive summaries with complete, currently eligible source proof are written. | | `source_trace` | `str` | `'okf-import'` | Stamped into memory_chunks.source_url when the chunk carries none. | | Output field | Type | Meaning | @@ -1116,7 +1116,7 @@ Write exported evidence chunks and digests into a namespace, re-embedded here an | `embedding_model` | `str` | The server embedder the imported rows were re-embedded with. | | `embedding_dim` | `int` | Vector length of the written embeddings; 0 when nothing was embedded (every item read as high, or none was written). | | `edited` | `list[str]` | Chunks whose body did not hash to the content_sha256 they arrived with. | -| `skipped` | `int` | Items not written: empty, unembeddable, or refused by a blacklist pattern (Refuse). | +| `skipped` | `int` | Items not written: empty, unembeddable, refused, or missing eligible source proof. | | `unchecked` | `int` | Of the skipped, those the blacklist could not check: the request's values are too long for its regexes; sent in a smaller request they are checked, unless one is too long alone, which must be shortened. | ## `memory.inspect` @@ -1842,7 +1842,7 @@ Scan (org, namespace) for orphaned duplicate rows and re-activate them. ## `memory.restore` -Bring back a deleted memory (within 14 days) or one Loom retired on its own. +Reverse automatic retirement when no deletion or erasure refusal blocks the value. `POST /v1/memory.restore` · MCP `loom_memory_restore` · role `writer` · idempotent · audited diff --git a/docs/reference/schema.md b/docs/reference/schema.md index 7d94cdf67d..534ec76a46 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -142,7 +142,7 @@ the `loom` tables each view reads, and, for each materialized view, its target, -196 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. +197 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. #### Tables @@ -276,6 +276,7 @@ the `loom` tables each view reads, and, for each materialized view, its target, | `memory_record_fields` | `ReplacingMergeTree(version)` | `(org, namespace, schema_name, field_name, record_id)` | | | 1 skip index (bloom_filter) | | `memory_record_versions` | `ReplacingMergeTree` | `(org, namespace, schema_name, memory_key, record_id, version)` | `toYYYYMM(created_at)` | | 1 skip index (bloom_filter) | | `memory_records` | `ReplacingMergeTree(version)` | `(org, namespace, schema_name, memory_key, record_id)` | | | 1 skip index (bloom_filter) | +| `memory_refusals` | `MergeTree` | `(org, scope_kind, scope_value, user_id, kind, units, digest)` | | | | | `memory_reinforcement` | `ReplacingMergeTree(version)` | `(org, namespace, agent, user_id, memory_key)` | | | | | `memory_schema_fields` | `ReplacingMergeTree(version)` | `(org, namespace, schema_id, field_name)` | | | | | `memory_schemas` | `ReplacingMergeTree(version)` | `(org, namespace, name)` | | | | diff --git a/llms.txt b/llms.txt index 3cc5487452..4b32f3b00c 100644 --- a/llms.txt +++ b/llms.txt @@ -166,6 +166,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Live connected-source search](docs/architecture/live-source-search.md): `search.query(live=true)` adds an ephemeral section from the asker's own connected Drive and Notion registrations. - [LLM and embedding architecture](docs/architecture/llm-architecture.md): Loom has two pluggable providers: an embedder (`server/embedding/`) that turns text into vectors, and an LLM client (`server/llm/`) that completes prompts. - [Memory architecture](docs/architecture/memory-architecture.md): The memory plane stores facts in `loom.memory`, makes them searchable through `loom.memory_embeddings`, and derives evidence, chunks, graph rows and statistics from them. +- [Memory consent and derived text](docs/architecture/memory-consent.md): A deletion contracts memory and records content refusal before its tombstone commits. - [n8n adapter](docs/architecture/n8n-adapter.md): The n8n community package stores conversation history and durable facts in a Loom deployment through the TypeScript SDK. - [OpenAI Agents conversation sessions](docs/architecture/openai-agents-sessions.md): `LoomSession` implements the native Python and TypeScript session interfaces over Loom episodic memories. - [Pipecat demonstration recording](docs/architecture/pipecat-demo-recording.md): The optional `pipecat-demo.yml` workflow renders a timestamped execution visualization of an authenticated native Pipecat text-memory check. diff --git a/loom.example.yaml b/loom.example.yaml index 523368cf27..55bec8b0d5 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -3237,8 +3237,8 @@ settings: # `offset` (wire cap stays 1000). 0 = no viewer cap. [restart] # memory_list_viewer_limit: 100 - # Transcript characters the session-digest call reads (~4 chars a token); 0 = the - # built-in 3000. [restart] + # Characters of complete source facts included in an extractive session digest; 0 = + # the built-in 3000. Facts are never clipped to fit. [restart] # memory_session_digest_max_input_chars: 3000 # content_excerpt ceiling for answer-bearing facts (question_topic set) when @@ -4987,10 +4987,10 @@ settings: # records none (the default; consent is given, not withdrawn); "query" only the legs # whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); # "all" also the legs carrying retrieved memories or conversation turns - # (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's - # `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also - # gates loom.search_trace_log.query. Token counts, cost, latency and quality scores - # are never gated. [restart] + # (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` + # overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates + # loom.search_trace_log.query. Token counts, cost, latency and quality scores are + # never gated. [restart] # langfuse_capture_io_level: "off" # Per-field cap on recorded prompt/completion text; truncation adds an explicit diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index dea7d18cc5..52d15f0da0 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -3606,7 +3606,7 @@ "all" ], "type": "string", - "description": "How much LLM prompt and completion text is recorded, by what the prompt holds: \"off\" records none (the default; consent is given, not withdrawn); \"query\" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); \"all\" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated." + "description": "How much LLM prompt and completion text is recorded, by what the prompt holds: \"off\" records none (the default; consent is given, not withdrawn); \"query\" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); \"all\" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated." }, "langfuse_circuit_cooldown_s": { "default": 30.0, @@ -5347,7 +5347,7 @@ "memory_session_digest_max_input_chars": { "default": 3000, "type": "integer", - "description": "Transcript characters the session-digest call reads (~4 chars a token); 0 = the built-in 3000." + "description": "Characters of complete source facts included in an extractive session digest; 0 = the built-in 3000. Facts are never clipped to fit." }, "memory_set_chunk_max_chunks": { "default": 48, diff --git a/schemas/migrations/115_memory_refusals.sql b/schemas/migrations/115_memory_refusals.sql new file mode 100644 index 0000000000..b51fd9b6b1 --- /dev/null +++ b/schemas/migrations/115_memory_refusals.sql @@ -0,0 +1,24 @@ +-- Durable consent fingerprints contain no removed plaintext and have no TTL. +CREATE TABLE IF NOT EXISTS loom.memory_refusals +( + org String, + namespace String, + agent String DEFAULT '-', + user_id String DEFAULT '-', + session_id String DEFAULT '-', + scope_kind LowCardinality(String) DEFAULT 'identity', + scope_value String, + actor String, + source_class LowCardinality(String) DEFAULT 'derived', + kind LowCardinality(String), + units UInt32, + digest FixedString(64), + created_at DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY (org, scope_kind, scope_value, user_id, kind, units, digest) +SETTINGS non_replicated_deduplication_window = 1000; + +-- A numeric version alone cannot distinguish independently written full states. +ALTER TABLE loom.delete_version_checks + ADD COLUMN IF NOT EXISTS checked_state_sha256 String DEFAULT ''; diff --git a/schemas/pg_migrations/014_memory_refusals.sql b/schemas/pg_migrations/014_memory_refusals.sql new file mode 100644 index 0000000000..2ab028c0b1 --- /dev/null +++ b/schemas/pg_migrations/014_memory_refusals.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS loom.memory_refusals ( + org text COLLATE "C" NOT NULL, + namespace text COLLATE "C" NOT NULL, + agent text COLLATE "C" NOT NULL DEFAULT '-', + user_id text COLLATE "C" NOT NULL DEFAULT '-', + session_id text COLLATE "C" NOT NULL DEFAULT '-', + scope_kind text NOT NULL DEFAULT 'identity', + scope_value text NOT NULL, + actor text NOT NULL, + source_class text NOT NULL DEFAULT 'derived', + kind text NOT NULL, + units integer NOT NULL, + digest text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (org, scope_kind, scope_value, user_id, kind, units, digest) +); diff --git a/schemas/schema.sql b/schemas/schema.sql index e833512461..2b8590f2d8 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -16,13 +16,13 @@ -- Do NOT hand-edit. Edit schemas/migrations/*.sql and re-dump. -- CI verifies this file matches what migrations would produce. -- --- Counts: 196 tables + 5 views + 19 materialized views. +-- Counts: 197 tables + 5 views + 19 materialized views. CREATE DATABASE IF NOT EXISTS `loom`; CREATE DATABASE IF NOT EXISTS `loom_telemetry`; -- ====================================================================== --- Tables (196) +-- Tables (197) -- ====================================================================== -- loom.acl_versions @@ -1609,7 +1609,8 @@ CREATE TABLE IF NOT EXISTS loom.delete_version_checks `tombstone` UInt64, `done` UInt8 DEFAULT 0, `created_at` DateTime DEFAULT now(), - `updated_at` DateTime64(3) DEFAULT now64(3) + `updated_at` DateTime64(3) DEFAULT now64(3), + `checked_state_sha256` String DEFAULT '' ) ENGINE = ReplacingMergeTree(updated_at) ORDER BY (org, namespace, agent, user_id, session_id, memory_key, tombstone) @@ -3032,6 +3033,27 @@ ENGINE = ReplacingMergeTree(version) ORDER BY (org, namespace, schema_name, memory_key, record_id) SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 1000; +-- loom.memory_refusals +CREATE TABLE IF NOT EXISTS loom.memory_refusals +( + `org` String, + `namespace` String, + `agent` String DEFAULT '-', + `user_id` String DEFAULT '-', + `session_id` String DEFAULT '-', + `scope_kind` LowCardinality(String) DEFAULT 'identity', + `scope_value` String, + `actor` String, + `source_class` LowCardinality(String) DEFAULT 'derived', + `kind` LowCardinality(String), + `units` UInt32, + `digest` FixedString(64), + `created_at` DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY (org, scope_kind, scope_value, user_id, kind, units, digest) +SETTINGS non_replicated_deduplication_window = 1000, index_granularity = 8192; + -- loom.memory_reinforcement CREATE TABLE IF NOT EXISTS loom.memory_reinforcement ( diff --git a/scripts/codegen/push_langfuse_prompts.py b/scripts/codegen/push_langfuse_prompts.py index a414324959..c6af194791 100644 --- a/scripts/codegen/push_langfuse_prompts.py +++ b/scripts/codegen/push_langfuse_prompts.py @@ -61,7 +61,6 @@ def registry() -> dict[str, str]: from server.memory.app import extract as extract_svc from server.memory.app import period_digest as _period_digest from server.memory.app import rerank - from server.memory.app import session_summary as sess from server.memory.domain import extraction_prompts as extract from server.memory.domain import search_prompts from server.memory.workers import entity_extractor, profile_consolidation @@ -88,7 +87,6 @@ def registry() -> dict[str, str]: # relation backfill, sent bare "loom-extract-relations": extract_svc._RELATION_BACKFILL_SYSTEM_PROMPT, # summarisation / workers - "loom-session-summary": sess._SESSION_SUMMARY_SYSTEM, "loom-period-digest": _period_digest._PERIOD_DIGEST_SYSTEM, "loom-conversation-title": conversation_titler._TITLE_SYSTEM, "loom-profile-fold": profile_consolidation._FOLD_SYSTEM, diff --git a/scripts/gates/function_length_baseline.json b/scripts/gates/function_length_baseline.json index d778beb05e..9c81dcec52 100644 --- a/scripts/gates/function_length_baseline.json +++ b/scripts/gates/function_length_baseline.json @@ -27,7 +27,6 @@ "server/memory/app/consolidate.py": 1, "server/memory/app/entity_resolution.py": 1, "server/memory/app/expiry.py": 1, - "server/memory/app/extract/handler.py": 1, "server/memory/app/extract/parse.py": 1, "server/memory/app/extract/sweep.py": 1, "server/memory/app/extract/typed.py": 1, diff --git a/sdk/python/loom/memory.py b/sdk/python/loom/memory.py index 1a67b33434..816de7822a 100644 --- a/sdk/python/loom/memory.py +++ b/sdk/python/loom/memory.py @@ -1386,8 +1386,10 @@ async def restore( if_superseded_by: str | None = None, only_if_superseded: bool = False, ) -> dict[str, Any]: - """Bring back a deleted memory (within the 14-day grace window) or one - Loom retired on its own (status `superseded`). + """Reverse an automatic tombstone within its 14-day grace window, or + an automatic supersession. Explicit deletion and clearing record a + permanent refusal: restoring that content raises VALIDATION_FAILED + with reason `forgotten`. `if_superseded_by` restores the row only while it is still retired behind that memory id (as a `memory.auto_supersede` record names it); @@ -1399,13 +1401,12 @@ async def restore( - `memory_id: str | None` — UUID of the restored row. - `version: int | None` — new version assigned. - `expired: bool` — True when the tombstone aged past the 14d - grace window. The data is being GC'd; restoration is no longer - possible. - - `from_status: str` — `"tombstoned"` (a delete undone) or + grace window; restoration is no longer available. + - `from_status: str` — `"tombstoned"` (automatic expiry reversed) or `"superseded"` (a retirement reversed); empty on a no-op. Preserves the original char_limit, pinned, tags, and entities - across the restore. An undone delete clears `expires_at`, so the row + across the restore. Reversing an automatic tombstone clears `expires_at`, so the row gets a fresh lease (its old expiry may have passed, which would tombstone it again at once); an undone retirement keeps it. """ @@ -1434,9 +1435,8 @@ async def clear( Returns `{cleared_at, rows_affected: {table: count}, total, locked_skipped}`; locked rows stay and are counted in `locked_skipped`. - Tombstone-not-hard-delete: cleared rows are recoverable for the - 14-day grace window by re-writing the same key. For - cross-system erasure use `erasure.request_user_id` or + History remains, but restoration and re-writing refused content are + blocked. For physical erasure across stores use `erasure.request_user_id` or `erasure.request_namespace`. """ payload: dict[str, Any] = { diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index a95526c354..9606382061 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -15586,7 +15586,7 @@ class MemoryResolveOrphanDuplicatesOutput(TypedDict, total=False): class MemoryRestoreInput(TypedDict, total=False): - "Bring back a deleted memory (within 14 days) or one Loom retired on its own." + "Reverse automatic retirement when no deletion or erasure refusal blocks the value." org: Required[str] namespace: Required[str] @@ -15600,7 +15600,7 @@ class MemoryRestoreInput(TypedDict, total=False): class MemoryRestoreBody(TypedDict, total=False): - "Bring back a deleted memory (within 14 days) or one Loom retired on its own." + "Reverse automatic retirement when no deletion or erasure refusal blocks the value." org: NotRequired[str] namespace: NotRequired[str] diff --git a/sdk/python/loom/profile.py b/sdk/python/loom/profile.py index 1798c27a1d..e2cbae7811 100644 --- a/sdk/python/loom/profile.py +++ b/sdk/python/loom/profile.py @@ -156,8 +156,10 @@ async def delete( user_id: str | None = None, session_id: str | None = None, ) -> bool: - """Tombstone a profile section. Soft-delete via memory.delete; - restoreable via `memory.restore` within the grace window.""" + """Tombstone a profile section and refuse its removed content. + + `memory.restore` cannot republish explicitly deleted content. + """ return await self._agent.memory.delete( memory_key=label, user_id=user_id, diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index cc44181142..aae290778f 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -781,7 +781,7 @@ def _cmd_import_clickgap(args: argparse.Namespace) -> int: from server.core.ch import get_client from server.embedding import get_embedder from server.memory.app.provenance import writer_actor - from server.memory.store import insert_memory_versions + from server.memory.store import insert_embedding_rows, insert_memory_versions async def _run() -> int: sqlite_path = args.sqlite @@ -860,8 +860,8 @@ async def _run() -> int: await insert_memory_versions(mem_batch, ch=ch) total_items += len(mem_batch) if emb_batch: - await ch.insert("loom.memory_embeddings", emb_batch) - total_emb += len(emb_batch) + admitted = await insert_embedding_rows(emb_batch, ch=ch) + total_emb += len(admitted) print( f" ✓ {total_items} lessons → memory ({total_emb} embeddings, " f"{w.polarity_neg} suppression/polarity:negative, " diff --git a/server/core/ch.py b/server/core/ch.py index 151e46a7b3..563ec14134 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -1365,7 +1365,13 @@ async def query_one( return None return dict(zip(result.column_names, rows[0], strict=False)) - async def _query_fail_open(self, op_name: str, sql: str, params: dict[str, Any] | None) -> Any: + async def _query_fail_open( + self, + op_name: str, + sql: str, + params: dict[str, Any] | None, + settings: dict[str, Any] | None = None, + ) -> Any: """Run a SELECT with the standard retry; if the server rejects the quantized-codes setting (a replica that does not know it), turn the setting off for this process and run the query once @@ -1376,7 +1382,7 @@ async def _q(text: str) -> Any: res = await self._statement_client().query( text, parameters=params or {}, - settings=statement_settings(text, cls="read"), + settings=statement_settings(text, settings, cls="read"), **_reader_auth(), ) note_result_summary(res) @@ -1408,8 +1414,10 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: - result = await self._query_fail_open("query_all", sql, params) + result = await self._query_fail_open("query_all", sql, params, settings) return [dict(zip(result.column_names, row, strict=False)) for row in result.result_rows] async def query_columns( diff --git a/server/core/repo_base.py b/server/core/repo_base.py index f68ea2cb2c..d9646b4563 100644 --- a/server/core/repo_base.py +++ b/server/core/repo_base.py @@ -29,6 +29,8 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = ..., + *, + settings: dict[str, Any] | None = ..., ) -> list[dict[str, Any]]: ... async def execute( self, diff --git a/server/erasure/app/saga_steps.py b/server/erasure/app/saga_steps.py index 3e77271375..78f89f3cd1 100644 --- a/server/erasure/app/saga_steps.py +++ b/server/erasure/app/saga_steps.py @@ -21,6 +21,7 @@ from server.core.principals import personal_name from server.core.repo_base import _CHClient from server.erasure.store import census as erasure_census +from server.erasure.store import refusals as refusal_steps from server.erasure.store import steps_graph as graph_steps from server.erasure.store import steps_ledgers as ledger_steps from server.erasure.store import steps_memory as memory_steps @@ -327,6 +328,7 @@ def authority_steps[StepT: tuple[str, Any]](steps: tuple[StepT, ...]) -> list[St # The step names are stored in the ledger's `steps_completed`: renaming one # makes a resumed saga run it again. USER_STEPS: tuple[UserStep, ...] = ( + ("memory.refusals", refusal_steps.capture_user), # First: the authority's copy, then the wait for its publishers, so every # ClickHouse copy the steps below look for has landed. ("pg_memory.delete", pg_steps.delete_pg_memory_by_user), @@ -386,6 +388,7 @@ def authority_steps[StepT: tuple[str, Any]](steps: tuple[StepT, ...]) -> list[St # The loom-side steps of a namespace erasure over (org, namespace or list), in # saga order. Names as for USER_STEPS. NAMESPACE_STEPS: tuple[NamespaceStep, ...] = ( + ("memory.refusals", refusal_steps.capture_namespace), ("knowledge_sources.retire", _step_retire_knowledge_sources_by_ns), ("knowledge_source_push.delete", ledger_steps.delete_source_push_by_namespace), ("pg_memory.delete", pg_steps.delete_pg_memory_by_namespace), diff --git a/server/erasure/store/census.py b/server/erasure/store/census.py index ce1551e109..236d052526 100644 --- a/server/erasure/store/census.py +++ b/server/erasure/store/census.py @@ -530,6 +530,8 @@ def owner(self, scope_type: str) -> str | None: ) STEP_ONLY_TABLES: dict[str, tuple[str, ...]] = { + # Captures retained hash-only consent policy; this auxiliary step erases no table. + "memory.refusals": (), "entity_canonical_map.tombstone": ("loom.entity_canonical_map",), "agent_events.scrub": ("loom.agent_events", "loom.agent_events_private"), "subject_last_seen.delete": ("loom.subject_last_seen",), @@ -767,6 +769,10 @@ def owner(self, scope_type: str) -> str | None: "profile built from one is a loom.memory row." ), "loom.blacklist": "consent patterns authored by admins; policy, not subject data.", + "loom.memory_refusals": ( + "durable deletion consent fingerprints; no removed plaintext or TTL. " + "Retained so erasure cannot permit the same value to be learned again." + ), # -- tracker plane: org-shared reference data mirrored from CI, not anyone's # memory; rebuilt from the source, never read into a memory answer. "loom.qa_test_daily": ( diff --git a/server/erasure/store/consent.py b/server/erasure/store/consent.py new file mode 100644 index 0000000000..b0d222fb78 --- /dev/null +++ b/server/erasure/store/consent.py @@ -0,0 +1,32 @@ +"""The consent policy the composition root supplies to erasure stores.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass + +from server.core.errors import BackendUnavailableError +from server.core.repo_base import _CHClient + + +@dataclass(frozen=True) +class ConsentPolicy: + fence_names: Callable[[str, str, str], list[str]] + lease_client: Callable[[_CHClient], _CHClient] + lock_postgres: Callable[..., Awaitable[None]] + record_clickhouse: Callable[..., Awaitable[None]] + record_postgres: Callable[..., Awaitable[None]] + + +_policy: ConsentPolicy | None = None + + +def use_policy(policy: ConsentPolicy) -> None: + global _policy + _policy = policy + + +def current_policy() -> ConsentPolicy: + if _policy is None: + raise BackendUnavailableError("erasure consent policy is not wired") + return _policy diff --git a/server/erasure/store/refusals.py b/server/erasure/store/refusals.py new file mode 100644 index 0000000000..e1dd9f9b57 --- /dev/null +++ b/server/erasure/store/refusals.py @@ -0,0 +1,251 @@ +"""Record refusal fingerprints before an erasure removes its source history.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from server.core import lease_writes, pg +from server.core.ch import sequential_reads, set_query_tag, write_limit_s +from server.core.ch_params import chunks +from server.core.repo_base import _CHClient, _default_client +from server.erasure.store.consent import current_policy +from server.erasure.store.predicates import ( + USER_CHUNKS, + USER_SUMMARIES, + USER_SUMMARY_CHUNKS, + USER_SUMMARY_VECTORS, +) + +_KEYS = ( + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "version", + "body_sha", +) +_CURSOR = "Tuple(String,String,String,String,String,UInt64,FixedString(64))" +_BODY_SHA = "hex(SHA256(toJSONString((value_json, value)))) AS body_sha" + + +def _sources() -> list[tuple[str, str]]: + identity = "org, namespace, agent, user_id, session_id, 'process:erasure' AS actor, " + version = "toUInt64(toUnixTimestamp64Nano(toDateTime64(created_at, 9))) AS version, " + sources = [ + ( + "loom.session_summaries", + identity + "concat('summary:', session_id) AS memory_key, version, " + "'{}' AS value_json, " + "toJSONString(arrayJoin(arrayConcat([summary_text], key_facts))) AS value", + ), + ( + "loom.memory_evidence", + identity + + "concat('evidence:', toString(evidence_id)) AS memory_key, " + + version + + "'{}' AS value_json, toJSONString(content) AS value", + ), + ( + "loom.memory_evidence", + identity + + "concat('evidence-metadata:', toString(evidence_id)) AS memory_key, " + + version + + "'{}' AS value_json, metadata AS value, 'metadata' AS capture_kind", + ), + ( + "loom.memory_chunks", + identity + "concat('chunk:', toString(chunk_index), ':', " + "content_sha256) AS memory_key, " + version + "'{}' AS value_json, " + "toJSONString(arrayJoin([content, caption, source_url, source_local_path])) AS value", + ), + ] + sources.append( + ( + "loom.memory_embeddings", + identity + "memory_key, version, '{}' AS value_json, " + "toJSONString(content_excerpt) AS value", + ) + ) + sources.extend( + [ + ( + "loom.memory_extraction_queue", + identity + "concat('queue:', queue_id) AS memory_key, version, " + "'{}' AS value_json, payload AS value, 'payload' AS capture_kind", + ), + ( + "loom.memory_extraction_queue", + identity + "concat('queue-error:', queue_id) AS memory_key, version, " + "'{}' AS value_json, toJSONString(last_error) AS value", + ), + ( + "loom.entity_summaries", + "org, namespace, agent, user_id, '-' AS session_id, 'process:erasure' AS actor, " + "concat('entity:', entity) AS memory_key, version, '{}' AS value_json, " + "toJSONString(arrayJoin(arrayConcat([summary_text], key_facts))) AS value", + ), + ] + ) + # Cleared native revision payloads survive in ClickHouse history. + sources.insert( + 0, + ( + "loom.memory_versions", + identity + "memory_key, version, toString(value_json) AS value_json, value", + ), + ) + return sources + + +def _predicate(table: str, kind: str) -> str: + if kind == "user_id": + if table == "loom.memory_embeddings": + return f"(user_id={{uid:String}} OR {USER_SUMMARY_VECTORS})" + if table == "loom.memory_chunks": + return f"({USER_CHUNKS} OR {USER_SUMMARY_CHUNKS})" + if table == "loom.memory_versions": + return f"(user_id={{uid:String}} OR {USER_SUMMARIES})" + return f"{kind}={{scope:String}}" + + +async def _capture(org: str, kind: str, value: str) -> int: + ch = _default_client() + refusals = current_policy() + names = refusals.fence_names(org, kind, value) + count = 0 + # The ledger blocks new intake. Drain admitted sends before paging history; + # capture can outlast a send lease without admitting new writers. + async with lease_writes.fenced_many( + names, ch=refusals.lease_client(ch), wait_s=write_limit_s() + ): + pass + with sequential_reads(): + for table, projection in _sources(): + count += await _capture_source(ch, table, projection, org, kind, value) + return count + + +async def _capture_source( + ch: _CHClient, table: str, projection: str, org: str, kind: str, value: str +) -> int: + refusals = current_policy() + count = 0 + after: tuple[Any, ...] | None = None + while True: + set_query_tag("erasure.memory_refusals") + predicates = [] + if after: + predicates.append(f"({', '.join(_KEYS)}) > {{after:{_CURSOR}}}") + if table != "loom.memory_versions": + predicates.append("value != {empty_text:String}") + cursor = "WHERE " + " AND ".join(predicates) if predicates else "" + rows = await ch.query_all( + f"SELECT *, {_BODY_SHA} FROM (SELECT {projection} FROM {table} " + f"WHERE org={{org:String}} AND {_predicate(table, kind)}) {cursor} " + f"ORDER BY {', '.join(_KEYS)} LIMIT 1000", + params={ + "org": org, + "scope": value, + "uid": value, + "after": after, + "empty_text": '""', + }, + ) + if not rows: + break + if pg.enabled(): + async with pg.transaction() as conn: + await refusals.record_postgres(conn, rows, scope_kind=kind, scope_value=value) + else: + await refusals.record_clickhouse(ch, rows, scope_kind=kind, scope_value=value) + count += len(rows) + after = tuple(rows[-1][c] for c in _KEYS) + return count + + +async def capture_user(org: str, user_id: str) -> int: + return await _capture(org, "user_id", user_id) + + +async def capture_namespace(org: str, namespace: str | Sequence[str]) -> int: + count = 0 + for ns in [namespace] if isinstance(namespace, str) else sorted(set(namespace)): + count += await _capture(org, "namespace", ns) + return count + + +async def capture_native_chunks(conn: Any, sources: list[dict[str, Any]], user_id: str) -> None: + keys = sorted( + { + tuple( + str(r.get(c) or "-") for c in ("org", "namespace", "agent", "user_id", "memory_key") + ) + for r in sources + } + ) + for part in chunks(keys, limit=16 * 1024): + await _capture_native_chunk_keys(conn, part, user_id) + + +async def _capture_native_chunk_keys( + conn: Any, sources: list[tuple[str, ...]], user_id: str +) -> None: + """Capture raw dependents by authoritative keys before native history disappears.""" + if not sources: + return + org = sources[0][0] + identities = sorted( + { + (ns, agent, uid, key) + for _, ns, agent, source_uid, key in sources + for uid in (source_uid, "-") + } + ) + legacy = sorted({(ns, key) for _, ns, _, source_uid, key in sources if source_uid == user_id}) + ch = _default_client() + for table, parent in ( + ("loom.memory_chunks", "parent_memory_key"), + ("loom.memory_embeddings", "memory_key"), + ): + await _capture_native_dependents(conn, org, user_id, identities, legacy, table, parent, ch) + + +async def _capture_native_dependents( + conn: Any, + org: str, + user_id: str, + identities: Sequence[tuple[str, ...]], + legacy: Sequence[tuple[str, ...]], + table: str, + parent: str, + ch: _CHClient, +) -> None: + projection = dict(_sources())[table] + refusals = current_policy() + after: tuple[Any, ...] | None = None + with sequential_reads(): + while True: + set_query_tag("erasure.memory_refusals") + cursor = f"AND ({', '.join(_KEYS)}) > {{after:{_CURSOR}}}" if after else "" + rows = await ch.query_all( + f"SELECT *, {_BODY_SHA} FROM (SELECT {projection} FROM {table} " + f"WHERE org={{org:String}} AND ((namespace, agent, user_id, {parent}) " + "IN {identities:Array(Tuple(String,String,String,String))} OR " + f"(user_id='-' AND (namespace, {parent}) " + "IN {legacy:Array(Tuple(String,String))}))) " + f"WHERE value != {{empty_text:String}} {cursor} " + f"ORDER BY {', '.join(_KEYS)} LIMIT 1000", + params={ + "org": org, + "identities": identities, + "legacy": legacy, + "after": after, + "empty_text": '""', + }, + ) + if not rows: + return + await refusals.record_postgres(conn, rows, scope_kind="user_id", scope_value=user_id) + after = tuple(rows[-1][c] for c in _KEYS) diff --git a/server/erasure/store/steps_pg.py b/server/erasure/store/steps_pg.py index 7fd6f091ef..410065db0e 100644 --- a/server/erasure/store/steps_pg.py +++ b/server/erasure/store/steps_pg.py @@ -135,6 +135,14 @@ async def _erase_once( # A large scope takes longer than a request's statement budget, on the client's # timeout as on the server's: its own connection, with neither. The scope's writes wait. async with pg.dedicated() as conn, conn.transaction(): + from server.erasure.store.consent import current_policy + + refusals = current_policy() + + for value in sorted(set(namespaces or [])) if user_id is None else [user_id]: + await refusals.lock_postgres( + conn, [], erasure=("namespace" if user_id is None else "user_id", org, value) + ) # Advancing each generation locks it: the scope's commits wait for this # transaction, and an admission that captured the old one checks again. scoped: list[str] | None = None # a user's scope reads the org's evidence @@ -170,6 +178,7 @@ async def _erase_once( "ORDER BY d.org, d.namespace, d.seq FOR UPDATE OF d", *args, ) + await _capture_history(conn, where_r, where_c, args, user_id) # The deliveries go with their revisions (ON DELETE CASCADE). This statement locks # them and reads their leases, any committed since the locks above included, and # moves the audit rows ClickHouse lacks to the outbox: the audit trail outlives the @@ -206,6 +215,50 @@ async def _erase_once( return int(revisions or 0) + int(heads or 0), [float(x) for x in leases] +async def _capture_history( + conn: Any, where_r: str, where_c: str, args: list[Any], user_id: str | None +) -> None: + """Keep payload fingerprints and dependent keys before native history is removed. + + Delivered revisions may retain only typed metadata. Their plaintext is captured + from ClickHouse history by the refusal step; their keys still find raw dependents. + """ + from server.erasure.store.consent import current_policy + from server.erasure.store.refusals import capture_native_chunks + + refusals = current_policy() + + async def capture(batch: list[dict[str, Any]]) -> None: + payloads = [source["row"] for source in batch if source["row"] is not None] + await refusals.record_postgres( + conn, + payloads, + scope_kind="namespace" if user_id is None else "user_id", + scope_value=user_id, + ) + if user_id is not None: + await capture_native_chunks( + conn, + [{key: value for key, value in row.items() if key != "row"} for row in batch], + user_id, + ) + + batch: list[dict[str, Any]] = [] + async for source in conn.cursor( + "SELECT r.row, r.org, r.namespace, coalesce(r.agent, r.row->>'agent', '') AS agent, " + "r.user_id, coalesce(r.memory_key, r.row->>'memory_key', '') AS memory_key " + f"FROM loom.memory_revisions r WHERE {where_r} " + "UNION ALL SELECT row, org, namespace, agent, user_id, memory_key " + f"FROM loom.memory_current WHERE {where_c}", + *args, + ): + batch.append(dict(source)) + if len(batch) >= 1000: + await capture(batch) + batch = [] + await capture(batch) + + async def _mark_step(conn: Any, org: str) -> None: """The running saga's step recorded in its ledger row, in this step's transaction (`pg_ledger.mark_step`); outside a saga, nothing.""" @@ -517,6 +570,7 @@ async def delete_pg_idempotency(org: str) -> int: } # The Postgres tables that hold none, with the reason. PG_NO_MEMORY_CONTENT: dict[str, str] = { + "loom.memory_refusals": "durable consent fingerprints, no removed plaintext; survives erasure", "loom.authority": "the deployment's recorded authority and epoch; no tenant data.", "loom.cell_first": "whether this cell is the deployment's first, and its name; no tenant data.", "loom.tenancy_mode": "the cell's tenancy mode (recording or enforced); no tenant data.", diff --git a/server/memory/api/models_crud.py b/server/memory/api/models_crud.py index 1e7b58452a..9a8c99b292 100644 --- a/server/memory/api/models_crud.py +++ b/server/memory/api/models_crud.py @@ -569,7 +569,7 @@ class MemoryDeleteVersionOutput(BaseModel): class MemoryRestoreInput(_Identity): - """Bring back a deleted memory (within 14 days) or one Loom retired on its own.""" + """Reverse automatic retirement when no deletion or erasure refusal blocks the value.""" # The key to restore, under this exact identity (no namespace-wide fallback). memory_key: str diff --git a/server/memory/api/models_evidence_transfer.py b/server/memory/api/models_evidence_transfer.py index 66646f77df..ad3cd381df 100644 --- a/server/memory/api/models_evidence_transfer.py +++ b/server/memory/api/models_evidence_transfer.py @@ -58,10 +58,8 @@ class MemoryExportEvidenceInput(BaseModel): # Rows per page (1–2000); `has_more` says whether another page follows. limit: int = Field(default=500, ge=1, le=2000) cursor: str = "" # opaque keyset cursor from a previous page - # Also export the chunks and digests behind a 'high' row or whose text holds a high - # shape (a credential, card number, SSN or stated PIN), and the chunks stored without - # a vector (one may hold part of a key). Off: a bundle reaches a wider audience than - # the namespace's grants. + # Include high chunks and history-held digests. A digest still needs complete, + # current, eligible source proof; this flag never overrides that requirement. include_high: bool = False @@ -89,8 +87,8 @@ class MemoryImportEvidenceInput(BaseModel): # one that reads as high is inserted without a vector, and a key cut between # two calls' chunks is not found. chunks: list[EvidenceChunk] = Field(default_factory=list, max_length=IMPORT_MAX_ITEMS) - # Session digests to re-embed and insert (≤256); empty summaries are skipped, and one - # whose text reads as high is inserted without a vector. + # Session digests to re-embed and insert (≤256); only exact extractive summaries + # with complete, currently eligible source proof are written. digests: list[EvidenceDigest] = Field(default_factory=list, max_length=IMPORT_MAX_ITEMS) # Stamped into memory_chunks.source_url when the chunk carries none. source_trace: str = Field(default="okf-import", max_length=512) @@ -110,7 +108,7 @@ class MemoryImportEvidenceOutput(BaseModel): embedding_dim: int = 0 # Chunks whose body did not hash to the content_sha256 they arrived with. edited: list[str] = Field(default_factory=list) - # Items not written: empty, unembeddable, or refused by a blacklist pattern (Refuse). + # Items not written: empty, unembeddable, refused, or missing eligible source proof. skipped: int = 0 # Of the skipped, those the blacklist could not check: the request's values are too # long for its regexes; sent in a smaller request they are checked, unless one is diff --git a/server/memory/app/chunk_projection.py b/server/memory/app/chunk_projection.py index 9106716ea3..faaadfc009 100644 --- a/server/memory/app/chunk_projection.py +++ b/server/memory/app/chunk_projection.py @@ -11,7 +11,6 @@ from contextvars import ContextVar from typing import Any -from server.core.ch_writes import committed_insert_settings from server.core.datetimes import naive_utc_now from server.embedding.documents import document_vector, document_vectors from server.llm import customer @@ -392,11 +391,9 @@ async def write_memory_chunks( # write must find them; a fire-and-forget insert would break that. Committed, # not batched: a mis-sized vector fails the shared part of a batched flush, # and with it every other insert in that flush. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) if provenance_rows: try: await ch.insert( diff --git a/server/memory/app/curation.py b/server/memory/app/curation.py index 4f0473a329..b59081a182 100644 --- a/server/memory/app/curation.py +++ b/server/memory/app/curation.py @@ -144,12 +144,16 @@ async def handle_memory_digests(input: MemoryDigestsInput) -> MemoryDigestsOutpu params["s"] = input.session_id where += scope_and(request_scope(), params) rows = await ch.query_all( - "SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, " + "SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, " + "valid_at, created_at, " "version FROM loom.session_summaries FINAL WHERE " + where + " ORDER BY created_at, session_id LIMIT {lim:UInt32}", params, ) + from server.memory.store.summary_sources import current_summary_sources + + rows = await current_summary_sources(ch, rows, org=input.org, namespace=input.namespace) page = rows[: int(input.limit)] # Checking every digest's text is CPU work; it runs off the event loop. return MemoryDigestsOutput( @@ -206,7 +210,8 @@ async def handle_memory_outline(input: MemoryOutlineInput) -> MemoryOutlineOutpu sessions_total = 0 if int(input.sessions_limit) > 0: digests = await ch.query_all( - "SELECT session_id, summary_text, length(key_facts) AS kf, length(derived_from) AS df, " + "SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, " + "length(key_facts) AS kf, length(derived_from) AS df, " "coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL " "WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL " "ORDER BY d DESC, session_id LIMIT {lim:UInt32}", @@ -218,6 +223,11 @@ async def handle_memory_outline(input: MemoryOutlineInput) -> MemoryOutlineOutpu base, ) sessions_total = int((total or [{}])[0].get("c") or 0) + from server.memory.store.summary_sources import current_summary_sources + + digests = await current_summary_sources( + ch, digests, org=input.org, namespace=input.namespace + ) page = digests[: int(input.sessions_limit)] # Checking every digest's text is CPU work; it runs off the event loop. headlines = await asyncio.to_thread( diff --git a/server/memory/app/deep_evidence.py b/server/memory/app/deep_evidence.py index 71b5d01b64..04a1611069 100644 --- a/server/memory/app/deep_evidence.py +++ b/server/memory/app/deep_evidence.py @@ -3,8 +3,8 @@ `memory.ask` synthesizes from fact excerpts (loom.memory, ≤2,000 chars each). A fact is a compression of the turns it came from, and some questions need the turns: the exact wording, the number the extractor rounded, the second -clause it dropped. The deep path recalls summaries, lets one LLM call pick the -relevant ones, follows the summary→event link to the raw text and answers again. +clause it dropped. The deep path selects relevant facts, follows their evidence +links to the supporting raw text and answers again. Loom stores the two tiers as `loom.memory` and `loom.memory_evidence`, linked by `loom.memory_belief_sources`; this module is the read that follows the link. @@ -358,145 +358,6 @@ async def fetch_raw_evidence( SUMMARY_KEY_PREFIX = "summary:" -_TOKEN_RE = re.compile(r"[a-z0-9]{4,}") - - -def _tokens(text: str) -> set[str]: - return set(_TOKEN_RE.findall((text or "").lower())) - - -def session_ids_of(memory_keys: list[str]) -> list[str]: - """Session ids behind `summary:` hits (the session-summary - leg, fetch_session_summaries), in order, de-duplicated.""" - - out: list[str] = [] - for key in memory_keys: - if key.startswith(SUMMARY_KEY_PREFIX): - sid = key[len(SUMMARY_KEY_PREFIX) :] - if sid and sid not in out: - out.append(sid) - return out - - -def window_turns_by_question( - rows: list[dict[str, Any]], - question: str, - *, - max_chars: int, - context_turns: int = 1, -) -> list[dict[str, Any]]: - """The turns of a session worth opening for `question`, bounded. - - A session summary is one hit for a whole session (tens of thousands of - characters), so it cannot be opened whole. Score each turn by the - distinct question tokens (4+ chars) it contains, keep the best-scoring - turns plus `context_turns` neighbours on each side for coherence, in - chronological order, until `max_chars` is spent. Zero-score turns are - only kept as neighbours. - """ - - if not rows or max_chars <= 0: - return [] - q = _tokens(question) - scored = [(len(q & _tokens(str(r.get("content") or ""))), i) for i, r in enumerate(rows)] - keep: set[int] = set() - spent = 0 - for score, i in sorted(scored, key=lambda p: (-p[0], p[1])): - if score <= 0: - break - block = range(max(0, i - context_turns), min(len(rows), i + context_turns + 1)) - cost = sum(len(str(rows[j].get("content") or "")) for j in block if j not in keep) - if spent + cost > max_chars: - # Take the scoring turn alone if it fits; otherwise stop. - solo = len(str(rows[i].get("content") or "")) - if i not in keep and spent + solo <= max_chars: - keep.add(i) - spent += solo - continue - for j in block: - keep.add(j) - spent += cost - return [rows[i] for i in sorted(keep)] - - -async def fetch_session_turns( - *, - org: str, - namespace: str, - session_ids: list[str], - question: str, - max_chars_per_session: int, - max_chars_total: int, - ch: Any | None = None, -) -> list[EvidenceTurn]: - """Open the sessions behind session-summary hits, windowed by the question. - - The summary leg ranks a whole session; this reads that session's turns - from `loom.memory_evidence` (a key-prefix read on org, namespace, then - session_id) and keeps the question-relevant window per session, capped - per session and in total. Labelled with the `summary:` key so the - synthesizer's citation points back at the hit. - """ - - if not session_ids: - return [] - from server.core.ch import get_client - - client = ch or get_client() - # Session ids repeat across users: a narrowed read opens only its own. - params: dict[str, Any] = {"org": org, "ns": namespace, "sids": list(session_ids)} - in_scope = scope_and(request_scope(), params) - # Versioned, not FINAL: an evidence row's id is minted per - # insert, so the only re-insert is a tombstone; the latest version by - # `created_at` decides, as FINAL would, without the query-time merge - # (same rows, a fraction of the read). - sql = f""" - SELECT session_id, role, toString(observed_at) AS observed_at, content - FROM ( - SELECT session_id, role, observed_at, content, evidence_id, tombstoned_at - FROM loom.memory_evidence - WHERE org = {{org:String}} - AND namespace = {{ns:String}} - AND session_id IN {{sids:Array(String)}} - AND content != ''{in_scope} - ORDER BY evidence_id, created_at DESC - LIMIT 1 BY evidence_id - ) - WHERE tombstoned_at IS NULL - ORDER BY session_id, observed_at ASC, evidence_id - LIMIT 2000 - """ - try: - rows = await client.query_all(sql, params=params) - except Exception as e: # noqa: BLE001 — fail-open - log.warning("deep-evidence session fetch failed (fail-open): %s", e) - return [] - # A turn that reads as high never reaches the synthesis prompt. - rows = await asyncio.to_thread(without_high, rows, lambda r: str(r.get("content") or "")) - by_sid: dict[str, list[dict[str, Any]]] = {} - for r in rows: - by_sid.setdefault(str(r.get("session_id") or ""), []).append(r) - out: list[EvidenceTurn] = [] - total = 0 - for sid in session_ids: - budget = min(max_chars_per_session, max_chars_total - total) - if budget <= 0: - break - for r in window_turns_by_question(by_sid.get(sid, []), question, max_chars=budget): - content = str(r.get("content") or "").strip() - if not content: - continue - total += len(content) - out.append( - EvidenceTurn( - memory_key=f"{SUMMARY_KEY_PREFIX}{sid}", - role=str(r.get("role") or ""), - observed_at=str(r.get("observed_at") or "")[:19], - content=content, - support_type="session", - ) - ) - return out def render_raw_evidence(turns: list[EvidenceTurn]) -> str: @@ -541,22 +402,7 @@ async def deep_evidence_pass( max_chars_total=max_chars_total, ch=ch, ) - # A picked session-summary hit opens its session, windowed by the - # question, with whatever budget the fact evidence left. - sids = session_ids_of(keys) - if sids: - left = max_chars_total - sum(len(t.content) for t in turns) - if left > 0: - turns = turns + await fetch_session_turns( - org=org, - namespace=namespace, - session_ids=sids, - question=question, - max_chars_per_session=max_chars_per_memory, - max_chars_total=left, - ch=ch, - ) - turns.sort(key=lambda t: (t.observed_at, t.memory_key)) + # A digest proves its source facts, not every raw turn from their session. block = render_raw_evidence(turns) return DeepEvidence( selected_keys=keys, @@ -575,10 +421,7 @@ async def deep_evidence_pass( "candidate_lines", "deep_evidence_pass", "fetch_raw_evidence", - "fetch_session_turns", "parse_picks", "render_raw_evidence", "select_candidates", - "session_ids_of", - "window_turns_by_question", ] diff --git a/server/memory/app/evidence.py b/server/memory/app/evidence.py index bdc4b49bd0..2b3709b209 100644 --- a/server/memory/app/evidence.py +++ b/server/memory/app/evidence.py @@ -83,11 +83,9 @@ async def write_evidence_rows( } ) out.append(EvidenceRow(evidence_id=evidence_id, content_sha256=content_sha)) - await ch.insert( - "loom.memory_evidence", - rows, - settings=batched_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_evidence", rows) return out diff --git a/server/memory/app/evidence_transfer.py b/server/memory/app/evidence_transfer.py index f21e7e3282..96607eae9d 100644 --- a/server/memory/app/evidence_transfer.py +++ b/server/memory/app/evidence_transfer.py @@ -146,6 +146,9 @@ async def handle_memory_export_evidence( """, params=params, ) + from server.memory.store.summary_sources import current_summary_sources + + rows = await current_summary_sources(ch, rows, org=input.org, namespace=input.namespace) digests = [ EvidenceDigest( session_id=str(r["session_id"]), @@ -379,7 +382,9 @@ async def handle_memory_import_evidence( ) ) if rows: - await ch.insert("loom.memory_chunks", rows, settings=committed_insert_settings()) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) out.chunks_written = len(rows) from server.memory.app.chunk_links import SOURCES_TABLE, import_link_rows @@ -432,10 +437,8 @@ async def handle_memory_import_evidence( ) ) if drows: - await ch.insert( - "loom.session_summaries", - drows, - settings=committed_insert_settings(), - ) - out.digests_written = len(drows) + from server.memory.store.summary_sources import insert_verified_summaries + + out.digests_written = await insert_verified_summaries(ch, drows) + out.skipped += len(drows) - out.digests_written return out diff --git a/server/memory/app/extract/handler.py b/server/memory/app/extract/handler.py index 4c062e55ef..da9088fcc7 100644 --- a/server/memory/app/extract/handler.py +++ b/server/memory/app/extract/handler.py @@ -456,7 +456,6 @@ async def _project_extraction( msgs: list[dict[str, Any]], state: Any, evidence_ids: list[str], - summary_draft_task: asyncio.Task[Any] | None, ) -> int: """The projections of a pipeline write — evidence links, verbatim chunks, the session summary — run together; returns the belief-source links @@ -535,32 +534,11 @@ async def _project_chunks() -> None: payload={"message_count": len(msgs)}, ) - # Session-summary plane: one cheap LLM call per session emits a dense - # dated digest, embedded with the SAME model/dim as the fact plane so - # cosine is comparable. Awaited before the response so an immediate - # post-ingest read sees it. Fail-open — the facts already landed. + # The committed facts, rather than the raw transcript, determine the digest. async def _project_session_summary() -> None: - draft: Any = None - if summary_draft_task is not None: - try: - draft = await summary_draft_task - except Exception as e: # noqa: BLE001 — a projection; the write stands - log.warning("session digest draft failed (non-fatal): %s", e) - await record_input_projection_failure( - input, - projection="session_summaries", - error=e, - memory_keys=written_memory_keys, - payload={"message_count": len(msgs), "stage": "draft"}, - ) - return - if draft is None: - # The model returned nothing (or the embedder did): the same - # skip the inline path takes, without a second model call. - return try: with traced("extract.session_summary", messages=len(msgs)): - await _write_session_summary(input, msgs, state, draft=draft) + await _write_session_summary(input, state) except Exception as e: # noqa: BLE001 — a projection; the write stands log.warning("session_summaries write failed (non-fatal): %s", e) await record_input_projection_failure( @@ -625,7 +603,14 @@ async def _set_from_messages( detect_updates_override.set(input.detect_updates) last_value_updates_linked.set(0) from server.memory.app.extract import ExtractionError + from server.memory.domain.refusals import transcript_content + from server.memory.store.refusals import check_intake + await check_intake( + get_client(), + input.model_dump(), + transcript_content(input.model_dump()), + ) if input.defer: return await _enqueue_deferred(input, rule_agent) if _deferred_by_allowlist(input): @@ -705,28 +690,10 @@ async def _set_from_messages( agent_learnings=learnings_on, **_extract_request_pins(input), ) - # The session digest is a function of the transcript, not of the extraction result, so - # its model call and embedding run WHILE the extractor runs; the row is written once the - # memory keys exist. A session that extracts nothing discards the draft (one cheap call, - # ~0.4% of sessions). - summary_draft_task: asyncio.Task[Any] | None = None - if msgs: - - async def _draft_digest() -> Any: - with traced("extract.session_summary_draft", messages=len(msgs)): - return await _draft_session_summary( - input, msgs, observation_date, operator_policy=operator_policy - ) - - summary_draft_task = create_deferred_task(_draft_digest()) - summary_draft_task.add_done_callback(_observe_task_result) try: with traced("extract.pipeline", steps=len(steps)): state = await run_pipeline(steps, state) except BaseException as e: - # Any failure or cancel of the extraction also stops the digest draft's LLM call. - if summary_draft_task is not None: - summary_draft_task.cancel() if not isinstance(e, ExtractionError): raise raise HTTPException( @@ -742,9 +709,7 @@ async def _draft_digest() -> Any: ) evidence_ids = await evidence_task _ing_clock.mark("evidence_write") # only what the pipeline did not cover - belief_source_links = await _project_extraction( - input, msgs, state, evidence_ids, summary_draft_task - ) + belief_source_links = await _project_extraction(input, msgs, state, evidence_ids) _ing_clock.mark("projections") ontology_warnings = await _ontology_warnings(input, state) @@ -817,16 +782,7 @@ def _observe_task_result(task: asyncio.Task[Any]) -> None: from server.memory.app.inline_records import ( # noqa: E402 (re-export) _write_inline_record_projection, ) -from server.memory.app.session_summary import ( # noqa: E402, F401 (re-export) - _SESSION_SUMMARY_MAX_INPUT_CHARS, - _SESSION_SUMMARY_MAX_KEY_FACTS, - _SESSION_SUMMARY_PROMPT_VERSION, - _SESSION_SUMMARY_SYSTEM, - _build_session_summary_user_prompt, - _draft_session_summary, - _top_key_facts, - _write_session_summary, -) +from server.memory.app.session_summary import _write_session_summary # noqa: E402 (re-export) __all__ = [ "handle_memory_bulk_set", @@ -839,12 +795,7 @@ def _observe_task_result(task: asyncio.Task[Any]) -> None: "_link_written_memories_to_evidence", "_write_source_evidence", "_write_inline_record_projection", - "_build_session_summary_user_prompt", - "_top_key_facts", - "_draft_session_summary", "_write_session_summary", - "_SESSION_SUMMARY_MAX_INPUT_CHARS", - "_SESSION_SUMMARY_MAX_KEY_FACTS", # Patch points: the session-summary helpers read these through this module. "get_client", "get_embedder", diff --git a/server/memory/app/extraction_queue.py b/server/memory/app/extraction_queue.py index 4adbda9fb7..b2130168fb 100644 --- a/server/memory/app/extraction_queue.py +++ b/server/memory/app/extraction_queue.py @@ -33,10 +33,12 @@ from datetime import timedelta from typing import Any, NamedTuple +from server.core.ch import sequential_reads from server.core.ch_names import split_table from server.core.datetimes import UNIX_EPOCH from server.core.datetimes import utc_now as _now -from server.core.errors import stored_caller_text +from server.core.errors import ValidationError, stored_caller_text +from server.core.hashing import sha256_text from server.identity.domain.read_scope import ( READ_SCOPE_KEY, ReadFilters, @@ -46,6 +48,7 @@ ) from server.llm import customer from server.memory.api.models_bulk import MemorySetFromMessagesInput +from server.memory.domain.refusals import transcript_content from server.memory.store.rows import next_version from server.settings import settings @@ -177,10 +180,92 @@ async def enqueue( "version": next_version(), "enqueued_by": str(json.loads(payload).get(ENQUEUED_BY_KEY) or ""), } - await ch.insert(TABLE, [row], settings=_SYNC) + await _insert_rows(ch, [row], skip_refused=False, create=True) return qid +async def _insert_rows( + ch: Any, rows: list[dict[str, Any]], *, skip_refused: bool = True, create: bool = False +) -> list[dict[str, Any]]: + """State transitions carry raw payloads, so they need intake admission too.""" + from server.memory.store.refusals import content_policy_row, publication + + written = [] + for row in rows: + content = [*transcript_content(json.loads(row.get("payload") or "{}"))] + if row.get("last_error"): + content.append(row["last_error"]) + try: + async with publication(ch, [content_policy_row(row, content)]): + if not create and not await _source_present(ch, row): + continue + await ch.insert(TABLE, [row], settings=_SYNC) + written.append(row) + except ValidationError as error: + if not skip_refused or error.extra.get("reason") != "forgotten": + raise + await _drop_refused(ch, row) + return written + + +class _QueueRevision(dict[str, Any]): + def __init__(self, values: dict[str, Any], source_version: int) -> None: + super().__init__(values) + self.source_version = source_version + + +async def _source_present(ch: Any, row: dict[str, Any]) -> bool: + if not isinstance(row, _QueueRevision): + raise ValueError("A queue transition requires its exact source revision") + with sequential_reads(): + found = await ch.query_all( + "SELECT 1 FROM loom.memory_extraction_queue FINAL " + "WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} " + "AND user_id={user_id:String} AND session_id={session_id:String} " + "AND queue_id={queue_id:String} AND version={source_version:UInt64} " + "AND hex(SHA256(payload))={payload_sha:String} LIMIT 1", + params={ + **{ + key: row[key] + for key in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "queue_id", + ) + }, + "source_version": row.source_version, + "payload_sha": sha256_text(str(row.get("payload") or "")).upper(), + }, + ) + return bool(found) + + +async def _drop_refused(ch: Any, row: dict[str, Any]) -> None: + """End an existing refused job without copying any of its erased payload.""" + if not isinstance(row, _QueueRevision): + raise ValueError("A refused transition requires its exact source revision") + await ch.execute( + "ALTER TABLE loom.memory_extraction_queue UPDATE status='failed', " + "last_error='dropped: refusal policy', lease_until=toDateTime64(0,3) " + "WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} " + "AND user_id={user_id:String} AND session_id={session_id:String} " + "AND queue_id={queue_id:String} AND version={source_version:UInt64} " + "AND hex(SHA256(payload))={payload_sha:String} " + "AND status IN ('pending','running')", + params={ + **{ + key: row[key] + for key in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + }, + "source_version": row.source_version, + "payload_sha": sha256_text(str(row.get("payload") or "")).upper(), + }, + ) + + def _mirror(row: dict[str, Any], **changes: Any) -> dict[str, Any]: """A new version of `row` with `changes` applied and every column carried forward — omitting one zeros it for readers under FINAL.""" @@ -197,7 +282,7 @@ def _mirror(row: dict[str, Any], **changes: Any) -> dict[str, Any]: for c in ("last_error", "payload", "enqueued_by"): if out.get(c) is None: out[c] = "" - return out + return _QueueRevision(out, int(row["version"])) def backoff_s(attempts: int) -> int: @@ -269,18 +354,15 @@ async def claim_batch( claimed = [ _mirror(dict(r), status=STATUS_RUNNING, lease_until=lease, last_error="") for r in rows ] - await ch.insert(TABLE, claimed, settings=_SYNC) - return claimed + return await _insert_rows(ch, claimed) async def mark_done(row: dict[str, Any], *, written: int, ch: Any | None = None) -> None: from server.core.ch import get_client ch = ch or get_client() - await ch.insert( - TABLE, - [_mirror(row, status=STATUS_DONE, written=int(written), lease_until=UNIX_EPOCH)], - settings=_SYNC, + await _insert_rows( + ch, [_mirror(row, status=STATUS_DONE, written=int(written), lease_until=UNIX_EPOCH)] ) @@ -299,8 +381,8 @@ async def mark_error( ch = ch or get_client() attempts = int(row.get("attempts") or 0) + 1 failed = attempts >= max(1, int(max_attempts)) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -312,7 +394,6 @@ async def mark_error( else _now() + timedelta(seconds=backoff_s(attempts)), ) ], - settings=_SYNC, ) return STATUS_FAILED if failed else STATUS_PENDING @@ -324,8 +405,8 @@ async def mark_unreachable(row: dict[str, Any], *, error: str, ch: Any | None = ch = ch or get_client() wait = backoff_s(int(row.get("attempts") or 0) + 1) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -334,7 +415,6 @@ async def mark_unreachable(row: dict[str, Any], *, error: str, ch: Any | None = lease_until=_now() + timedelta(seconds=wait), ) ], - settings=_SYNC, ) @@ -348,8 +428,8 @@ async def mark_held( from server.core.ch import get_client ch = ch or get_client() - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -360,7 +440,6 @@ async def mark_held( enqueued_by=str(row.get("enqueued_by") or "") or enqueued_actor(row), ) ], - settings=_SYNC, ) @@ -423,13 +502,12 @@ async def drop_waiting( ch=ch, ) why = f"{DROPPED_LABEL}model calls switched off, waited over {older_than_days} days" - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror(dict(r), status=STATUS_FAILED, last_error=why, lease_until=UNIX_EPOCH) for r in rows ], - settings=_SYNC, ) return len(rows) @@ -482,8 +560,8 @@ async def drop_expiring( why = ( f"{DROPPED_LABEL}still waiting after {_LAST_DAY} days, a day before the {TTL_DAYS}-day TTL" ) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( dict(r), @@ -493,7 +571,6 @@ async def drop_expiring( ) for r in rows ], - settings=_SYNC, ) return len(rows) diff --git a/server/memory/app/handlers.py b/server/memory/app/handlers.py index a4efe78689..7124edf928 100644 --- a/server/memory/app/handlers.py +++ b/server/memory/app/handlers.py @@ -24,7 +24,7 @@ from server.core.hashing import served_content_sha256 from server.embedding import get_embedder from server.identity.app.request_scope import request_scope -from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.identity.domain.read_scope import WHOLE_NAMESPACE, ReadFilters, narrower from server.memory.api.models_crud import ( CodeLink, MemoryClearInput, @@ -55,7 +55,9 @@ from server.memory.domain.served import served_totals from server.memory.domain.webhook_events import memory_change from server.memory.store import key_write_lock +from server.memory.store.delete_checks import checked_state_sha256 from server.memory.store.inserts import insert_memory_versions +from server.memory.store.pg_memory import AdmissionContendedError, unchanged from server.memory.store.rows import Identity, next_version, version_over from server.observability import degradations from server.observability.budget_shape import note_budget_shape @@ -603,10 +605,11 @@ async def handle_memory_unpin(input: MemoryUnpinInput) -> MemoryUnpinOutput: async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: - """Soft-delete: write a tombstone row with a higher version, so RMT - collapse picks it and `memory.get`'s `tombstoned_at IS NULL` filter - excludes it. Copies the live row's columns forward. Preserves the row - for the grace window; avoids expensive CH mutations.""" + """Refuse the exact live value, then carry it into a higher-version tombstone. + + The complete row remains available for history; ordinary reads exclude its + tombstone, and later intake cannot restore the removed content. + """ import server.memory.store as memory_repo from server.observability.otel import traced @@ -617,10 +620,18 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: user_id=input.user_id, session_id=input.session_id, ) + requested_scope = request_scope() + if requested_scope.narrowed and requested_scope.user_id != input.user_id: + return MemoryDeleteOutput(tombstoned=False, found=False, version=next_version()) + delete_scope = ReadFilters( + scope=narrower(requested_scope.scope, "own_user"), + user_id=input.user_id, + session_id=requested_scope.session_id, + ) with traced("ch.query.memory.precheck", op="tombstone_lookup"): # fresh: a delete right after a set must not no-op on a replica lag miss. existing = await memory_repo.get_by_key( - identity, input.memory_key, fresh=True, strong=True, scope=request_scope() + identity, input.memory_key, fresh=True, strong=True, scope=delete_scope ) if existing is None and identity.agent != "*": # A delete names a key, not the identity that wrote it (hooks write under their own @@ -634,7 +645,7 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: session_id=input.session_id, ) existing = await memory_repo.get_by_key( - wide, input.memory_key, fresh=True, strong=True, scope=request_scope() + wide, input.memory_key, fresh=True, strong=True, scope=delete_scope ) if existing is not None and not same_user(existing, identity): existing = None # the wide form ignores user_id; another user's row is not ours @@ -646,7 +657,7 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: # The version is drawn here, right before the insert: the vector retire skips rows at # or above it, so the later it is drawn the fewer late vector writes it leaves live. version = version_over(existing) - await _write_tombstone_row(existing, version=version) + await _write_tombstone_row(existing, version=version, refuse_value=True) try: tombstoned, downgraded = await _retire_derived( existing, version=version, op="memory.delete" @@ -748,13 +759,22 @@ async def _delete_version_locked( "memory_key": existing.memory_key, "checked": int(input.version), "tombstone": version, + "checked_state_sha256": checked_state_sha256(existing), } try: await memory_repo.record_delete_check(check, done=False) await _write_tombstone_row( - existing, version=version, trace=delete_version_trace(input.version) + existing, version=version, trace=delete_version_trace(input.version), refuse_value=True ) return await finish_delete_check(existing, check) + except AdmissionContendedError: + # Admission refused before sending or recording a refusal. Reconcile the + # newer head rather than treating it as an uncertain tombstone insert. + try: + answer = await rerun_delete_check(check) + return answer or MemoryDeleteVersionOutput(tombstoned=False, found=False) + except Exception as e: # a recorded check must remain owed on an uncertain result + raise _check_deferred(e) from e except Exception as e: # a retry finishes it, or the worker once the check is recorded raise _check_deferred(e) from e @@ -852,21 +872,85 @@ async def rerun_delete_check(check: dict[str, Any]) -> MemoryDeleteVersionOutput ) mine = {r.version: r for r in rows if (r.agent, r.user_id, r.session_id) == ident} removed = mine.get(checked) + committed = mine.get(tombstone) + if committed is not None and ( + committed.tombstoned_at is None or committed.source_trace != delete_version_trace(checked) + ): + committed = None + if committed is not None: + # History can lose the checked state to a same-version collision. The + # committed tombstone carries the content this deletion actually removed. + removed = committed.model_copy(update={"version": checked, "tombstoned_at": None}) if removed is None: await memory_repo.record_delete_check(check, done=True) return None - if tombstone not in mine: - same_delete, others = await _after_checked(removed, checked=checked, tombstone=tombstone) + if committed is None: + recovered = await _recover_missing_tombstone(removed, check, tombstone) + if isinstance(recovered, MemoryDeleteVersionOutput): + return recovered + tombstone = recovered + if tombstone != int(check["tombstone"]): + committed = await _version_row(removed, tombstone) + if committed.tombstoned_at is None or committed.source_trace != delete_version_trace( + checked + ): + raise BackendUnavailableError("the recovered tombstone no longer matches its check") + removed = committed.model_copy(update={"version": checked, "tombstoned_at": None}) + return await finish_delete_check(removed, check, tombstone=tombstone) + + +async def _recover_missing_tombstone( + removed: Any, check: dict[str, Any], tombstone: int +) -> int | MemoryDeleteVersionOutput: + """Retry known pre-send races; an uncertain send leaves the check owed.""" + import server.memory.store as memory_repo + + for _ in range(4): + same_delete, others = await _after_checked( + removed, checked=int(check["checked"]), tombstone=tombstone + ) if same_delete: - tombstone = max(same_delete) # a retry's, or a late one: the delete's that stands - elif not others: - # Nothing came after the removed version: the delete is still owed, above any - # write a fence lifted over a late commit of the original. - tombstone = _above_lift_band(tombstone) + return max(same_delete) + if others: + return tombstone + expected = check.get("checked_state_sha256", "") + if not expected or checked_state_sha256(removed) != expected: + # No write can recover the requested full state from a numeric version. + # Close this terminal check so it cannot starve newer recovery jobs. + await memory_repo.record_delete_check(check, done=True) + return MemoryDeleteVersionOutput(tombstoned=False, reason="version_changed") + current = await memory_repo.get_by_key( + Identity( + org=removed.org, + namespace=removed.namespace, + agent=removed.agent, + user_id=removed.user_id, + session_id=removed.session_id, + ), + removed.memory_key, + fresh=True, + strong=True, + include_tombstoned=True, + scope=WHOLE_NAMESPACE, + ) + # A same-version collision is invisible to the history's version range. + if not unchanged(removed, current): + await memory_repo.record_delete_check(check, done=True) + if current is None or current.tombstoned_at is not None: + return MemoryDeleteVersionOutput(tombstoned=False, found=False) + return MemoryDeleteVersionOutput(tombstoned=False, reason="version_changed") + try: + replacement = _above_lift_band(tombstone) await _write_tombstone_row( - removed, version=tombstone, trace=delete_version_trace(checked) + removed, + version=replacement, + trace=delete_version_trace(check["checked"]), + refuse_value=True, ) - return await finish_delete_check(removed, check, tombstone=tombstone) + return replacement + except AdmissionContendedError: + continue + raise AdmissionContendedError("the memory kept changing during deletion recovery") async def _key_moved_on( @@ -887,7 +971,9 @@ async def _key_moved_on( row = await _version_row(existing, max(others)) gone = row.tombstoned_at is not None # another remover took the key top = max([tombstone, *same_delete]) - if row.version < top and not (gone and await _head_carries(existing, top, row)): + if row.version == top or ( + row.version < top and not (gone and await _head_carries(existing, top, row)) + ): # A sweep's marker stays on the sweep's tombstone, which keeps the writes it caught. trace = delete_version_trace(checked) if gone else None await _write_above(row, tombstone=top, source_trace=trace) @@ -1035,7 +1121,7 @@ async def _after_checked( same_delete: list[int] = [] others: list[int] = [] for v, deleted, target in above: - if v != tombstone: + if v != tombstone or not deleted or target != checked: (same_delete if deleted and target == checked else others).append(v) return same_delete, others @@ -1084,7 +1170,14 @@ async def _write_above(row: Any, *, tombstone: int, source_trace: str | None = N await insert_memory_versions([back], ch=get_client()) -async def _write_tombstone_row(existing: Any, *, version: int, trace: str = "") -> None: +async def _write_tombstone_row( + existing: Any, + *, + version: int, + trace: str = "", + refuse_value: bool = False, + exact: bool = False, +) -> None: """The loom.memory tombstone of `existing` at `version`, `trace` as its source_trace.""" import server.memory.store as memory_repo from server.observability.otel import traced @@ -1102,7 +1195,13 @@ async def _write_tombstone_row(existing: Any, *, version: int, trace: str = "") ) with traced("ch.insert.memory.tombstone", table="loom.memory"): # Over the head it read: a lock taken since is never dropped by the tombstone. - await memory_repo.insert_versions_over_heads([tombstone_row], [existing], ch=get_client()) + await memory_repo.insert_versions_over_heads( + [tombstone_row], + [existing], + ch=get_client(), + exact=exact or refuse_value, + refuse_values=refuse_value, + ) async def _retire_derived( diff --git a/server/memory/app/move.py b/server/memory/app/move.py index f4a969958e..6c069c942d 100644 --- a/server/memory/app/move.py +++ b/server/memory/app/move.py @@ -22,7 +22,7 @@ from server.core.datetimes import naive_utc_now from server.core.hashing import served_content_sha256 from server.identity.domain.read_scope import WHOLE_NAMESPACE -from server.memory.api.models_crud import MemoryDeleteInput, MemorySetInput +from server.memory.api.models_crud import MemorySetInput from server.memory.api.models_move import MemoryMoveInput, MemoryMoveOutput from server.memory.app import handlers as _mem from server.memory.app import set_chunks as _set_chunks @@ -32,7 +32,9 @@ from server.memory.domain import provenance as _prov from server.memory.domain.delete_marks import carried_trace from server.memory.domain.memory_types import is_searchable_type +from server.memory.domain.webhook_events import memory_change from server.memory.store.rows import version_over +from server.observability.budget_shape import note_budget_shape if TYPE_CHECKING: from server.memory.store import MemoryRow @@ -144,7 +146,7 @@ async def _redirect_source(source: MemoryRow, dest_id: str, ch: Any) -> None: duplicate_of=dest_id, updated_at=now, ) - await memory_repo.insert_versions_over_heads([row], [source], ch=ch) + await memory_repo.insert_versions_over_heads([row], [source], ch=ch, exact=True) # The source key now reads as a redirect to the destination. await webhooks.announce([("memory.revised", row)]) try: @@ -171,16 +173,13 @@ async def _redirect_source(source: MemoryRow, dest_id: str, ch: Any) -> None: async def _tombstone_source(source: MemoryRow) -> None: """`keep_redirect: false` — the delete path, which carries the mirror and the chunk tombstones itself.""" - await _mem.handle_memory_delete( - MemoryDeleteInput( - org=source.org, - namespace=source.namespace, - agent=source.agent, - user_id=source.user_id, - session_id=source.session_id, - memory_key=source.memory_key, - ) - ) + version = version_over(source) + await _mem._write_tombstone_row(source, version=version, exact=True) + try: + await _mem._retire_derived(source, version=version, op="memory.move") + finally: + if await webhooks.announce([("memory.tombstoned", memory_change(source, version))]): + note_budget_shape("webhook") async def _destination_id(set_out: Any, set_input: Any) -> str: diff --git a/server/memory/app/ops.py b/server/memory/app/ops.py index 8d117fa353..cc838707d7 100644 --- a/server/memory/app/ops.py +++ b/server/memory/app/ops.py @@ -363,10 +363,21 @@ async def handle_memory_reindex(input: MemoryReindexInput) -> MemoryReindexOutpu # would; a key without axis text gets memory.set's zero vector at dim 0. embedding_rows = [] for i, r in enumerate(rows): + from server.core.hashing import hash_text, served_row_value, sha256_value + key_vec = key_vecs.get(i, []) + canonical_hash = sha256_value(served_row_value({"value": r["content_excerpt"]})) embedding_rows.append( memory_repo.build_embedding_row( - **{k: v for k, v in r.items() if k not in ("memory_version", "content_excerpt")}, + **{ + k: v + for k, v in r.items() + if k not in ("memory_version", "content_excerpt", "content_sha256", "verified") + }, + content_sha256=canonical_hash, + verified=int( + bool(r.get("verified")) and hash_text(r.get("content_sha256")) == canonical_hash + ), content_excerpt=texts[i][0], embedding_model=target_model, embedding_dim=embedder.dim, @@ -376,13 +387,13 @@ async def handle_memory_reindex(input: MemoryReindexInput) -> MemoryReindexOutpu embed_attempts=1, ) ) - await memory_repo.insert_embedding_rows(embedding_rows, ch=ch) + admitted = await memory_repo.insert_embedding_rows(embedding_rows, ch=ch) await _recheck_reindexed_heads(rows, embedding_rows, ch=ch, model=target_model) # One above versions read before the embed call: under the BM25 watermark. - await memory_repo.index_late_rows(embedding_rows, ch=ch) + await memory_repo.index_late_rows(admitted, ch=ch) return MemoryReindexOutput( - processed=len(embedding_rows), + processed=len(admitted), skipped_already_current=0, target_model=target_model, target_dim=embedder.dim, diff --git a/server/memory/app/period_digest.py b/server/memory/app/period_digest.py index f42d25a535..3d05173f29 100644 --- a/server/memory/app/period_digest.py +++ b/server/memory/app/period_digest.py @@ -34,6 +34,7 @@ import re import uuid from collections import defaultdict +from collections.abc import AsyncIterator from dataclasses import dataclass, field from datetime import datetime from typing import Any @@ -262,6 +263,7 @@ def build_period_memory_rows( source_trace=f"{SOURCE_TRACE}|prompt={PERIOD_DIGEST_PROMPT_VERSION}", actor=writer_actor(fallback="process:period_digest"), source_class="derived", + status="proposed", tags=tags, entities=ents, content_sha256=sha, @@ -291,7 +293,8 @@ def build_period_memory_rows( tags=tags, entities=ents, valid_at=valid_at, - status="active", + status="proposed", + content_sha256=sha, ) return memory_row, embedding_row @@ -342,28 +345,80 @@ async def _fetch_namespaces(ch: Any, cfg: PeriodDigestConfig) -> list[dict[str, return list(rows or []) -async def _fetch_digests(ch: Any, ident: dict[str, Any]) -> list[dict[str, Any]]: - rows = await ch.query_all( - """ - SELECT org, namespace, agent, user_id, session_id, summary_text, derived_from, - coalesce(valid_at, created_at) AS at - FROM loom.session_summaries FINAL - WHERE org = {org:String} AND namespace = {ns:String} - AND agent = {agent:String} AND user_id = {uid:String} - AND tombstoned_at IS NULL - ORDER BY at, session_id - """, - params={ - "org": ident["org"], - "ns": ident["namespace"], - "agent": ident["agent"], - "uid": ident["user_id"], - }, - ) - return list(rows or []) +async def _fetch_digests( + ch: Any, + ident: dict[str, Any], + *, + page_size: int = 256, + after: tuple[datetime, str] | None = None, +) -> AsyncIterator[list[dict[str, Any]]]: + """Stream ordered raw windows without retaining the identity's whole history.""" + from server.core.ch import set_query_tag + + while True: + set_query_tag("cli.period_digest.sources") + cursor = ( + " AND (coalesce(valid_at, created_at), session_id) > {after:Tuple(DateTime,String)}" + if after + else "" + ) + rows = await ch.query_all( + "SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, " + "derived_from, coalesce(valid_at, created_at) AS at " + "FROM loom.session_summaries FINAL " + "WHERE org={org:String} AND namespace={ns:String} " + "AND agent={agent:String} AND user_id={uid:String} AND tombstoned_at IS NULL " + + cursor + + " ORDER BY at, session_id LIMIT {limit:UInt32}", + params={ + "org": ident["org"], + "ns": ident["namespace"], + "agent": ident["agent"], + "uid": ident["user_id"], + "after": after, + "limit": page_size, + }, + ) + if not rows: + return + yield list(rows) + if len(rows) < page_size: + return + after = (rows[-1]["at"], str(rows[-1]["session_id"])) -async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) -> set[str]: +async def _plan_digest_pages( + ch: Any, ident: dict[str, Any], cfg: PeriodDigestConfig +) -> tuple[list[Period], int]: + """Keep one page, an incomplete window, and at most the requested output cap.""" + from dataclasses import replace + + k, cap = max(2, cfg.sessions_per_period), max(0, cfg.max_periods_per_identity) + planned: list[Period] = [] + tail: list[dict[str, Any]] = [] + skipped = 0 + if not cap: + return planned, skipped + async for page in _fetch_digests(ch, ident, page_size=max(256, k)): + combined = tail + page + complete = len(combined) // k * k + tail = combined[complete:] + periods, count = await _plan_identity( + ch, + combined[:complete], + ident, + replace(cfg, max_periods_per_identity=cap - len(planned)), + ) + planned.extend(periods) + skipped += count + if len(planned) == cap: + break + return planned, skipped + + +async def _skipped_period_keys( + ch: Any, ident: dict[str, Any], keys: list[str], *, rewrite: bool +) -> set[str]: """On ClickHouse, the identity's period keys a pass skips: every key with a version, or under a rewrite only those taken out (deleted, redacted or no longer active: an Undo of a retirement brings a redacted version back as the live head).""" @@ -375,7 +430,7 @@ async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {uid:String} AND session_id = '-' - AND startsWith(memory_key, {pfx:String}) + AND startsWith(memory_key, {pfx:String}) AND memory_key IN {keys:Array(String)} """, params={ "org": ident["org"], @@ -383,6 +438,7 @@ async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) "agent": ident["agent"], "uid": ident["user_id"], "pfx": PERIOD_KEY_PREFIX, + "keys": keys, }, ) return {str(r["memory_key"]) for r in rows if not rewrite or r["taken_out"]} @@ -575,15 +631,31 @@ async def _plan_identity( """(the identity's periods to write, how many were skipped as written or taken out). Under Postgres the authority's heads decide which are written, ClickHouse's otherwise; the cap applies to what is left, so a period skipped takes no place in it.""" + from server.memory.store.summary_sources import current_summary_sources + k, rewrite = cfg.sessions_per_period, cfg.rewrite_existing + windows, _ = plan_periods(digests, sessions_per_period=k) + if not windows: + return [], 0 if pg.enabled(): - windows, _ = plan_periods(digests, sessions_per_period=k) writable = await _writable_in_authority(windows, rewrite=rewrite) skipped = len(windows) - len(writable) else: - skip = await _skipped_period_keys(ch, ident, rewrite=rewrite) + skip = await _skipped_period_keys( + ch, ident, [p.memory_key for p in windows], rewrite=rewrite + ) writable, skipped = plan_periods(digests, sessions_per_period=k, skip_keys=skip) - return writable[: max(0, cfg.max_periods_per_identity)], skipped + if not writable: + return [], skipped + qualified = await current_summary_sources( + ch, + [d for p in writable for d in p.digests], + org=ident["org"], + namespace=ident["namespace"], + ) + allowed = {str(d["session_id"]) for d in qualified} + verified = [p for p in writable if all(str(d["session_id"]) in allowed for d in p.digests)] + return verified[: max(0, cfg.max_periods_per_identity)], skipped async def run_period_digest( @@ -615,8 +687,7 @@ async def run_period_digest( result.identities_scanned = len(identities) for ident in identities: try: - digests = await _fetch_digests(ch, ident) - periods, skipped = await _plan_identity(ch, digests, ident, cfg) + periods, skipped = await _plan_digest_pages(ch, ident, cfg) result.periods_planned += len(periods) result.periods_skipped += skipped if dry_run: diff --git a/server/memory/app/pg_delivery.py b/server/memory/app/pg_delivery.py index 045afeee08..e5883049fa 100644 --- a/server/memory/app/pg_delivery.py +++ b/server/memory/app/pg_delivery.py @@ -133,7 +133,7 @@ async def _publish_audits(rows: list[dict[str, Any]], start_by: float) -> Except return None -async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: +async def _publish(claims: list[Claimed], *, stub: bool = False) -> int: """One namespace's claims: memory rows and audit rows (`pg_publish.publish_rows`), then the search documents still owed, the old vectors' retirements, then the deleted decisions' records.""" @@ -169,14 +169,22 @@ async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: [c for c in owed if pg_publish.document_id(c.row) not in held], deadline ) docs = await _search_documents(build, stub=stub) + admitted: list[dict[str, Any]] = [] if docs: with pg_lease.leased(deadline): - await memory_repo.insert_embedding_rows(docs, ch=ch) - after_write.index_late_rows(docs, ch=ch) + admitted = await memory_repo.insert_embedding_rows(docs, ch=ch) + after_write.index_late_rows(admitted, ch=ch) + received = {(row["org"], row["namespace"], *pg_publish.document_id(row)) for row in admitted} + withheld = { + _key(c) + for c, row in zip(build, docs, strict=True) + if (row["org"], row["namespace"], *pg_publish.document_id(row)) not in received + } + settled = [c for c in owed if _key(c) not in withheld] await pg_memory.complete( - [(c.org, c.namespace, c.seq) for c in owed], + [_key(c) for c in settled], pg_memory.SEARCH_DOC, - tokens=[c.lease_token for c in owed], + tokens=[c.lease_token for c in settled], ) old_vectors = await _retire_old_vectors( [c for c in claims if c.targets & pg_memory.OLD_VECTORS], deadline, ch=ch @@ -193,6 +201,9 @@ async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: failed = refused or old_vectors or unretired if failed is not None: raise failed # an attempt that failed, once the rest is in + if withheld: + await pg_memory.release([c for c in owed if _key(c) in withheld]) + return len(claims) - len(withheld) async def _still_heads(claims: list[Claimed], deadline: float) -> list[Claimed]: @@ -333,8 +344,7 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: batch = await pg_memory.renew([c for c in group if not c.taken_over and c.attempts < limit]) if batch: try: - await _publish_marked(batch, run) - run.landed += len(batch) + run.landed += await _publish_marked(batch, run) batch = [] except pg.PostgresUnavailableError: raise @@ -346,8 +356,7 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: log.warning("pg_delivery: a batch of %s failed (%s); one at a time", _ns(batch[0]), why) async for c in _one_by_one([*alone, *batch], run): try: - await _publish_marked([c], run) - run.landed += 1 + run.landed += await _publish_marked([c], run) except pg.PostgresUnavailableError: raise except Exception as e: # one revision must not hold back the rest @@ -361,19 +370,20 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: run.failed += await pg_memory.record_failure([c], e) -async def _publish_marked(claims: list[Claimed], run: _Pass, *, stub: bool = False) -> None: +async def _publish_marked(claims: list[Claimed], run: _Pass, *, stub: bool = False) -> int: """`_publish`, with the claims marked as running while it does: a worker's pass cancelled in it keeps them held. A publish that ends, either way, clears the mark.""" keys = {_key(c) for c in claims} if run.worker else set() run.running |= keys try: - await _publish(claims, stub=stub) + landed = await _publish(claims, stub=stub) except asyncio.CancelledError: raise except BaseException: run.running -= keys raise run.running -= keys + return landed async def _last_attempt(c: Claimed, error: Exception | None, run: _Pass) -> None: @@ -419,8 +429,7 @@ async def _publish_stub(c: Claimed, run: _Pass) -> bool: if held.targets & pg_memory.SEARCHED != pg_memory.SEARCH_DOC: await pg_memory.release(renewed) return False - await _publish_marked([held], run, stub=True) - return True + return bool(await _publish_marked([held], run, stub=True)) def _ns(c: Claimed) -> tuple[str, str]: diff --git a/server/memory/app/pipeline.py b/server/memory/app/pipeline.py index 0c768308bc..2cdcc84a33 100644 --- a/server/memory/app/pipeline.py +++ b/server/memory/app/pipeline.py @@ -401,26 +401,22 @@ async def _fetch_for_forget_requests( async def _retire_forgotten(state: WorkflowState, keys: set[str]) -> int: """Tombstone the live row of every key the user asked to forget, under this identity's org/namespace and any agent/session (the row may have - been written in an earlier session). Fails open per key; returns how many - were retired.""" + been written in an earlier session). Refusal policy failures stop extraction.""" from server.memory.app.handlers import handle_memory_delete n = 0 for key in sorted(keys): - try: - out = await handle_memory_delete( - MemoryDeleteInput( - org=state.org, - namespace=state.namespace, - agent="*", - user_id=state.user_id or "-", - session_id="-", - memory_key=key, - ) + out = await handle_memory_delete( + MemoryDeleteInput( + org=state.org, + namespace=state.namespace, + agent="*", + user_id=state.user_id or "-", + session_id="-", + memory_key=key, ) - n += 1 if getattr(out, "tombstoned", False) else 0 - except Exception as e: # noqa: BLE001 — a forget that fails is logged, never fatal - log.warning("forget request: retiring %r failed (open): %s", key, e) + ) + n += 1 if getattr(out, "tombstoned", False) else 0 return n diff --git a/server/memory/app/profile_search.py b/server/memory/app/profile_search.py index 1541552fab..89f3664da6 100644 --- a/server/memory/app/profile_search.py +++ b/server/memory/app/profile_search.py @@ -354,8 +354,8 @@ async def queue_profile_reembed( ) for r in rows ] - await memory_repo.insert_embedding_rows(stubs, ch=ch) - return len(stubs) + admitted = await memory_repo.insert_embedding_rows(stubs, ch=ch) + return len(admitted) except Exception as e: # noqa: BLE001 — the read-side check heals it later log.warning("profile re-embed stub write failed: %s", e) return 0 diff --git a/server/memory/app/promote.py b/server/memory/app/promote.py index 70f59e20a2..798a4daa4b 100644 --- a/server/memory/app/promote.py +++ b/server/memory/app/promote.py @@ -6,15 +6,17 @@ import logging import uuid from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager from datetime import datetime, timedelta from typing import Any from fastapi import HTTPException from server.core import leases +from server.core.ch import get_client from server.core.datetimes import naive_utc_now, to_naive_utc from server.core.errors import ( + AuthInsufficientError, DefinitionCeilingError, DefinitionNameTakenError, DefinitionPromotionRefusedError, @@ -23,7 +25,7 @@ SensitiveDefinitionRefusedError, ValidationError, ) -from server.core.hashing import served_content_sha256 +from server.core.hashing import served_content_sha256, sha256_value from server.embedding import get_embedder from server.embedding.documents import document_vector from server.embedding.embedder import Embedder @@ -49,8 +51,8 @@ from server.memory.domain.promotion_source import promotion_source from server.memory.domain.search_types import SHARED_AGENT from server.memory.domain.trust import verification_state +from server.memory.store import refusals from server.memory.store import shared as shared_repo -from server.memory.store import source_review from server.memory.store.rows import MemoryRow, next_version, version_over from server.observability.budget_shape import note_budget_shape from server.settings import settings @@ -187,7 +189,7 @@ async def handle_memory_promote( if embedder is None else _shared_vector_row(src, target_key, excerpt, embedder, embedding, now_naive) ) - async with source_review.publication(src) as mark: + async with _promotion_admission(src, target_key, definition=is_definition_promotion) as mark: if is_definition_promotion: await _write_definition(input.org, input.namespace, names, row, vector_row, mark) else: @@ -201,6 +203,20 @@ async def handle_memory_promote( ) +@asynccontextmanager +async def _promotion_admission( + source: MemoryRow, target_key: str, *, definition: bool +) -> AsyncIterator[Callable[[bool], None]]: + # Demotion holds this claim while borrowing its source connection too. + async with AsyncExitStack() as stack: + if definition: + await stack.enter_async_context(_definition_claim(source.org, source.namespace)) + mark = await stack.enter_async_context( + refusals.source_publication(get_client(), source, target_key) + ) + yield mark + + async def _write_shared( row: dict[str, Any], vector: dict[str, Any] | None, mark: Callable[[bool], None] ) -> None: @@ -265,7 +281,10 @@ async def _approved_definition_names( def _refuse_unshareable(src: MemoryRow) -> None: """Refuse a row the shared layer cannot carry: a block row or a 'high' one.""" - source_review.require_reviewed(src) + from server.memory.domain.proposal_review import pending_review + + if pending_review(src): + raise AuthInsufficientError("An unreviewed proposal cannot be promoted") # Block rows are per-agent working memory; loom.shared has no char_limit, # so promoting one would silently drop its overflow cap. Refuse. if src.memory_type == "profile": @@ -320,6 +339,7 @@ def _shared_vector_row( "memory_type": src.memory_type, "scope": "shared", "content_excerpt": excerpt, + "content_sha256": sha256_value(src.value), "embedding_model": embedder.model_name, "embedding_dim": embedder.dim if embedding else 0, "embedding": embedding, # [] when embed failed @@ -332,7 +352,7 @@ def _shared_vector_row( # counts only while it binds the promoted content's hash. "source_class": src.source_class, "verified": int( - verification_state(src.verified_by, src.verified_sha256, src.content_sha256) + verification_state(src.verified_by, src.verified_sha256, sha256_value(src.value)) in ("human", "process") ), } @@ -349,18 +369,17 @@ async def _write_definition( """Write the definition's shared row and its vector unless the namespace is at its ceiling or another shared definition holds one of its names, under the namespace's claim: of a definition's promotions and demotes, the last to run decides.""" - async with _definition_claim(org, namespace): - key = row["memory_key"] - await _check_shared_definitions(org, namespace, names, key) - # Above the row and the vector: a demote cut off between the two left a retired - # vector above the live row. - head = await shared_repo.fetch_shared(org, namespace, key, strong=True) - version = await _definition_version(org, namespace, key, head) - mark(True) - await shared_repo.insert_shared_row({**row, "version": version}) - if vector_row is not None: - await shared_repo.insert_shared_embedding({**vector_row, "version": version}) - mark(False) + key = row["memory_key"] + await _check_shared_definitions(org, namespace, names, key) + # Above the row and the vector: a demote cut off between the two left a retired + # vector above the live row. + head = await shared_repo.fetch_shared(org, namespace, key, strong=True) + version = await _definition_version(org, namespace, key, head) + mark(True) + await shared_repo.insert_shared_row({**row, "version": version}) + if vector_row is not None: + await shared_repo.insert_shared_embedding({**vector_row, "version": version}) + mark(False) async def _definition_version( diff --git a/server/memory/app/reembed.py b/server/memory/app/reembed.py index 31484469fa..15d0451731 100644 --- a/server/memory/app/reembed.py +++ b/server/memory/app/reembed.py @@ -22,7 +22,7 @@ from collections.abc import Awaitable, Callable from typing import Any -from server.core.ch import sequential_reads +from server.core.ch import sequential_reads, set_query_tag from server.core.ch_params import chunks from server.llm import customer from server.memory.app.model_attribution import attribution_for, memory_writers @@ -148,6 +148,16 @@ def in_database_sql( the column type, model/dim columns are rewritten, the version column is bumped the way `rewrite_row` bumps it, every other column is copied. Pure so it can be tested without a server.""" + insert_cols, select = _database_projection(table, cols, model, dim, credentials) + return ( + f"INSERT INTO loom.{table} ({', '.join(insert_cols)})\n" + f"SELECT {', '.join(select)}\nFROM loom.{table} FINAL\nWHERE {where_sql}" + ) + + +def _database_projection( + table: str, cols: list[str], model: str, dim: int, credentials: str +) -> tuple[list[str], list[str]]: cfg = _TABLES[table] m = model.replace("'", "\\'") cred = credentials.replace("'", "\\'") @@ -185,12 +195,72 @@ def emb(text_col: str) -> str: select.append("version + 1") # as rewrite_row: never above a later write else: select.append(c) + return insert_cols, select + + +def _database_memory_page_sql( + cols: list[str], model: str, dim: int, credentials: str, where: str +) -> str: + names, expressions = _database_projection("memory_embeddings", cols, model, dim, credentials) + projected = ", ".join( + f"{expression} AS {name}" for name, expression in zip(names, expressions, strict=True) + ) + order = ", ".join(_KEYS["memory_embeddings"]) + # Filtering uses the original model; the outer aliases cannot shadow its predicate. return ( - f"INSERT INTO loom.{table} ({', '.join(insert_cols)})\n" - f"SELECT {', '.join(select)}\nFROM loom.{table} FINAL\nWHERE {where_sql}" + f"SELECT {projected}, _k FROM (SELECT *, tuple({order}) AS _k " + f"FROM loom.memory_embeddings FINAL WHERE {where} " + f"ORDER BY {order} LIMIT {{page:UInt32}}) ORDER BY _k" ) +def _memory_vector_row(row: dict[str, Any]) -> dict[str, Any]: + from server.memory.store.rows import EMBEDDING_PERSISTED_COLS, build_embedding_row + + known = {key: value for key, value in row.items() if key in EMBEDDING_PERSISTED_COLS} + # Carry columns added by a newer build alongside the complete known row. + return {**row, **build_embedding_row(**known)} + + +async def _publish_database_memory( + ch: Any, + cols: list[str], + model: str, + dim: int, + credentials: str, + where: str, + params: dict[str, Any], + left_orgs: set[str] | None, +) -> int: + from server.memory.store import insert_embedding_rows + + after = None + written = 0 + order = ", ".join(_KEYS["memory_embeddings"]) + while True: + set_query_tag("memory.reembed") + page_where = where + page_params = {**params, "page": 200} + if after is not None: + marks = ", ".join(f"{{k{i}:String}}" for i in range(len(after))) + page_where += f" AND ({order}) > ({marks})" + page_params.update({f"k{i}": str(value) for i, value in enumerate(after)}) + rows = await ch.query_all( + _database_memory_page_sql(cols, model, dim, credentials, page_where), + params=page_params, + settings=in_database_settings(200), + ) + if not rows: + return written + out = [_memory_vector_row(_without_key(row)) for row in rows] + admitted = await insert_embedding_rows(out, ch=ch) + _held_publications(out, admitted, left_orgs) + written += len(admitted) + after = rows[-1]["_k"] + if len(rows) < 200: + return written + + def in_database_settings(rows: int) -> dict[str, Any]: """Session settings for one in-database re-embed statement: the experimental gate, batches of 100 texts per request, and per-query @@ -220,11 +290,12 @@ async def reembed_table_in_database( namespace: str | None = None, dry_run: bool = False, exclude_orgs: frozenset[str] = frozenset(), + left_orgs: set[str] | None = None, log: Callable[..., None] = print, ) -> int: - """Re-embed one table's off-model rows inside ClickHouse. - One INSERT … SELECT per table; ClickHouse batches the embedding calls. - Returns the off-model row count the statement covered.""" + """Re-embed off-model rows with aiEmbed. Memory vectors publish through admission + in bounded pages; other tables use one INSERT … SELECT. Return acknowledged memory + vectors, or the off-model row count covered for the other tables.""" cfg = _TABLES[table] cols = await _cols(ch, table) where = [f"embedding_model != '{model}'", "embedding_dim > 0"] @@ -252,6 +323,10 @@ async def reembed_table_in_database( ) if dry_run or total == 0: return 0 + if table == "memory_embeddings": + return await _publish_database_memory( + ch, cols, model, dim, credentials, where_sql, params, left_orgs + ) _ = cfg sql = in_database_sql( table, cols, model=model, dim=dim, credentials=credentials, where_sql=where_sql @@ -365,10 +440,7 @@ async def reread_page(read: dict[int, dict[str, Any]]) -> dict[int, dict[str, An if left: if left_orgs is not None: left_orgs.update(left) - log( - f" {table}: {sum(left.values())} rows of an org switched off during the run " - "left for later" - ) + log(f" {table}: {sum(left.values())} unpublished rows left for later") after = [str(x) for x in rows[-1]["_k"]] progress = f"{written}/{total}" + (f" ({held} held as high)" if held else "") progress += f" ({changed} removed or rewritten since read, skipped)" if changed else "" @@ -491,12 +563,36 @@ async def _rewrite_page( for i, r in kept.items() ] if out: - await ch.insert( - f"loom.{table}", out, settings={"async_insert": 0, "wait_for_async_insert": 1} - ) + if table == "memory_embeddings": + from server.memory.store import insert_embedding_rows + + out = [_memory_vector_row(row) for row in out] + admitted = await insert_embedding_rows(out, ch=ch) + for org, count in _held_publications(out, admitted).items(): + left_by_org[org] = left_by_org.get(org, 0) + count + else: + await ch.insert( + f"loom.{table}", out, settings={"async_insert": 0, "wait_for_async_insert": 1} + ) return left_by_org +def _held_publications( + rows: list[dict[str, Any]], + admitted: list[dict[str, Any]], + left_orgs: set[str] | None = None, +) -> dict[str, int]: + keys = _KEY_COLUMNS["memory_embeddings"] + accepted = {tuple(row.get(key) for key in keys) for row in admitted} + held: dict[str, int] = {} + for row in rows: + if tuple(row.get(key) for key in keys) not in accepted: + held[row["org"]] = held.get(row["org"], 0) + 1 + if left_orgs is not None: + left_orgs.update(held) + return held + + # Each table's sort key, the keyset a run pages through and resumes after, and the # same as strings (the Enum `scope` by name). _KEY_COLUMNS: dict[str, tuple[str, ...]] = { @@ -649,6 +745,7 @@ async def reembed_all( namespace=namespace, dry_run=dry_run, exclude_orgs=off, + left_orgs=left_orgs, log=log, ) else: diff --git a/server/memory/app/restore.py b/server/memory/app/restore.py index 30d452a7ab..cf582f3aeb 100644 --- a/server/memory/app/restore.py +++ b/server/memory/app/restore.py @@ -17,6 +17,7 @@ from server.core.datetimes import naive_utc_now from server.core.error_summary import error_summary from server.core.errors import BackendUnavailableError, LoomServerError +from server.core.hashing import served_content_sha256 from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory.api.models_crud import ( MemoryRestoreInput, @@ -619,7 +620,7 @@ async def _restore_embedding_mirror( entities=list(existing.entities or []), # Carried forward so restore doesn't zero them under FINAL. files=list(getattr(existing, "files", None) or []), - content_sha256=str(getattr(existing, "content_sha256", "") or ""), + content_sha256=served_content_sha256(existing.value), concepts=list(getattr(existing, "concepts", None) or []), status=str(existing.status or "active"), temporal_anchor=existing.temporal_anchor, @@ -637,20 +638,19 @@ async def _restore_embedding_mirror( mentioned_at=existing.mentioned_at, source_resource_id=existing.source_resource_id or "", ) - await ch.insert( - "loom.memory_embeddings", - [restore_emb_row], - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) + from server.memory.store import insert_embedding_rows + + await insert_embedding_rows([restore_emb_row], ch=ch) async def handle_memory_restore(input: MemoryRestoreInput) -> MemoryRestoreOutput: - """Undo a delete, or an automatic retirement. + """Reverse an automatic tombstone or supersession when consent permits it. + + Within the 14-day grace window, a restorable tombstone is copied forward with + a higher version and tombstoned_at=NULL on memory and its search mirror. + An older tombstone answers expired=True. Explicit deletion or clearing leaves + permanent refusal, so its content cannot be republished. - A delete within the 14-day grace window: copy the tombstoned row forward with a - higher version and tombstoned_at=NULL so RMT collapses the pair live, on both - loom.memory and loom.memory_embeddings. A tombstone past the 14-day TTL returns - expired=True. A retired (`superseded`) row is served again by `_restore_superseded`, whose two preconditions hold it to the retirement on screen: `if_superseded_by` (still diff --git a/server/memory/app/retrieval/summaries.py b/server/memory/app/retrieval/summaries.py index 27d901621c..5abdf216db 100644 --- a/server/memory/app/retrieval/summaries.py +++ b/server/memory/app/retrieval/summaries.py @@ -167,7 +167,7 @@ async def fetch_session_summaries( -- supertype for String, DateTime" — silently swallowed by the -- except below, killing the whole leg under any as_of. toString(coalesce(valid_at, toDateTime(0))) AS valid_at_iso, - derived_from + derived_from, key_facts FROM loom.session_summaries FINAL WHERE {where_sql} -- No `LIMIT 1 BY` and no `session_id` tiebreak here: diff --git a/server/memory/app/runs_manifest.py b/server/memory/app/runs_manifest.py index 2c481ebe38..3e6afb1fc7 100644 --- a/server/memory/app/runs_manifest.py +++ b/server/memory/app/runs_manifest.py @@ -204,8 +204,8 @@ async def fetch_manifest_rows( if sids: digests = await ch.query_all( f""" - SELECT session_id, substring(summary_text, 1, {_DIGEST_CHARS * 2}) AS summary_text, - length(derived_from) AS derived_n + SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, + derived_from, version, length(derived_from) AS derived_n FROM loom.session_summaries FINAL WHERE {scope} AND session_id IN {{sids:Array(String)}} @@ -214,6 +214,9 @@ async def fetch_manifest_rows( """, params={**params, "sids": sids}, ) + from server.memory.store.summary_sources import current_summary_sources + + digests = await current_summary_sources(ch, digests, org=org, namespace=namespace) return merge_rows(list(observations or []), list(runs or []), list(digests or []), limit=limit) diff --git a/server/memory/app/sensitivity_sweep.py b/server/memory/app/sensitivity_sweep.py index 3eda4a04da..f4b864e627 100644 --- a/server/memory/app/sensitivity_sweep.py +++ b/server/memory/app/sensitivity_sweep.py @@ -550,9 +550,10 @@ async def _tag_embeddings(c: SweepCandidate, *, ch: _CHClient) -> int: ) ) if out: - await insert_embedding_rows(out, ch=ch) + admitted = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) - return len(out) + return len(admitted) + return 0 async def _close_mined_facts( diff --git a/server/memory/app/session_outcomes.py b/server/memory/app/session_outcomes.py index 5068ebc361..805ad6079c 100644 --- a/server/memory/app/session_outcomes.py +++ b/server/memory/app/session_outcomes.py @@ -239,7 +239,7 @@ async def record_outcome(ch: Any, **fields: Any) -> None: _UNLABELLED_SQL = """ SELECT s.org AS org, s.namespace AS namespace, s.agent AS agent, s.user_id AS user_id, s.session_id AS session_id, - s.summary_text AS summary_text, s.key_facts AS key_facts + s.summary_text AS summary_text, s.key_facts AS key_facts, s.derived_from AS derived_from FROM loom.session_summaries AS s FINAL LEFT ANTI JOIN ( SELECT org, namespace, agent, user_id, session_id FROM loom.session_outcomes_by_user FINAL @@ -276,6 +276,9 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR "off": sorted(await customer.switched_off_orgs()), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + rows = await verified_summary_batch(ch, rows) result.sessions_scanned = len(rows) # The judge's call is for the session's writer, else its namespace's agent. writers = await _writers(ch, rows) @@ -331,11 +334,12 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR SELECT o.org AS org, o.namespace AS namespace, o.agent AS agent, o.session_id AS session_id, o.reason AS reason, o.confidence AS confidence, s.summary_text AS summary_text, s.summary_embedding AS summary_embedding, - s.embedding_model AS embedding_model, s.user_id AS user_id + s.embedding_model AS embedding_model, s.user_id AS user_id, + s.key_facts AS key_facts, s.derived_from AS derived_from FROM loom.session_outcomes_by_user AS o FINAL INNER JOIN ( SELECT org, namespace, agent, session_id, summary_text, summary_embedding, - embedding_model, user_id + embedding_model, user_id, key_facts, derived_from FROM loom.session_summaries FINAL WHERE tombstoned_at IS NULL AND embedding_dim > 0 ) AS s ON s.org = o.org AND s.namespace = o.namespace AND s.agent = o.agent @@ -356,7 +360,9 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR """ _NEAREST_POSITIVE_SQL = """ - SELECT s.session_id AS session_id, s.summary_text AS summary_text, + SELECT s.org AS org, s.namespace AS namespace, s.agent AS agent, s.user_id AS user_id, + s.key_facts AS key_facts, s.derived_from AS derived_from, + s.session_id AS session_id, s.summary_text AS summary_text, 1 - cosineDistance(s.summary_embedding, {q:Array(Float32)}) AS similarity FROM loom.session_summaries AS s FINAL INNER JOIN ( @@ -400,27 +406,32 @@ async def write_contrast_reflection( "contradicts": [neg_session], "pair_similarity": round(float(pair_similarity), 4), } - await handle_memory_set( - MemorySetInput( - org=org, - namespace=namespace, - agent=agent or "-", - user_id=user_id or "-", - session_id=neg_session, - memory_key=key, - memory_type="reflection", - value=value, - tags=[ - "reflection", - "contrast", - f"session:{neg_session}", - f"session:{pos_session}", - "status:failed", - ], - confidence=float(max(0.0, min(1.0, confidence))), - source_trace=f"contrast:{neg_session}<{pos_session}", + from server.memory.domain.provenance import pipeline_actor + + with pipeline_actor("process:session_contrast"): + await handle_memory_set( + MemorySetInput( + org=org, + namespace=namespace, + agent=agent or "-", + user_id=user_id or "-", + session_id=neg_session, + memory_key=key, + memory_type="reflection", + proposed=True, + source_class="derived", + value=value, + tags=[ + "reflection", + "contrast", + f"session:{neg_session}", + f"session:{pos_session}", + "status:failed", + ], + confidence=float(max(0.0, min(1.0, confidence))), + source_trace=f"contrast:{neg_session}<{pos_session}", + ) ) - ) return key @@ -439,6 +450,9 @@ async def distil_contrasts( "off": sorted(await customer.switched_off_orgs()), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + negatives = await verified_summary_batch(ch, negatives) result.contrast_candidates = len(negatives) writers = await _writers(ch, negatives) for neg in negatives: @@ -483,12 +497,16 @@ async def _contrast_pair(ch: Any, llm: Any, cfg: DistillConfig, neg: Any, writer "uid": str(neg.get("user_id") or "-"), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + qualified = await verified_summary_batch(ch, [pos] if pos else []) + pos = qualified[0] if qualified else None sim = float((pos or {}).get("similarity") or 0.0) if not pos or sim < float(cfg.min_pair_similarity): return False system, user = contrast_prompt( str(neg.get("summary_text") or ""), - str(neg.get("reason") or ""), + "failed", str(pos.get("summary_text") or ""), ) with llm_stage("session_contrast"): @@ -509,7 +527,7 @@ async def _contrast_pair(ch: Any, llm: Any, cfg: DistillConfig, neg: Any, writer neg_session=sid, pos_session=str(pos.get("session_id") or ""), lessons=lessons, - what_failed=what_failed or str(neg.get("reason") or ""), + what_failed=what_failed, confidence=min(float(neg.get("confidence") or 0.0), sim), pair_similarity=sim, ) diff --git a/server/memory/app/session_summary.py b/server/memory/app/session_summary.py index 7548245c94..7c7ed72b02 100644 --- a/server/memory/app/session_summary.py +++ b/server/memory/app/session_summary.py @@ -1,10 +1,7 @@ -"""Session-summary plane (write side) for memory.set_from_messages.""" +"""Session digests built only from authoritative, committed facts.""" from __future__ import annotations -import json -import logging -from dataclasses import dataclass from typing import Any from server.core.ch import get_client @@ -12,266 +9,94 @@ from server.embedding import get_embedder from server.embedding._provider import MalformedEmbedding from server.embedding.documents import document_vector -from server.memory.domain.extraction_policy import with_policy -from server.memory.domain.sensitivity import reads_high, value_reads_high +from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.summary_provenance import render_summary, source_text from server.observability.genai import staged -from server.settings.budgets import llm_max_tokens +from server.settings import settings -log = logging.getLogger(__name__) - -# After extraction, compress a session into one dense dated digest, embed it -# with the fact-plane embedder, and store it in loom.session_summaries. The -# read leg retrieves these by vector as source="summary" hits — a compact, -# high-recall stand-in for a whole session. - -# Bumped when the prompt text changes, so an operator can tell which prompt -# produced a given digest. -_SESSION_SUMMARY_PROMPT_VERSION = "ss-3" - -# Bounds the digest call's input cost. ~3000 chars ≈ 750 tokens. -_SESSION_SUMMARY_MAX_INPUT_CHARS = 3000 - - -# How many fact values ride along as key_facts — a dedup signal for the -# assembler (which facts a summary subsumes); never independently embedded. _SESSION_SUMMARY_MAX_KEY_FACTS = 6 -# Calibrated to emit dated, fact-only sentences. -_SESSION_SUMMARY_SYSTEM = ( - "You are compressing one conversation session into a dense, factual digest " - "for later retrieval. Write 1-3 sentences stating only the concrete facts, " - "decisions, events, and stated values from THIS session. Prefer dated " - "phrasing ('on 2024-03-11, the user ...'). No greetings, no meta-commentary, " - "no 'the user discussed'. If a date is given, attribute facts to it. Date " - "each EVENT at the time the speaker placed it, resolved against the session " - "date: 'dyed my hair last week' in a session dated 2022-04-15 is 'around " - "2022-04-08', never 'on 2022-04-15'; 'yesterday' is the day before the " - "session date. Only what happened during the session itself is dated at " - "the session date. State every fact as the speakers gave it: the digest is " - "their record, so never judge a statement true or false, never call one " - "incorrect or a misstatement, and never replace a stated value with the " - "real-world one. Output the digest only." -) - -def _digest_max_input_chars() -> int: - from server.settings import settings +@staged("session_summary") +async def _write_session_summary(input: Any, state: Any) -> None: + """Persist an extractive digest with enough lineage to reconstruct every byte.""" + import server.memory.store as memory_repo + from server.memory.app.digest_chain import link_digest - try: - return max( - 500, - int(settings.memory_session_digest_max_input_chars or _SESSION_SUMMARY_MAX_INPUT_CHARS), + ch = get_client() + identity = tuple( + str(getattr(input, axis, "-") or "-") for axis in ("agent", "user_id", "session_id") + ) + keys: list[str] = [] + facts: list[str] = [] + cap = int(settings.memory_session_digest_max_input_chars) + if cap == 0: + cap = 3000 # Zero selects the documented built-in character budget. + cap = max(500, cap) + for written in state.written or []: + memory_id = str(written.get("memory_id") or "") + version = int(written.get("version") or 0) + if not memory_id or not version: + continue + source = await memory_repo.get_by_id( + memory_id, + org=input.org, + namespace=input.namespace, + scope=WHOLE_NAMESPACE, + ch=ch, + strong=True, + fresh=True, + include_usage_totals=False, ) - except (TypeError, ValueError): - return _SESSION_SUMMARY_MAX_INPUT_CHARS - - -def _top_key_facts( - extracted_facts: list[dict[str, Any]], - *, - limit: int = _SESSION_SUMMARY_MAX_KEY_FACTS, -) -> list[str]: - """Top-N non-empty fact values as strings, order-preserving + deduped. A fact whose text - reads as `high` is left out: the digest is listed by memory.digests and sent to - the outcome judge, so its key facts never carry such a value.""" - out: list[str] = [] - seen: set[str] = set() - for fact in extracted_facts or []: - if not isinstance(fact, dict): + if source is None or int(source.version) != version: continue - raw = fact.get("value") - if isinstance(raw, str): - text = raw.strip() - elif raw is None: - text = "" - else: - text = json.dumps(raw, default=str) - text = text.strip() - if not text or text in seen: + source_identity = (source.agent or "-", source.user_id or "-", source.session_id or "-") + if source_identity != identity or source.memory_key in keys: continue - if value_reads_high(raw): + text = source_text(source, now=naive_utc_now()) + if not text or len(render_summary([*facts, text])) > cap: continue - seen.add(text) - out.append(text) - if len(out) >= limit: + keys.append(source.memory_key) + facts.append(text) + if len(facts) >= _SESSION_SUMMARY_MAX_KEY_FACTS: break - return out - - -def _build_session_summary_user_prompt( - msgs: list[dict[str, str]], - observation_date: str | None, -) -> str: - """Role-prefixed, length-bounded transcript with a session-date header.""" - transcript = "\n".join( - f"[{m.get('role', 'user')}] {m.get('content', '')}" for m in (msgs or []) - ) - cap = _digest_max_input_chars() - if len(transcript) > cap: - transcript = transcript[:cap] - return ( - f"Session date: {observation_date or 'unknown'}\n\nConversation:\n{transcript}\n\nDigest:" - ) - - -@dataclass(frozen=True) -class SessionSummaryDraft: - """The model's digest and its vector, drafted before the row exists. - - The digest depends on the transcript only, so it is drafted while the extraction - pipeline runs and joined to the written keys after. - """ - - summary_text: str - embedding: list[float] - embedding_model: str - tokens_in: int = 0 - tokens_out: int = 0 - - -@staged("session_summary") -async def _draft_session_summary( - input: Any, - msgs: list[dict[str, str]], - observation_date: str | None, - *, - operator_policy: str, -) -> SessionSummaryDraft | None: - """One LLM call + one embedding: everything about the digest that does not - need the extraction result. None when the model returns nothing or the - embedder returns an invalid vector (the read leg needs a valid embedding); - a digest that reads as high is drafted with no vector. The digest is - written from the raw turns, so it follows the operator's extraction policy as - the extractor does.""" - from server.llm.client import get_llm_client - - if not msgs: - return None - user_prompt = _build_session_summary_user_prompt(msgs, observation_date or None) - llm = get_llm_client() - result = await llm.complete( - with_policy(_SESSION_SUMMARY_SYSTEM, operator_policy), - user_prompt, - max_tokens=llm_max_tokens("session_summary"), - ) - summary_text = (result.text or "").strip() - if not summary_text: - log.info("session_summaries: empty digest for session=%r; skipping", input.session_id) - return None + if not facts: + return + summary = render_summary(facts) + if value_reads_high(summary): + return embedder = get_embedder() - # A digest that reads as high is written without a vector: the summary leg - # reads only digests that have one, and the text never reaches the embedder. - embedding: list[float] = [] - if not reads_high(summary_text): - try: - embedding = await document_vector(embedder, summary_text) - except MalformedEmbedding: - log.info( - "session_summaries: invalid embedding for session=%r; skipping", input.session_id - ) - return None - return SessionSummaryDraft( - summary_text=summary_text, - embedding=embedding, - embedding_model=embedder.model_name, - tokens_in=int(getattr(result, "tokens_in", 0) or 0), - tokens_out=int(getattr(result, "tokens_out", 0) or 0), - ) - - -async def _insert_session_summary( - input: Any, - state: Any, # WorkflowState - draft: SessionSummaryDraft, -) -> None: - """Join a draft to the extraction result and insert the row.""" - from server.memory.store import build_session_summary_row - - extracted_facts = list(getattr(state, "extracted_facts", None) or []) - observation_date = getattr(state, "observation_date", None) or None - ch = get_client() - row = build_session_summary_row( + try: + embedding = await document_vector(embedder, summary) + except MalformedEmbedding: + return + row = memory_repo.build_session_summary_row( org=input.org, namespace=input.namespace, - agent=getattr(input, "agent", "-") or "-", - user_id=getattr(input, "user_id", "-") or "-", - session_id=getattr(input, "session_id", "") or "", - # The digest belongs to the run that wrote it. + agent=identity[0], + user_id=identity[1], + session_id=identity[2], run_id=str(getattr(input, "run_id", "") or "-"), - summary_text=draft.summary_text, - summary_embedding=draft.embedding, - embedding_model=draft.embedding_model, - embedding_dim=len(draft.embedding), - key_facts=_top_key_facts(extracted_facts), - # Lineage: the memory keys this digest compressed. - derived_from=sorted( - { - str(w.get("memory_key") or "") - for w in (getattr(state, "written", None) or []) - if w.get("memory_key") - } - ), - # valid_at = the observation date, surfaced as a per-line LABEL (never a - # reorder key); parse_iso_dt fail-opens to NULL, matching the column DEFAULT. - valid_at=parse_iso_dt(observation_date), + summary_text=summary, + summary_embedding=embedding, + embedding_model=embedder.model_name, + embedding_dim=len(embedding), + key_facts=facts, + derived_from=keys, + valid_at=parse_iso_dt(getattr(state, "observation_date", None)), created_at=naive_utc_now(), ) - await ch.insert( - "loom.session_summaries", - [row], - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) - log.info( - "session_summaries: wrote digest for session=%r (%d chars, prompt=%s)", - input.session_id, - len(draft.summary_text), - _SESSION_SUMMARY_PROMPT_VERSION, - ) - # The digest chain: this digest is linked to the previous and next digest of the same - # identity by event time, as `precedes` edges in loom.operator_edges. Model-free, two - # indexed reads and one insert; fail-open, after the digest is in place. - from server.memory.app.digest_chain import link_digest + from server.memory.store.summary_sources import insert_verified_summaries + if not await insert_verified_summaries(ch, [row]): + return await link_digest( ch, org=input.org, namespace=input.namespace, - agent=str(row.get("agent") or "-"), - user_id=str(row.get("user_id") or "-"), - session_id=str(row.get("session_id") or ""), + agent=identity[0], + user_id=identity[1], + session_id=identity[2], at=row.get("valid_at") or row["created_at"], ) - - -@staged("session_summary") -async def _write_session_summary( - input: Any, - msgs: list[dict[str, str]], - state: Any, # WorkflowState - *, - draft: SessionSummaryDraft | None = None, -) -> None: - """Generate + embed + insert one session-summary row. Fail-open. - - Skips a session with no extracted facts. One LLM call produces the digest; - the fact-plane embedder embeds it so the read leg's cosine is comparable; - inserted synchronously so the immediate post-ingest read sees it. A - `draft` made ahead of time skips the model and embedder calls; a session that extracted - nothing discards it. - """ - extracted_facts = list(getattr(state, "extracted_facts", None) or []) - if not extracted_facts: - return - if not msgs: - return - if draft is None: - observation_date = getattr(state, "observation_date", None) or None - draft = await _draft_session_summary( - input, - msgs, - observation_date, - operator_policy=state.operator_policy, - ) - if draft is None: - return - await _insert_session_summary(input, state, draft) diff --git a/server/memory/app/set_chunks.py b/server/memory/app/set_chunks.py index ec299a2f4d..089a4a179f 100644 --- a/server/memory/app/set_chunks.py +++ b/server/memory/app/set_chunks.py @@ -30,7 +30,6 @@ from datetime import UTC, datetime from typing import Any -from server.core.ch_writes import committed_insert_settings from server.core.datetimes import naive_utc_now from server.embedding.documents import document_vector, document_vectors from server.memory.app.chunk_projection import chunk_text_with_ranges, chunk_text_with_truncation @@ -218,11 +217,9 @@ def _chunk_state_row( async def _insert_chunk_states(ch: Any, rows: list[dict[str, Any]]) -> None: # Synchronous: the caller's next search must not lift the parent # through a chunk of the value it just replaced or deleted. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) async def tombstone_set_chunks_fail_open( @@ -497,11 +494,9 @@ async def write_set_chunks( # Synchronous for the reason the fact insert is: the chunk plane is read # on the default search path, and a query landing before the chunks embed # must not find the head of the value and miss its body. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) # A newer write may have made the key 'high' (or unsearchable) or deleted it while # these chunks were embedding. It inserts its row before tombstoning the key's # chunks, so either its tombstone saw these or this strong read sees its row. A diff --git a/server/memory/domain/promotion_source.py b/server/memory/domain/promotion_source.py index 07a8576533..3c7f93a921 100644 --- a/server/memory/domain/promotion_source.py +++ b/server/memory/domain/promotion_source.py @@ -8,6 +8,8 @@ from __future__ import annotations +PromotionKey = tuple[str, str, str, str, str, str] + _FIELDS = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") @@ -25,3 +27,75 @@ def parse_promotion_source(entry: str) -> dict[str, str] | None: if len(parts) != len(_FIELDS) or not parts[-1]: return None return dict(zip(_FIELDS, parts, strict=True)) + + +# not a setting: fail-closed work ceiling for ambiguous legacy source identities. +_CONSENT_WORK_LIMIT = 250_000 + + +def promotion_users(entries: list[str], org: str, namespace: str) -> set[str] | None: + """Every possible source user, or no proof when complete denial checking is too large. + + The known organization and namespace fix the prefix. Slashes in any remaining + field leave several possible users; consent checks must cover all of them. + """ + prefix = f"{org}/{namespace}/" + users: set[str] = set() + work = 0 + for entry in entries: + if not isinstance(entry, str) or not entry.startswith(prefix): + return None + work += len(entry) + if work > _CONSENT_WORK_LIMIT: + return None + parts = entry[len(prefix) :].split("/") + if len(parts) < 4: + return None + found = False + for start in range(1, len(parts) - 2): + for stop in range(start + 1, len(parts) - 1): + user = "/".join(parts[start:stop]) + key = "/".join(parts[stop + 1 :]) + work += len(user) + len(key) + stop - start + if work > _CONSENT_WORK_LIMIT: + return None + if key: + users.add(user) + found = True + if not found: + return None + return users or None + + +def promotion_keys(entries: list[str], org: str, namespace: str) -> list[PromotionKey] | None: + """All exact identities a legacy entry may name, with complete bounded verification.""" + from itertools import combinations + + prefix = f"{org}/{namespace}/" + keys: set[PromotionKey] = set() + work = 0 + for entry in entries: + if not isinstance(entry, str) or not entry.startswith(prefix): + return None + work += len(entry) + if work > _CONSENT_WORK_LIMIT: + return None + parts = entry[len(prefix) :].split("/") + if len(parts) < 4: + return None + found = False + for first, second, third in combinations(range(1, len(parts)), 3): + fields = tuple( + "/".join(parts[a:b]) + for a, b in ((0, first), (first, second), (second, third), (third, len(parts))) + ) + work += sum(map(len, fields)) + len(parts) + if work > _CONSENT_WORK_LIMIT: + return None + agent, user, session, key = fields + if key: + keys.add((org, namespace, agent, user, session, key)) + found = True + if not found: + return None + return sorted(keys) or None diff --git a/server/memory/domain/provenance.py b/server/memory/domain/provenance.py index 28197c585d..76f2fce6c4 100644 --- a/server/memory/domain/provenance.py +++ b/server/memory/domain/provenance.py @@ -110,6 +110,8 @@ def cli_actor(command: str) -> str: # server/ is one of these, or a `loom` command's (tests/unit/test_model_call_writers.py). PIPELINE_ACTORS: Final = frozenset( { + "process:memory_refusal", + "process:session_contrast", EXTRACTION_WORKER_ACTOR, "process:budget_warning", "process:code_coverage_notify", diff --git a/server/memory/domain/refusals.py b/server/memory/domain/refusals.py new file mode 100644 index 0000000000..df6ffe59c1 --- /dev/null +++ b/server/memory/domain/refusals.py @@ -0,0 +1,178 @@ +"""Fingerprints for refusing deleted values without storing their plaintext.""" + +from __future__ import annotations + +import re +import unicodedata +from collections.abc import Iterable, Mapping +from typing import Any + +from server.core.errors import ValidationError +from server.core.hashing import sha256_value + +Fingerprint = tuple[str, int, str] +_WORDS = re.compile(r"\w+|[^\w\s]") +# not a setting: fail-closed CPU safety ceiling for complete consent verification. +_WORK_LIMIT = 250_000 + + +def tokens(text: str) -> list[str]: + return _WORDS.findall(unicodedata.normalize("NFKC", text).casefold()) + + +def fingerprint(value: Any) -> list[Fingerprint]: + """One canonical hash and, for a string, a normalized literal fingerprint.""" + out = [("value", 0, sha256_value(value))] + if isinstance(value, str) and (words := tokens(value)): + out.append(("literal", len(words), sha256_value(words))) + return out + + +def inspect(value: Any) -> tuple[set[Fingerprint], list[list[str]]]: + """Canonical subtrees and normalized string leaves, bounded by input work.""" + exact: set[Fingerprint] = set() + text: list[list[str]] = [] + pending = [value] + work = 0 + while pending: + part = pending.pop() + work += 1 + exact.add(("value", 0, sha256_value(part))) + if isinstance(part, dict): + pending.extend(part.values()) + pending.extend(part.keys()) + elif isinstance(part, list): + pending.extend(part) + elif isinstance(part, str): + words = tokens(part) + text.append(words) + work += len(words) + if work > _WORK_LIMIT: + raise ValidationError("The value is too large to verify against refusal policy") + return exact, text + + +def candidates( + inspected: tuple[set[Fingerprint], list[list[str]]], sizes: Iterable[int] +) -> set[Fingerprint]: + exact, texts = inspected + result = set(exact) + work = 0 + sizes = set(sizes) + for words in texts: + for size in sizes: + work += max(0, len(words) - size + 1) * size + if size <= 0 or work > _WORK_LIMIT: + raise ValidationError("The value is too large to verify against refusal policy") + result.update( + ("literal", size, sha256_value(words[i : i + size])) + for i in range(len(words) - size + 1) + ) + return result + + +def matches(value: Any, patterns: Iterable[Fingerprint]) -> bool: + """Check complete JSON subtrees and literal passages; refuse an incomplete scan.""" + patterns = set(patterns) + return bool( + patterns + and patterns & candidates(inspect(value), [p[1] for p in patterns if p[0] == "literal"]) + ) + + +EXTRACTION_CONTENT_FIELDS = frozenset( + { + "messages", + "last_messages", + "session_values", + "extra_tags", + "recalled_keys", + "extraction_instructions", + "record_schema", + "pipeline_steps", + "observation_date", + "memory_key", + } +) +EXTRACTION_PROTOCOL_FIELDS = frozenset( + { + "org", + "namespace", + "agent", + "user_id", + "session_id", + "run_id", + "max_tokens", + "judge_extraction", + "llm_cache", + "detect_updates", + "coverage_sweep", + "agent_learnings", + "defer", + "valid_at", + } +) + + +EXTRACTION_PROTOCOL_STEPS = frozenset( + { + "loom.fetch_namespace_categories", + "loom.fetch_existing_memories", + "loom.llm_extract", + "loom.drop_recall_echoes", + "loom.drop_session_echoes", + "loom.drop_activity_narration", + "loom.drop_non_durable", + "loom.verify_durable", + "loom.hash_dedup", + "loom.bulk_write", + } +) + + +def transcript_content(payload: Mapping[str, Any]) -> list[Any]: + """Caller data, excluding the request's and nested models' protocol defaults.""" + result: list[Any] = [] + for field in ("messages", "last_messages"): + for message in payload.get(field) or []: + result.extend(message[k] for k in ("content", "name") if message.get(k)) + role = message.get("role") + if role and role not in { + "user", + "assistant", + "system", + "developer", + "tool", + "function", + }: + result.append(role) + for field in ("session_values", "extra_tags", "recalled_keys"): + result.extend(payload.get(field) or []) + for field in ("extraction_instructions", "observation_date", "memory_key"): + if payload.get(field): + result.append(payload[field]) + for name, spec in (payload.get("record_schema") or {}).items(): + result.append(name) + if not isinstance(spec, str) or spec not in {"string", "number", "integer", "boolean"}: + result.append(spec) + for step in payload.get("pipeline_steps") or []: + step_id = step.get("step_id") + if "step_id" in step and not ( + isinstance(step_id, str) and step_id in EXTRACTION_PROTOCOL_STEPS + ): + result.append(step_id) + if "config" in step: + result.append(step["config"]) + arguments = {k: v for k, v in step.items() if k not in {"step_id", "config"}} + if arguments: + result.append(arguments) + return result + + +def payload_fingerprints(payload: Mapping[str, Any]) -> set[Fingerprint]: + result: set[Fingerprint] = set() + for value in transcript_content(payload): + exact, texts = inspect(value) + result.update(exact) + result.update(("literal", len(words), sha256_value(words)) for words in texts if words) + return result diff --git a/server/memory/domain/summary_provenance.py b/server/memory/domain/summary_provenance.py new file mode 100644 index 0000000000..3ce89d7d20 --- /dev/null +++ b/server/memory/domain/summary_provenance.py @@ -0,0 +1,57 @@ +"""Extractive digests reconstructed from their authoritative sources.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.temporal_sql import SERVED_STATUSES + + +def source_text( + row: Any, *, now: datetime, as_of: datetime | None = None, as_of_basis: str = "valid_time" +) -> str | None: + """Only a complete, currently served source can contribute digest text.""" + if ( + row.tombstoned_at is not None + or row.status not in SERVED_STATUSES + or getattr(row, "requires_review", False) + or row.sensitivity == "high" + or row.value_truncated + or value_reads_high(row.value) + ): + return None + for expiry in (row.expires_at, row.valid_to): + if expiry is not None and _utc(expiry) <= _utc(now): + return None + if as_of is not None: + at = _utc(as_of) + if as_of_basis == "knowledge_time": + elapsed = at - datetime(1970, 1, 1) + cutoff = ( + elapsed.days * 86400 + elapsed.seconds + ) * 1_000_000_000 + elapsed.microseconds * 1000 + if int(row.version) > cutoff: + return None + else: + lower = row.valid_at or row.temporal_anchor or row.created_at + if lower is None or _utc(lower) > at: + return None + if row.valid_to is not None and _utc(row.valid_to) <= at: + return None + return ( + row.value + if isinstance(row.value, str) + else json.dumps(row.value, ensure_ascii=False, sort_keys=True, default=str) + ) + + +def render_summary(facts: list[str]) -> str: + """Every factual byte comes from the source texts, in their recorded order.""" + return "\n".join(facts) + + +def _utc(value: datetime) -> datetime: + return value.astimezone(UTC).replace(tzinfo=None) if value.tzinfo else value diff --git a/server/memory/store/delete_checks.py b/server/memory/store/delete_checks.py index 283bc6f9ae..b43a5424ac 100644 --- a/server/memory/store/delete_checks.py +++ b/server/memory/store/delete_checks.py @@ -7,9 +7,12 @@ from __future__ import annotations +import json from typing import Any +from server.core.hashing import sha256_text from server.core.repo_base import _CHClient, _default_client +from server.memory.store.rows import MemoryRow, memory_head_state from server.memory.store.sql import _SEQUENTIAL, read_your_writes_enabled TABLE = "loom.delete_version_checks" @@ -28,6 +31,11 @@ ) +def checked_state_sha256(row: MemoryRow) -> str: + """Bind recovery to the full checked state, including same-version collisions.""" + return sha256_text(json.dumps(memory_head_state(row), sort_keys=True, default=str)) + + async def record_delete_check( check: dict[str, Any], *, done: bool, ch: _CHClient | None = None ) -> None: @@ -37,6 +45,7 @@ async def record_delete_check( row = {k: check[k] for k in _CHECK_COLUMNS} row["checked"], row["tombstone"] = int(row["checked"]), int(row["tombstone"]) row["done"] = 1 if done else 0 + row["checked_state_sha256"] = check.get("checked_state_sha256", "") await ch.insert(TABLE, [row], settings={"async_insert": 0, "wait_for_async_insert": 1}) @@ -56,7 +65,8 @@ async def find_delete_check( ch = ch or _default_client() rows = await ch.query_all( f""" - SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, done + SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, + checked_state_sha256, done FROM {TABLE} FINAL WHERE org = {{org:String}} AND namespace = {{ns:String}} AND agent = {{agent:String}} AND user_id = {{uid:String}} AND session_id = {{sid:String}} @@ -86,7 +96,8 @@ async def pending_delete_checks( ch = ch or _default_client() return await ch.query_all( f""" - SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone + SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, + checked_state_sha256 FROM {TABLE} FINAL WHERE done = 0 AND created_at <= now() - toIntervalSecond({{age:UInt32}}) ORDER BY created_at diff --git a/server/memory/store/embeddings.py b/server/memory/store/embeddings.py index 27fdbf46ca..78c57ef653 100644 --- a/server/memory/store/embeddings.py +++ b/server/memory/store/embeddings.py @@ -13,7 +13,9 @@ from server.core import pg from server.core.ch_writes import committed_insert_settings, partition_blocks from server.core.datetimes import naive_utc_now +from server.core.hashing import hash_text from server.core.repo_base import _CHClient, _default_client, query_all_sliced +from server.memory.domain.redaction_lineage import is_content_hash from server.memory.domain.temporal_sql import SERVED_STATUSES from server.memory.store.inserts import insert_embedding_rows from server.memory.store.rows import ( @@ -125,7 +127,7 @@ async def mirror_status_to_embeddings( overrides["tombstoned_at"] = m.get("tombstoned_at") if judged: overrides["version"] = drawn - shas[key] = str(m.get("content_sha256") or "") + shas[key] = hash_text(m.get("content_sha256")) elif m.get("version"): overrides["version"] = int(m["version"]) wanted[key] = overrides @@ -143,7 +145,11 @@ async def mirror_status_to_embeddings( continue for r in rows: want = wanted.get(identity_key(r)) - if judged and str(r.get("content_sha256") or "") != shas.get(identity_key(r)): + if ( + judged + and hash_text(r.get("content_sha256")) != shas.get(identity_key(r)) + and not _unproven_retirement(r, want) + ): continue # another revision's vector if judged and str(r.get("status") or "") not in SERVED_STATUSES: continue # a retire (a refines supersede) won meanwhile @@ -160,7 +166,10 @@ async def mirror_status_to_embeddings( return len(out) if out: try: - await insert_embedding_rows(out, ch=ch) + candidates = out + out = await insert_embedding_rows(candidates, ch=ch) + if not judged: + out += await _retire_held_status_copies(candidates, out, wanted, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) except Exception as e: # a mirror, never the write; strict re-raises if strict: @@ -170,6 +179,34 @@ async def mirror_status_to_embeddings( return len(out) +async def _retire_held_status_copies( + candidates: list[dict[str, Any]], + written: list[dict[str, Any]], + wanted: Mapping[IdentityKey, dict[str, Any]], + *, + ch: _CHClient, +) -> list[dict[str, Any]]: + """Retire an older, unpublishable vector at its canonical retirement's pin.""" + + def receipt(row: Mapping[str, Any]) -> tuple[Any, ...]: + return tuple(row[c] for c in (*_BARE_TOMBSTONE_KEPT, "version")) + + acknowledged = {receipt(r) for r in written} + now = naive_utc_now() + bare = [] + for row in candidates: + overrides = wanted[identity_key(row)] + pin = overrides.get("version") + if ( + pin is not None + and int(row["version"]) == int(pin) + and overrides.get("status", "active") not in SERVED_STATUSES + and receipt(row) not in acknowledged + ): + bare.append(_bare_tombstone(row, int(pin), now)) + return await insert_embedding_rows(bare, ch=ch) if bare else [] + + def status_divergence_sql() -> str: """Identity rows (org, namespace, agent, user_id, session_id, memory_key) whose latest live memory status differs from the latest live embeddings @@ -380,7 +417,7 @@ async def reemit_embedding_rows_for_keys( if row is not None: out.append(row) if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) written += len(out) return written @@ -446,6 +483,14 @@ async def _live_rows_for_keys( ) +def _unproven_retirement(row: Mapping[str, Any], overrides: Mapping[str, Any] | None) -> bool: + return bool( + overrides is not None + and overrides.get("status", "active") not in SERVED_STATUSES + and not is_content_hash(hash_text(row.get("content_sha256"))) + ) + + def _copy_row(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any] | None: """A new version of read row `r` with `overrides`: at read + 1, or at the `version` the overrides pin (a retire's, or the expiry sweep's demote). None when the row is @@ -454,7 +499,10 @@ def _copy_row(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any] | pinned = overrides.get("version") if pinned is not None and int(r.get("version") or 0) >= int(pinned): return None - return _reemitted(r, overrides) + copy = _reemitted(r, overrides) + if _unproven_retirement(r, overrides): + return _bare_tombstone(r, int(copy["version"]), naive_utc_now()) + return copy def _reemitted(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]: @@ -475,7 +523,7 @@ async def reemit_rows( the rows written.""" out = [row for r in rows if (row := _copy_row(r, overrides)) is not None] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) @@ -528,8 +576,7 @@ async def retire_copies_under_unembedded_heads( for r in copies if identity_key(r) in stale ] - await insert_embedding_rows(tombstones, ch=ch) - return len(tombstones) + return len(await insert_embedding_rows(tombstones, ch=ch)) except Exception as e: # noqa: BLE001 — the copy landed; a missed check is logged, not raised _log.warning("vector copy: head re-check failed for %d rows: %s", len(copies), e) return 0 @@ -719,8 +766,7 @@ async def _bare_tombstones(items: list[tuple[IdentityKey, int]], *, ch: _CHClien ] # A retirement across many keys can span more partitions than an insert takes. for block in partition_blocks(out, created_month): - await insert_embedding_rows(block, ch=ch) - n += len(out) + n += len(await insert_embedding_rows(block, ch=ch)) return n @@ -752,8 +798,7 @@ async def reject_embedding_rows( out.append(_reemitted(r, {"status": "rejected", "version": version})) # A retirement across many keys can span more partitions than an insert takes. for block in partition_blocks(out, created_month): - await insert_embedding_rows(block, ch=ch) - n += len(out) + n += len(await insert_embedding_rows(block, ch=ch)) return n @@ -789,8 +834,8 @@ async def fence_vectors( ) for key, version in items ] - await insert_embedding_rows(rows, ch=ch) - return len(rows) + admitted = await insert_embedding_rows(rows, ch=ch) + return len(admitted) def version_second(version: int) -> datetime: @@ -892,7 +937,7 @@ async def unsupersede_embeddings( if r.get("status") != status or r.get("invalid_at") is not None ] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) @@ -939,7 +984,7 @@ async def restamp_embedding_rows( overrides = {"status": status, "version": int(version)} out = [row for r in rows if (row := _copy_row(r, overrides)) is not None] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) diff --git a/server/memory/store/entities.py b/server/memory/store/entities.py index 8e037d5f16..68a9966c32 100644 --- a/server/memory/store/entities.py +++ b/server/memory/store/entities.py @@ -304,9 +304,9 @@ async def _copy_mirrors( ) for r in emb_rows ] - await insert_embedding_rows(copies, ch=ch) - await retire_copies_under_unembedded_heads(copies, ch=ch) - return len(emb_rows) + acknowledged = await insert_embedding_rows(copies, ch=ch) + await retire_copies_under_unembedded_heads(acknowledged, ch=ch) + return len(acknowledged) def _shows(mirror: Mapping[str, Any], head_sha256: str) -> bool: @@ -375,6 +375,6 @@ async def heal_divergent_entities(*, limit: int = 200, ch: _CHClient | None = No ) for r in emb_rows ] - await insert_embedding_rows(copies, ch=ch) - await retire_copies_under_unembedded_heads(copies, ch=ch) - return len(emb_rows) + acknowledged = await insert_embedding_rows(copies, ch=ch) + await retire_copies_under_unembedded_heads(acknowledged, ch=ch) + return len(acknowledged) diff --git a/server/memory/store/inserts.py b/server/memory/store/inserts.py index 0fb1453301..bd4ec0d679 100644 --- a/server/memory/store/inserts.py +++ b/server/memory/store/inserts.py @@ -4,7 +4,6 @@ from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from contextlib import nullcontext from typing import Any from server.core.repo_base import _CHClient, _default_client @@ -42,6 +41,7 @@ async def insert_memory_versions( expected_heads: Mapping[IdentityKey, MemoryRow | None] | None = None, on_send: Callable[[], None] | None = None, on_inserted: Callable[[], None] | None = None, + refuse_values: bool = False, ) -> int: """The one loom.memory write: every row is checked against MEMORY_PERSISTED_COLS before the insert (see `check_memory_rows`), and @@ -64,19 +64,26 @@ async def insert_memory_versions( admission = pg_memory.Admission(generations=dict(policy)) if policy else None try: if publish_now: - await pg_publish.commit_and_publish(rows, ch=ch, admission=admission) + await pg_publish.commit_and_publish( + rows, ch=ch, admission=admission, refuse_values=refuse_values + ) else: - await pg_publish.commit(rows, publisher_lease=False, admission=admission) + await pg_publish.commit( + rows, publisher_lease=False, admission=admission, refuse_values=refuse_values + ) except pg_memory.AdmissionMoved: raise pg_memory.AdmissionContendedError( "the write policy changed while it was checked" ) from None return len(rows) ch = ch or _default_client() + from server.memory.store import refusals from server.memory.store.write_fence import checked_write guard = ( - nullcontext() if pg.enabled() else checked_write(rows, expected_heads=expected_heads, ch=ch) + refusals.publication(ch, rows, committed=True) + if pg.enabled() + else checked_write(rows, expected_heads=expected_heads, ch=ch, refuse_values=refuse_values) ) async with guard: if on_send is not None: @@ -96,7 +103,7 @@ async def insert_embedding_rows( rows: list[dict[str, Any]], *, ch: _CHClient | None = None, -) -> None: +) -> list[dict[str, Any]]: """Insert N memory_embedding rows, committed. Synchronous because this is the row that makes a memory findable @@ -104,14 +111,29 @@ async def insert_embedding_rows( fire-and-forget insert would let `memory.set` return before its memory can be found. The cost is tens of milliseconds per write. - Empty `rows` is a no-op. + Return the input rows whose publication or bounded orphan cleanup was acknowledged. """ if not rows: - return + return [] ch = ch or _default_client() - await ch.insert( - "loom.memory_embeddings", - rows, - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) - pg_pipeline.record_insert("memory_embeddings", len(rows)) + from server.memory.store.shared import repair_shared_embeddings + + shared = [r for r in rows if r.get("scope") == "shared" and r.get("tombstoned_at") is None] + acknowledged = [] + if shared: + acknowledged = await repair_shared_embeddings(shared, ch=ch) + rows = [r for r in rows if r not in shared] + if not rows: + return acknowledged + from server.memory.store import refusals + + retired: list[dict[str, Any]] = [] + async with refusals.publication(ch, rows, embeddings=True, retired=retired) as admitted: + if admitted: + await ch.insert( + "loom.memory_embeddings", + [dict(r) for r in admitted], + settings={"async_insert": 0, "wait_for_async_insert": 1}, + ) + pg_pipeline.record_insert("memory_embeddings", len(admitted)) + return acknowledged + [dict(row) for row in admitted] + retired diff --git a/server/memory/store/integrity.py b/server/memory/store/integrity.py index e7c77282e0..13bc4dcf60 100644 --- a/server/memory/store/integrity.py +++ b/server/memory/store/integrity.py @@ -160,15 +160,15 @@ async def repair_dim0_rows( if dry_run or not out: return len(out) ch = ch or _default_client() - await insert_embedding_rows(out, ch=ch) + admitted = await insert_embedding_rows(out, ch=ch) from server.memory.store.embeddings import retire_copies_under_unembedded_heads await retire_copies_under_unembedded_heads(out, ch=ch) # A row that was dim 0 was never counted, and its + 1 sits under the BM25 watermark. from server.memory.store.bm25_stats import index_late_rows - await index_late_rows(out, ch=ch) - return len(out) + await index_late_rows(admitted, ch=ch) + return len(admitted) async def drift_sample( diff --git a/server/memory/store/lifecycle.py b/server/memory/store/lifecycle.py index 8052eeeeb2..98e99233c6 100644 --- a/server/memory/store/lifecycle.py +++ b/server/memory/store/lifecycle.py @@ -316,6 +316,7 @@ async def _insert_clickhouse_copies( landed: Callable[[], Awaitable[None]] | None, on_commit: Callable[[], None] | None, exact: bool, + refuse_values: bool, ) -> int: inserted = not rows count = 0 @@ -337,7 +338,11 @@ async def follow() -> int: {identity_key(r): h for r, h in zip(rows, read, strict=True)} if exact else None ) await insert_memory_versions( - rows, ch=ch, expected_heads=expected, on_inserted=mark_inserted + rows, + ch=ch, + expected_heads=expected, + on_inserted=mark_inserted, + refuse_values=refuse_values, ) finally: # A successful memory insert still owes its follow-ups when holder @@ -355,12 +360,12 @@ async def insert_versions_over_heads( then: Callable[[], Awaitable[int | None]] | None = None, landed: Callable[[], Awaitable[None]] | None = None, documented: Sequence[bool] | None = None, - settle_held: bool = False, sources: Sequence[tuple[str, int]] = (), live_sources: bool = False, policy: Mapping[tuple[str, str], int] | None = None, on_commit: Callable[[], None] | None = None, exact: bool = False, + refuse_values: bool = False, ) -> int: """Write `rows`, each a copy of the authority's head beside it in `read`, drawn above it (`rows.version_over`) or, to yield to a write since, before the read: a copy below @@ -376,16 +381,22 @@ async def insert_versions_over_heads( of it is raised once the commit is published. `landed` runs once the rows are in, outside the lease: on ClickHouse right after the insert, before `then`; under Postgres after the publish, whatever `then` did. `documented`: the rows that owe a live search - document, which `then` writes; under Postgres they are settled once `then` returns - (with `settle_held`, only those ClickHouse holds at their row's version, for a `then` - that copies what it finds), and the delivery worker builds one it did not. + document, which `then` writes; under Postgres only documents ClickHouse holds at + their row's version are settled, and the delivery worker builds one it did not. `sources`: the revisions the rows copy (`Admission.sources`), with `live_sources` each still its key's live head. `on_commit` is called once the rows are in, before holder cleanup or publication can fail. A commit whose publication is pending is done.""" if not pg.enabled() or not rows: return await _insert_clickhouse_copies( - rows, read, ch=ch, then=then, landed=landed, on_commit=on_commit, exact=exact + rows, + read, + ch=ch, + then=then, + landed=landed, + on_commit=on_commit, + exact=exact, + refuse_values=refuse_values, ) check_memory_rows(rows) heads = {identity_key(r): e for r, e in zip(rows, read, strict=True)} @@ -398,7 +409,9 @@ async def insert_versions_over_heads( plain = pg_memory.MEMORY_ROW | pg_memory.AUDIT_ROW owed = [plain | (pg_memory.SEARCH_DOC if d else 0) for d in documented or [False] * len(rows)] try: - committed = await pg_publish.commit(rows, targets=owed, admission=admission) + committed = await pg_publish.commit( + rows, targets=owed, admission=admission, refuse_values=refuse_values + ) except pg_memory.HeadMoved: raise pg_memory.AdmissionContendedError("the memory changed while it was read") from None except pg_memory.AdmissionMoved: @@ -413,7 +426,7 @@ async def insert_versions_over_heads( with pg_lease.leased(committed.deadline), sequential_reads(): answer = await then() or 0 if documented: - await _settle_documents(committed, documented, held=settle_held, ch=ch) + await _settle_documents(committed, documented, ch=ch) except pg_lease.PublicationPending: _log.warning("a copy's follow-up was left: its commit's lease ended first") except Exception as e: # noqa: BLE001 — raised once the commit is published @@ -460,30 +473,27 @@ async def insert_documented_versions( async def _settle_documents( - committed: Any, documented: Sequence[bool], *, held: bool = False, ch: _CHClient | None = None + committed: Any, documented: Sequence[bool], *, ch: _CHClient | None = None ) -> None: - """Clear the search documents written under the commit's lease; with `held`, only - those that ClickHouse holds at their row's version (`held_documents`). Failing that, + """Clear only documents ClickHouse holds at their row's version (`held_documents`). + Failing that, the delivery worker settles one it finds there and builds again one written below it (an Undo's rows, re-emitted at their own).""" owed = [(r, q) for r, q, d in zip(committed.rows, committed.seqs, documented, strict=True) if d] try: - if held: - client = ch or _default_client() - by_scope: dict[tuple[str, str], list[tuple[Any, ...]]] = {} - for r, _ in owed: - by_scope.setdefault((r["org"], r["namespace"]), []).append( - pg_publish.document_id(r) - ) - held_ids: set[tuple[Any, ...]] = set() - for (org, ns), ids in by_scope.items(): - docs = await pg_publish.held_documents(org, ns, ids, ch=client) - held_ids |= {(org, ns, *d) for d in docs} - owed = [ - (r, q) - for r, q in owed - if (r["org"], r["namespace"], *pg_publish.document_id(r)) in held_ids - ] + client = ch or _default_client() + by_scope: dict[tuple[str, str], list[tuple[Any, ...]]] = {} + for r, _ in owed: + by_scope.setdefault((r["org"], r["namespace"]), []).append(pg_publish.document_id(r)) + held_ids: set[tuple[Any, ...]] = set() + for (org, ns), ids in by_scope.items(): + docs = await pg_publish.held_documents(org, ns, ids, ch=client) + held_ids |= {(org, ns, *d) for d in docs} + owed = [ + (r, q) + for r, q in owed + if (r["org"], r["namespace"], *pg_publish.document_id(r)) in held_ids + ] await pg_memory.complete([q for _, q in owed], pg_memory.SEARCH_DOC) except Exception as e: # noqa: BLE001 — the commit stands; the worker settles or builds them _log.warning("documents left owed for the delivery worker: %s", error_summary(e)) @@ -884,7 +894,6 @@ def on_commit() -> None: ch=ch, then=copy_vectors, documented=documented, - settle_held=True, on_commit=on_commit, ) except pg_memory.AdmissionContendedError: @@ -1037,8 +1046,24 @@ async def _tombstone_ch_heads( ) for head in page ] - for block in partition_blocks(rows, created_month): - await insert_memory_versions(block, ch=ch) + blocks = [ + block[start : start + 16] + for block in partition_blocks(rows, created_month) + for start in range(0, len(block), 16) + ] + for block in blocks: + expected = { + identity_key(h): _hydrate(h) + for h in page + if identity_key(h) in {identity_key(r) for r in block} + } + try: + await insert_memory_versions( + block, ch=ch, expected_heads=expected, refuse_values=True + ) + except pg_memory.AdmissionContendedError: + # Re-page at the original cutoff: a new value or lock is not ours to erase. + break heads = await memory_rows_by_identity_keys( [identity_key(r) for r in block], ch=ch, include_tombstoned=True, sequential=True ) @@ -1066,7 +1091,8 @@ async def _tombstone_ch_heads( ], ch=ch, ) - after = key_in_namespace(page[-1]) + else: + after = key_in_namespace(page[-1]) async def _tombstone_where( @@ -1160,6 +1186,7 @@ async def _tombstone_heads( try: committed = await pg_publish.commit( rows, + refuse_values=True, admission=pg_memory.Admission( recheck=pg_memory.heads_unchanged(heads, exact=True) ), diff --git a/server/memory/store/pg_batch.py b/server/memory/store/pg_batch.py index 538f150b4a..98ecc28ce7 100644 --- a/server/memory/store/pg_batch.py +++ b/server/memory/store/pg_batch.py @@ -437,6 +437,7 @@ async def _publish(group: list[_Entry]) -> None: deadline = min(e.committed.deadline for e in group) rows = [r for e in group if not e.rows_sent for r in e.committed.rows] docs = [d for e in group for d in served.get(id(e), {}).values()] + acknowledged: set[tuple[Any, ...]] = set() async def send_rows() -> None: for block in partition_blocks(rows, created_month): @@ -444,7 +445,8 @@ async def send_rows() -> None: async def send_documents() -> None: for block in partition_blocks(docs, created_month): - await inserts.insert_embedding_rows(block, ch=ch) + admitted = await inserts.insert_embedding_rows(block, ch=ch) + acknowledged.update(_doc_key(row) for row in admitted) with pg_lease.leased(deadline): rows_error, audit_error, docs_error = await asyncio.gather( @@ -464,6 +466,18 @@ async def send_documents() -> None: ) rows_in = not isinstance(rows_error, BaseException) documents_in = not isinstance(docs_error, BaseException) + if documents_in: + for e in group: + attempted = served.get(id(e), {}) + served[id(e)] = { + key: row for key, row in attempted.items() if _doc_key(row) in acknowledged + } + withheld = attempted.keys() - served[id(e)].keys() + e.stale.extend( + seq + for seq, row in zip(e.committed.seqs, e.committed.rows, strict=True) + if _doc_key(row) in withheld + ) await pg_memory.complete_each( _landed( group, diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 7ef14e58de..28057dd362 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -17,8 +17,7 @@ import logging import math import time -from collections.abc import AsyncIterator, Callable, Mapping, Sequence -from contextlib import asynccontextmanager +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import UTC, datetime from typing import Any @@ -700,6 +699,7 @@ async def commit_versions( request_id: str = "", idempotency: tuple[str, str, int] | None = None, publisher_lease: bool = True, + refuse_values: bool = False, ) -> Committed: """Commit `rows` (full loom.memory version rows): heads, revisions with `audits[i]` (row i's hash-only loom.audit_log row), deliveries (`targets`: one @@ -725,6 +725,9 @@ async def commit_versions( per_scope, scopes, heads, keys = _batch_indexes(stored) async with pg.transaction() as conn: await _bound_lock_waits(conn) + from server.memory.store import refusals + + await refusals.lock_postgres(conn, stored, forgetting=refuse_values) if admission is not None: await _check_policy(conn, admission, per_scope) bumped = await conn.fetch( @@ -735,6 +738,7 @@ async def commit_versions( # runs, and before the head recheck: an erased memory is a 404, not a moved head. await _refuse_stale_rows(conn, stored, admission) await _refuse_moved_sources(conn, scopes, admission) + await refusals.check_postgres(conn, stored) if admission is not None: moved = { s: last[s] - per_scope[s] @@ -755,6 +759,8 @@ async def commit_versions( recheck(k, head) if _replaces_vectors(stored[heads[k]], head): per_row[heads[k]] |= OLD_VECTORS + if refuse_values: + await refusals.record_postgres(conn, stored) seqs: list[tuple[str, str, int]] = [] next_seq = {s: last[s] - per_scope[s] for s in scopes} for r in stored: @@ -1618,7 +1624,11 @@ async def settled_head_versions(keys: Sequence[IdentityKey]) -> dict[IdentityKey async def heads_by_keys( - keys: Sequence[IdentityKey], *, include_tombstoned: bool = False, durable: bool = False + keys: Sequence[IdentityKey], + *, + include_tombstoned: bool = False, + durable: bool = False, + max_value_chars: int | None = None, ) -> dict[IdentityKey, dict[str, Any]]: """The heads, expired ones included, of exact keys (a deleted one too with `include_tombstoned`); a key with none is absent. `durable`: read for ClickHouse @@ -1629,24 +1639,41 @@ async def heads_by_keys( sql = _HEADS_BY_KEYS if not include_tombstoned: sql += " WHERE c.row->>'tombstoned_at' IS NULL" - rows = await _read(sql, *_key_arrays(keys), durable=durable) + args = _key_arrays(keys) + if max_value_chars is not None: + sql += (" AND " if " WHERE " in sql else " WHERE ") + ( + "length(coalesce(c.row->>'value','')) <= $7 AND " + "length(coalesce(c.row->>'value_json','{}')) <= $7" + ) + bound: list[Any] = [*args, max_value_chars] + else: + bound = list(args) + rows = await _read(sql, *bound, durable=durable) return {identity_key(r["row"]): decode_row(r["row"]) for r in rows} -@asynccontextmanager -async def locked_source(source: MemoryRow) -> AsyncIterator[None]: - """Hold an unchanged durable source while its shared projection is published.""" - key = identity_key(source) - durable = await heads_by_keys([key], include_tombstoned=True, durable=True) - if not unchanged(source, durable.get(key)): - raise AdmissionContendedError("The promotion's source changed") - async with pg.rolled_back() as conn: - await _bound_lock_waits(conn) - rows = await conn.fetch(_LOCK_HEADS, *_key_arrays([key])) - current = decode_row(rows[0]["row"]) if rows else None - if not unchanged(source, current): - raise AdmissionContendedError("The promotion's source changed") - yield +async def heads_on_connection( + conn: Any, + keys: Sequence[IdentityKey], + *, + lock: bool = False, + max_value_chars: int | None = None, +) -> dict[IdentityKey, dict[str, Any]]: + """Recheck heads while the caller holds scope locks on this connection.""" + if not keys: + return {} + sql = _HEADS_BY_KEYS + args: list[Any] = list(_key_arrays(keys)) + if max_value_chars is not None: + sql += ( + " WHERE length(coalesce(c.row->>'value','')) <= $7 AND " + "length(coalesce(c.row->>'value_json','{}')) <= $7" + ) + args.append(max_value_chars) + if lock: + sql += " " + _KEY_ORDER + " FOR UPDATE OF c" + rows = await conn.fetch(sql, *args) + return {identity_key(r["row"]): decode_row(r["row"]) for r in rows} async def heads_owing( diff --git a/server/memory/store/pg_publish.py b/server/memory/store/pg_publish.py index 806185b8c1..69cf5c031f 100644 --- a/server/memory/store/pg_publish.py +++ b/server/memory/store/pg_publish.py @@ -95,6 +95,7 @@ async def commit( admission: Admission | None = None, idempotent: bool = False, publisher_lease: bool = True, + refuse_values: bool = False, ) -> Committed: """The commit alone; `publish` sends it. `idempotent`: these are the op's own rows, which the request's idempotency key records as committed (another write the request @@ -114,6 +115,7 @@ async def commit( request_id=rid, idempotency=idempotency.bound_key() if idempotent else None, publisher_lease=publisher_lease, + refuse_values=refuse_values, ) @@ -338,8 +340,9 @@ async def commit_and_publish( *, ch: _CHClient | None = None, admission: Admission | None = None, + refuse_values: bool = False, ) -> Committed: """`insert_memory_versions` under the Postgres authority, held to its admission.""" - committed = await commit(rows, admission=admission) + committed = await commit(rows, admission=admission, refuse_values=refuse_values) await publish(committed, ch=ch) return committed diff --git a/server/memory/store/reads.py b/server/memory/store/reads.py index c535913d2b..5987266061 100644 --- a/server/memory/store/reads.py +++ b/server/memory/store/reads.py @@ -383,6 +383,7 @@ async def memory_rows_by_identity_keys( ch: _CHClient | None = None, sequential: bool = False, include_tombstoned: bool = False, + max_value_chars: int | None = None, ) -> dict[IdentityKey, MemoryRow]: """`get_by_key` for N exact identity keys, in statements whose keys each fit one form field (`query_all_sliced`): the same projection under FINAL, the full sort key in @@ -393,19 +394,29 @@ async def memory_rows_by_identity_keys( if not keys: return {} if pg.enabled(): - heads = await pg_memory.heads_by_keys(keys, include_tombstoned=include_tombstoned) + heads = await pg_memory.heads_by_keys( + keys, include_tombstoned=include_tombstoned, max_value_chars=max_value_chars + ) return {k: _hydrate(h) for k, h in heads.items()} ch = ch or _default_client() # Bound a slice at a time: one parameter holds as many keys as a form field does. + where = [] + if not include_tombstoned: + where.append("tombstoned_at IS NULL") + if max_value_chars is not None: + where.append( + "length(value) <= {value_cap:UInt64} AND " + "length(toString(value_json)) <= {value_cap:UInt64}" + ) rows = await query_all_sliced( ch, f"""{_MEMORY_FINAL_PROJECTION} PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN {{keys:{_IDENTITY_KEY_TUPLE}}} - {"" if include_tombstoned else "WHERE tombstoned_at IS NULL"} + {("WHERE " + " AND ".join(where)) if where else ""} {_SEQUENTIAL if sequential else ""} """, - {"keys": list(keys)}, + {"keys": list(keys), "value_cap": max_value_chars}, "keys", ) return {identity_key(r): _hydrate(r) for r in rows} diff --git a/server/memory/store/refusals.py b/server/memory/store/refusals.py new file mode 100644 index 0000000000..488243a7d1 --- /dev/null +++ b/server/memory/store/refusals.py @@ -0,0 +1,697 @@ +"""Durable refusal policy and admission fences shared by writers and erasers.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from contextlib import AsyncExitStack, asynccontextmanager +from contextvars import ContextVar +from datetime import UTC +from typing import Any + +from server.core import lease_writes, pg +from server.core.ch import sequential_reads, set_query_tag, write_limit_s +from server.core.ch_params import chunks +from server.core.ch_writes import batched_insert_settings, committed_insert_settings +from server.core.datetimes import naive_utc_now +from server.core.errors import BackendUnavailableError, ValidationError +from server.core.hashing import hash_text, served_row_value, sha256_value +from server.core.repo_base import _CHClient +from server.memory.domain import refusals + +# Protocol constant: erasers acquire every stripe; every publisher acquires one. +_STRIPES = 32 +_writer: Callable[[], str] | None = None +_shared_permit: ContextVar[tuple[str, str, str, str] | None] = ContextVar( + "shared_refusal_permit", default=None +) + + +_CH_SCOPE = ( + "((scope_kind IN ('identity','erased_identity') AND scope_value={ns:String} " + "AND user_id={uid:String}) " + "OR (scope_kind='namespace' AND scope_value={ns:String}) " + "OR (scope_kind='user_id' AND scope_value={uid:String}))" +) +_PG_SCOPE = ( + "((scope_kind IN ('identity','erased_identity') AND scope_value=$2 AND user_id=$3) " + "OR (scope_kind='namespace' AND scope_value=$2) " + "OR (scope_kind='user_id' AND scope_value=$3))" +) + +_CH_ERASURE_SCOPE = _CH_SCOPE.replace("('identity','erased_identity')", "('erased_identity')") +_PG_ERASURE_SCOPE = _PG_SCOPE.replace("('identity','erased_identity')", "('erased_identity')") + + +def _lease_client(ch: _CHClient) -> _CHClient: + return ch + + +def _admission_wait_s() -> float: + return write_limit_s() + + +def use_writer(writer: Callable[[], str]) -> None: + global _writer + _writer = writer + + +def _scope(kind: str, org: str, value: str) -> str: + return json.dumps([kind, org, value], separators=(",", ":")) + + +def _name(scope: str, stripe: int) -> str: + return "memory_refusal/" + hashlib.sha256(scope.encode()).hexdigest() + f"/{stripe}" + + +def fence_names(rows: Sequence[Mapping[str, Any]], *, forgetting: bool = False) -> list[str]: + names: set[str] = set() + for row in rows: + org, ns, uid = str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-") + encoded = json.dumps( + [org, ns, row.get("agent"), uid, row.get("session_id"), row.get("memory_key")], + separators=(",", ":"), + default=str, + ) + stripe = int.from_bytes(hashlib.sha256(encoded.encode()).digest()[:4]) % _STRIPES + names.add(_name(_scope("org", org, ""), stripe)) + names.add(_name(_scope("namespace", org, ns), stripe)) + user_scope = _scope("user_id", org, uid) + names.update(_name(user_scope, n) for n in (range(_STRIPES) if forgetting else [stripe])) + return sorted(names) + + +def erasure_fence_names(org: str, kind: str, value: str) -> list[str]: + return sorted( + _name(scope, n) + for scope in (_scope("org", org, ""), _scope(kind, org, value)) + for n in range(_STRIPES) + ) + + +def assert_shared_publication(row: Mapping[str, Any], *, vector: bool = False) -> None: + if row.get("tombstoned_at") is not None: + return + digest = row.get("content_sha256") if vector else sha256_value(served_row_value(row)) + if _shared_permit.get() != (row["org"], row["namespace"], row["memory_key"], digest): + raise ValidationError("A promotion requires its live source's admission") + + +@asynccontextmanager +async def source_publication( + ch: _CHClient, source: Any, target_key: str, *, refusal_users: set[str] | None = None +) -> AsyncIterator[Callable[[bool], None]]: + from server.memory.domain.temporal_sql import SERVED_STATUSES + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import build_memory_version_row, identity_key + + if ( + source.status not in SERVED_STATUSES + or source.requires_review + or source.tombstoned_at + or source.sensitivity == "high" + ): + raise ValidationError("Only an approved live source can be promoted") + row = build_memory_version_row(source) + key = identity_key(row) + async with lease_writes.fenced_many( + fence_names([row]), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark: + async with AsyncExitStack() as stack: + conn = None + if pg.enabled(): + durable = await pg_memory.heads_by_keys( + [key], include_tombstoned=True, durable=True + ) + if not pg_memory.unchanged(source, durable.get(key)): + raise pg_memory.AdmissionContendedError("The promotion's source changed") + conn = await stack.enter_async_context(pg.transaction()) + await lock_postgres(conn, [row]) + heads = await pg_memory.heads_on_connection(conn, [key], lock=True) + await check_postgres(conn, [row]) + else: + heads = { + k: build_memory_version_row(h) + for k, h in ( + await memory_rows_by_identity_keys( + [key], ch=ch, sequential=True, include_tombstoned=True + ) + ).items() + } + await check_clickhouse(ch, [row]) + if refusal_users is not None and ( + not refusal_users + or await _shared_values_refused( + ch, + conn, + [{**row, "user_id": user or "-"} for user in refusal_users], + erasure_only=False, + ) + ): + raise _refused() + if not pg_memory.unchanged(source, heads.get(key)): + raise pg_memory.AdmissionContendedError("The promotion's source changed") + token = _shared_permit.set( + (source.org, source.namespace, target_key, sha256_value(source.value)) + ) + try: + yield mark + finally: + _shared_permit.reset(token) + + +@asynccontextmanager +async def shared_retirement(ch: _CHClient, row: Mapping[str, Any]) -> AsyncIterator[bool]: + """A delayed shared tombstone cannot restore bytes removed by erasure.""" + from server.memory.domain.promotion_source import promotion_users + + users = promotion_users(list(row.get("promoted_from") or []), row["org"], row["namespace"]) + base = content_policy_row(row, served_row_value(row)) + sources = [{**base, "user_id": user or "-"} for user in sorted(users)] if users else [base] + async with ( + lease_writes.fenced_many( + fence_names([base]), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark, + AsyncExitStack() as stack, + ): + conn = await stack.enter_async_context(pg.transaction()) if pg.enabled() else None + if conn is not None: + await lock_postgres(conn, [base]) + erased = await _shared_values_refused(ch, conn, sources) + from server.memory.store.shared import fetch_shared + + head = await fetch_shared( + row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch + ) + publish = bool(users) and head is not None and not erased + mark(True) + if not publish: + await _delete_old_shared(ch, row) + mark(False) + yield publish + mark(False) + + +async def _delete_old_shared(ch: _CHClient, row: Mapping[str, Any]) -> None: + set_query_tag("memory.retire_orphan_shared") + await ch.execute( + "DELETE FROM loom.shared WHERE org={org:String} " + "AND namespace={namespace:String} AND memory_key={memory_key:String} " + "AND version < {version:UInt64}", + params={c: row[c] for c in ("org", "namespace", "memory_key", "version")}, + settings={"lightweight_deletes_sync": 2}, + ) + + +async def _shared_values_refused( + ch: _CHClient, conn: Any, sources: Sequence[Mapping[str, Any]], *, erasure_only: bool = True +) -> bool: + refused = False + inspected = [refusals.inspect(served_row_value(sources[0]))] + keys = {(str(row["org"]), str(row["namespace"]), str(row["user_id"])) for row in sources} + with sequential_reads(): + for key in sorted(keys): + set_query_tag("memory.shared_consent") + refused |= ( + await _matches_postgres(conn, key, inspected, erasure_only=erasure_only) + if conn is not None + else await _matches_clickhouse(ch, key, inspected, erasure_only=erasure_only) + ) + return refused + + +def _policy( + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> list[dict[str, Any]]: + if _writer is None: + raise RuntimeError("Refusal policy writer is not wired") + result = [] + for row in rows: + if row.get("value_truncated"): + raise ValidationError("A complete value is required to record refusal policy") + value = served_row_value(row) + if row.get("capture_kind") == "metadata" and value in (None, "", {}, []): + continue + fingerprints = ( + refusals.payload_fingerprints(value) + if row.get("capture_kind") == "payload" + else refusals.fingerprint(value) + ) + for kind, units, digest in sorted(fingerprints): + result.append( + { + **{ + c: str(row.get(c) or "-") + for c in ("org", "namespace", "agent", "user_id", "session_id") + }, + "scope_kind": scope_kind, + "scope_value": scope_value + if scope_value is not None + else str(row["namespace"]), + "actor": _writer() or "process:memory_refusal", + "source_class": "derived", + "kind": kind, + "units": units, + "digest": digest, + "created_at": naive_utc_now(), + } + ) + return result + + +def _recording_policy(rows: Sequence[Mapping[str, Any]], **scope: Any) -> list[dict[str, Any]]: + policy = _policy(rows, **scope) + if scope.get("scope_kind") == "user_id": + others = [r for r in rows if str(r.get("user_id") or "-") != scope.get("scope_value")] + policy += _policy(others, scope_kind="erased_identity") + return policy + + +def _live_groups(rows: Sequence[Mapping[str, Any]]) -> dict[tuple[str, str, str], list[Any]]: + groups: dict[tuple[str, str, str], list[Any]] = {} + for row in rows: + if row.get("tombstoned_at") is None: + key = (str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-")) + groups.setdefault(key, []).append(refusals.inspect(served_row_value(row))) + return groups + + +def _candidate_batch(inspected: list[Any], sizes: list[int]) -> list[refusals.Fingerprint]: + if -1 in sizes: + raise BackendUnavailableError("Memory intake waits for the organization's erasure") + found: set[refusals.Fingerprint] = set() + for value in inspected: + found.update(refusals.candidates(value, sizes)) + return sorted(found) + + +def _max_units(inspected: list[Any]) -> int: + return max((len(words) for _, texts in inspected for words in texts), default=0) + + +def _refused() -> ValidationError: + return ValidationError("The value is refused by a previous deletion", reason="forgotten") + + +def _check_groups( + rows: Sequence[Mapping[str, Any]], +) -> Iterator[tuple[tuple[str, str, str], list[Any], bool]]: + yield from ((key, values, False) for key, values in _live_groups(rows).items()) + dead = [{**r, "tombstoned_at": None} for r in rows if r.get("tombstoned_at") is not None] + yield from ((key, values, True) for key, values in _live_groups(dead).items()) + + +async def _matches_clickhouse( + ch: _CHClient, key: tuple[str, str, str], inspected: list[Any], *, erasure_only: bool = False +) -> bool: + scope = _CH_ERASURE_SCOPE if erasure_only else _CH_SCOPE + params = {"org": key[0], "ns": key[1], "uid": key[2], "max": _max_units(inspected)} + lengths = await ch.query_all( + "SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals " + "WHERE org={org:String} AND " + scope + " AND kind='literal' AND units <= {max:UInt32} " + "UNION ALL SELECT toInt64(-1) AS units FROM loom.erasure_requests FINAL " + "WHERE org={org:String} AND status IN ('queued', 'in_progress', 'failed') " + "AND NOT startsWith(failure_reason, 'max_attempts (') LIMIT 1", + params=params, + ) + candidates = _candidate_batch(inspected, [int(r["units"]) for r in lengths]) + for batch in chunks(candidates): + match = await ch.query_all( + "SELECT 1 FROM loom.memory_refusals WHERE org={org:String} AND " + + scope + + " AND (kind, units, digest) IN " + "{candidates:Array(Tuple(String,UInt32,FixedString(64)))} LIMIT 1", + params={**params, "candidates": batch}, + ) + if match: + return True + return False + + +async def check_clickhouse(ch: _CHClient, rows: Sequence[Mapping[str, Any]]) -> None: + with sequential_reads(): + for key, inspected, erasure_only in _check_groups(rows): + if await _matches_clickhouse(ch, key, inspected, erasure_only=erasure_only): + raise _refused() + + +async def record_clickhouse( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> None: + policy = _recording_policy(rows, scope_kind=scope_kind, scope_value=scope_value) + if policy: + await ch.insert( + "loom.memory_refusals", policy, settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + + +async def lock_postgres( + conn: Any, + rows: Sequence[Mapping[str, Any]], + *, + forgetting: bool = False, + erasure: tuple[str, str, str] | None = None, +) -> None: + """Lock scope admission before generation rows, including empty/new namespaces.""" + scopes: dict[str, bool] = {} + orgs = {str(row["org"]) for row in rows} + if erasure is not None: + orgs.add(erasure[1]) + # Every transaction takes organization locks first, including multi-namespace erasure. + for org in sorted(orgs): + fn = "pg_advisory_xact_lock" if erasure is not None else "pg_advisory_xact_lock_shared" + await conn.execute(f"SELECT {fn}(hashtextextended($1, 0))", _scope("org", org, "")) + for row in rows: + org = str(row["org"]) + scopes[_scope("namespace", org, str(row["namespace"]))] = False + scopes[_scope("user_id", org, str(row.get("user_id") or "-"))] = forgetting + if erasure is not None: + scopes[_scope(*erasure)] = True + for name, exclusive in sorted(scopes.items()): + fn = "pg_advisory_xact_lock" if exclusive else "pg_advisory_xact_lock_shared" + await conn.execute(f"SELECT {fn}(hashtextextended($1, 0))", name) + + +async def _matches_postgres( + conn: Any, key: tuple[str, str, str], inspected: list[Any], *, erasure_only: bool = False +) -> bool: + scope = _PG_ERASURE_SCOPE if erasure_only else _PG_SCOPE + lengths = await conn.fetch( + "SELECT DISTINCT units::bigint AS units FROM loom.memory_refusals " + "WHERE org=$1 AND " + scope + " AND kind='literal' AND units<=$4 " + "UNION ALL SELECT -1::bigint FROM loom.erasure_ledger " + "WHERE org=$1 AND completed_at IS NULL AND abandoned_at IS NULL", + *key, + _max_units(inspected), + ) + candidates = _candidate_batch(inspected, [int(r["units"]) for r in lengths]) + for start in range(0, len(candidates), 2000): + batch = candidates[start : start + 2000] + match = await conn.fetchval( + "SELECT 1 FROM loom.memory_refusals r JOIN unnest($4::text[], $5::integer[], " + "$6::text[]) AS k(kind, units, digest) USING(kind, units, digest) " + "WHERE r.org=$1 AND " + scope + " LIMIT 1", + *key, + [p[0] for p in batch], + [p[1] for p in batch], + [p[2] for p in batch], + ) + if match: + return True + return False + + +async def check_postgres( + conn: Any, rows: Sequence[Mapping[str, Any]], *, erasure_only: bool = False +) -> None: + for key, inspected, retired in _check_groups(rows): + if await _matches_postgres(conn, key, inspected, erasure_only=erasure_only or retired): + raise _refused() + + +async def record_postgres( + conn: Any, + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> None: + policy = _recording_policy(rows, scope_kind=scope_kind, scope_value=scope_value) + if policy: + for row in policy: + row["created_at"] = row["created_at"].replace(tzinfo=UTC) + columns = tuple(policy[0]) + placeholders = ",".join(f"${n}" for n in range(1, len(columns) + 1)) + await conn.executemany( + f"INSERT INTO loom.memory_refusals ({','.join(columns)}) VALUES ({placeholders}) " + "ON CONFLICT DO NOTHING", + [tuple(row[c] for c in columns) for row in policy], + ) + + +def content_policy_row(row: Mapping[str, Any], value: Any) -> dict[str, Any]: + return { + **{ + c: str(row.get(c) or "-") + for c in ("org", "namespace", "agent", "user_id", "session_id") + }, + "memory_key": str(row.get("memory_key") or row.get("parent_memory_key") or "-"), + "value_json": json.dumps(value, default=str), + "value": json.dumps(value, default=str), + "tombstoned_at": row.get("tombstoned_at"), + } + + +async def check_intake(ch: _CHClient, row: Mapping[str, Any], value: Any) -> None: + policy = [content_policy_row(row, value)] + if pg.enabled(): + async with pg.transaction() as conn: + await lock_postgres(conn, policy) + await check_postgres(conn, policy) + else: + await check_clickhouse(ch, policy) + + +def _raw_content_values(row: Mapping[str, Any]) -> list[Any]: + values = [ + row[c] + for c in ("content", "caption", "source_url", "source_local_path") + if row.get(c) not in (None, "") + ] + metadata = row.get("metadata") + if isinstance(metadata, str): + try: + metadata = json.loads(metadata) + except (ValueError, TypeError): + pass + if metadata not in (None, "", {}, []): + values.append(metadata) + return values + + +async def insert_content_rows(ch: _CHClient, table: str, rows: Sequence[Mapping[str, Any]]) -> None: + """Refuse raw content under the same policy fences as canonical memory.""" + if table not in {"loom.memory_evidence", "loom.memory_chunks"}: + raise ValueError("unsupported refusal-protected content table") + policy = [content_policy_row(row, _raw_content_values(row)) for row in rows] + if policy: + async with publication(ch, policy): + if table == "loom.memory_evidence": + await ch.insert( + "loom.memory_evidence", + [dict(r) for r in rows], + settings=batched_insert_settings(), + ) + else: + await ch.insert( + "loom.memory_chunks", + [dict(r) for r in rows], + settings=committed_insert_settings(), + ) + + +async def _embedding_sources( + ch: _CHClient, rows: Sequence[Mapping[str, Any]] +) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]: + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import build_memory_version_row, identity_key + + keys = [identity_key(r) for r in rows] + if pg.enabled(): + heads = await pg_memory.heads_by_keys(keys, include_tombstoned=True, durable=True) + else: + found = ( + await memory_rows_by_identity_keys( + keys, ch=ch, sequential=True, include_tombstoned=True + ) + if keys + else {} + ) + heads = {k: build_memory_version_row(h) for k, h in found.items()} + accepted: list[Mapping[str, Any]] = [] + sources: list[Mapping[str, Any]] = [] + for row in rows: + head = heads.get(identity_key(row)) + if row.get("tombstoned_at") is not None: + accepted.append(row) + if head is not None: + sources.append(head) + continue + if ( + head is not None + and head.get("tombstoned_at") is None + and hash_text(row.get("content_sha256")) == sha256_value(served_row_value(head)) + and row.get("status", "active") == head.get("status", "active") + ): + accepted.append(row) + sources.append(head) + return accepted, sources + + +def _publication_values( + sources: Sequence[Mapping[str, Any]], admitted: Sequence[Mapping[str, Any]], *, embeddings: bool +) -> list[Mapping[str, Any]]: + from server.memory.store.rows import identity_key + + admitted_keys = {identity_key(r) for r in admitted} + values = [s for s in sources if not embeddings or identity_key(s) in admitted_keys] + if embeddings: + values.extend( + content_policy_row(row, row["content_excerpt"]) + for row in admitted + if row.get("tombstoned_at") is None and row.get("content_excerpt") + ) + return values + + +@asynccontextmanager +async def publication( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + *, + embeddings: bool = False, + committed: bool = False, + retired: list[dict[str, Any]] | None = None, +) -> AsyncIterator[Sequence[Mapping[str, Any]]]: + """Hold the policy admission through the complete memory/vector send.""" + async with lease_writes.fenced_many( + fence_names(rows), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark: + if pg.enabled(): + from server.memory.store import pg_memory + from server.memory.store.rows import identity_key + + # A durable read must finish before any bytes it returned can be published. + admitted, sources = await _embedding_sources(ch, rows) if embeddings else (rows, rows) + async with pg.transaction() as conn: + await lock_postgres(conn, rows) + if embeddings: + current = await pg_memory.heads_on_connection( + conn, [identity_key(r) for r in rows], lock=True + ) + unchanged = { + identity_key(r) for r in sources if current.get(identity_key(r)) == r + } + admitted = [ + r + for r in admitted + if identity_key(r) in unchanged + or ( + r.get("tombstoned_at") is not None + and identity_key(r) not in current + and identity_key(r) not in {identity_key(s) for s in sources} + ) + ] + sources = [r for r in sources if identity_key(r) in unchanged] + if embeddings: + admitted = await _embedding_retirements( + ch, admitted, sources, mark, retired, conn=conn + ) + await check_postgres( + conn, + _publication_values(sources, admitted, embeddings=embeddings), + # Authority commits enforce point refusal before becoming history. + erasure_only=committed and not embeddings, + ) + mark(True) + yield admitted + mark(False) + else: + admitted, sources = await _embedding_sources(ch, rows) if embeddings else (rows, rows) + if embeddings: + admitted = await _embedding_retirements(ch, admitted, sources, mark, retired) + await check_clickhouse( + ch, _publication_values(sources, admitted, embeddings=embeddings) + ) + mark(True) + yield admitted + mark(False) + + +async def _embedding_retirements( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + sources: Sequence[Mapping[str, Any]], + mark: Callable[[bool], None], + retired: list[dict[str, Any]] | None, + *, + conn: Any = None, +) -> list[Mapping[str, Any]]: + from server.memory.domain.redaction_lineage import is_content_hash + from server.memory.store.embeddings import _bare_tombstone + from server.memory.store.rows import identity_key + + heads = {identity_key(r): r for r in sources} + accepted: list[Mapping[str, Any]] = [] + for row in rows: + if row.get("tombstoned_at") is None: + accepted.append(row) + continue + head = heads.get(identity_key(row)) + values = [served_row_value(head)] if head is not None else [] + if row.get("content_excerpt"): + values.append(row["content_excerpt"]) + inspected = refusals.inspect(values) + digest = hash_text(row.get("content_sha256")) + if is_content_hash(digest): + inspected[0].add(("value", 0, digest)) + key = (str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-")) + with sequential_reads(): + erased = ( + await _matches_postgres(conn, key, [inspected], erasure_only=True) + if conn is not None + else await _matches_clickhouse(ch, key, [inspected], erasure_only=True) + ) + if head is None or erased: + mark(True) + await _delete_old_vector(ch, row) + mark(False) + if retired is not None: + retired.append(dict(row)) + continue + refused = ( + await _matches_postgres(conn, key, [inspected]) + if conn is not None + else await _matches_clickhouse(ch, key, [inspected]) + ) + accepted.append( + _bare_tombstone(dict(row), int(row["version"]), naive_utc_now()) if refused else row + ) + return accepted + + +async def _delete_old_vector(ch: _CHClient, row: Mapping[str, Any]) -> None: + """Clean an orphan without introducing a row after a physical erasure.""" + set_query_tag("memory.retire_orphan_vector") + await ch.execute( + "DELETE FROM loom.memory_embeddings WHERE org={org:String} " + "AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} " + "AND session_id={session_id:String} AND scope={scope:String} " + "AND embedding_model={embedding_model:String} AND memory_key={memory_key:String} " + "AND version < {version:UInt64}", + params={ + c: row[c] + for c in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + "version", + ) + }, + settings={"lightweight_deletes_sync": 2}, + ) diff --git a/server/memory/store/shared.py b/server/memory/store/shared.py index 475bb95872..d70c4bcdf2 100644 --- a/server/memory/store/shared.py +++ b/server/memory/store/shared.py @@ -12,6 +12,7 @@ from __future__ import annotations +import json from datetime import datetime from typing import Any @@ -63,6 +64,16 @@ async def insert_shared_row( f"missing={missing} unknown={unknown}" ) ch = ch or _default_client() + from server.memory.store.refusals import assert_shared_publication, shared_retirement + + if row.get("tombstoned_at") is not None: + async with shared_retirement(ch, row) as publish: + if publish: + await ch.insert( + "loom.shared", [row], settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + return + assert_shared_publication(row) await ch.insert( "loom.shared", [row], @@ -86,6 +97,9 @@ async def insert_shared_embedding( from server.memory.store.rows import build_embedding_row ch = ch or _default_client() + from server.memory.store.refusals import assert_shared_publication + + assert_shared_publication(row, vector=True) await ch.insert( "loom.memory_embeddings", [build_embedding_row(**row)], @@ -93,6 +107,94 @@ async def insert_shared_embedding( ) +async def shared_embedding_source( + row: dict[str, Any], *, ch: _CHClient +) -> tuple[dict[str, Any], Any, set[str]] | None: + """The complete approved source behind this shared cache row, if it still agrees.""" + from server.core import pg + from server.core.hashing import served_row_value, sha256_value + from server.memory.domain.excerpt import build_excerpt + from server.memory.domain.promotion_source import promotion_keys, promotion_users + from server.memory.domain.temporal_sql import SERVED_STATUSES + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import _hydrate + + shared = await fetch_shared(row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch) + if not shared or shared.get("tombstoned_at") is not None: + return None + entries = shared.get("promoted_from") or [] + users = promotion_users(entries, row["org"], row["namespace"]) + if users is None: + return None + keys = promotion_keys(entries, row["org"], row["namespace"]) + if keys is None: + return None + if pg.enabled(): + raw_heads = await pg_memory.heads_by_keys(keys, include_tombstoned=True, durable=True) + heads = {key: _hydrate(head) for key, head in raw_heads.items()} + else: + heads = await memory_rows_by_identity_keys( + keys, include_tombstoned=True, sequential=True, ch=ch + ) + digest = sha256_value(served_row_value(shared)) + for key in keys: + source = heads.get(key) + if source is None or sha256_value(source.value) != digest: + continue + if ( + source.status not in SERVED_STATUSES + or source.requires_review + or source.tombstoned_at + or source.sensitivity == "high" + ): + continue + if row.get("content_excerpt") == build_excerpt( + json.dumps(source.value, default=str), memory_key=source.memory_key + ): + return shared, source, users + return None + + +async def repair_shared_embeddings( + rows: list[dict[str, Any]], *, ch: _CHClient +) -> list[dict[str, Any]]: + """Repair a shared mirror only while its full promoted source still qualifies.""" + from server.core.hashing import sha256_value + from server.memory.domain.trust import verification_state + from server.memory.store.refusals import source_publication + + written = [] + for row in rows: + proof = await shared_embedding_source(row, ch=ch) + if proof is None: + continue + shared, source, users = proof + digest = sha256_value(source.value) + async with source_publication(ch, source, row["memory_key"], refusal_users=users) as mark: + current = await fetch_shared( + row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch + ) + if current != shared: + continue + mark(True) + await insert_shared_embedding( + { + **row, + "content_sha256": digest, + "source_class": source.source_class, + "verified": int( + verification_state(source.verified_by, source.verified_sha256, digest) + in ("human", "process") + ), + }, + ch=ch, + ) + mark(False) + written.append(row) + return written + + async def fetch_shared( org: str, namespace: str, diff --git a/server/memory/store/source_review.py b/server/memory/store/source_review.py deleted file mode 100644 index 7fb4134110..0000000000 --- a/server/memory/store/source_review.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Keep promotion tied to the reviewed source through its shared writes.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager - -from server.core import lease_writes, pg -from server.core.errors import AuthInsufficientError -from server.core.repo_base import _default_client -from server.identity.domain.read_scope import WHOLE_NAMESPACE -from server.memory.domain.proposal_review import pending_review -from server.memory.store import pg_memory, write_fence -from server.memory.store.rows import Identity, MemoryRow, identity_key - - -def require_reviewed(source: MemoryRow) -> None: - if pending_review(source): - raise AuthInsufficientError("An unreviewed proposal cannot be promoted") - - -@asynccontextmanager -async def publication(source: MemoryRow) -> AsyncIterator[Callable[[bool], None]]: - import server.memory.store as memory_repo - - require_reviewed(source) - ch = _default_client() - async with lease_writes.fenced_many( - [write_fence.lease_name(identity_key(source))], - ch=write_fence._lease_client(ch), - wait_s=write_fence._admission_wait_s(), - ) as mark: - if pg.enabled(): - async with pg_memory.locked_source(source): - yield mark - else: - current = await memory_repo.get_by_key( - Identity( - org=source.org, - namespace=source.namespace, - agent=source.agent, - user_id=source.user_id, - session_id=source.session_id, - ), - source.memory_key, - scope=WHOLE_NAMESPACE, - strong=True, - fresh=True, - ) - if not pg_memory.unchanged(source, current): - raise pg_memory.AdmissionContendedError("The promotion's source changed") - yield mark diff --git a/server/memory/store/summary_sources.py b/server/memory/store/summary_sources.py index 2be31f0c33..7748cbb238 100644 --- a/server/memory/store/summary_sources.py +++ b/server/memory/store/summary_sources.py @@ -1,24 +1,97 @@ -"""Source-state checks for the session-summary shortlist.""" +"""Verify every digest byte against its authoritative, currently served sources.""" from __future__ import annotations +from collections import defaultdict +from collections.abc import Mapping +from contextlib import AsyncExitStack from typing import Any -from server.core.repo_base import _CHClient, query_all_sliced -from server.memory.domain.temporal_sql import ( - as_of_predicate, - status_predicate, - tombstone_predicate, -) +from server.core import lease_writes, pg +from server.core.ch import sequential_reads +from server.core.datetimes import naive_utc_now +from server.core.repo_base import _CHClient +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.summary_provenance import render_summary, source_text +from server.memory.store import pg_memory +from server.memory.store.reads import memory_rows_by_identity_keys +from server.memory.store.rows import IdentityKey, MemoryRow, _hydrate -SourceKey = tuple[str, str, str, str] +def _identity(row: Mapping[str, Any]) -> tuple[str, str, str]: + return tuple(str(row.get(c) or "-") for c in ("agent", "user_id", "session_id")) # type: ignore[return-value] -def _sources(row: dict[str, Any]) -> list[SourceKey]: - agent, user, session = ( - str(row.get(axis) or "-") for axis in ("agent", "user_id", "session_id") + +def _complete(row: dict[str, Any]) -> bool: + keys, facts = row.get("derived_from"), row.get("key_facts") + return bool( + isinstance(keys, list) + and isinstance(facts, list) + and keys + and len(keys) == len(facts) + and all(isinstance(k, str) and k for k in keys) + and len(set(keys)) == len(keys) + and all(isinstance(f, str) and f for f in facts) + and row.get("summary_text") == render_summary(facts) + and not value_reads_high(row["summary_text"]) + ) + + +async def _source_heads( + ch: _CHClient, + rows: list[dict[str, Any]], + *, + org: str, + namespace: str, + durable: bool = False, + conn: Any = None, +) -> dict[IdentityKey, MemoryRow]: + candidates = [r for r in rows if _complete(r)] + keys = sorted( + {(org, namespace, *_identity(r), key) for r in candidates for key in r["derived_from"]} ) - return [(agent, user, session, str(key)) for key in row.get("derived_from", []) if key] + if not keys: + return {} + # A JSON string may use two UTF-16 escapes per code point, plus its quotes. + cap = 12 * max(len(f) for r in candidates for f in r["key_facts"]) + 2 + if pg.enabled(): + if conn is not None: + found = await pg_memory.heads_on_connection(conn, keys, lock=True, max_value_chars=cap) + else: + found = await pg_memory.heads_by_keys(keys, durable=durable, max_value_chars=cap) + return {key: _hydrate(row) for key, row in found.items()} + with sequential_reads(): + return await memory_rows_by_identity_keys(keys, ch=ch, sequential=True, max_value_chars=cap) + + +def _verified( + rows: list[dict[str, Any]], + heads: Mapping[IdentityKey, MemoryRow], + *, + org: str, + namespace: str, + as_of: Any = None, + as_of_basis: str = "valid_time", +) -> list[dict[str, Any]]: + now = naive_utc_now() + result = [] + for row in rows: + if not _complete(row): + continue + texts = [] + for key in row["derived_from"]: + source = heads.get((org, namespace, *_identity(row), key)) + text = ( + source_text(source, now=now, as_of=as_of, as_of_basis=as_of_basis) + if source + else None + ) + if text is None: + break + texts.append(text) + if texts == row["key_facts"]: + result.append(row) + return result async def current_summary_sources( @@ -30,55 +103,77 @@ async def current_summary_sources( as_of: Any = None, as_of_basis: str = "valid_time", ) -> list[dict[str, Any]]: - """Withhold digests of known unserved or rewritten sources, within the full identity. - - A digest has no source revision hashes, so a later version conservatively invalidates - it. Missing lineage remains session evidence; it is not proof of a current fact. - """ - sources = sorted({key for row in rows for key in _sources(row)}) - if not sources: - return rows - replaying = as_of is not None - state = [ - tombstone_predicate(as_of_basis, replaying=replaying), - status_predicate(as_of_basis, replaying=replaying), - "sensitivity != 'high'", - "(expires_at IS NULL OR expires_at > now())", - "(valid_to IS NULL OR valid_to > now())", - ] - params: dict[str, Any] = {"org": org, "ns": namespace, "sources": sources} - if replaying: - params["as_of"] = as_of - state.append(as_of_predicate(as_of_basis)) - parents = await query_all_sliced( - ch, - f""" - SELECT agent, if(user_id = '', '-', user_id) AS source_user, - if(session_id = '', '-', session_id) AS source_session, memory_key, - version, {" AND ".join(state)} AS source_served - FROM loom.memory FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND (agent, if(user_id = '', '-', user_id), if(session_id = '', '-', session_id), - memory_key) IN {{sources:Array(Tuple(String,String,String,String))}} - """, - params, - "sources", + """Withhold missing, unverifiable and rewritten lineage, including legacy digests.""" + heads = await _source_heads(ch, rows, org=org, namespace=namespace) + return _verified( + rows, heads, org=org, namespace=namespace, as_of=as_of, as_of_basis=as_of_basis ) - heads: dict[SourceKey, tuple[bool, int]] = {} - for p in parents: - key = ( - str(p["agent"]), - str(p["source_user"]), - str(p["source_session"]), - str(p["memory_key"]), - ) - served, version = heads.get(key, (True, 0)) - heads[key] = (served and bool(p["source_served"]), max(version, int(p["version"]))) - return [ - row - for row in rows - if all( - key in heads and heads[key][0] and heads[key][1] <= int(row["version"]) - for key in _sources(row) + + +async def verified_summary_batch(ch: _CHClient, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Verify a worker batch without mixing source identities across tenants.""" + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + if row.get("org") and row.get("namespace"): + groups[(str(row["org"]), str(row["namespace"]))].append(row) + verified: set[int] = set() + for (org, namespace), batch in groups.items(): + verified.update( + id(r) for r in await current_summary_sources(ch, batch, org=org, namespace=namespace) ) - ] + return [row for row in rows if id(row) in verified] + + +async def insert_verified_summaries(ch: _CHClient, rows: list[dict[str, Any]]) -> int: + """Publish only current proven text, fenced against deletion and erasure.""" + from server.memory.store import refusals + + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + groups[(str(row["org"]), str(row["namespace"]))].append(row) + written = 0 + for (org, namespace), batch in groups.items(): + policy = [ + refusals.content_policy_row(r, [r["summary_text"], *r["key_facts"]]) for r in batch + ] + async with lease_writes.fenced_many( + refusals.fence_names(policy), + ch=refusals._lease_client(ch), + wait_s=refusals._admission_wait_s(), + ) as mark: + async with AsyncExitStack() as stack: + if pg.enabled(): + durable = await _source_heads( + ch, batch, org=org, namespace=namespace, durable=True + ) + conn = await stack.enter_async_context(pg.transaction()) + await refusals.lock_postgres(conn, policy) + current = await _source_heads( + ch, batch, org=org, namespace=namespace, conn=conn + ) + heads = { + key: row + for key, row in current.items() + if key in durable and pg_memory.unchanged(durable[key], row) + } + else: + heads = await _source_heads(ch, batch, org=org, namespace=namespace) + admitted = _verified(batch, heads, org=org, namespace=namespace) + admitted_policy = [ + refusals.content_policy_row(r, [r["summary_text"], *r["key_facts"]]) + for r in admitted + ] + if pg.enabled(): + await refusals.check_postgres(conn, admitted_policy) + else: + await refusals.check_clickhouse(ch, admitted_policy) + if admitted: + mark(True) + await ch.insert( + "loom.session_summaries", + admitted, + settings={"async_insert": 0, "wait_for_async_insert": 1}, + ) + mark(False) + written += len(admitted) + return written diff --git a/server/memory/store/write_fence.py b/server/memory/store/write_fence.py index 0ec927f9a2..ced15c7ba2 100644 --- a/server/memory/store/write_fence.py +++ b/server/memory/store/write_fence.py @@ -36,6 +36,7 @@ async def checked_write( *, expected_heads: Mapping[IdentityKey, MemoryRow | None] | None, ch: _CHClient, + refuse_values: bool = False, ) -> AsyncIterator[None]: """Every value writer takes these fences too, closing the check/insert window. @@ -45,7 +46,13 @@ async def checked_write( keys = {identity_key(r) for r in rows} if expected_heads is not None and set(expected_heads) != keys: raise ValueError("expected heads must name exactly the inserted keys") - names = [lease_name(key) for key in sorted(keys)] + from server.memory.store import refusals + + names = sorted( + set( + [lease_name(key) for key in keys] + refusals.fence_names(rows, forgetting=refuse_values) + ) + ) async with lease_writes.fenced_many( names, ch=_lease_client(ch), wait_s=_admission_wait_s() ) as mark: @@ -60,6 +67,9 @@ async def checked_write( for key, head in expected_heads.items() ): raise AdmissionContendedError("the memory changed while it was read") + await refusals.check_clickhouse(ch, rows) mark(True) + if refuse_values: + await refusals.record_clickhouse(ch, rows) yield mark(False) diff --git a/server/memory/workers/embedding.py b/server/memory/workers/embedding.py index 5dfac20a09..373898bd36 100644 --- a/server/memory/workers/embedding.py +++ b/server/memory/workers/embedding.py @@ -289,13 +289,19 @@ async def _stubs_still_served( rows: list[dict[str, Any]], stub_embedding: list[float], now: datetime ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """The stubs whose head may still keep a vector, and tombstones (two above the - stub, its excerpt cleared) for the rest. Shared-scope stubs have no head of - their own and are kept.""" + stub, its excerpt cleared) for the rest. Shared stubs require their promoted + source proof before any provider work; an unbacked cache row is withheld.""" + from server.core.ch import get_client from server.memory.store import read_your_writes_enabled + from server.memory.store.shared import shared_embedding_source + shared = { + id(row) + for row in rows + if row.get("scope") == "shared" + and await shared_embedding_source(row, ch=get_client()) is not None + } own = [r for r in rows if str(r.get("scope") or "") != "shared"] - if not own: - return rows, [] sequential = read_your_writes_enabled() heads = await memory_repo.head_sensitivities( sorted({memory_repo.identity_key(r) for r in own}), sequential=sequential @@ -303,7 +309,7 @@ async def _stubs_still_served( def served(r: dict[str, Any]) -> bool: if str(r.get("scope") or "") == "shared": - return True + return id(r) in shared head = heads.get(memory_repo.identity_key(r)) return head != "high" and (head is not None or not sequential) @@ -317,7 +323,7 @@ def served(r: dict[str, Any]) -> bool: embedding=stub_embedding, tombstoned_at=now, ) - for r in rows + for r in own if not served(r) ] return keep, dropped @@ -407,6 +413,29 @@ def _outcome(result: EmbeddingWorkerResult, *, idle: bool, rows_written: int) -> } +def _record_fill_outcomes( + replacements: list[dict[str, Any]], retries: list[dict[str, Any]], result: EmbeddingWorkerResult +) -> None: + result.embedded = [ + {"namespace": row["namespace"], "memory_key": row["memory_key"]} + for row in replacements + if row.get("tombstoned_at") is None and row.get("embedding_dim", 0) > 0 + ] + result.failed, result.poisoned = [], [] + for row in retries: + if row.get("embed_last_error") == HELD: + continue + target = result.poisoned if row.get("embed_failed_at") is not None else result.failed + target.append( + { + "namespace": row["namespace"], + "memory_key": row["memory_key"], + "attempts": row.get("embed_attempts", 0), + "error": row.get("embed_last_error", ""), + } + ) + + async def run_pass( cfg: EmbeddingWorkerConfig | None = None, ) -> EmbeddingWorkerResult: @@ -448,7 +477,7 @@ async def run_pass( since_seconds=_orphan_scan_window(), ) if orphans: - await memory_repo.insert_embedding_rows( + reconciled = await memory_repo.insert_embedding_rows( [ memory_repo.build_embedding_row( **{k: v for k, v in o.items() if k != "created_at"}, @@ -465,12 +494,12 @@ async def run_pass( ] ) result.reconciled = [ - {"namespace": o["namespace"], "memory_key": o["memory_key"]} for o in orphans + {"namespace": o["namespace"], "memory_key": o["memory_key"]} for o in reconciled ] log.warning( "embedding worker: reconciled %d memories that had NO embedding " "row (unsearchable until now)", - len(orphans), + len(reconciled), ) except Exception as e: # noqa: BLE001 — repair is additive; never break the pass # The pass goes on, the supervisor hears about it. @@ -493,7 +522,7 @@ async def run_pass( # to the embedding provider: its excerpt may hold the high text. It is retired. rows, dropped = await _stubs_still_served(rows, stub_embedding, now) if dropped: - await memory_repo.insert_embedding_rows(dropped) + dropped = await memory_repo.insert_embedding_rows(dropped) writers = await memory_writers(rows) if rows and identity.reads_writer() else {} if rows: embedded = await _embed_rows(embedder, rows, writers) @@ -552,31 +581,18 @@ async def run_pass( ) continue replacements.extend(_replacement_rows(r, vec, embedder, stub_embedding, now)) - result.embedded.append( - { - "namespace": r["namespace"], - "memory_key": r["memory_key"], - } - ) - # Entities can advance between the stub read and this insert - # (the entity extractor's apply_entities_batch re-emits version rows); - # a re-emit built from the stale snapshot would resurrect the - # old list under FINAL. - # Re-read at insert time to shrink that window from - # minutes (embed round-trips) to milliseconds. + # Refresh mutable facets, then report only acknowledged publications. await _refresh_entities(replacements + retries) - await memory_repo.insert_embedding_rows(replacements) - # A stub drawn from a read before a 'high' write landed can sit above - # its fence: the fill re-checks its heads like every vector copy. + replacements = await memory_repo.insert_embedding_rows(replacements) await memory_repo.retire_copies_under_unembedded_heads(replacements) - # A fill sits at its stub's version + 1, usually far under the BM25 watermark. await memory_repo.index_late_rows(replacements) - await memory_repo.insert_embedding_rows(retries) + retries = await memory_repo.insert_embedding_rows(retries) + _record_fill_outcomes(replacements, retries, result) rec.outcome = _outcome( result, idle=not rows and not result.reconciled, - rows_written=len(replacements) + len(retries) + len(result.reconciled), + rows_written=len(replacements) + len(retries) + len(result.reconciled) + len(dropped), ) log.info( diff --git a/server/memory/workers/usage_fold.py b/server/memory/workers/usage_fold.py index e6c78c2034..9d17c28aae 100644 --- a/server/memory/workers/usage_fold.py +++ b/server/memory/workers/usage_fold.py @@ -111,7 +111,11 @@ async def query_one( ) -> dict[str, Any] | None: ... async def query_all( - self, sql: str, params: dict[str, Any] | None = None + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: ... async def execute( diff --git a/server/observability/langfuse_prompts.lock.json b/server/observability/langfuse_prompts.lock.json index a107fc7b03..0eb1c0d9b8 100644 --- a/server/observability/langfuse_prompts.lock.json +++ b/server/observability/langfuse_prompts.lock.json @@ -38,10 +38,6 @@ "name": "loom-period-digest", "version": 1 }, - "84e51221d19f": { - "name": "loom-session-summary", - "version": 7 - }, "87e36f72aac0": { "name": "loom-code-query-expand", "version": 2 diff --git a/server/settings/catalog.py b/server/settings/catalog.py index aa657c59b5..d1734acb01 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -2780,8 +2780,8 @@ class MemorySettings(BaseModel): # memory.list rows per call for a token below writer on the namespace; it # pages with `offset` (wire cap stays 1000). 0 = no viewer cap. memory_list_viewer_limit: int = 100 - # Transcript characters the session-digest call reads (~4 chars a token); - # 0 = the built-in 3000. + # Characters of complete source facts included in an extractive session digest; + # 0 = the built-in 3000. Facts are never clipped to fit. memory_session_digest_max_input_chars: int = 3000 # content_excerpt ceiling for answer-bearing facts (question_topic set) # when memory_capture_structured_answers is on, so an enumerated answer @@ -4443,7 +4443,7 @@ class ObservabilitySettings(BaseModel): # "off" records none (the default; consent is given, not withdrawn); "query" only # the legs whose prompt is the user's own search text (hyde, query_rewrite, # enrichment, …); "all" also the legs carrying retrieved memories or conversation - # turns (inline_rerank, iterative_judge, extract, session_summary, …). + # turns (inline_rerank, iterative_judge, extract, …). # A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ # all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token # counts, cost, latency and quality scores are never gated. @@ -6793,8 +6793,6 @@ class TrackerSettings(BaseModel): # session.brief: one window's revision (about 600-1,500 tokens); a cut-off answer is # salvaged key by key. "session_brief_revision": 3000, - # The session summary: one to three dated sentences. - "session_summary": 160, # memory.ask: the judge of an answer against its evidence. "ask_consistency": 200, # memory.ask: the split of a question into sub-questions. diff --git a/server/wiring.py b/server/wiring.py index bf716e334a..9918b6d7f8 100644 --- a/server/wiring.py +++ b/server/wiring.py @@ -21,11 +21,12 @@ def wire() -> None: from server.code.domain.index import repos from server.code.domain.index.compiler_tiers import TIERS from server.core.ch import add_insert_observer + from server.erasure.store.consent import ConsentPolicy, use_policy from server.identity.app.directory import IdentityDirectory from server.knowledge.domain.live_chunks import LIVE_CHUNK_SQL from server.memory.app import provenance, recent_writes from server.memory.domain.memory_types import NON_SEARCHABLE_TYPES - from server.memory.store import pg_publish + from server.memory.store import pg_publish, refusals from server.memory.store.usage import AGENT_LABEL_EVENT_TYPES from server.observability.budget_notifications import use_observer from server.observability.state_inputs import StateInputs, use_state_inputs @@ -35,6 +36,16 @@ def wire() -> None: use_directory(IdentityDirectory()) use_observer(schedule) pg_publish.use_writer(provenance.writer_actor) + refusals.use_writer(provenance.writer_actor) + use_policy( + ConsentPolicy( + fence_names=lambda org, kind, value: refusals.erasure_fence_names(org, kind, value), + lease_client=lambda ch: refusals._lease_client(ch), + lock_postgres=lambda *args, **kwargs: refusals.lock_postgres(*args, **kwargs), + record_clickhouse=lambda *args, **kwargs: refusals.record_clickhouse(*args, **kwargs), + record_postgres=lambda *args, **kwargs: refusals.record_postgres(*args, **kwargs), + ) + ) use_state_inputs( StateInputs( live_chunk_sql=LIVE_CHUNK_SQL, diff --git a/tests/analyzer/cases/erasure.py b/tests/analyzer/cases/erasure.py index e3994a49f5..881361356e 100644 --- a/tests/analyzer/cases/erasure.py +++ b/tests/analyzer/cases/erasure.py @@ -170,6 +170,14 @@ async def _exports(ch: RecordingCH, c: dict[str, Any]) -> None: await erasure_exports.export_audit_by_subject_substring(ORG, "alice", limit=100, ch=ch) +async def _consent_capture(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.erasure.store import refusals + + for kind, value in (("namespace", "n"), ("user_id", "alice")): + for table, projection in refusals._sources(): + await refusals._capture_source(ch, table, projection, ORG, kind, value) + + def cases() -> list[Case]: steps: list[Case] = [] for module in STEP_MODULES: @@ -199,6 +207,7 @@ def cases() -> list[Case]: ] return [ *steps, + Case("erasure.consent_capture", "server.erasure.store.refusals", _consent_capture, {}), Case( "erasure.census_and_ledger", CENSUS_MODULE, diff --git a/tests/analyzer/cases/legs.py b/tests/analyzer/cases/legs.py index e0e78e805b..ab7e017aa8 100644 --- a/tests/analyzer/cases/legs.py +++ b/tests/analyzer/cases/legs.py @@ -593,7 +593,7 @@ async def _entity_write(ch: RecordingCH, c: dict[str, Any]) -> None: async def _deep_evidence(ch: RecordingCH, c: dict[str, Any]) -> None: - from server.memory.app.deep_evidence import fetch_raw_evidence, fetch_session_turns + from server.memory.app.deep_evidence import fetch_raw_evidence await fetch_raw_evidence( org=ORG, @@ -603,15 +603,6 @@ async def _deep_evidence(ch: RecordingCH, c: dict[str, Any]) -> None: max_chars_total=4000, ch=ch, ) - await fetch_session_turns( - org=ORG, - namespace=NS, - session_ids=["s1"], - question="q", - max_chars_per_session=800, - max_chars_total=4000, - ch=ch, - ) async def _stale_links(ch: RecordingCH, c: dict[str, Any]) -> None: diff --git a/tests/analyzer/cases/workers.py b/tests/analyzer/cases/workers.py index 075661a0b2..5fdfc76abd 100644 --- a/tests/analyzer/cases/workers.py +++ b/tests/analyzer/cases/workers.py @@ -292,7 +292,10 @@ async def _period_keys(ch: RecordingCH, c: dict[str, Any]) -> None: from server.memory.app import period_digest ident = {"org": ORG, "namespace": NS, "agent": "bot", "user_id": "alice"} - await period_digest._skipped_period_keys(ch, ident, rewrite=False) + await period_digest._skipped_period_keys(ch, ident, ["period_digest_first_last"], rewrite=False) + for after in (None, (datetime(2026, 1, 1), "s1")): + async for _ in period_digest._fetch_digests(ch, ident, after=after): + pass async def _conflict_resolver(ch: RecordingCH, c: dict[str, Any]) -> None: @@ -399,8 +402,53 @@ async def _source_retirements(ch: RecordingCH, c: dict[str, Any]) -> None: await queue.write(item, pending=False, ch=ch) +async def _refusal_publication(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.memory.app import extraction_queue as queue + from server.memory.domain.refusals import inspect + from server.memory.store import refusals + + key = (ORG, NS, "alice") + await refusals._matches_clickhouse(ch, key, [inspect("a removed passage")]) + await refusals._matches_clickhouse(ch, key, [inspect("a removed passage")], erasure_only=True) + await refusals._delete_old_vector( + ch, + { + "org": ORG, + "namespace": NS, + "agent": "a", + "user_id": "alice", + "session_id": "s", + "memory_key": "k", + "scope": "private", + "embedding_model": "m", + "version": 7, + }, + ) + await refusals._delete_old_shared( + ch, {"org": ORG, "namespace": NS, "memory_key": "k", "version": 7} + ) + row = queue._mirror( + { + "org": ORG, + "namespace": NS, + "agent": "a", + "user_id": "alice", + "session_id": "s", + "queue_id": "q", + "payload": '{"messages":[]}', + "version": 6, + }, + status="done", + ) + await queue._source_present(ch, row) + await queue._drop_refused(ch, row) + + def cases() -> list[Case]: return [ + Case( + "memory.refusal_publication", "server.memory.store.refusals", _refusal_publication, {} + ), Case( "workers.source_retirements", "server.knowledge.store.source_retirements", diff --git a/tests/analyzer/corpus.py b/tests/analyzer/corpus.py index b9e57739b2..a6311aa0cc 100644 --- a/tests/analyzer/corpus.py +++ b/tests/analyzer/corpus.py @@ -163,7 +163,11 @@ def _answer(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: return [r if isinstance(r, _Row) else _Row(r) for r in self._rows(sql, params)] async def query_all( - self, sql: str, params: dict[str, Any] | None = None + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: p = dict(params or {}) self.calls.append(("query_all", sql, p)) diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index dd8f3abfe7..a86c073dbf 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -131,6 +131,7 @@ 0ec5f4fd6da5 SELECT max(version) AS version FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = ? AND session_id = ? AND scope = ? AND memory_k 0ef6f4537013 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org 0f371a9d43dd SELECT memory_key, support_type, char_start, char_end, toString(evidence_id) AS evidence_id FROM ( SELECT memory_key, support_type, char_start, char_end, evidence_id, tombstoned_at FROM loom.memory_be +0f510c19553c SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, ? AS value_json, toJSONString(content_exc 0f5951ba0297 SELECT DISTINCT kind, start_line, end_line FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND tombstoned_at IS NULL 0f5fb44818c9 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, value, ? AS value_truncated, confidence, version, pinned, char_limit, created_at, updated_at, expires_at, valid_a 0fd1cbc1eecb SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ @@ -140,6 +141,7 @@ 1030febb1e77 ALTER TABLE loom.chunks UPDATE doc_superseded_at = {now:DateTime} WHERE (org = {org:String} AND user_id = {uid:String} AND doc_superseded_at IS NULL) AND ({erasure_mutation_tag:String} = {erasure_muta 1032019497c6 SELECT value, memory_type, confidence, promoted_from, promotion_event, promoted_from_candidate_id, promoted_by, rationale, promoted_at, reversible_until, tombstoned_at, version FROM loom.shared FINAL 1036d9982eb4 SELECT concat(?, c.session_id, ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, c.content AS content_excerpt, toString(ss.session_date) AS created_at, toString(ss.session_date) AS valid_at +1069a110039a SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 107be5b763da SELECT count() AS n FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 109db307d85f SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND user_id = {uid:String} AND doc_superseded_at IS NULL 10ff90cbf7b6 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -198,6 +200,7 @@ 17612729b3e2 SELECT * FROM loom.learnings_queue FINAL WHERE org = {org:String} AND candidate_id IN {cids:Array(UUID)} 177807b65c1e SELECT f AS path, uniqExact(pr_number) AS prs FROM loom.code_pr_history FINAL ARRAY JOIN files AS f WHERE org = {org:String} AND namespace = {ns:String} AND merged_at > now() - INTERVAL {days:UInt32} 1799e5db040e SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g +17c5bb172f19 SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value= 17d1d3c496e4 SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)}) 17dac6a35956 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 17e1defc0d07 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -209,6 +212,7 @@ 1891df92bb07 SELECT DISTINCT org FROM ( SELECT org FROM loom.namespaces FINAL WHERE owner_email = {s:String} AND active = ? UNION ALL SELECT org FROM loom.namespace_grants FINAL WHERE lower(subject) = lower({s:Str 18beffc43c3d SELECT memory_key_a, memory_key_b, allowed_at, allowed_by, reason FROM loom.conflict_allowlist FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL ORDER BY allowed_at 1904c89aac43 SELECT uniqExact(path) AS n FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL AND startsWith(path, {prefix +1906e06d78a5 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, ? AS value_json, toJSONString(content_exc 190702c74f85 SELECT ts, actor, subject FROM loom.audit_log WHERE org = ? AND namespace = ? AND event_type = {event:String} AND ({name:String} = ? OR JSONExtractString(subject, ?) = {name:String}) ORDER BY ts DESC 190758ec6ba2 SELECT count() AS n FROM loom.memory_drops WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 19091cfef8cd SELECT memory_key, memory_type, content_excerpt, valid_at, created_at, namespace, count() OVER () AS total_matching FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:Strin @@ -237,7 +241,6 @@ 1bfe4bc713e1 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, 1c34dca1df62 SELECT org, namespace, kind, number, source_id, component, is_open, created_at, updated_at, embedding, embedding_dim, embedding_model, text_sha256, tombstoned_at, version FROM loom.qa_item_embeddings 1c545b956083 SELECT uniqExact(issue_number) AS issues, count() AS pairs, max(computed_at) AS last_computed FROM loom.qa_link_candidates FINAL WHERE org = {org:String} AND qa_namespace = {ns:String} -1c7cbb1b18f8 SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL AN 1ca42e357807 ALTER TABLE loom.entity_triple_rows DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 1ca47c4c11f3 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a 1cb430ccdf4d SELECT ? FROM (SELECT * FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND thread_id = {tid:String} AND tombstoned_at IS NULL AND msg_id IN {ids:Array(Str @@ -287,6 +290,7 @@ 225194c2f6a6 DELETE FROM loom.entity_vectors WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync = ? 22612cf3a394 SELECT namespace, source_uri, public_namespace, public_uri FROM (SELECT namespace, source_uri, indexOf({bindings:Array(Tuple(String, String))}, (namespace, right(source_uri, ?))) AS binding, arrayElem 2261d5bd6945 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND namespace = {ns:String} AND content_sha256 IN {hashes:Array(String)} AND source_uri != {ex:String} AND doc_superseded_at IS NULL AND ( +22e0a45e948a SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(chunk_index), ?, content_sha256) AS memory_ 22e4b395ef49 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} 22eb12673f82 SELECT count() AS n FROM loom.memory_extraction_queue FINAL WHERE org = {org:String} AND user_id = {uid:String} 2303571b09cb ALTER TABLE loom.memory_versions DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -304,6 +308,7 @@ 2448faf35e3e SELECT DISTINCT src_path FROM loom.code_test_map FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL 244b76b58819 ALTER TABLE loom.memory UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta 246241942bf7 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to +2490acfcecb6 DELETE FROM loom.shared WHERE org={org:String} AND namespace={namespace:String} AND memory_key={memory_key:String} AND version < {version:UInt64} 24c3fd0e6b29 SELECT path, chunk_id FROM loom.code_chunks WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} GROUP BY path, chunk_id HAVING argMax(embedding_dim, version) = ? 24d4d8775291 SELECT start_line, end_line, scope, substring(content, greatest(?, position(content, {lit:String}) - ?), ?) AS excerpt FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} 24df34974019 SELECT ts, org, namespace, agent, user_id, session_id, operation, request_id, trace_id, latency_ms, input_tokens, output_tokens, embedding_tokens, upstream_cost_usd, bytes_written, success, error_code @@ -396,6 +401,7 @@ 2ed1e4c14bc5 SELECT path, scope, symbol_ids, kind, start_line, end_line, content, shingles, uniqExact(path) OVER () AS admitted_files FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chu 2ed709d1c719 SELECT count() AS n FROM loom.memory_belief_sources FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 2eeae7d3e832 SELECT path, scope, kind, start_line, end_line, content FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND +2f64af3a35d9 SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombston 2f87519d6c67 SELECT count() AS n FROM loom.memory_records FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 2fab97e447f4 SELECT count() AS n FROM loom.entity_vectors FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 2fc0cc735773 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a @@ -405,7 +411,6 @@ 30406949d371 SELECT memory_id, version, agent, user_id, session_id, memory_key, created_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_key IN {keys:Array(String)} AND tom 30e3abcc719b ALTER TABLE loom.query_nodes DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 30ef01ea55bc SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND user_id = {uid:String} AND ? -3112e7749090 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 311579a423b9 SELECT created_at, memory_key FROM loom.memory WHERE memory_id = {mid:UUID} AND org = {org:String} AND namespace = {in_ns:String} LIMIT ? 3117b6749ab2 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam 311c3ee7bb8b SELECT j.source_uri AS source_uri, if(j.open_jobs > ?, j.open_ingestion_id, j.latest_ingestion_id) AS ingestion_id, if(j.open_jobs > ?, j.open_filename, j.latest_filename) AS filename, if(j.open_jobs @@ -448,7 +453,6 @@ 342f026b3008 SELECT count() AS n FROM loom.memory_code_link_queue WHERE org = {org:String} AND namespace = {ns:String} AND ? 343b7c396165 SELECT count() AS n FROM loom.session_brief_revisions WHERE org = {org:String} AND namespace IN {ns:Array(String)} 345bb4f0bdc6 SELECT count() AS n FROM loom.learnings_supporters WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? -346a0c5da453 -- schemas/schema.sql -- -- Reference dump of `loom.*` and `loom_telemetry.*`: what replaying schemas/migrations/*.sql -- produces, auto-generated by scripts/ops/dump_schema.py from a live -- ClickHou 348ab5a553ad SELECT org, namespace, source_uri, chunk_idx, content, tags, doc_metadata, chunk_metadata, ingested_at, doc_superseded_at, embedding_model, embedding_dim, embedding, created_at, embed_attempts, embed_ 34912ecbbcb8 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))} 349d1b15dfc4 SELECT count() AS n FROM loom.dataset_rows WHERE org = {org:String} AND user_id = {uid:String} @@ -504,6 +508,7 @@ 3b05a28d96f2 SELECT count() AS cnt, sum(numeric_value) AS s, avg(numeric_value) AS a, min(occurred_at) AS first_at, max(occurred_at) AS last_at FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns: 3b1abae99635 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 3b1ae278b7f6 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { +3b32757af980 SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at FROM loom.session_summaries FINAL WHERE org={org:String} AND namespace={n 3b4b4580cc9a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 3ba235828531 ALTER TABLE loom.documents UPDATE superseded_at = {now:DateTime} WHERE (org = {org:String} AND user_id != {uid:String} AND superseded_at IS NULL AND cityHash64(namespace, source_uri) IN {docs:Array(UI 3bab9ca71efb SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND ((user_id = {uid:String} OR (user_id = ? AND (namespace, parent_memory_key) IN (SELECT namespace, memory_key FROM loom.me @@ -687,6 +692,7 @@ 525d9e3b3774 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL 5287f5e8fe20 SELECT operation, sumMerge(cost_usd_state) AS cost_usd, sumMerge(tokens_state) AS tokens, sumMerge(embed_tokens_state) AS embed_tokens, countMerge(ops_state) AS ops, sumMerge(errors_state) AS errors, 528a44452aa5 SELECT namespace, agent, user_id, session_id, memory_key, toString(status) AS row_status, duplicate_of AS canonical_id FROM loom.memory FINAL WHERE org = {org:String} AND (namespace, agent, user_id, s +52c2e4a12bb9 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 52d32818a393 SELECT namespace, toString(memory_id) AS canonical_id, memory_key, memory_type, value, tags, version, agent, session_id, user_id, source_class, sensitivity, toString(created_at) AS created_at, toStrin 52ef9d28c739 SELECT org, namespace, agent, user_id, session_id, memory_key, memory_type, embedding_model, embedding, version, content_sha256, status FROM loom.memory_embeddings FINAL WHERE (org, namespace, agent, 52f36dc110b9 SELECT memory_id, memory_key, agent, value, created_at FROM loom.memory FINAL WHERE org = {org:String} AND memory_id = {mid:UUID} AND tombstoned_at IS NULL LIMIT ? @@ -702,7 +708,6 @@ 54082f4a0bda SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER 541a4c06fca2 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, created_at, valid_at, version, status FROM ( SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, t 542242ea89a0 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to -54325369241a SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, done FROM loom.delete_version_checks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:St 544bda275e23 ALTER TABLE loom.memory_code_link_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 548b3de5bbd4 SELECT memory_key, toString(memory_id) AS mid, content_sha256, agent, memory_type, -- a high value is readable by key only if(sensitivity = ?, ?, substring(value, ?, ?)) AS preview, sensitivity, tags, 549dd1314271 SELECT count() AS n FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL @@ -728,6 +733,7 @@ 566af30aabf5 SELECT count() AS n FROM (SELECT ? FROM loom.memory LIMIT ?) SETTINGS select_sequential_consistency = ? 569c9b218f83 ALTER TABLE loom.conflict_allowlist DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = 56a045b3692e SELECT kind, number, source_id, component, is_open, created_at, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.qa_item_embeddings FINAL WHERE org = {org:String} AND namespace = {n +56ab4d25e910 DELETE FROM loom.memory_embeddings WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} AND session_id={session_id:String} AND scope={scope:Str 56f4c4320d80 SELECT f AS path, uniqExact(pr_number) AS prs FROM loom.code_pr_history FINAL ARRAY JOIN files AS f WHERE org = {org:String} AND namespace = {ns:String} AND merged_at > now() - INTERVAL {days:UInt32} 56f6ba832916 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 56f6eca8bb7f SELECT count() AS n FROM loom.session_briefs WHERE org = {org:String} AND user_id = {uid:String} @@ -747,6 +753,7 @@ 57fc18cfd1a4 SELECT pr_number, any(title) AS title, any(files) AS files, any(author) AS author, any(merge_commit_sha) AS merge_sha, min(merged_at) AS merged_at FROM loom.code_pr_history FINAL WHERE org = {org:Stri 5805dc626bcc SELECT countIf(NOT startsWith(path, ?)) AS source_files, countIf(NOT startsWith(path, ?) AND path NOT IN ( SELECT src_path FROM loom.code_test_map FINAL WHERE org = {org:String} AND namespace = {ns:St 580b700ab41c SELECT name, kind, path FROM ( SELECT tier, qualified_name, symbol_id, name, kind, path, argMax(tombstoned_at IS NULL, version) AS is_live FROM loom.code_symbols WHERE org = {org:String} AND namespace +5832b8ab5ff9 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 5834a5a32d06 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom 5835eff36a44 SELECT count() AS n FROM loom.memory_evidence FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 5837a3e90ebe ALTER TABLE loom.context_blobs DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -770,6 +777,7 @@ 599725704d75 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 59b30916ba34 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 59b3f6e281d9 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? +59b463c0ef4b SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, p 59bc4767d952 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 59cfa551438a ALTER TABLE loom.learnings_supporters DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 59de9570d846 ALTER TABLE loom.context_blobs UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_muta @@ -792,7 +800,6 @@ 5c174b8803f6 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 5c2243636c81 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 5c5bd0cdac42 SELECT count() AS n, if(n = ?, NULL, max(ingested_at)) AS latest FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND superseded_at IS NULL -5c68e73d0f88 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 5c7221c42763 SELECT to_key AS memory_key, relation, toString(created_at) AS created_at FROM loom.operator_edges FINAL WHERE org = {o:String} AND namespace = {n:String} AND from_key = {k:String} AND tombstoned_at I 5c93fe003e33 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org 5caa1782814d SELECT ts, org, namespace, agent, operation, latency_ms, success, error_code, request_id, trace_id FROM (SELECT raw.ts AS ts, raw.org AS org, raw.namespace AS namespace, raw.agent AS agent, raw.user_i @@ -921,6 +928,7 @@ 6befffe2772b SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 6c17c9f6e702 SELECT org, namespace, agent, thread_id, msg_id, role, content, reasoning, user_id, run_id, metadata, tokens, ts FROM loom.conversations WHERE org = {org:String} AND namespace IN {nss:Array(String)} A 6c1cb0579e58 SELECT entity, community_id FROM loom.entity_communities FINAL WHERE org = {org:String} AND namespace IN {nss:Array(String)} AND level = ? AND entity IN {ents:Array(String)} AND (namespace, computed_a +6c2e2e39e107 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, p 6c4a272aa608 SELECT (SELECT count() FROM loom.entity_triples WHERE org = {org:String} AND namespace = {ns:String} AND adapter_name = {adapter:String} AND confidence < {conf:Float32}) AS t_count, (SELECT count() FR 6c4d6b79f018 SELECT agent, count() AS rows, count() OVER () AS total, sum(count()) OVER () AS total_rows FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND st 6cf657fa1bf5 SELECT count() AS n, lower(hex(SHA256(arrayStringConcat(arraySort(groupArray(toString(row_hash))))))) AS digest FROM loom.audit_log WHERE ts >= toDateTime64({ws:DateTime}, ?, ?) AND ts < toDateTime64( @@ -928,14 +936,13 @@ 6d1913594aa8 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasur 6d2ff1c4506a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 6d37b84f121a ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasu -6d42ff967211 SELECT org, namespace, agent, user_id, session_id, memory_key, sensitivity FROM loom.memory FINAL PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN {keys:Array(Tuple(String, String, 6d57c5550379 SELECT symbol_id FROM loom.code_symbol_stats FINAL WHERE org = {org:String} AND namespace = {ns:String} AND fan_in_calls > ? 6d918add4f31 ALTER TABLE loom.memory_categories DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 6d98dc2e3672 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 6da9f7f60075 SELECT count() AS n FROM loom.entity_canonical_map FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 6daa2655ee73 SELECT count() AS n FROM loom.entity_degree WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 6e07272917e3 SELECT count() AS n FROM loom.memory_versions WHERE org = {org:String} AND namespace = {ns:String} AND ? -6e2bd7b72f84 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key, scope, embedding_model FROM loom.memory_embeddings WHERE (org, namespace, agent, user_id, session_id, memory_key) IN {keys:Array +6e1154b224c2 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, t 6e5e297bbb30 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND status != ? AND tombstoned_at IS NULL AND status != ? AND agent = {agent:String} AND user_id = {uid: 6e894706d089 SELECT count() AS n FROM loom.code_search_outcome WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 6e90fc205193 SELECT org, namespace, agent, user_id, session_id, memory_key FROM ( SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND (arrayExist @@ -996,11 +1003,10 @@ 7584931ee488 SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU 7585095cfe72 SELECT ? AS x FROM loom.scim_users FINAL WHERE id = {id:String} AND active = ? 7588988984a6 ALTER TABLE loom.conversations DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -75cbcf76345b SELECT * FROM loom.memory_embeddings FINAL PREWHERE (org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key) IN {keys:Array(Tuple(String, String, String, String, String, String 75d1ab9d987d SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi 75e9410435ef SELECT path, scope, symbol_ids, kind, start_line, end_line, content, shingles, uniqExact(path) OVER () AS admitted_files FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chu 75f3fd72118d ALTER TABLE loom.memory_term_freq DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -762c79ce2e4f SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias +7615ac826d0c SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni 764e065d440e SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND user_id = {uid:String} AND ? 764e0e259701 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 765404bd5d8d SELECT toDate(coalesce(temporal_anchor, valid_at)) AS event_date, cosineDistance(embedding, {q:Array(Float32)}) AS distance, substring(content_excerpt, ?, ?) AS excerpt, memory_key, temporal_anchor IS @@ -1009,6 +1015,7 @@ 76f47ab9df72 SELECT ? AS present FROM loom.code_caller_guards WHERE org = {org:String} AND namespace = {ns:String} LIMIT ? 77424b91d322 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 778a3bb04c67 SELECT namespace, source_uri, title, description, tags, ingested_at, superseded_at, blob_path, blob_size, toString(blob_content_type) AS blob_content_type, original_filename, content_preview FROM loom +7799bb45ee57 SELECT ? FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?,?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value={ns:String}) OR (scope_kind=? 77abb6560e82 ALTER TABLE loom.memory_code_links DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 77b5d76bd16d SELECT count() AS n FROM loom.entity_triple_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 77be1ad28338 SELECT DISTINCT agent FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL ORDER BY agent @@ -1041,7 +1048,7 @@ 7a543640c602 ALTER TABLE loom.working_memory DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 7a8545148669 SELECT memory_key, memory_type, content_excerpt, valid_at, created_at, namespace, count() OVER () AS total_matching FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:Strin 7a87bbd25387 SELECT org, namespace, agent, user_id, thread_id, title, metadata, run_id, created_at, updated_at, tombstoned_at FROM loom.conversation_threads FINAL WHERE org = {org:String} AND user_id = {scope:Stri -7a9d47bbc467 SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL OR +7aab1cf5e970 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, session_id) AS memory_key, version, ? AS value_json, 7aad3247f4a7 SELECT count() AS n FROM loom.session_summaries FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL 7ac14ad61019 WITH ( SELECT max(mirror_marked) FROM (SELECT raw.ts AS ts, raw.org AS org, raw.namespace AS namespace, raw.agent AS agent, raw.user_id AS user_id, raw.session_id AS session_id, raw.run_id AS run_id, 7ada090d2235 ALTER TABLE loom.shared UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta @@ -1093,6 +1100,7 @@ 7e726e6f73c0 SELECT model, count() AS turns, uniqExact(session_id) AS sessions, sum(toUInt64(input_tokens)) AS input_tokens, sum(toUInt64(cache_read_tokens)) AS cache_read_tokens, sum(toUInt64(cache_creation_5m_to 7eccab6dfda3 SELECT org, namespace, name, source_id, kind, source_kind, config_json, items_json, credential_enc, secret_enc, access, credential_mode, owner_subject, connection_id, created_by, active, pending_delet 7ed9e5f6ceb3 ALTER TABLE loom.memory_chunk_sources UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasu +7eea2ee7caf9 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str 7ef1d533ac81 SELECT count() AS n FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND ? 7f22d53d8b16 SELECT uniqExact(k) AS n FROM ( SELECT k FROM (SELECT arrayJoin(memories_used) AS k FROM loom.agent_runs FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND start 7f3000230a53 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { @@ -1315,8 +1323,10 @@ 998d3948018f ALTER TABLE loom.session_brief_revisions DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 99a41729ee1c SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 99b2b4b77f2a SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi +99b329fe464e SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 99bb0fb8f03c ALTER TABLE loom.session_summaries UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_m 99d86c8fa8ea SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g +99ebc0e5d139 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 99ebe72c9d4f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 9a356be92fff SELECT path, commit_sha, run_at, lines, tests FROM loom.code_coverage_files FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL 9a3d8fb32435 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} @@ -1324,6 +1334,7 @@ 9a9428a61df0 INSERT INTO loom.chunk_supersessions (org, namespace, embedding_model, source_uri, doc_version, superseded_at) SELECT DISTINCT org, namespace, embedding_model, source_uri, doc_version, {when:DateTime} 9a9a371beffd SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 9a9aaad3e6e4 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, +9ab0f1177c93 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(chunk_index), ?, content_sha256) AS memory_ 9adfa8216cee ALTER TABLE loom.memory_code_link_queue DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 9ae3708317b3 SELECT if(consumer = ?, args[?], consumer) AS consumer_key, count() AS calls, countIf(NOT success) AS errors, countIf(NOT success AND error_code = ?) AS server_errors, round(errors / calls, ?) AS erro 9b1e4cc93d63 ALTER TABLE loom.memory_exposures DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -1373,11 +1384,13 @@ 9fad0b356a81 SELECT namespace, countIf(status=?) AS pending, countIf(status=?) AS promoted, countIf(status=?) AS rejected, if(countIf(status IN (?,?)) > ?, countIf(status=?) / countIf(status IN (?,?)), ?) AS appro 9ff4c37ed354 WITH spend AS ( SELECT org, namespace, sum(upstream_cost_usd) AS spent, sumIf(upstream_cost_usd, operation = {op:String}) AS own FROM (SELECT ts, org, namespace, agent, user_id, session_id, run_id, op 9fff0b609d85 SELECT toUInt32(quantileExact(?)(failing_tests)) AS median_tests, count() AS hours FROM loom.qa_fail_hours FINAL WHERE org = {org:String} AND qa_namespace = {ns:String} AND hour >= {h0:DateTime} AND h +a010bc956c57 SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, checked_state_sha256, done FROM loom.delete_version_checks FINAL WHERE org = {org:String} AND namespace = {ns:String} a055584d8e91 SELECT toString(f.failure_id) AS failure_id, org, namespace, agent, projection, source_operation, source_resource_id, memory_keys, error_type, error_message, payload, status, created_at, resolved_at F a0585852a421 SELECT memory_id, memory_key, value, pinned, agent, user_id, session_id, run_id, valid_at FROM loom.memory FINAL PREWHERE org = {org:String} AND namespace = {ns:String} AND (agent, user_id, session_id a0706937823b SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { a08014de0c22 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g a09ec2d88823 SELECT pr_number, title, author, labels, files, merged_at, opened_at, linked_issues, lexemes FROM loom.code_pr_history FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS N +a0b4dab9a372 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, session_id) AS memory_key, version, ? AS value_json, a0be1210dc79 SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {org:String} AND namespace = {ns:String} AND ? a0e30a489601 SELECT * EXCEPT (embedding) FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL a0f9368cb508 SELECT count() AS n FROM loom.code_answers WHERE org = {org:String} AND hasAny(memory_keys, arrayConcat((SELECT groupUniqArray(memory_key) FROM loom.memory WHERE org = {org:String} AND user_id = {uid: @@ -1430,6 +1443,7 @@ a543652b324c SELECT org, namespace, agent, user_id, session_id, run_id, goal, us a55966ee4380 SELECT f.lang AS lang, toString(s.tier) AS tier, count() AS symbols, max(s.created_at) AS last_at FROM ( SELECT path, tier, created_at FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespa a5598bacc84d SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g a563ab145bc6 SELECT org, namespace, agent, thread_id, msg_id, role, content, reasoning, user_id, run_id, metadata, tokens, ts FROM loom.conversations WHERE org = {org:String} AND namespace IN {nss:Array(String)} A +a56fae89e202 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni a5a4769e8d84 ALTER TABLE loom.memory DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? a5c02075396f SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU a5e98e7d16d8 SELECT m.memory_id, m.memory_key, m.value, m.pinned, m.agent, m.user_id, m.session_id, m.run_id, m.valid_at, e.dist AS dist FROM loom.memory m FINAL JOIN ( SELECT agent, user_id, session_id, memory_ke @@ -1441,7 +1455,6 @@ a627d0337788 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:Str a63dc159ea03 SELECT count() AS n, argMax(role, ts) AS role, argMax(content, ts) AS content, argMax(run_id, ts) AS run_id, max(ts) AS last_ts, max(saved_at) AS last_saved_at, groupUniqArray(user_id) AS owners, coun a64937812d3d SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las a661d7b26199 ALTER TABLE loom.memory_extraction_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -a662cd95ca59 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str a67dc6a1b98a SELECT path, start_line, end_line, content_hash FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND start_l a68b12d4c3b3 SELECT memory_type, count() AS c FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL AND status = ? AND memory_key NOT LIKE ? GROUP BY memory_type a6ae128a7aef SELECT callee_name, path, line, from_symbol_id, argMax(from_qualified_name, version) AS from_qualified_name, argMax(call_text, version) AS call_text, argMax(disposition, version) AS disposition, argMa @@ -1509,8 +1522,10 @@ acdf77ff85a4 SELECT count() AS n FROM loom.shared FINAL WHERE org = {org:String} acfa8ad8dceb SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist ad229cd4a4cf SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g ad2a1b0821de DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync +ad2f3798c352 SELECT ? FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value={ns:String}) OR (scope_kind=? A ad39ba238825 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} ad5724799b14 SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, content, context, tags, chunk_metadata, ingested_at, agent, session_id, toString(source_class) AS +ad575f71f510 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, ? AS session_id, ? AS actor, concat(?, entity) AS memory_key, version, ? AS value_json ad69a81ba9d2 SELECT queue_id, agent, user_id, session_id, status, attempts, message_count, content_chars, written, last_error, enqueued_at, updated_at FROM loom.memory_extraction_queue FINAL WHERE org = {org:Strin ad717da278a8 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND (user_id = {uid:String} OR (namespace, memory_id) IN (SELECT toString(namespace), memory_id FROM (SELECT namespace, agent, u ad8e16687ea3 SELECT count() AS n FROM loom.query_history_sources WHERE org = {org:String} AND namespace = {ns:String} @@ -1524,7 +1539,6 @@ ae7d2da05030 SELECT count() AS n FROM loom.memory_extraction_queue WHERE org = { ae99c20b3d1e SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user af399844e950 ALTER TABLE loom.memory_record_fields DELETE WHERE (org = {org:String} AND record_id IN {ids:Array(UUID)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = af8aae57db93 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} -af9585af4ffb SELECT session_id, role, toString(observed_at) AS observed_at, content FROM ( SELECT session_id, role, observed_at, content, evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:Strin afdcda2bf23a SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {org:String} AND hasAny(arrayConcat(window_keys, cited_keys), arrayConcat((SELECT groupUniqArray(memory_key) FROM loom.memory WHERE org aff1e632ca9c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, e.agent AS agent, e.user_id AS user_id, e.session_id AS b0198f2a1337 SELECT uniqExact(path) AS n FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL AND startsWith(path, {prefix @@ -1580,7 +1594,6 @@ b517057ddbb9 SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {or b546d3bd8553 SELECT count() AS n FROM loom.entity_edge_memories WHERE org = {org:String} AND namespace = {ns:String} AND ? b556883f0824 ALTER TABLE loom.entity_triple_memories DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? b57226087b36 ALTER TABLE loom.memory DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -b583fd3cb60b SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias b599a544d8e9 SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND superseded_at IS NULL b5c1f249d43b ALTER TABLE loom.delete_version_checks DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? b5cfdd2b8732 DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND scope = ? AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta @@ -1634,6 +1647,7 @@ ba28d3f48cd1 SELECT ts, org, namespace, agent, user_id, session_id, operation, r ba3aa58d4a58 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g ba505cefcdfd SELECT subject, namespace, role, granted_at, pending_role, is_owner FROM ( SELECT subject, namespace, toString(role) AS role, granted_at, pending_role, ? AS is_owner FROM loom.namespace_grants FINAL W ba7611490c4d ALTER TABLE loom.search_misses DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? +ba79c569a2f7 SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at FROM loom.session_summaries FINAL WHERE org={org:String} AND namespace={n bb02b0e3dee7 SELECT symbol_id, qualified_name, name, signature, path, start_line, end_line, tier, kind FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND bb04945353ff SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g bb4566fa0eb8 SELECT namespace, toString(owner_email) AS owner, active, tags, password_hash = ? AS sso FROM loom.namespaces FINAL WHERE org = {org:String} AND contact_email = {ident:String} @@ -1657,8 +1671,10 @@ bccb3b408622 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = { bcd0d6a01148 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND namespace IN {ns:Array(String)} bcf5fd2c0231 SELECT count() AS n FROM ( SELECT evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND hasAnyTokens(content_lc, {tokens:Array(String)}) ORDER B bd095b55bf5d SELECT count() AS n FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND session_id = {s:String} AND memory_key = {k:Str +bd1ff98ced0a SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL WHE bd4266cbfe15 SELECT count() AS n FROM loom.conflict_allowlist FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} bda43b1f632f WITH arrayConcat(splitByNonAlpha(context_lc), splitByNonAlpha(content_lc)) AS words SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, cosineDistance +bdde77a5c044 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni bde7c432bca1 ALTER TABLE loom.entity_communities DELETE WHERE (org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)})) AND ({ bde98d358a2b SELECT path, uniqExact(facet_kind, name) AS matched, groupArray((facet_kind, name, ordinal, line, confidence)) AS hits, length(arrayIntersect(groupUniqArrayIf(ordinal, facet_kind = {k0:String} AND nam be29d7515a0a SELECT count() AS n FROM loom.memory_doc_stats WHERE org = {org:String} AND namespace = {ns:String} @@ -1673,6 +1689,7 @@ bef8fe4e58b7 SELECT tier, edge_kind, to_symbol_id, from_symbol_id, path, line, a bf09992e73fc SELECT memory_key, toString(memory_id) AS id, version, memory_type, toString(status) AS status, source_class, agent, user_id, session_id, actor, updated_at, sensitivity, org, namespace, redacted_at FR bf0c10f9a95a SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision FROM loom.chunks WHERE org = {org:String} AND namespace IN {nss:Array(String)} AND embedding_model bf193ac927b4 SELECT count() AS n FROM loom.memory_record_conflicts FINAL WHERE org = {org:String} AND namespace = {ns:String} +bf1ce8fef88c SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias bf4ffadd242a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g bfbdb1dc9399 SELECT namespace, count() AS turns, uniqExact(session_id) AS sessions, sum(toUInt64(input_tokens)) AS input_tokens, sum(toUInt64(cache_read_tokens)) AS cache_read_tokens, sum(toUInt64(cache_creation_5 bfc004b68fb9 DELETE FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND session_id = {s:String} AND memory_key = {k:String} AND (((c @@ -1716,7 +1733,6 @@ c46246e37ab2 SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String c464d4b1053c SELECT count() AS n FROM loom.knowledge_uploads WHERE org = {org:String} AND user_id = {uid:String} AND ? c466198a7fda WITH (SELECT toFloat64(count()) FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL) AS n_keys, (SELECT mapFromArrays(groupArray(s), groupA c4831b86cc45 SELECT now64(?) AS now, groupArray((token, started_at, finished)) AS keys FROM loom.graph_projections WHERE token IN {tokens:Array(String)} -c4c23cde5373 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias c4cee4797736 SELECT count() AS sources, countIf(j.has_document = ? AND NOT j.replaced) AS without_document, countIf(j.replaced) AS replaced, sum(j.pending) AS pending, sum(j.processing) AS processing, sum(j.ready) c4d11f8d928a SELECT count() AS n FROM loom.memory_record_fields FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} c4d8e27d9015 SELECT tier, qualified_name, symbol_id, name, path, start_line, end_line FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespace = {ns:String} AND (tier, qualified_name, symbol_id) IN ( SE @@ -1825,6 +1841,7 @@ cec3f5ad060e SELECT org, namespace, agent, user_id, session_id, memory_id, memor cee7e1df6fac SELECT jti, revoked_at, expires_at, reason, revoked_by, issued_before, version FROM loom.revoked_tokens FINAL WHERE jti = {j:String} AND expires_at > now() cf06ff759823 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g cf0cc9e2222f SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns:String} +cf104f8e96d5 SELECT ? FROM loom.memory_extraction_queue FINAL WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} AND session_id={session_id:String} AND qu cf1e114983a0 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND has(tags, ?) AND hasAny(tags, {src:Array(String)}) AND hasAny(arrayMap(t -> (namespace, t), cf1f27acd065 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, version FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns: cf3d0a421f45 SELECT s.org AS org, s.namespace AS namespace, s.schema_json AS schema_json, s.schema_sha256 AS schema_sha256 FROM loom.profile_schemas AS s FINAL WHERE s.tombstoned_at IS NULL AND NOT has({off:Array( @@ -1874,7 +1891,6 @@ d3ec29699853 SELECT count() AS n FROM loom.code_answers WHERE org = {org:String} d41fb49bc0a1 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user d44a207c193a SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user d455eae6762a SELECT count() AS n FROM loom.procedure_runs WHERE org = {org:String} AND namespace = {ns:String} -d462d47f669a SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias d469073eeef7 SELECT subject, relation, object, sum(co_occur) AS co_occur, max(last_seen) AS last_seen, -- The strongest tier behind the aggregated triple. -- Not `AS provenance`: with the provenance filter in WHER d4a86d775cfd SELECT path, chunk_id, argMax(tombstoned_at IS NULL, version) AS is_live FROM loom.code_chunks WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND path IN {pa d4dce40507f9 SELECT state_key, state_value, version FROM loom.code_index_state FINAL WHERE org = ? AND namespace = {ns:String} AND state_key IN {keys:Array(String)} @@ -1899,8 +1915,10 @@ d749ae65d41a SELECT toString(memory_id) AS id, argMax(status, version) AS status d76186bb002a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g d77696150f16 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom d7794951d874 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND namespace = {ns:String} +d782528717e8 ALTER TABLE loom.memory_extraction_queue UPDATE status=?, last_error=?, lease_until=toDateTime64(?,?) WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user d791a50120b8 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} d795664c37b5 WITH reversals AS ( SELECT erasure_id, count() AS n FROM loom.erasure_reversals FINAL WHERE org = {org:String} GROUP BY erasure_id ) SELECT e.id AS id, e.scope_type AS scope_type, e.scope_value AS sco +d7ac29f6b87f SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni d7ad5fb147fc SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ d7c28db7e874 SELECT org, code_namespace, path, qualified_name, namespace, memory_id FROM loom.memory_code_links WHERE org >= {a0:String} AND (org > {a0:String} OR (code_namespace >= {a1:String} AND (code_namespace d7cfaad88c70 SELECT toString(memory_id) AS memory_id, version, value, tags, valid_at, pinned, lock_mode, lock_until, sensitivity FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND user @@ -1967,6 +1985,7 @@ dd9185239ef3 ALTER TABLE loom.memory_extraction_queue UPDATE status = ?, attempt dd94ab4c37d1 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL dda464c89047 SELECT neighbor, role, sum(co_occur) AS co_occur FROM ( SELECT object AS neighbor, ? AS role, sum(co_occur) AS co_occur FROM loom.entity_triples WHERE org = {org:String} AND namespace = {ns:String} AN ddb1a5ea55c9 SELECT count() AS n FROM loom.memory_record_fields FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL +ddc00b268be8 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, t dde8208a93a2 SELECT count() AS n FROM loom.entity_triple_memories WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? ddfb30beb525 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g de25c2cb1919 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -2042,6 +2061,7 @@ e4e978084474 SELECT namespace, kind, toString(visibility) AS visibility, title, e5024546eff6 SELECT count() AS n FROM loom.knowledge_group_members WHERE org = {org:String} AND namespace = {ns:String} e51463feccae SELECT namespace, sum(upstream_cost_usd) AS cost_usd, sum(toUInt64(input_tokens) + toUInt64(output_tokens)) AS tokens, sum(toUInt64(embedding_tokens)) AS embed_tokens, count() AS ops, countIf(mirror_m e528e5544515 SELECT count() AS n FROM loom.document_blobs WHERE org = {org:String} AND namespace = {ns:String} AND ? +e52c0bc8a1ba SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, ? AS session_id, ? AS actor, concat(?, entity) AS memory_key, version, ? AS value_json e583636bd338 ALTER TABLE loom.entity_edge_memories DELETE WHERE (org = {org:String} AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:St e5977d678a52 ALTER TABLE loom.code_search_trace DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? e5a6b0c13d38 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description FROM loom.code_chunks FINAL WHERE (org, namespace, embedding_model, path, chunk_id) IN {keys:Array(Tu @@ -2052,6 +2072,7 @@ e5df581175ec SELECT count() AS n FROM loom.knowledge_gaps WHERE org = {org:Strin e5e482e9d615 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g e60347067264 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, e67a89c4f6c6 SELECT group_id, members FROM loom.principal_groups FINAL WHERE org = {org:String} AND NOT deleted +e68412b65969 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value e6990fcc8c82 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam e6f6f6bc74e2 SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} e7331ee31d1b SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user @@ -2175,11 +2196,11 @@ f516a9cc81d3 SELECT * FROM loom.entity_vectors FINAL WHERE org = {org:String} AN f51d6af32512 SELECT e.org AS org, e.namespace AS namespace, e.agent AS agent, e.user_id AS user_id, e.session_id AS session_id, e.memory_key AS memory_key FROM (SELECT org, namespace, agent, user_id, session_id, m f51e0ce42dcd SELECT count() OVER () AS total, memory_key, toString(memory_id) AS memory_id, expires_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND (emp f52e30d82733 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g -f5622add735c SELECT session_id, summary_text, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:St f5b6a3a81817 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org f5ddaf977ab5 SELECT state_key, state_value AS value FROM loom.code_index_state FINAL WHERE org = {org:String} AND namespace = {ns:String} AND state_key IN {keys:Array(String)} f60a04215196 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user f617d9867707 SELECT count() AS n FROM loom.memory_code_links WHERE org = {org:String} AND namespace IN {ns:Array(String)} +f6389b5cd8c4 -- schemas/schema.sql -- -- Reference dump of `loom.*` and `loom_telemetry.*`: what replaying schemas/migrations/*.sql -- produces, auto-generated by scripts/ops/dump_schema.py from a live -- ClickHou f646cb42e8e3 ALTER TABLE loom.derived_facts DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? f65fc198c24d ALTER TABLE loom.memory_record_conflicts UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:Strin f6669612c23e SELECT p.pr_number AS pr_number, p.merge_commit_sha AS merge_commit_sha, p.pack_tokens AS pack_tokens, p.created_at AS created_at, ifNull(h.title, ?) AS title, ifNull(h.author, ?) AS author, h.merged_ @@ -2244,6 +2265,7 @@ fcb2b681c3cc SELECT pr_number, title, author, files, merged_at, linked_issues, a fcf47e494773 SELECT count() AS n FROM loom.knowledge_group_members WHERE org = {org:String} AND namespace IN {ns:Array(String)} fd4b9de98c6f SELECT count() AS live_docs, countIf(ifNull(c.live, ?) = ?) AS without_chunks, groupArrayIf(?)(d.source_uri, ifNull(c.live, ?) = ?) AS without_chunks_uris, countIf(ifNull(c.live, ?) = ? AND d.ingestio fd68c20c1c4c SELECT content_sha256, count() AS doc_count, groupArray(?)((doc_id, source_uri, title, ingested_at)) AS docs FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND supersed +fd694d19d0d4 SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?,?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_valu fd8bd0425be6 INSERT INTO loom.knowledge_ingest_cancels (org, namespace, ingestion_id, requested_at) SELECT org, namespace, ingestion_id, requested_at FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND fd94b00f3b5d SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user fd95e7268232 ALTER TABLE loom.events DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -2264,3 +2286,5 @@ ff23e7932268 ALTER TABLE loom.working_memory DELETE WHERE (org = {org:String} AN ff438b330e4f SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND status != ? AND session_id = {sid:String} AND startsWith(memory_key, {p:St ffa5c94364aa SELECT DISTINCT namespace FROM ( SELECT namespace FROM loom.memory WHERE org = {org:String} AND (namespace, agent, user_id, session_id, memory_key) IN ( SELECT namespace, agent, user_id, session_id, m ffa9b9e1eddc SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU +ffbcd8654cca SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value +ffd2717a1e1a SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombston diff --git a/tests/integration/_memory.py b/tests/integration/_memory.py new file mode 100644 index 0000000000..af1fe26048 --- /dev/null +++ b/tests/integration/_memory.py @@ -0,0 +1,49 @@ +"""Canonical retirement fixtures that do not express permanent content refusal.""" + +from server.core.ch import get_client +from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.memory.api.models_crud import MemoryDeleteInput, MemoryDeleteOutput +from server.memory.store.rows import Identity, version_over + + +async def retire_reversibly(input: MemoryDeleteInput) -> MemoryDeleteOutput: + """Seed an automatic or legacy tombstone through the real canonical write path. + + Restore tests use this state to keep covering resurrection without cancelling + the permanent refusal expressed by an explicit deletion. + """ + import server.memory.store as memory_repo + from server.memory.app.handlers import _retire_derived, _write_tombstone_row + + ch = get_client() + connected_here = not ch.connected + if connected_here: + await ch.connect() + try: + source = await memory_repo.get_by_key( + Identity( + org=input.org, + namespace=input.namespace, + agent=input.agent, + user_id=input.user_id, + session_id=input.session_id, + ), + input.memory_key, + fresh=True, + strong=True, + scope=WHOLE_NAMESPACE, + ch=ch, + ) + assert source is not None, "the reversible retirement fixture needs a live source" + version = version_over(source) + await _write_tombstone_row(source, version=version, refuse_value=False) + tombstoned, downgraded = await _retire_derived(source, version=version, op="memory.expire") + return MemoryDeleteOutput( + tombstoned=True, + version=version, + dependents_tombstoned=tombstoned, + dependents_downgraded=downgraded, + ) + finally: + if connected_here: + await ch.disconnect() diff --git a/tests/integration/same_key/_rows.py b/tests/integration/same_key/_rows.py index eeb8aee21d..64cb1c47da 100644 --- a/tests/integration/same_key/_rows.py +++ b/tests/integration/same_key/_rows.py @@ -853,18 +853,20 @@ async def _open_edges(ch: Any, org: str, want: set[str]) -> list[str]: async def _retired_embedding(ch: Any, row: dict[str, Any]) -> None: """`row`'s search row as its retirement left it: superseded, with a vector.""" + from server.core.hashing import served_row_value, sha256_value from server.memory import store as M - cols = ("org", "namespace", "agent", "user_id", "session_id", "memory_key", "content_sha256") + cols = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") emb = M.build_embedding_row( **{c: row[c] for c in cols}, + content_sha256=sha256_value(served_row_value(row)), embedding_model="itest", embedding_dim=3072, embedding=[0.01] * 3072, status="superseded", invalid_at=naive_utc_now(), ) - await M.insert_embedding_rows([emb], ch=ch) + assert await M.insert_embedding_rows([emb], ch=ch) == [emb] async def _embedding_status(ch: Any, row: dict[str, Any]) -> str: diff --git a/tests/integration/test_blacklist.py b/tests/integration/test_blacklist.py index 173ab3edb5..c126b51a5d 100644 --- a/tests/integration/test_blacklist.py +++ b/tests/integration/test_blacklist.py @@ -63,6 +63,8 @@ async def test_matching_write_rejected_with_422( from loom import Agent from loom.errors import LoomError + from tests.integration._eventually import eventually + ns = f"bl-match-{uuid.uuid4().hex[:6]}" tok = mint_test_token(org=ORG, namespaces=[ns], roles=["admin"]) async with Agent( @@ -90,9 +92,14 @@ async def test_matching_write_rejected_with_422( ok = await a.memory.set("other", "unrelated content", memory_type="semantic") assert ok["content_sha256"] - # List reports the hit_count incremented - rows = await a.blacklist.list() - ours = [r for r in rows if r["pattern_id"] == added["pattern_id"]] + # The hit counter is an asynchronous mutation. + async def counted(): + rows = await a.blacklist.list() + ours = [r for r in rows if r["pattern_id"] == added["pattern_id"]] + assert len(ours) == 1 + return ours if int(ours[0]["hit_count"]) >= 1 else None + + ours = await eventually(counted, description="blacklist hit count to be published") assert len(ours) == 1 assert int(ours[0]["hit_count"]) >= 1 @@ -173,6 +180,16 @@ async def persisted(table): after = [await persisted(table) for table in ("memory", "memory_embeddings")] assert after == before, "a refusal changes no persisted memory or search field" await agent.memory.set("allowed", "unrelated value", memory_type="semantic") - await agent.memory.delete("allowed") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=ORG, + namespace=ns, + agent="t", + memory_key="allowed", + ) + ) restored = await agent.memory.restore("allowed") assert restored["restored"] is True diff --git a/tests/integration/test_conflict_resolver_ch.py b/tests/integration/test_conflict_resolver_ch.py index cefd4c4708..8be366cf3c 100644 --- a/tests/integration/test_conflict_resolver_ch.py +++ b/tests/integration/test_conflict_resolver_ch.py @@ -3253,14 +3253,16 @@ async def statuses() -> dict[str, tuple[str, str]]: MemoryStatusesInput(org=org, namespace="elsewhere", memory_ids=[loser]) ) assert other.rows == [], "another namespace holds none of them" - # A delete, then its restore: the row reads deleted, then live again. + # Explicit deletion remains deleted when restoration is refused. key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s2"} await handle_memory_delete(MemoryDeleteInput(**key, memory_key="office_location")) assert (await statuses())[winner][0] == "deleted" - assert ( + from server.core.errors import ValidationError + + with pytest.raises(ValidationError) as refused: await handle_memory_restore(MemoryRestoreInput(**key, memory_key="office_location")) - ).restored - assert (await statuses())[winner] == ("active", "") + assert refused.value.extra["reason"] == "forgotten" + assert (await statuses())[winner][0] == "deleted" async def test_a_retried_undo_finishes_a_restore_that_landed_after_its_503( @@ -3710,6 +3712,28 @@ async def down(*a: Any, **kw: Any) -> Any: assert len(llm.calls) > before, "the pause ended: rows are judged again" +async def _undelete_after_uncertain_sends_settle(input: Any) -> Any: + from server.core import lease_writes, rmt_carry + from server.core.ch import write_limit_s + from server.core.lease_writes import FenceUnavailableError + from server.memory.app import restore + from tests.integration._eventually import eventually + + async def retry() -> Any: + try: + return await restore.handle_memory_restore(input) + except FenceUnavailableError: + return None + + # Visibility of the canonical row cannot certify that duplicate sends have ended. + return await eventually( + retry, + max_wait=lease_writes.lease_ttl_s(rmt_carry.send_bound_s()) + 2 * write_limit_s(), + interval=1.0, + description="the uncertain restore's writers to settle", + ) + + async def test_an_undelete_whose_insert_answer_was_lost_finishes(ch, wired, monkeypatch) -> None: """The undelete's insert lands with its answer lost: the re-read finds it, so the search row and the facts come back as on any undelete, and it answers restored.""" @@ -3717,13 +3741,13 @@ async def test_an_undelete_whose_insert_answer_was_lost_finishes(ch, wired, monk from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "home2", "Ana lives in Porto", relations=[("ana", "lives_in", "porto")]) await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="home2")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="home2")) await asyncio.sleep(1.0) real = ch.insert @@ -3736,6 +3760,7 @@ async def lands_then_fails(table: str, rows: Any, **kw: Any) -> None: m.setattr(ch, "insert", lands_then_fails) out = await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home2")) assert out.restored and out.from_status == "tombstoned" + await _undelete_after_uncertain_sends_settle(MemoryRestoreInput(**key, memory_key="home2")) search = await ch.query_all( "SELECT tombstoned_at FROM loom.memory_embeddings FINAL WHERE org = {o:String}" " AND memory_key = 'home2' SETTINGS select_sequential_consistency = 1", @@ -3839,13 +3864,13 @@ async def test_an_undelete_whose_answer_and_reread_were_lost_finishes_on_retry( from server.core.errors import BackendUnavailableError from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "home3", "Ana lives in Braga") await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="home3")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="home3")) await asyncio.sleep(1.0) real = ch.insert @@ -3863,7 +3888,9 @@ async def unread(*a: Any, **kw: Any) -> None: with pytest.raises(BackendUnavailableError) as lost: await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home3")) assert lost.value.extra["outcome_unknown"] - out = await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home3")) + out = await _undelete_after_uncertain_sends_settle( + MemoryRestoreInput(**key, memory_key="home3") + ) assert out.restored and out.from_status == "tombstoned" search = await ch.query_all( "SELECT tombstoned_at FROM loom.memory_embeddings FINAL WHERE org = {o:String}" @@ -3920,13 +3947,13 @@ async def test_a_high_row_restored_after_a_lost_undelete_answers_not_restored( from server.core.errors import BackendUnavailableError from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "vault4", "The vault code is 4417", sensitivity="high") await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="vault4")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="vault4")) await asyncio.sleep(1.0) real = ch.insert diff --git a/tests/integration/test_conflict_resolver_invariants_ch.py b/tests/integration/test_conflict_resolver_invariants_ch.py index b8465affa6..dcbaba3b0c 100644 --- a/tests/integration/test_conflict_resolver_invariants_ch.py +++ b/tests/integration/test_conflict_resolver_invariants_ch.py @@ -4,7 +4,7 @@ - no user write or pin is lost: each key holds what its last user write left; - every retired row has a `memory.auto_supersede` record and can be undone; -- no flag, clear or retirement below 0.90; +- no model action below 0.90; dead flags clear without a model judgement; - no text of another user or another tenant reaches a prompt; - nothing is written in an org whose erasure was requested while a call was in flight; - once the writes stop and a pass drains, the next pass makes no call. @@ -140,10 +140,17 @@ async def write(self, keys: list[str]) -> None: self.wrote(key, session, out.version, ("value", value, False)) elif op == "pin" and head is not None and head["tombstoned_at"] is None: value = _text(head["value"]) - out = await handle_memory_pin( - MemorySetInput(**ident, session_id=session, memory_key=key, value=value) - ) - self.wrote(key, session, out.version, ("value", value, True)) + from server.core.errors import ValidationError + + try: + out = await handle_memory_pin( + MemorySetInput(**ident, session_id=session, memory_key=key, value=value) + ) + except ValidationError as error: + if error.extra.get("reason") != "forgotten": + raise + else: + self.wrote(key, session, out.version, ("value", value, True)) elif op == "delete": gone = await handle_memory_delete( MemoryDeleteInput(**ident, session_id=session, memory_key=key) @@ -206,6 +213,7 @@ async def test_the_resolver_keeps_its_invariants_under_concurrent_writes( from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import after_write, write from server.memory.app.restore import handle_memory_restore + from server.memory.store import conflict_resolver as queue from server.memory.workers.conflict_resolver import run_pass from server.settings import settings @@ -214,6 +222,22 @@ async def test_the_resolver_keeps_its_invariants_under_concurrent_writes( writer = _Writer(ch, org, seed) erasing = seed % 3 == 0 stored: dict[str, set[int]] = {} + dead_peers: set[str] = set() + read_dead_flags = queue.dead_flags + + async def observed_dead_flags( + read_org: str, + namespace: str, + identity: tuple[str, str, str, str], + *, + ch: Any = None, + ) -> list[str]: + peers = await read_dead_flags(read_org, namespace, identity, ch=ch) + if (read_org, namespace) == (org, NS): + dead_peers.update(peers) + return peers + + monkeypatch.setattr(queue, "dead_flags", observed_dead_flags) async def in_call(entries: list[dict[str, Any]]) -> None: acts = any(e["confidence"] >= 0.9 and e["same_entity"] for e in entries) @@ -276,7 +300,17 @@ async def slow_embed(w: Any, embedder: Any = None) -> Any: assert not any(_SECRET in prompt for prompt in calls), "another user's or tenant's text" for record in await _audit(ch, org, "memory.conflict_resolver"): - assert all(p["confidence"] >= 0.90 for p in record["pairs"]), record + assert record["pairs"], record + if record["model"]: + assert all(p["confidence"] >= 0.90 for p in record["pairs"]), record + else: + for pair in record["pairs"]: + assert pair == { + "peer_id": pair["peer_id"], + "action": "clear", + "reason": "peer_not_live", + }, record + assert pair["peer_id"] in dead_peers, record recorded = { e["memory_id"] for r in await _audit(ch, org, "memory.auto_supersede") for e in r["retired"] diff --git a/tests/integration/test_conflict_resolver_long_session_ch.py b/tests/integration/test_conflict_resolver_long_session_ch.py index 1f47852963..c3afe36c62 100644 --- a/tests/integration/test_conflict_resolver_long_session_ch.py +++ b/tests/integration/test_conflict_resolver_long_session_ch.py @@ -6,8 +6,8 @@ through `memory.set` (odd sessions) and `memory.bulk_set` (even sessions), each a week apart. Six attributes change once or twice, each change under a new key and in new words; other facts are restated, elaborated, or sit near a different fact; -one pinned fact is contradicted later; one fact is deleted and brought back -with Undo; the rest are unrelated (`bench/conflict_resolver_scenario.py`, the +one pinned fact is contradicted later; one fact is deleted and stays absent; +the rest are unrelated (`bench/conflict_resolver_scenario.py`, the scenario and its ground truth). A scenario embedder gives every text a fixed direction, so the write path's own check decides which rows the resolver owes a judgement, and the judge answers each (row, peer) pair from the scenario's @@ -37,8 +37,10 @@ from bench.conflict_resolver_scenario import ( ATTRIBUTES, BY_KEY, + CHECK_DELETED_IN, CURRENT, DELETE_IN, + DELETED, FACTS, FLAGGED_AT_WRITE, JUDGE, @@ -46,7 +48,6 @@ LOW_CONFIDENCE, MALFORMED, OLDER, - RESTORE_IN, SESSIONS, SOLO, relation_of, @@ -146,11 +147,40 @@ async def query_all(self, sql: str, params: Any = None) -> Any: return await self.ch.query_all(sql, params={**(params or {}), "only_org": self.org}) +async def _insert_queue_vector(ch: Any, row: dict[str, Any]) -> None: + from server.core.hashing import sha256_value + from server.memory.store.inserts import insert_embedding_rows, insert_memory_versions + from server.memory.store.rows import build_first_version_row + + value = row["org"] + row["content_sha256"] = sha256_value(value) + row["content_excerpt"] = value + source = build_first_version_row( + { + **{ + k: row[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + "value": json.dumps(value), + "value_json": "{}", + "memory_type": row["memory_type"], + "confidence": 1.0, + "content_sha256": row["content_sha256"], + "status": row["status"], + "version": row["version"], + "actor": "process:itest", + "source_class": "asserted", + }, + now=naive_utc_now(), + ) + await insert_memory_versions([source], ch=ch) + assert await insert_embedding_rows([row], ch=ch) == [row] + + @pytest.mark.parametrize("leg", ["owed", "unjudged"]) async def test_org_queue_scope_precedes_the_candidate_limit(ch, leg: str) -> None: from server.core.vectors.codes_state import stored_width from server.memory.store import conflict_resolver as queue - from server.memory.store.inserts import insert_embedding_rows from server.memory.store.rows import build_embedding_row orgs = [unique_org("queue-scope") for _ in range(2)] @@ -184,9 +214,9 @@ def vector(org: str) -> dict[str, Any]: ) try: - await insert_embedding_rows([vector(foreign)], ch=ch) + await _insert_queue_vector(ch, vector(foreign)) assert await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, own)) == [] - await insert_embedding_rows([vector(own)], ch=ch) + await _insert_queue_vector(ch, vector(own)) rows = await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, own)) assert [(row["org"], row["memory_key"]) for row in rows] == [(own, "pending")] rows = await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, foreign)) @@ -318,10 +348,11 @@ async def _write_session(org: str, session: int) -> None: await after_write.drain() -async def _delete_or_restore(org: str, session: int) -> None: - from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput +async def _delete_and_check_absent(org: str, session: int) -> None: + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory import store as memory_repo + from server.memory.api.models_crud import MemoryDeleteInput from server.memory.app.handlers import handle_memory_delete - from server.memory.app.restore import handle_memory_restore if session in DELETE_IN: key = DELETE_IN[session] @@ -329,12 +360,16 @@ async def _delete_or_restore(org: str, session: int) -> None: MemoryDeleteInput(**_identity(org, BY_KEY[key].session), memory_key=key) ) assert out.tombstoned, key - if session in RESTORE_IN: - key = RESTORE_IN[session] - back = await handle_memory_restore( - MemoryRestoreInput(**_identity(org, BY_KEY[key].session), memory_key=key) + if session in CHECK_DELETED_IN: + key = CHECK_DELETED_IN[session] + current = await memory_repo.get_by_key( + memory_repo.Identity(**_identity(org, BY_KEY[key].session)), + key, + fresh=True, + strong=True, + scope=WHOLE_NAMESPACE, ) - assert (back.restored, back.from_status) == (True, "tombstoned"), key + assert current is None, key async def _drain(run: Run) -> None: @@ -434,7 +469,7 @@ async def test_a_long_run_of_sessions_ends_with_one_live_value_per_attribute(ch, org = run.org for session in range(1, SESSIONS + 1): await _write_session(org, session) - await _delete_or_restore(org, session) + await _delete_and_check_absent(org, session) rows = await _rows(ch, org) for f in FACTS: if f.session == session: @@ -442,7 +477,7 @@ async def test_a_long_run_of_sessions_ends_with_one_live_value_per_attribute(ch, await _drain(run) rows = await _rows(ch, org) - assert set(rows) == {f.key for f in FACTS} + assert set(rows) == {f.key for f in FACTS} - DELETED assert run.judge.unexpected == [], "a cluster held a peer the scenario did not place" # Each attribute: its current value is live and unflagged; every older value is diff --git a/tests/integration/test_embedding_worker.py b/tests/integration/test_embedding_worker.py index 47116e43d6..315639b2d6 100644 --- a/tests/integration/test_embedding_worker.py +++ b/tests/integration/test_embedding_worker.py @@ -13,6 +13,7 @@ sys.path.insert(0, str(ROOT / "sdk" / "python")) sys.path.insert(0, str(ROOT)) +from server.core.hashing import sha256_value # noqa: E402 from server.core.vectors.placeholders import placeholder # noqa: E402 from server.memory.workers.embedding import ( # noqa: E402 EmbeddingWorkerConfig, @@ -88,14 +89,19 @@ async def test_stub_inserted_directly_gets_embedded( """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'pending_k', - 'semantic', 'private', 'clickhouse columnar speed', + 'semantic', 'private', 'clickhouse columnar speed', {sha:FixedString(64)}, 'fake-ngram-1024', 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now() - INTERVAL 1 MINUTE) """, - parameters={"org": ORG, "ns": ns, "ph": PLACEHOLDER}, + parameters={ + "org": ORG, + "ns": ns, + "ph": PLACEHOLDER, + "sha": sha256_value("clickhouse columnar speed"), + }, ) assert _embedding_dim(ch_client, ns, "pending_k") == 0 # stub @@ -127,7 +133,6 @@ async def _seed_memory_row(ns: str, key: str, value: str) -> None: """The memory row an async-mode memory.set writes beside its stub: the worker fills only stubs whose head may keep a vector, and a stub with no head is an orphan.""" - import hashlib import json from datetime import UTC, datetime @@ -147,7 +152,7 @@ async def _seed_memory_row(ns: str, key: str, value: str) -> None: "memory_type": "semantic", "value": json.dumps(value), "value_json": "{}", - "content_sha256": hashlib.sha256(value.encode()).hexdigest(), + "content_sha256": sha256_value(value), "confidence": 1.0, "actor": "process:itest", "source_class": "asserted", @@ -176,13 +181,18 @@ async def test_stub_invisible_to_search_until_embedded( """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'pending', - 'semantic', 'private', 'distinctive query content here', + 'semantic', 'private', 'distinctive query content here', {sha:FixedString(64)}, 'fake-ngram-1024', 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now()) """, - parameters={"org": ORG, "ns": ns, "ph": PLACEHOLDER}, + parameters={ + "org": ORG, + "ns": ns, + "ph": PLACEHOLDER, + "sha": sha256_value("distinctive query content here"), + }, ) async with Agent(org=ORG, namespace=ns, agent="w", base_url=control_plane_url) as a: @@ -257,11 +267,11 @@ def _insert(text: str, version: int) -> None: """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at, version) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'fenced_k', - 'semantic', 'private', {text:String}, {model:String}, + 'semantic', 'private', {text:String}, {sha:FixedString(64)}, {model:String}, 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now() - INTERVAL 1 MINUTE, {v:UInt64}) """, @@ -269,6 +279,7 @@ def _insert(text: str, version: int) -> None: "org": ORG, "ns": ns, "text": text, + "sha": sha256_value(text), "model": model, "v": version, "ph": PLACEHOLDER, diff --git a/tests/integration/test_erasure.py b/tests/integration/test_erasure.py index e19d614955..c4232cfa3d 100644 --- a/tests/integration/test_erasure.py +++ b/tests/integration/test_erasure.py @@ -93,6 +93,13 @@ async def test_user_id_erasure_tombstones_memory(admin_agent, ch_client) -> None # user_id filter). bob's True would merge into alice's row # and never reach loom.memory under bob's user_id. await a.memory.set("prefers_yaml", "yaml-please", memory_type="preference", user_id="bob") + # The ClickHouse tombstone assertions require the committed seeds to be published. + listed = await a.memory.list(user_id=target_user) + assert {"prefers_json", "favorite_table"} <= { + row["memory_key"] for row in listed["entries"] + } + control = await a.memory.list(user_id="bob") + assert "prefers_yaml" in {row["memory_key"] for row in control["entries"]} # Run erasure for alice result = await admin_agent.erasure.request_user_id(target_user, requester="ops@example.com") @@ -340,6 +347,8 @@ async def test_namespace_erasure_tombstones_only_target(admin_agent, ch_client) source_uri=f"src://{ns}", tags=["test"], ) + listed = await a.memory.list() + assert any(row["memory_key"] == "k1" for row in listed["entries"]) # Erase ns_target result = await admin_agent.erasure.request_namespace(ns_target, requester="ops@example.com") diff --git a/tests/integration/test_erasure_replicated_ch.py b/tests/integration/test_erasure_replicated_ch.py index 04da3cc513..341501c8f1 100644 --- a/tests/integration/test_erasure_replicated_ch.py +++ b/tests/integration/test_erasure_replicated_ch.py @@ -36,6 +36,8 @@ "documents", "memory_code_links", "memory_versions", + "memory_refusals", + "erasure_requests", "shared", ) # loom.entity_vectors is left out: this fixture's copy of its quantized codec column is diff --git a/tests/integration/test_inference_identity_workers.py b/tests/integration/test_inference_identity_workers.py index d11cca3218..3172ff74db 100644 --- a/tests/integration/test_inference_identity_workers.py +++ b/tests/integration/test_inference_identity_workers.py @@ -59,7 +59,13 @@ async def test_an_older_worker_reclaims_a_hold_when_its_lease_ends( assert response.status_code == 200, response.text assert response.json()["pending"] == 1 monkeypatch.setattr(queue, "_now", lambda: datetime.now(UTC) - timedelta(seconds=1)) - await queue.mark_held(claimed[0], reason="refused", seconds=0, ch=ch) + current = await ch.query_one( + f"SELECT {', '.join(queue.COLUMNS)} FROM loom.memory_extraction_queue FINAL " + "WHERE org={org:String} AND queue_id={qid:String}", + params={"org": org, "qid": claimed[0]["queue_id"]}, + ) + assert current is not None + await queue.mark_held(current, reason="refused", seconds=0, ch=ch) assert len(await ch.query_all(old_selection, params={"org": org})) == 1 diff --git a/tests/integration/test_mcp_session_run_scope.py b/tests/integration/test_mcp_session_run_scope.py index 9a5ccebedb..5a75638281 100644 --- a/tests/integration/test_mcp_session_run_scope.py +++ b/tests/integration/test_mcp_session_run_scope.py @@ -70,6 +70,19 @@ async def _require_auth(c: httpx.AsyncClient) -> None: pytest.skip("the server does not enforce auth") +async def _published_memories(ch_client: Any, org: str, expected: int) -> None: + async def published() -> bool: + return ( + ch_client.query( + "SELECT count() FROM loom.memory FINAL WHERE org = {org:String}", + parameters={"org": org}, + ).result_rows[0][0] + == expected + ) + + await eventually(published, description="the MCP memories to be published") + + @pytest.mark.asyncio async def test_a_session_scoped_tokens_run_is_filed_under_its_session( control_plane_url: str, ch_client: Any @@ -92,6 +105,7 @@ async def test_a_session_scoped_tokens_run_is_filed_under_its_session( assert len(runs) == 1 run_id, user, row_session, status = runs[0] assert (user, row_session, status) == (USER, APP_SESSION, "completed") + await _published_memories(ch_client, org, 3) stamped = ch_client.query( "SELECT DISTINCT run_id FROM loom.memory FINAL WHERE org = {org:String}", parameters={"org": org}, @@ -120,6 +134,7 @@ async def test_two_tokens_sending_one_session_id_hold_two_runs( ).result_rows ) assert runs == {"ns-a": "active", "ns-b": "completed"} + await _published_memories(ch_client, org, 4) stamped = ch_client.query( "SELECT namespace, uniqExact(run_id) FROM loom.memory FINAL WHERE org = {org:String} " "GROUP BY namespace ORDER BY namespace", diff --git a/tests/integration/test_memory_code_links.py b/tests/integration/test_memory_code_links.py index 53aae5460b..8f99de6a5e 100644 --- a/tests/integration/test_memory_code_links.py +++ b/tests/integration/test_memory_code_links.py @@ -1988,7 +1988,7 @@ async def test_a_link_written_after_an_erasure_is_dropped(env, monkeypatch, ch_c await _pass() assert queue.erased.status == "completed" and _all_links(ch_client) == [] # the recheck: moving a link after the erasure - await _remember(MEM, "e", "`erased_value` returns one") + await _remember(MEM, "e", "`erased_value` yields one") await _pass() assert _all_links(ch_client) with monkeypatch.context() as m: @@ -2580,15 +2580,15 @@ async def test_imports_and_member_assignments_in_a_rename(env, path, before, aft async def test_a_delayed_cleanup_keeps_links_made_after_a_restore(env) -> None: from server.core.ch import get_client from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput - from server.memory.app.handlers import handle_memory_delete from server.memory.app.restore import handle_memory_restore from server.memory.store import code_links as store + from tests.integration._memory import retire_reversibly await _drop_fetch_rows(env, TRUNK) ident = {"org": ORG, "namespace": MEM, "agent": "claude-code", "memory_key": "back"} mid = await _remember(MEM, "back", "`fetch_rows` returns n") await _pass() - await handle_memory_delete(MemoryDeleteInput(**ident)) + await retire_reversibly(MemoryDeleteInput(**ident)) class _RestoreBeforeDelete: def __init__(self, real: Any) -> None: diff --git a/tests/integration/test_memory_promote.py b/tests/integration/test_memory_promote.py index 54ac639d8a..c9277b2191 100644 --- a/tests/integration/test_memory_promote.py +++ b/tests/integration/test_memory_promote.py @@ -451,7 +451,8 @@ async def approved(body: str) -> str: " AND memory_key = {k:String}", parameters=params, ).result_rows[0] - assert vectors[1] and bool(vectors[0]) == bool(live_row), (live_row, vectors) + # Retired mirrors may be physically removed when their source is absent. + assert bool(vectors[0]) == bool(live_row), (live_row, vectors) @pytest.mark.asyncio diff --git a/tests/integration/test_memory_read_lifecycle.py b/tests/integration/test_memory_read_lifecycle.py index d4cb4519cb..4d3cc130dc 100644 --- a/tests/integration/test_memory_read_lifecycle.py +++ b/tests/integration/test_memory_read_lifecycle.py @@ -12,6 +12,7 @@ from server.core.ch import CHClient from server.core.datetimes import naive_utc_now from server.memory.app.retrieval.summaries import fetch_session_summaries +from server.memory.domain.summary_provenance import render_summary from server.memory.store import lifecycle from server.memory.store.lifecycle import bulk_tombstone_scope from server.memory.store.rows import MEMORY_PERSISTED_COLS, next_version @@ -93,13 +94,14 @@ async def test_session_summaries_inherit_known_source_state_and_version( vector = [1.0] + [0.0] * 3071 states = ["active", "conflicted", "rejected", "proposed", "superseded", "tombstoned"] for status in [*states, "rewritten", "high", "expired", "past", "standalone", "missing"]: - fields: dict[str, Any] = {"version": 100, "session_id": status} + fields: dict[str, Any] = {"version": 100, "session_id": status, "value": json.dumps(status)} if status in states: fields["status"] = status if status == "tombstoned": fields["tombstoned_at"] = naive_utc_now() if status == "rewritten": fields["version"] = 300 + fields["value"] = json.dumps("a replacement fact") if status == "high": fields["sensitivity"] = "high" if status in ("past", "expired"): @@ -115,7 +117,8 @@ async def test_session_summaries_inherit_known_source_state_and_version( "user_id": "reader", "session_id": status, "version": 200, - "summary_text": status, + "summary_text": render_summary([status]), + "key_facts": [status], "summary_embedding": vector, "embedding_model": "fixture-model", "embedding_dim": len(vector), @@ -140,7 +143,7 @@ async def test_session_summaries_inherit_known_source_state_and_version( ) finally: await ch.disconnect() - assert {h.content_excerpt for h in hits} == {"active", "conflicted", "standalone"} + assert {h.content_excerpt for h in hits} == {"active", "conflicted"} async def test_clear_keeps_a_complete_tombstone_in_version_history( @@ -452,7 +455,15 @@ async def test_nearest_stale_summary_does_not_hide_the_next_valid_digest( ("stale", "rejected", query), ("fresh", "active", [0.7, 0.2] + [0.0] * 3070), ): - _memory(ch_client, me, "source", session_id=session, status=status, version=100) + _memory( + ch_client, + me, + "source", + session_id=session, + status=status, + version=100, + value=json.dumps(session), + ) row = { "org": me["org"], "namespace": me["namespace"], @@ -460,7 +471,8 @@ async def test_nearest_stale_summary_does_not_hide_the_next_valid_digest( "user_id": "reader", "session_id": session, "version": 200, - "summary_text": session, + "summary_text": render_summary([session]), + "key_facts": [session], "summary_embedding": vector, "embedding_dim": 3072, "embedding_model": "fixture-model", diff --git a/tests/integration/test_memory_refusal_capture_live_ch.py b/tests/integration/test_memory_refusal_capture_live_ch.py new file mode 100644 index 0000000000..89153943e1 --- /dev/null +++ b/tests/integration/test_memory_refusal_capture_live_ch.py @@ -0,0 +1,124 @@ +"""Erasure captures consent scopes and raw metadata before the content disappears.""" + +from __future__ import annotations + +import json +import uuid + +import pytest + +from server.core.ch import get_client +from server.core.errors import ValidationError +from server.erasure.store import refusals as capture +from server.memory.store import refusals as policy +from tests.integration._resources import unique_org + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def ch(monkeypatch): + client = get_client() + await client.connect() + monkeypatch.setattr(capture, "_default_client", lambda: client) + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + yield client + await client.disconnect() + + +def _row(org, ns, uid, value, *, key="source", **fields): + return { + "org": org, + "namespace": ns, + "agent": "writer", + "user_id": uid, + "session_id": "-", + "memory_key": key, + "memory_type": "fact", + "tags": [], + "value": json.dumps(value), + "version": 1, + **fields, + } + + +async def _refused(ch, row): + with pytest.raises(ValidationError, match="previous deletion"): + await policy.check_clickhouse(ch, [row]) + + +async def test_namespace_capture_refuses_every_user_only_in_its_namespace(ch): + org = unique_org("refusal-namespace") + values = [f"retired namespace fact {uuid.uuid4().hex}" for _ in range(2)] + await ch.insert( + "loom.memory_versions", + [ + _row(org, "closed", uid, value) + for uid, value in zip(("alice", "bob"), values, strict=True) + ], + ) + assert await capture.capture_namespace(org, "closed") == 2 + for value in values: + await _refused(ch, _row(org, "closed", "future-user", value)) + await policy.check_clickhouse(ch, [_row(org, "elsewhere", "alice", value)]) + await policy.check_clickhouse(ch, [_row(org, "closed", "alice", "unrelated fact")]) + + +async def test_user_capture_reaches_future_namespaces_and_derived_writer_identity(ch): + org = unique_org("refusal-user") + parent = _row(org, "first", "alice", "original private fact") + derived = _row( + org, + "first", + "-", + "consolidated private fact", + key="summary", + tags=["consolidated", "src:source"], + ) + other = _row(org, "second", "alice", "second private fact") + await ch.insert("loom.memory", [parent, derived, other]) + assert await capture.capture_user(org, "alice") == 3 + for value in ("original private fact", "consolidated private fact", "second private fact"): + await _refused(ch, _row(org, "future", "alice", value)) + await policy.check_clickhouse(ch, [_row(org, "future", "bob", value)]) + + +async def test_capture_decodes_structured_metadata_and_keeps_literal_null(ch): + org = unique_org("refusal-metadata") + await ch.insert( + "loom.memory_evidence", + [ + { + "org": org, + "namespace": "facts", + "agent": "writer", + "user_id": "alice", + "session_id": "-", + "evidence_id": uuid.uuid4(), + "content": "null", + "metadata": json.dumps({"credential": "copper lark secret"}), + } + ], + ) + assert await capture.capture_user(org, "alice") == 2 + for value in ("null", {"credential": "copper lark secret"}): + await _refused(ch, _row(org, "future", "alice", value)) + await policy.check_clickhouse(ch, [_row(org, "future", "alice", {"other": "allowed"})]) + + +async def test_a_large_source_value_never_becomes_a_cursor_parameter(ch, monkeypatch): + org = unique_org("refusal-cursor") + value = "x" * 150_000 + await ch.insert("loom.memory_versions", [_row(org, "facts", "alice", value)]) + cursors = [] + query = ch.query_all + + async def observe(sql, params=None): + if params and params.get("after"): + cursors.append(params["after"]) + return await query(sql, params=params) + + monkeypatch.setattr(ch, "query_all", observe) + assert await capture.capture_user(org, "alice") == 1 + assert cursors and all(len(str(cursor)) < 1000 for cursor in cursors) + await _refused(ch, _row(org, "future", "alice", value)) diff --git a/tests/integration/test_memory_refusal_policy.py b/tests/integration/test_memory_refusal_policy.py new file mode 100644 index 0000000000..da733d9ffc --- /dev/null +++ b/tests/integration/test_memory_refusal_policy.py @@ -0,0 +1,191 @@ +"""Deletion refuses later intake while preserving unrelated identities and values.""" + +from __future__ import annotations + +import json +import uuid + +import httpx +import pytest + +from tests.integration._resources import unique_org + + +@pytest.mark.asyncio +async def test_wildcard_forgetting_refuses_only_the_requested_users_value( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal-users") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = {"org": org, "namespace": "facts", "agent": "writer"} + values = {"alice": "A copper lark stays near the lake", "bob": "A silver heron stays upstream"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for uid, value in values.items(): + wrote = await client.post( + "/v1/memory.set", + json={ + **identity, + "user_id": uid, + "memory_key": "shared-key", + "value": value, + "memory_type": "episodic", + }, + ) + assert wrote.status_code == 200, wrote.text + deleted = await client.post( + "/v1/memory.delete", + json={ + **identity, + "agent": "*", + "user_id": "alice", + "memory_key": "shared-key", + }, + ) + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text + for uid, expected in (("alice", 422), ("bob", 200)): + got = await client.post( + "/v1/memory.get", + json={ + **identity, + "user_id": uid, + "memory_key": "shared-key", + }, + ) + assert got.status_code == 200, got.text + assert bool(got.json()["found"]) is (uid == "bob") + replay = await client.post( + "/v1/memory.set", + json={ + **identity, + "user_id": uid, + "memory_key": "replayed", + "value": values[uid], + "memory_type": "episodic", + }, + ) + assert replay.status_code == expected, replay.text + + +@pytest.mark.asyncio +async def test_deleted_content_cannot_return_through_another_intake( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal") + ns = "facts" + token = mint_test_token(org=org, namespaces=[ns], roles=["writer"]) + identity = {"org": org, "namespace": ns, "agent": "writer", "user_id": "subject"} + value = f"The retired phrase is copper lark {uuid.uuid4().hex}" + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + + async def call(op: str, **fields): + return await client.post(f"/v1/{op}", json={**identity, **fields}) + + wrote = await call("memory.set", memory_key="source", value=value) + assert wrote.status_code == 200, wrote.text + deleted = await call("memory.delete", memory_key="source") + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text + restored = await call("memory.restore", memory_key="source") + assert restored.status_code == 422, restored.text + assert restored.json()["detail"]["reason"] == "forgotten" + for new_value in (value, f"Reminder: {value.upper()}.", {"wrapped": value}): + refused = await call("memory.set", memory_key="returned", value=new_value) + assert refused.status_code == 422, refused.text + assert refused.json()["detail"]["reason"] == "forgotten" + deferred = await call( + "memory.set_from_messages", + defer=True, + messages=[{"role": "user", "content": value}], + ) + assert deferred.status_code == 422, deferred.text + unrelated = await call( + "memory.set", memory_key="unrelated", value="A separate current fact" + ) + assert unrelated.status_code == 200, unrelated.text + other = await call("memory.set", memory_key="other", user_id="another", value=value) + assert other.status_code == 200, other.text + + +@pytest.mark.asyncio +async def test_structured_deleted_values_are_refused_canonically( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal-object") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = {"org": org, "namespace": "facts", "agent": "writer", "user_id": "subject"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for op, fields, expected in ( + ("memory.set", {"memory_key": "source", "value": {"a": "one", "b": "two"}}, 200), + ("memory.delete", {"memory_key": "source"}, 200), + ("memory.set", {"memory_key": "returned", "value": {"b": "two", "a": "one"}}, 422), + ): + result = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert result.status_code == expected, result.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_type", ["user_id", "namespace"]) +@pytest.mark.parametrize("residue", [False, True]) +async def test_delayed_shared_retirement_cannot_recreate_erased_content( + control_plane_url, mint_test_token, scope_type, residue +): + from server.core.ch import get_client + from server.core.datetimes import naive_utc_now + from server.memory.store import shared + from server.memory.store.rows import next_version + + org, namespace = unique_org("refusal-shared-retirement"), "facts" + token = mint_test_token(org=org, namespaces=["*"], roles=["org_admin", "admin", "writer"]) + identity = {"org": org, "namespace": namespace, "agent": "writer", "user_id": "alice"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for op, fields in ( + ( + "memory.set", + {"memory_key": "source", "value": f"Private copper lark {uuid.uuid4().hex}"}, + ), + ("memory.promote", {"memory_key": "source"}), + ): + result = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert result.status_code == 200, result.text + ch = get_client() + await ch.connect() + try: + original = await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) + assert original is not None + original = { + "superseded_by": None, + **original, + "org": org, + "namespace": namespace, + "memory_key": "source", + } + erased = await client.post( + "/v1/erasure.request", + json={ + "org": org, + "scope_type": "user_id" if scope_type == "user_id" else "namespace_list", + "scope_value": "alice" + if scope_type == "user_id" + else json.dumps({"namespaces": [namespace], "purge": True}), + "requester": "operator", + "purge": True, + }, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) is None + if residue: + # An old writer's admitted send leaves a copy for the retirement to remove. + await ch.insert("loom.shared", [original], settings={"async_insert": 0}) + await shared.insert_shared_row( + {**original, "version": next_version(), "tombstoned_at": naive_utc_now()}, ch=ch + ) + assert await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) is None + finally: + await ch.disconnect() diff --git a/tests/integration/test_memory_webhooks.py b/tests/integration/test_memory_webhooks.py index 1120d72dc5..662cfca04e 100644 --- a/tests/integration/test_memory_webhooks.py +++ b/tests/integration/test_memory_webhooks.py @@ -318,7 +318,16 @@ async def test_each_write_path_leaves_its_event_and_version( await ops("memory.bulk_set", entries=[{"memory_key": "k/two", "value": "bulk new"}], **a) by_id = await ops("memory.update_by_id", memory_id=first["memory_id"], value="by id") deleted = await ops("memory.delete", memory_key="k/one", **a) - restored = await ops("memory.restore", memory_key="k/one", **a) + refused = await ops("memory.restore", memory_key="k/one", expect=422, **a) + assert refused["detail"]["reason"] == "forgotten" + automatic = await ops("memory.set", memory_key="k/auto", value="automatic retirement", **a) + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput(org=ORG, namespace=ops.ns, agent="assistant", memory_key="k/auto") + ) + restored = await ops("memory.restore", memory_key="k/auto", **a) head = await ops("memory.get", memory_key="k/two", **a) dv = await ops("memory.delete_version", memory_key="k/two", version=head["version"], **a) redacted = await ops( @@ -338,7 +347,8 @@ async def test_each_write_path_leaves_its_event_and_version( ("memory.revised", "k/one", versions["k/one"]), ("memory.revised", "k/one", by_id["new_version"]), ("memory.tombstoned", "k/one", deleted["version"]), - ("memory.restored", "k/one", restored["version"]), + ("memory.written", "k/auto", automatic["version"]), + ("memory.restored", "k/auto", restored["version"]), ("memory.tombstoned", "k/two", dv["version"]), ("memory.redacted", "k/one", first["version"]), # The restatement copies the redacted text, so the redaction scrubs it too. diff --git a/tests/integration/test_okf_sync.py b/tests/integration/test_okf_sync.py index 7450e66eaf..9006eb5183 100644 --- a/tests/integration/test_okf_sync.py +++ b/tests/integration/test_okf_sync.py @@ -199,12 +199,12 @@ def get(agent: str, key: str) -> dict[str, Any]: assert get("okf-sync", keys[0])["found"] assert get("bot", "bot-note")["found"] assert get("okf-sync", "bot-as-sync")["found"] - # A prune is a tombstone: memory.restore brings the copy back. - restored = _op( - url, - writer, - DST_ORG, - "memory.restore", - {"namespace": ns, "agent": "okf-sync", "memory_key": keys[2]}, + # Pruning explicitly refuses the removed copy, including restoration. + restored = httpx.post( + f"{url}/v1/memory.restore", + headers={"Authorization": f"Bearer {writer}"}, + json={"org": DST_ORG, "namespace": ns, "agent": "okf-sync", "memory_key": keys[2]}, ) - assert restored["restored"] and get("okf-sync", keys[2])["found"] + assert restored.status_code == 422, restored.text + assert restored.json()["detail"]["reason"] == "forgotten" + assert not get("okf-sync", keys[2])["found"] diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index fdae7ce954..dbf13b682d 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -30,6 +30,127 @@ _SEQUENTIAL = {"select_sequential_consistency": 1} +@pytest.mark.parametrize("scope_type", ["user_id", "namespace"]) +async def test_erasure_refuses_old_values_after_native_revision_payload_cleanup( + pg_conn, call, org, ch_client, scope_type +): + from server.core import pg + from server.core.ch import get_client + from server.memory.store import history, pg_memory + from server.memory.workers.pg_payload_clear import effective_horizon_minutes + + namespace, user = f"cleared-{uuid.uuid4().hex[:8]}", "alice" + values = [f"retired secret {uuid.uuid4().hex}", f"replacement secret {uuid.uuid4().hex}"] + for value in values: + response = await call( + "memory.set", _set(namespace, "secret", value, user_id=user, classify=False) + ) + assert response.status_code == 200, response.text + await _published(pg_conn, org) + await pg_conn.execute("DELETE FROM loom.memory_deliveries WHERE org=$1 AND settled", org) + await pg_conn.execute( + "UPDATE loom.memory_revisions SET committed_at=now()-interval '30 days', " + "delivered_at=now()-interval '30 days' WHERE org=$1", + org, + ) + ch = get_client() + await ch.connect() + try: + horizon = effective_horizon_minutes() * 60.0 + offered = [ + r for r in await pg_memory.clearable_revisions(horizon, None, 100_000) if r.org == org + ] + assert len(offered) == 2 + assert await pg_memory.clear_payloads(horizon, await history.versions_held(offered)) == 2 + finally: + await ch.disconnect() + await pg.close_pool() + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_revisions WHERE org=$1 AND row IS NULL", org + ) + == 2 + ) + erased = await call( + "erasure.request", + { + "scope_type": scope_type, + "scope_value": user if scope_type == "user_id" else namespace, + "requester": "operator", + }, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert ( + await pg_conn.fetchval("SELECT count(*) FROM loom.memory_revisions WHERE org=$1", org) == 0 + ) + target = "future" if scope_type == "user_id" else namespace + for value in values: + returned = await call( + "memory.set", _set(target, uuid.uuid4().hex, value, user_id=user, classify=False) + ) + assert returned.status_code == 422, returned.text + assert returned.json()["detail"]["reason"] == "forgotten" + unrelated = await call( + "memory.set", + _set(target, "unrelated", "A separate current fact", user_id=user, classify=False), + ) + assert unrelated.status_code == 200, unrelated.text + + +async def test_erasure_keeps_consent_for_native_dependents_without_a_clickhouse_parent( + pg_conn, call, org, ch_client +): + """Authoritative keys capture another writer's raw quote despite projection lag.""" + private = f"private source {uuid.uuid4().hex}" + quote = f"derived quote {uuid.uuid4().hex}" + for key, user, value, tags in ( + ("source", "alice", private, []), + ("summary", "bob", "A consolidated private source", ["consolidated", "src:source"]), + ): + response = await call( + "memory.set", _set("facts", key, value, user_id=user, tags=tags, classify=False) + ) + assert response.status_code == 200, response.text + await _published(pg_conn, org) + ch_client.insert( + "loom.memory_chunks", + [[org, "facts", "a", "bob", "-", "summary", 0, quote, [0.0] * 3072, 3072]], + column_names=[ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "parent_memory_key", + "chunk_index", + "content", + "embedding", + "embedding_dim", + ], + ) + for table in ("memory", "memory_versions"): + ch_client.command( + f"DELETE FROM loom.{table} WHERE org={{org:String}}", parameters={"org": org} + ) + erased = await call( + "erasure.request", + {"scope_type": "user_id", "scope_value": "alice", "requester": "operator"}, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert await pg_conn.fetchval("SELECT count(*) FROM loom.memory_current WHERE org=$1", org) == 0 + for value in (private, quote, "A consolidated private source"): + returned = await call( + "memory.set", _set("future", uuid.uuid4().hex, value, user_id="alice", classify=False) + ) + assert returned.status_code == 422, returned.text + assert returned.json()["detail"]["reason"] == "forgotten" + unrelated = await call( + "memory.set", + _set("future", "unrelated", "An unrelated current fact", user_id="alice", classify=False), + ) + assert unrelated.status_code == 200, unrelated.text + + @pytest.fixture async def pg_authority(control_plane_url: str) -> str: """The DSN of the Postgres authority the server under test runs on; skips otherwise.""" @@ -69,8 +190,9 @@ def call(control_plane_url: str, mint_test_token: Any, org: str) -> Any: token = mint_test_token(org=org, namespaces=["*"], roles=["writer", "admin", "org_admin"]) async def _call(op: str, body: dict[str, Any], **headers: str) -> httpx.Response: - # An erasure on a remote ClickHouse runs longer than a write. - async with httpx.AsyncClient(base_url=control_plane_url, timeout=120) as c: + # Erasure waits out publishers before waiting for ClickHouse mutations. + timeout = httpx.Timeout(120, read=600 if op == "erasure.request" else 120) + async with httpx.AsyncClient(base_url=control_plane_url, timeout=timeout) as c: return await c.post( f"/v1/{op}", json={"org": org, **body}, @@ -164,8 +286,8 @@ async def test_a_bulk_write_of_bytes_another_key_holds_is_judged_on_the_authorit ) -> None: """Identity dedup holds an entry whose bytes a live key of the identity holds, even one whose delivery ClickHouse is still owed (a write that answered at its commit); - a key deleted or rewritten since holds them no more, whatever ClickHouse holds yet, - and the entry is written.""" + a key rewritten since holds them no more, whatever ClickHouse holds yet. An explicit + deletion refuses those bytes under another key too.""" from server.core.hashing import sha256_value from server.memory.store import pg_memory @@ -206,6 +328,19 @@ async def test_a_bulk_write_of_bytes_another_key_holds_is_judged_on_the_authorit "memory.bulk_set", {"namespace": ns, "agent": "a", "entries": [{"memory_key": "b", "value": value}]}, ) + if keeper == "deleted": + assert out.status_code == 422, out.text + assert out.json()["detail"]["reason"] == "forgotten" + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_current " + "WHERE org = $1 AND namespace = $2 AND memory_key = 'b'", + org, + ns, + ) + == 0 + ) + return assert out.status_code == 200, out.text written = [r["memory_key"] for r in out.json()["written"] if r.get("version")] assert written == ([] if keeper == "undelivered" else ["b"]), out.text @@ -775,6 +910,7 @@ async def test_a_redact_reaches_a_revision_by_its_metadata( whose payload is cleared through its columns, which it then marks.""" ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) at = "WHERE org = $1 AND namespace = $2 AND version = $3" @@ -1231,6 +1367,8 @@ async def test_a_relabel_refused_for_a_head_written_since_its_read_fences_nothin ns = f"pg-{uuid.uuid4().hex[:8]}" scanned = (await call("memory.set", _set(ns, "k", "the vault password is hunter2"))).json() + await _published(pg_conn, org) + assert _live_sweep_vectors(ch_client, org, ns, "a") >= 1 key = (org, ns, "a", "-", "-", "k") candidate = sensitivity_sweep.SweepCandidate( memory_id=scanned["memory_id"], @@ -1558,6 +1696,7 @@ async def test_a_user_erasure_leaves_a_consolidated_row_another_namespace_built( _set(theirs, "profile", "uses emacs", **tags), ): assert (await call("memory.set", body)).status_code == 200 + await _published(pg_conn, org) r = await call( "erasure.request", {"scope_type": "user_id", "scope_value": user, "requester": "t"} ) @@ -2034,6 +2173,10 @@ async def test_a_redaction_leaves_another_key_holding_the_same_id_alone( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the badge number is 99"))).json() copy = {**await _head_row(pg_conn, org, ns), "agent": "b", "version": rows.next_version()} + try: + await pg_memory.commit_versions([copy], [audit_row_for(copy, request_id="", writer="")]) + finally: + await pg.close_pool() gone = await call("memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k"}) assert gone.status_code == 200, gone.text tombstone = await pg_conn.fetchval( @@ -2042,7 +2185,6 @@ async def test_a_redaction_leaves_another_key_holding_the_same_id_alone( ns, ) try: - await pg_memory.commit_versions([copy], [audit_row_for(copy, request_id="", writer="")]) _, copies = await pg_memory.redact( (org, ns, "a", "-", "-", "k"), first["memory_id"], @@ -2420,6 +2562,8 @@ async def test_a_reject_is_the_authoritys_head_and_clickhouses_row( and ClickHouse's memory row both read rejected, with the reason.""" ns = f"pg-{uuid.uuid4().hex[:8]}" assert (await call("memory.set", _set(ns, "k", "a fact to reject"))).status_code == 200 + await _published(pg_conn, org) + assert _live_sweep_vectors(ch_client, org, ns, "a") >= 1 before = int((await _head_row(pg_conn, org, ns, "k"))["version"]) body = {"namespace": ns, "agent": "a", "memory_key": "k", "rejection_reason": "wrong"} rejected = await call("memory.reject", body) @@ -2615,6 +2759,7 @@ async def test_an_erasure_takes_an_in_flight_requests_claim( """A claim a request still holds is fenced by the org's erasure: its commit then finds no claim under its token and is refused, so it stores no answer.""" from server.core import pg + from server.core.hashing import sha256_value from server.memory.store import pg_memory ns = f"pg-{uuid.uuid4().hex[:8]}" @@ -2640,6 +2785,8 @@ async def test_an_erasure_takes_an_in_flight_requests_claim( == 2 ) row["memory_id"], row["version"] = str(uuid.uuid4()), int(row["version"]) + 1 + value = "A different request must still honor the fenced claim" + row["value"], row["content_sha256"] = json.dumps(value), sha256_value(value) try: with pytest.raises(pg_memory.IdempotencyFenced): await pg_memory.commit_versions([row], [{}], idempotency=(org, key, 1)) @@ -2834,6 +2981,7 @@ async def test_a_redaction_fences_the_claim_still_sending_the_old_text( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) seq = await pg_conn.fetchval( @@ -3116,6 +3264,7 @@ async def test_a_user_erasure_run_again_waits_out_what_an_earlier_run_removed( """A user's scope reads the org's evidence: a run that finds the user's rows gone still waits out the publishers the earlier run removed.""" from server.core import pg + from server.core.ch import get_client from server.erasure.store import steps_pg from server.memory.store import pg_memory @@ -3133,13 +3282,18 @@ async def test_a_user_erasure_run_again_waits_out_what_an_earlier_run_removed( seq, pg_memory.MEMORY_ROW, ) + ch = get_client() + await ch.connect() try: removed, leases = await steps_pg._erase_once(org, None, user) assert removed == 2 and 15 < max(leases) <= 20 removed, leases = await steps_pg._erase_once(org, None, user) assert removed == 0 and 15 < max(leases) <= 20 finally: - await pg.close_pool() + try: + await ch.disconnect() + finally: + await pg.close_pool() @pytest.mark.usefixtures("pg_authority") @@ -3381,8 +3535,29 @@ def memory_status(key: str) -> str: tombstoned = {"tombstoned_at": "2026-01-01T00:00:00"} await _unpublished(pg_conn, org, ns, "deleted", tombstoned, owed, an_hour) for k in ("deleted", "erased", "published"): - await memory_repo.reemit_embedding_rows_for_keys( - [(ident[k], {"status": "superseded"})], ch=ch + vectors = await memory_repo.live_embedding_rows_for_key( + ch, org=org, namespace=ns, agent="a", user_id="-", session_id="-", memory_key=k + ) + assert vectors + # Corrupt the projection directly; publication admission refuses this mismatch. + await ch.insert( + "loom.memory_embeddings", + [ + memory_repo.build_embedding_row( + **( + { + c: v + for c, v in vector.items() + if c in memory_repo.EMBEDDING_PERSISTED_COLS + } + | { + "status": "superseded", + "version": memory_repo.reemit_version(vector), + } + ) + ) + for vector in vectors + ], ) await pg_conn.execute( "DELETE FROM loom.memory_current WHERE org = $1 AND namespace = $2 " @@ -3497,8 +3672,19 @@ async def test_a_restores_failed_search_document_is_left_to_the_worker( ns = f"pg-{uuid.uuid4().hex[:8]}" assert (await call("memory.set", _set(ns, "k", "a fact to bring back"))).status_code == 200 - gone = await call("memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k"}) - assert gone.status_code == 200, gone.text + # Automatic retirement has no permanent refusal, so a restore remains permitted. + from server.memory.store.pg_publish import audit_row_for + + original = rows._hydrate(await _head_row(pg_conn, org, ns)) + retired = rows.build_memory_version_row( + original, version=rows.version_over(original), tombstoned_at=datetime.now(UTC) + ) + try: + await pg_memory.commit_versions( + [retired], [audit_row_for(retired, request_id="", writer="")], publisher_lease=False + ) + finally: + await pg.close_pool() head = rows._hydrate(await _head_row(pg_conn, org, ns)) row = rows.build_memory_version_row( head, version=rows.version_over(head), tombstoned_at=None, expires_at=None @@ -3901,6 +4087,7 @@ async def test_a_fenced_holder_is_waited_out_after_its_delivery_moves_on( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) seq = await pg_conn.fetchval( @@ -4126,8 +4313,11 @@ async def test_a_retry_that_took_over_a_fenced_claim_stores_no_residue( from server.erasure.store.steps_pg import pg_residue ns = f"pg-{uuid.uuid4().hex[:8]}" - body, key = _set(ns, "k", "a fact in flight"), uuid.uuid4().hex - assert (await call("memory.set", body, **{"Idempotency-Key": key})).status_code == 200 + original = _set(ns, "k", "a fact in flight") + assert (await call("memory.set", original)).status_code == 200 + body, key = {"namespace": ns, "agent": "a", "memory_key": "k"}, uuid.uuid4().hex + deleted = await call("memory.delete", body, **{"Idempotency-Key": key}) + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text # The request as it was before the erasure: claimed long ago, still running. await pg_conn.execute( "UPDATE loom.idempotency SET state = 'running', response_body = NULL, " @@ -4139,11 +4329,16 @@ async def test_a_retry_that_took_over_a_fenced_claim_stores_no_residue( "erasure.request", {"scope_type": "namespace", "scope_value": ns, "requester": "t"} ) assert r.json()["status"] == "completed", r.text - retry = await call("memory.set", body, **{"Idempotency-Key": key}) + retry = await call("memory.delete", body, **{"Idempotency-Key": key}) assert retry.status_code == 200, retry.text + assert not retry.json()["tombstoned"], retry.text assert ( await pg_conn.fetchval("SELECT state FROM loom.idempotency WHERE org = $1", org) == "done" ) + refused = await call("memory.set", original) + assert refused.status_code == 422 and refused.json()["detail"]["reason"] == "forgotten", ( + refused.text + ) try: residue = await pg_residue(org, "namespace", ns) finally: @@ -4429,10 +4624,7 @@ async def test_erasing_a_namespace_leaves_its_restored_copy_writable( async def test_a_restore_leaves_out_the_memories_an_erasure_removed_while_evidence_lasts( call: Any, pg_conn: Any, org: str ) -> None: - """A restore into the namespace writes the snapshot's memories but those an erasure - removed from it, counts the ones left out, and is done for that snapshot. The - evidence expires by its age, not by the sweep: a snapshot first restored after it - writes the memory, and an erasure of it again starts its evidence again.""" + """A restore skips recent erasures and still refuses them after evidence expiry.""" from server.core import pg from server.memory.store import pg_memory from server.settings import settings @@ -4475,10 +4667,10 @@ async def _age_evidence() -> None: ) await _erase() - gone = await call( - "memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k2", "user_id": "other"} + rejected = await call( + "memory.reject", {"namespace": ns, "agent": "a", "memory_key": "k2", "user_id": "other"} ) - assert gone.status_code == 200, gone.text + assert rejected.status_code == 200, rejected.text sid = snapshot.json()["snapshot_id"] restored = await _restore(sid) assert restored["restored"] is True and restored["counts"]["erased_skipped"] == 1 @@ -4496,14 +4688,11 @@ async def _age_evidence() -> None: replay = await _restore(sid) assert replay["restored"] is False and replay["counts"]["erased_skipped"] == 1 assert (await _get("k1", erased_user)).get("found") is False - # Another snapshot id restores it: its commit is no longer refused. - again = await _restore(str(uuid.uuid4())) - assert again["counts"]["erased_skipped"] == 0 - assert (await _get("k1", erased_user))["memory_id"] == mid - # Erased again while the old evidence row is still there: the evidence restarts. - await _erase() - assert await pg_memory.erased_ids(org, ns, [mid]) == {mid} - await _age_evidence() + payload = {**snapshot.json(), "snapshot_id": str(uuid.uuid4())} + again = await call("namespace.restore", {"target_namespace": ns, "payload": payload}) + assert again.status_code == 422, again.text + assert again.json()["detail"]["reason"] == "forgotten" + assert (await _get("k1", erased_user)).get("found") is False await pg_memory.sweep_erasure_evidence(10_000) finally: await pg.close_pool() diff --git a/tests/integration/test_profile.py b/tests/integration/test_profile.py index 038e454b15..0826e498c3 100644 --- a/tests/integration/test_profile.py +++ b/tests/integration/test_profile.py @@ -159,7 +159,17 @@ async def test_profile_append_respects_existing_cap(agent) -> None: @pytest.mark.asyncio async def test_profile_restore_preserves_char_limit(agent) -> None: await agent.profile.set("revivable", "Stable persona text.", char_limit=1024) - await agent.profile.delete("revivable") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=agent.org, + namespace=agent.namespace, + agent=agent.agent, + memory_key="revivable", + ) + ) await agent.memory.restore("revivable") got = await agent.profile.get("revivable") assert got is not None @@ -226,7 +236,17 @@ async def test_memory_restore_preserves_pinned(agent) -> None: got = await agent.memory.get(memory_key="important") assert got is not None and got.get("pinned") is True, "pin failed to take" - await agent.memory.delete(memory_key="important") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=agent.org, + namespace=agent.namespace, + agent=agent.agent, + memory_key="important", + ) + ) await agent.memory.restore("important") got = await agent.memory.get(memory_key="important") diff --git a/tests/integration/test_ratelimit.py b/tests/integration/test_ratelimit.py index a0005c0b3d..9bef4ee7b9 100644 --- a/tests/integration/test_ratelimit.py +++ b/tests/integration/test_ratelimit.py @@ -177,7 +177,12 @@ async def test_memory_class_more_generous_than_knowledge_add( async with httpx.AsyncClient( base_url=control_plane_url, timeout=10.0, headers=_auth_headers() ) as c: - responses = await asyncio.gather(*[c.post("/v1/memory.set", json=body) for _ in range(15)]) + responses = await asyncio.gather( + *[ + c.post("/v1/memory.set", json={**body, "memory_key": f"k{i}", "value": i}) + for i in range(15) + ] + ) statuses = [r.status_code for r in responses] assert all(s == 200 for s in statuses), f"memory.set should not throttle at 15 reqs: {statuses}" diff --git a/tests/integration/test_raw_evidence_sensitivity.py b/tests/integration/test_raw_evidence_sensitivity.py index 4a83fc6ee0..492595bd03 100644 --- a/tests/integration/test_raw_evidence_sensitivity.py +++ b/tests/integration/test_raw_evidence_sensitivity.py @@ -198,9 +198,22 @@ def test_a_chunk_an_earlier_build_embedded_is_held_back_by_its_text( def test_a_session_digest_that_reads_as_a_credential_has_no_vector( control_plane_url: str, ch_client, token: str ) -> None: - """Imported so the digest's text is known: a plain digest is embedded and found, one - that reads as high is stored as written with no vector and never a hit.""" + """A proven plain digest is found; credential-bearing digest text is withheld.""" plain, secret = f"s-{uuid.uuid4().hex[:8]}", f"s-{uuid.uuid4().hex[:8]}" + for session, text in ((plain, _PLAIN_TURN[:300]), (secret, _SECRET_TURN[:300])): + source = _post( + control_plane_url, + token, + NS_DIGEST, + "memory.set", + { + "session_id": session, + "memory_key": "digest-source", + "memory_type": "episodic", + "value": text, + }, + ) + assert source.status_code == 200, source.text r = _post( control_plane_url, token, @@ -208,12 +221,18 @@ def test_a_session_digest_that_reads_as_a_credential_has_no_vector( "memory.import_evidence", { "digests": [ - {"session_id": plain, "summary_text": _PLAIN_TURN[:300]}, - {"session_id": secret, "summary_text": _SECRET_TURN[:300]}, + { + "session_id": session, + "summary_text": text, + "key_facts": [text], + "derived_from": ["digest-source"], + } + for session, text in ((plain, _PLAIN_TURN[:300]), (secret, _SECRET_TURN[:300])) ] }, ) - assert r.status_code == 200 and r.json()["digests_written"] == 2, r.text + assert r.status_code == 200 and r.json()["digests_written"] == 1, r.text + assert r.json()["skipped"] == 1, r.text dims = dict( ch_client.query( "SELECT session_id, embedding_dim FROM loom.session_summaries FINAL " @@ -221,7 +240,7 @@ def test_a_session_digest_that_reads_as_a_credential_has_no_vector( parameters={"o": ORG, "n": NS_DIGEST}, ).result_rows ) - assert int(dims[secret]) == 0 and int(dims[plain]) > 0, dims + assert secret not in dims and int(dims[plain]) > 0, dims r = _search(control_plane_url, token, NS_DIGEST, "nightly export billing job key") assert f"summary:{secret}" not in _keys(r) assert _API_KEY not in r.text and _AWS_KEY_ID not in r.text diff --git a/tests/integration/test_read_scope_isolation.py b/tests/integration/test_read_scope_isolation.py index 18f698c3b4..2b0d4a6cbc 100644 --- a/tests/integration/test_read_scope_isolation.py +++ b/tests/integration/test_read_scope_isolation.py @@ -348,7 +348,11 @@ def _bounded_writes() -> list[Any]: names = {n for n, why in NARROW_SAFE_OPS.items() if why.startswith("a write")} names |= {n for n in SCOPED_OPS if _writes(OPERATIONS_BY_NAME[n])} - return [OPERATIONS_BY_NAME[n] for n in sorted(names)] + # A permanent delete runs after writes that reuse this walk's canary value. + return [ + OPERATIONS_BY_NAME[n] + for n in sorted(names, key=lambda name: (name == "memory.delete", name)) + ] def _refused_ops() -> list[Any]: @@ -992,6 +996,18 @@ def test_writes_aimed_at_vs_runs_threads_records_and_blobs_leave_them_alone( "memory.set", {**v, "memory_key": f"pinned/{tag}", "value": pinned_text, "pinned": True}, ) + wait_until( + lambda: ( + ch_client.query( + "SELECT count() FROM loom.memory_embeddings FINAL " + "WHERE org = {org:String} AND namespace = {ns:String} " + "AND memory_key = {key:String} AND tombstoned_at IS NULL", + parameters={"org": seed.org, "ns": seed.ns, "key": f"pinned/{tag}"}, + ).result_rows[0][0] + == 1 + ), + description="the pinned conflict peer to be published", + ) near = {"memory_key": f"near/{tag}", "value": pinned_text.replace(code, "unknown")} # Control: an unbounded writer is blocked by the pinned row, so the probe can see it. control = _write( diff --git a/tests/integration/test_repositories_live_ch.py b/tests/integration/test_repositories_live_ch.py index 4a663f29b4..664cfe0947 100644 --- a/tests/integration/test_repositories_live_ch.py +++ b/tests/integration/test_repositories_live_ch.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import os import uuid from datetime import UTC, datetime @@ -13,6 +14,7 @@ import pytest +from server.core.hashing import sha256_value from server.identity.domain.read_scope import WHOLE_NAMESPACE pytestmark = pytest.mark.skipif( @@ -87,6 +89,7 @@ async def _seed_memory( "memory_key": memory_key, "memory_type": memory_type, "value": json.dumps(value, default=str), + "content_sha256": sha256_value(value), "confidence": confidence, "version": version, "pinned": 1 if pinned else 0, @@ -97,6 +100,44 @@ async def _seed_memory( return version +async def _seed_promoted( + ch: Any, org: str, ns: str, key: str, value: str, memory_type: str = "semantic" +) -> Any: + from server.memory import store as memory_repo + from server.memory.domain.promotion_source import promotion_source + from server.memory.store import shared + from server.memory.store.refusals import source_publication + + identity = memory_repo.Identity(org=org, namespace=ns, agent="itest-bot") + await _seed_memory(ch, identity, key, memory_type, value) + source = await memory_repo.get_by_key(identity, key, ch=ch, fresh=True, scope=WHOLE_NAMESPACE) + assert source is not None + now = datetime.now(UTC).replace(tzinfo=None) + row = { + "org": org, + "namespace": ns, + "memory_key": key, + "value": json.dumps(value), + "memory_type": memory_type, + "confidence": 0.9, + "promoted_from": [ + promotion_source(org, ns, source.agent, source.user_id, source.session_id, key) + ], + "promotion_event": str(uuid.uuid4()), + "promoted_from_candidate_id": None, + "promoted_by": "process:test", + "rationale": "fixture", + "promoted_at": now, + "reversible_until": now, + "superseded_by": None, + "tombstoned_at": None, + "version": memory_repo.next_version(), + } + async with source_publication(ch, source, key): + await shared.insert_shared_row(row, ch=ch) + return source + + async def _tombstone_memory(ch: Any, identity: Any, memory_key: str) -> None: await ch.execute( """ @@ -472,33 +513,9 @@ async def test_shared_repo_insert_fetch_promoted( org, ns = "itest-org", isolated_id key = "itest-shared-key" - now = datetime.now(UTC).replace(tzinfo=None) - promo_id = str(uuid.uuid4()) - await shared_repo.insert_shared_row( - { - "org": org, - "namespace": ns, - "memory_key": key, - "value": "v1", - "memory_type": "semantic", - "confidence": 0.9, - "promoted_from": [], - "promotion_event": promo_id, - "promoted_from_candidate_id": None, - "promoted_by": "itest", - "rationale": "itest", - "promoted_at": now, - "reversible_until": now, - "superseded_by": None, - "tombstoned_at": None, - "version": 1, - }, - ch=ch, - ) - + await _seed_promoted(ch, org, ns, key, "v1") row = await shared_repo.fetch_shared(org, ns, key, ch=ch) - assert row is not None - assert row["value"] == "v1" + assert row is not None and json.loads(row["value"]) == "v1" @pytest.mark.asyncio @@ -699,6 +716,10 @@ async def test_embedding_worker_reconciles_shared_stub( org = "itest-org" ns = isolated_id memory_key = f"shared-orphan-{uuid.uuid4().hex[:6]}" + from server.memory.domain.excerpt import build_excerpt + + value = "this is the content the worker will embed" + await _seed_promoted(ch, org, ns, memory_key, value, "preference") # tz-aware UTC matches the worker's INSERT path (collapses on FINAL) now = datetime.now(UTC) # stub must use the worker's model_name or rows never collapse on FINAL @@ -720,7 +741,8 @@ async def test_embedding_worker_reconciles_shared_stub( "memory_key": memory_key, "memory_type": "preference", "scope": "shared", - "content_excerpt": "this is the content the worker will embed", + "content_excerpt": build_excerpt(json.dumps(value), memory_key=memory_key), + "content_sha256": sha256_value(value), "embedding_model": embedder_model, "embedding_dim": 0, "embedding": _wide([]), diff --git a/tests/integration/test_restore_reopens_facts_ch.py b/tests/integration/test_restore_reopens_facts_ch.py index bdf66d74fc..9779ae69e4 100644 --- a/tests/integration/test_restore_reopens_facts_ch.py +++ b/tests/integration/test_restore_reopens_facts_ch.py @@ -52,15 +52,15 @@ async def _paris_open(ch: Any, org: str, want: bool) -> bool: async def test_undoing_a_delete_reopens_the_facts_the_row_states(ch) -> None: from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput - from server.memory.app.handlers import handle_memory_delete from server.memory.app.restore import handle_memory_restore + from tests.integration._memory import retire_reversibly org = unique_org("restore-facts") fact = ("user", "live_in", "paris") a, b = f"a-{uuid.uuid4().hex[:6]}", f"b-{uuid.uuid4().hex[:6]}" await _set(org, a, "I live in Paris", [fact]) await _set(org, b, "Home for me is the city of Paris, France", [fact]) - await handle_memory_delete(MemoryDeleteInput(org=org, namespace=NS, agent="bot", memory_key=a)) + await retire_reversibly(MemoryDeleteInput(org=org, namespace=NS, agent="bot", memory_key=a)) # B still states it, so it stays open; B's revision drops it, and nothing holds it. assert await _paris_open(ch, org, want=True) await _set(org, b, "Weekends go to sailing lessons on the lake", []) diff --git a/tests/integration/test_scim_provider_flows.py b/tests/integration/test_scim_provider_flows.py index d6121905ec..4f63f2e5ec 100644 --- a/tests/integration/test_scim_provider_flows.py +++ b/tests/integration/test_scim_provider_flows.py @@ -78,7 +78,7 @@ def test_okta_spec_test(scim_server) -> None: # Deactivated by a path-less replace (Okta OIN) or a full PUT (Okta AIW). off = _patch(base, user["id"], {"op": "replace", "value": {"active": False}}) - assert off.status_code == 200 and off.json()["active"] is False + assert off.status_code == 200 and off.json()["active"] is False, off.text on = _scim(base, "PUT", f"/Users/{user['id']}", json={**okta, "active": True}) assert on.status_code == 200 and on.json()["active"] is True assert _scim(base, "GET", "/Groups", params={"startIndex": "1", "count": "100"}).json() == { @@ -131,7 +131,7 @@ def test_entra_provisioning_and_validator(scim_server) -> None: ) assert edits.status_code == 200, edits.text off = _patch(base, uid, {"op": "Replace", "path": "active", "value": "False"}) - assert off.status_code == 200 and off.json()["active"] is False + assert off.status_code == 200 and off.json()["active"] is False, off.text assert _scim(base, "GET", f"/Users/{uid}").json()["active"] is False # still listed # The aadOptscim062020 dialect: a path-less replace with dotted keys. on = _patch( diff --git a/tests/integration/test_sensitivity_high_canary.py b/tests/integration/test_sensitivity_high_canary.py index 3f8d25c74f..6821bcf854 100644 --- a/tests/integration/test_sensitivity_high_canary.py +++ b/tests/integration/test_sensitivity_high_canary.py @@ -285,12 +285,17 @@ def test_payment_and_identity_numbers_are_labelled_on_write( "recall_by_agent": [], "sensitivity": level, }, r.text[:500] - stored = ch_client.query( - "SELECT memory_id, org, namespace, agent, user_id, session_id, sensitivity " - "FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} " - "AND memory_key = {k:String} AND tombstoned_at IS NULL", - parameters={"o": ORG, "n": NAMESPACE, "k": key}, - ).result_rows + + def stored_rows(): + return ch_client.query( + "SELECT memory_id, org, namespace, agent, user_id, session_id, sensitivity " + "FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} " + "AND memory_key = {k:String} AND tombstoned_at IS NULL", + parameters={"o": ORG, "n": NAMESPACE, "k": key}, + ).result_rows + + _wait(lambda: bool(stored_rows())) + stored = stored_rows() assert [tuple(map(str, row)) for row in stored] == [ (got["memory_id"], ORG, NAMESPACE, "canary", "-", "-", level) ] diff --git a/tests/integration/test_vector_retire_race.py b/tests/integration/test_vector_retire_race.py index 9571ced922..291594413c 100644 --- a/tests/integration/test_vector_retire_race.py +++ b/tests/integration/test_vector_retire_race.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import hashlib import json import os import uuid @@ -19,6 +18,7 @@ import pytest +from server.core.hashing import sha256_value from server.settings import settings from tests.integration._eventually import eventually @@ -83,8 +83,8 @@ def _memory_row(ns: str, version: int, value: str, sensitivity: str) -> dict[str "memory_key": KEY, "memory_type": "semantic", "value": json.dumps(value), - "value_json": json.dumps({"v": value}), - "content_sha256": hashlib.sha256(value.encode()).hexdigest(), + "value_json": "{}", + "content_sha256": sha256_value(value), "confidence": 1.0, "actor": "process:itest", "source_class": "asserted", @@ -119,6 +119,7 @@ async def _embedded_write(ch: Any, ns: str, value: str, entity: str) -> None: memory_type="semantic", scope="private", content_excerpt=value, + content_sha256=sha256_value(value), embedding_model="fake", embedding_dim=_DIM, embedding=_wide([0.1, 0.2, 0.3]), @@ -870,10 +871,11 @@ async def live_chunks() -> int: async def test_a_deleted_memory_restores_searchable(ch: Any) -> None: - """set, delete, restore: the restored key has a live vector again (the restore's + """Set, automatic retirement, restore: the restored key has a live vector again (the restore's embeddings read once failed on every call with ILLEGAL_AGGREGATION).""" from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput, MemorySetInput - from server.memory.app import handlers, restore, write + from server.memory.app import restore, write + from tests.integration._memory import retire_reversibly ns = f"itest-{uuid.uuid4().hex[:12]}" key = f"restore_live_{uuid.uuid4().hex[:6]}" @@ -892,7 +894,7 @@ async def live_vectors() -> int: value = f"the {key} vault is audited each quarter" await write.handle_memory_set(MemorySetInput(**ident, memory_key=key, value=value)) assert await live_vectors() == 1 - out = await handlers.handle_memory_delete(MemoryDeleteInput(**ident, memory_key=key)) + out = await retire_reversibly(MemoryDeleteInput(**ident, memory_key=key)) assert out.tombstoned and await live_vectors() == 0 res = await restore.handle_memory_restore(MemoryRestoreInput(**ident, memory_key=key)) assert res.restored @@ -1279,6 +1281,7 @@ async def query_all(self, sql: str, *a: Any, **kw: Any) -> Any: and "head" not in self.fired and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql and "FROM loom.memory FINAL" in sql + and "WHERE tombstoned_at IS NULL" in sql ): self.fired.append("head") await self._before_head() @@ -1355,6 +1358,7 @@ async def query_all(self, sql: str, *a: Any, **kw: Any) -> Any: and "head" not in self.fired and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql and "FROM loom.memory FINAL" in sql + and "WHERE tombstoned_at IS NULL" in sql ): self.fired.append("head") elif ( @@ -1607,7 +1611,8 @@ async def _high_key_with_row_above_its_fence( created_at=datetime.now(UTC).replace(tzinfo=None), version=int(head["version"]) + 10, ) - await M.insert_embedding_rows([row], ch=ch) + # Seed a legacy cache write already admitted before its source became high. + await ch.insert("loom.memory_embeddings", [row], settings={"async_insert": 0}) return secret, int(head["version"]) @@ -1636,8 +1641,8 @@ async def test_a_high_candidate_whose_head_moved_retires_nothing(ch: Any) -> Non newer = scanned | { "version": M.next_version(), "value": json.dumps(text), - "value_json": json.dumps({"v": text}), - "content_sha256": hashlib.sha256(text.encode()).hexdigest(), + "value_json": "{}", + "content_sha256": sha256_value(text), } await M.insert_memory_versions([newer], ch=ch) embedder = get_embedder() @@ -1651,6 +1656,7 @@ async def test_a_high_candidate_whose_head_moved_retires_nothing(ch: Any) -> Non memory_type="semantic", scope="private", content_excerpt=text, + content_sha256=newer["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1702,6 +1708,7 @@ async def test_a_high_candidate_is_labelled_whatever_other_key_holds_its_id(ch: memory_type="semantic", scope="private", content_excerpt="the vault password", + content_sha256=scanned["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1757,6 +1764,7 @@ async def test_the_sweep_labels_a_row_stored_with_no_user_or_session(ch: Any) -> memory_type="semantic", scope="private", content_excerpt="the badge PIN", + content_sha256=row["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1879,6 +1887,7 @@ def late_row(key: str, *, ago_s: int = 30) -> dict[str, Any]: memory_type="semantic", scope="private", content_excerpt=f"the {word} runbook", + content_sha256=sha256_value(f"the {word} runbook"), embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1886,6 +1895,12 @@ def late_row(key: str, *, ago_s: int = 30) -> dict[str, Any]: version=time.time_ns() - ago_s * 1_000_000_000, # drawn ago_s seconds ago ) + async def publish(rows: list[dict[str, Any]]) -> None: + for row in rows: + canonical = _memory_row(ns, row["version"], row["content_excerpt"], "") + await M.insert_memory_versions([{**canonical, "memory_key": row["memory_key"]}], ch=ch) + assert len(await M.insert_embedding_rows(rows, ch=ch)) == len(rows) + async def marker_indexed() -> bool: await refresh() row = await ch.query_one( @@ -1898,10 +1913,10 @@ async def marker_indexed() -> bool: try: # A row drawn now: once a pass has indexed it, the watermarks are past it and so # past the late rows below, drawn 30 s ago, whatever other namespaces wrote. - await M.insert_embedding_rows([late_row("marker", ago_s=0)], ch=ch) + await publish([late_row("marker", ago_s=0)]) await eventually(marker_indexed, max_wait=60.0, interval=1.0, description="the marker") indexed, control = late_row("slow_write"), late_row("slow_write_unindexed") - await M.insert_embedding_rows([indexed, control], ch=ch) + await publish([indexed, control]) assert await M.index_late_rows([indexed], ch=ch) == 1 await refresh() stats = await ch.query_all( @@ -1917,7 +1932,13 @@ async def marker_indexed() -> bool: ) assert int(df["n"]) == 2 # the marker and the indexed late row, not the control finally: - for table in ("memory_embeddings", "memory_doc_stats", "memory_term_freq"): + for table in ( + "memory", + "memory_versions", + "memory_embeddings", + "memory_doc_stats", + "memory_term_freq", + ): await ch.execute( f"DELETE FROM loom.{table} WHERE org = {{o:String}} AND namespace = {{n:String}}", params={"o": ORG, "n": ns}, diff --git a/tests/unit/bulk_stubs.py b/tests/unit/bulk_stubs.py index 615ca5e014..90195c76da 100644 --- a/tests/unit/bulk_stubs.py +++ b/tests/unit/bulk_stubs.py @@ -36,7 +36,11 @@ def __init__(self, lookup_rows: list[dict[str, Any]] | None = None) -> None: async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: - if "FROM loom.memory" in sql: + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store import identity_key + + return [r for r in self.lookup_rows if identity_key(r) in (params or {})["keys"]] + if "FROM loom.memory " in sql: return self.lookup_rows return [] diff --git a/tests/unit/ch_fake.py b/tests/unit/ch_fake.py index ebdbe73779..743756e26d 100644 --- a/tests/unit/ch_fake.py +++ b/tests/unit/ch_fake.py @@ -88,7 +88,14 @@ def _read(self, kind: str, sql: str, params: dict[str, Any] | None) -> Rows: raise AssertionError(f"unrouted {kind}: {head!r}; routes: {known}") return [] - async def query_all(self, sql: str, params: dict[str, Any] | None = None, **_: Any) -> Rows: + async def query_all( + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, + **_: Any, + ) -> Rows: return self._read("query_all", sql, params) async def query_one( diff --git a/tests/unit/clickhouse_local.py b/tests/unit/clickhouse_local.py index f16aaf703a..04c90801ba 100644 --- a/tests/unit/clickhouse_local.py +++ b/tests/unit/clickhouse_local.py @@ -126,6 +126,13 @@ def __init__(self, path: Path, *setup: str) -> None: def create(self, *statements: str) -> None: """Run DDL (with any session `SET` it needs) in one session.""" + created = {m.group(1) for statement in statements for m in _CREATED.finditer(statement)} + if "loom.memory" in created: + # Canonical writers read consent and erasure state even when both are empty. + dependencies = ("memory_refusals", "erasure_requests") + statements += tuple( + ddl(table) for table in dependencies if f"loom.{table}" not in created | self.tables + ) run(list(statements), "SELECT 1", path=self.path) self.tables |= {m.group(1) for s in statements for m in _CREATED.finditer(s)} diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 96cd97172f..0577304b9d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -76,6 +76,11 @@ async def capture(org): monkeypatch.setattr(blacklist_guard, "capture_policy", capture) monkeypatch.setattr(blacklist_guard, "enforce_rows", allow) + async def no_refusal(ch, rows): + return None + + monkeypatch.setattr("server.memory.store.refusals.check_clickhouse", no_refusal) + @pytest.fixture(autouse=True) def auth_account_lifecycle(monkeypatch): @@ -1210,11 +1215,13 @@ def memory_insert_keeper(monkeypatch): Business-domain ClickHouse fakes need no Keeper SQL implementation. Tests of contention and failure can change this store or replace the fence explicitly. """ - from server.memory.store import write_fence + from server.memory.store import refusals, write_fence from tests.unit.claims_fake import FakeLockTable keeper = FakeLockTable() monkeypatch.setattr(write_fence, "_lease_client", lambda ch: keeper) + monkeypatch.setattr(refusals, "_lease_client", lambda ch: keeper) + monkeypatch.setattr(refusals, "_admission_wait_s", lambda: 0.0) # Domain tests fail fast on deliberate contention; waiting has explicit coverage. monkeypatch.setattr(write_fence, "_admission_wait_s", lambda: 0.0) return keeper diff --git a/tests/unit/memory_store.py b/tests/unit/memory_store.py index e3fda9da92..47599a4d8c 100644 --- a/tests/unit/memory_store.py +++ b/tests/unit/memory_store.py @@ -347,3 +347,37 @@ async def before(row: mr.MemoryRow, *, ch: Any = None) -> str: monkeypatch.setattr(mr, "status_before", before) return asked + + +def seed_summary_sources(monkeypatch: pytest.MonkeyPatch, rows: list[dict[str, Any]]) -> None: + """Give model-policy fixtures real, complete extractive digests and source heads.""" + heads = {} + for row in rows: + key = (row.get("derived_from") or [f"source:{row['session_id']}"])[0] + for column in ("agent", "user_id", "session_id"): + row.setdefault(column, "-") + text = row["summary_text"] + row.update(derived_from=[key], key_facts=[text]) + source = mr.MemoryRow( + memory_id=str(uuid.uuid4()), + org=row["org"], + namespace=row["namespace"], + agent=row["agent"], + user_id=row["user_id"], + session_id=row["session_id"], + memory_key=key, + memory_type="semantic", + value=text, + confidence=1, + version=1, + pinned=False, + created_at=datetime.now(UTC), + expires_at=None, + ) + heads[mr.identity_key(source)] = source + + async def read(keys: Any, **kwargs: Any) -> dict[Any, Any]: + assert kwargs["sequential"] + return {key: heads[key] for key in keys if key in heads} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) diff --git a/tests/unit/session_content_senders.json b/tests/unit/session_content_senders.json index 2ab7d3f24c..a88a417f63 100644 --- a/tests/unit/session_content_senders.json +++ b/tests/unit/session_content_senders.json @@ -406,7 +406,7 @@ "sdk/python/loom/integrations/langgraph.py": "5d68e6ab1645d868477a13b14aa6d734c72f72057464cac9a21aa707add17e7b", "sdk/python/loom/integrations/llamaindex.py": "06eaa071115cad752d27bdc9bf7c4d5b6f5d7d39d9896267fcd459a4e06bf661", "sdk/python/loom/integrations/openai_agents.py": "143fe1df060b96550a33f77fa44416e291c2154552e433f8309275f61c8fca2a", - "sdk/python/loom/memory.py": "e1384e84ddf58b503dc430eca1b2e734eb5e5680ac6904a64785bb50a63316d1", + "sdk/python/loom/memory.py": "ce7a8616b3d079d7c5241d932f04d85b5c6da6f4843d68d3938e52a3ddb6e8ee", "sdk/python/loom/run.py": "1d47d5f4d197f78de347d2cf66b9f59f4292846d214f7116c444035971c6ab42", "sdk/python/loom/session.py": "5f0fd61fcfdbf9d60888b111887fca8f32e24eb51ca9ff04a256901657c3202b", "sdk/typescript/src/integrations/claude-memory.ts": "c846f0011d7cf1ba4b30e99fbfd8adb6b29abff56856d0dbe14fba620007f6bc", diff --git a/tests/unit/test_bm25_late_rows.py b/tests/unit/test_bm25_late_rows.py index c32c235946..7c9943a0eb 100644 --- a/tests/unit/test_bm25_late_rows.py +++ b/tests/unit/test_bm25_late_rows.py @@ -174,7 +174,11 @@ async def _noop(*_a: Any, **_k: Any) -> Any: monkeypatch.setattr(memory_repo, "index_late_rows", _index) monkeypatch.setattr(memory_repo, "insert_memory_versions", _noop) - monkeypatch.setattr(memory_repo, "insert_embedding_rows", _noop) + + async def _insert(rows: Any, **_: Any) -> Any: + return rows + + monkeypatch.setattr(memory_repo, "insert_embedding_rows", _insert) monkeypatch.setattr(memory_repo, "retire_vectors_for_keys", _noop) state = _BulkWriteState() state.memory_rows.append({**_row(), "memory_type": "semantic", "sensitivity": "", "version": 1}) @@ -204,9 +208,10 @@ async def _stubs(**_k: Any) -> list[dict[str, Any]]: async def _none(**_k: Any) -> list[Any]: return [] - async def _insert(rows: Any, **_k: Any) -> None: + async def _insert(rows: Any, **_k: Any) -> Any: if rows: order.append("insert") + return rows async def _index_after(rows: Any, **_k: Any) -> int: order.append("index") @@ -251,11 +256,11 @@ async def _index(rows: Any, **_k: Any) -> int: handed.extend(r["memory_key"] for r in rows) return 0 - async def _noop(*_a: Any, **_k: Any) -> Any: - return 0 + async def _insert(rows: Any, **_k: Any) -> Any: + return rows monkeypatch.setattr(bm25_stats, "index_late_rows", _index) - monkeypatch.setattr(memory_repo, "insert_embedding_rows", _noop) + monkeypatch.setattr(memory_repo, "insert_embedding_rows", _insert) await integrity.repair_dim0_rows( [{**_row(memory_key="dim0"), "embedding": [0.1, 0.2], "version": 1}], ch=object() ) diff --git a/tests/unit/test_cell_scan_tables.py b/tests/unit/test_cell_scan_tables.py index c558ec5e27..0305405d41 100644 --- a/tests/unit/test_cell_scan_tables.py +++ b/tests/unit/test_cell_scan_tables.py @@ -21,6 +21,7 @@ "loom.analysis_canary", "loom.audit_log", "loom.blacklist", + "loom.memory_refusals", "loom.blob_uploads", "loom.categories", "loom.chunk_supersessions", diff --git a/tests/unit/test_ch_client_settings.py b/tests/unit/test_ch_client_settings.py index 9a3bd3eaa3..7287cdbc43 100644 --- a/tests/unit/test_ch_client_settings.py +++ b/tests/unit/test_ch_client_settings.py @@ -14,6 +14,28 @@ from server.settings import settings +async def test_query_all_keeps_its_statement_settings_through_the_driver(monkeypatch): + seen = [] + + class Driver: + async def query(self, sql, parameters, settings, **kwargs): + seen.append((sql, parameters, settings)) + return SimpleNamespace(column_names=["v"], result_rows=[(1,)], summary={}) + + async def guarded(*args, **kwargs): + return None + + client = ch_mod.CHClient() + monkeypatch.setattr(type(client), "client", property(lambda self: Driver())) + monkeypatch.setattr(ch_mod, "guard_pending_schema", guarded) + result = await client.query_all( + "SELECT {v:UInt8} AS v", params={"v": 1}, settings={"allow_experimental_ai_functions": 1} + ) + assert result == [{"v": 1}] + assert seen[0][1] == {"v": 1} + assert seen[0][2]["allow_experimental_ai_functions"] == 1 + + @pytest.mark.asyncio async def test_session_settings_enable_filesystem_cache(monkeypatch: pytest.MonkeyPatch) -> None: captured: dict[str, Any] = {} diff --git a/tests/unit/test_chunk_parent_identity.py b/tests/unit/test_chunk_parent_identity.py index 6acff7aa7a..368651ecad 100644 --- a/tests/unit/test_chunk_parent_identity.py +++ b/tests/unit/test_chunk_parent_identity.py @@ -9,7 +9,7 @@ async def _project(monkeypatch, user, session): - ch = SimpleNamespace(insert=AsyncMock()) + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) async def embedded(text, ranges, chunks, embed): return [SimpleNamespace(written=True, vector=[1.0, 0.0]) for _ in chunks] diff --git a/tests/unit/test_clickhouse_mutation_inventory.py b/tests/unit/test_clickhouse_mutation_inventory.py index 7bf18dcb36..d4c18a6a4a 100644 --- a/tests/unit/test_clickhouse_mutation_inventory.py +++ b/tests/unit/test_clickhouse_mutation_inventory.py @@ -29,8 +29,8 @@ # written meanwhile holds (rare: only in that race), and end_triples_now, # which ends now a dropped fact that ends later (only when one does). ("server/memory/app/projections.py", "entity_triples", "UPDATE"): 3, - # In-place retries cannot recreate a transcript erased before a delayed command lands. - ("server/memory/app/extraction_queue.py", "memory_extraction_queue", "UPDATE"): 1, + # In-place retries and refusal termination never copy an erased transcript. + ("server/memory/app/extraction_queue.py", "memory_extraction_queue", "UPDATE"): 2, ("server/memory/store/blacklist.py", "blacklist", "UPDATE"): 1, ("server/memory/store/conflicts.py", "conflict_allowlist", "UPDATE"): 1, ("server/knowledge/store/conversations.py", "conversation_threads", "UPDATE"): 1, diff --git a/tests/unit/test_code_member_writes.py b/tests/unit/test_code_member_writes.py index 507d4ad159..eb006c7d36 100644 --- a/tests/unit/test_code_member_writes.py +++ b/tests/unit/test_code_member_writes.py @@ -175,16 +175,25 @@ def _row(actor: str, **over: Any) -> memory_repo.MemoryRow: class _CH: - def __init__(self) -> None: + def __init__(self, existing=None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} + if existing is not None: + row = memory_repo.build_memory_version_row(existing) + self.heads[memory_repo.identity_key(row)] = row async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update((memory_repo.identity_key(r), dict(r)) for r in rows) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql, params=None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql: + keys = set((params or {}).get("keys", self.heads)) + return [r for key, r in self.heads.items() if key in keys] return [] async def execute(self, *_: Any, **__: Any) -> None: @@ -206,7 +215,7 @@ def _wire(monkeypatch: pytest.MonkeyPatch, existing: memory_repo.MemoryRow | Non from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod - ch = _CH() + ch = _CH(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_dedup_ops.py b/tests/unit/test_dedup_ops.py index c4cfeb3d5e..f651a35f09 100644 --- a/tests/unit/test_dedup_ops.py +++ b/tests/unit/test_dedup_ops.py @@ -8,6 +8,7 @@ import pytest from fastapi import HTTPException +from server.core.hashing import sha256_value from server.memory.domain.webhook_events import memory_change from server.memory.store.dedup import CanonicalRef, DuplicateCluster from server.memory.store.rows import MemoryRow @@ -557,17 +558,28 @@ async def query_all(self, sql, params=None): if "FROM loom.memory_refusals" in sql: return [] if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: - return [mr.build_memory_version_row(existing)] - emb = {c: None for c in mr.EMBEDDING_PERSISTED_COLS} - identity = { - "org": "src-org", - "namespace": "src-ns", - "agent": "a", - "user_id": "u", - "session_id": "s", - "memory_key": "k", - } - return [emb | identity | {"status": "duplicate"}] + return next( + (rows for table, rows in reversed(self.inserts) if table == "loom.memory"), + [mr.build_memory_version_row(existing)], + ) + return [ + mr.build_embedding_row( + **{ + c: getattr(existing, c) + for c in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + ) + }, + status="duplicate", + version=existing.version, + content_sha256=sha256_value(existing.value), + ) + ] async def insert(self, table, rows, settings=None): self.inserts.append((table, rows)) diff --git a/tests/unit/test_delete_by_key.py b/tests/unit/test_delete_by_key.py index 10be6744d0..ec6851f344 100644 --- a/tests/unit/test_delete_by_key.py +++ b/tests/unit/test_delete_by_key.py @@ -8,6 +8,7 @@ import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from datetime import datetime from types import SimpleNamespace from typing import Any @@ -44,11 +45,16 @@ def test_home_namespace_prefers_the_callers_own() -> None: class _CH: def __init__(self) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.read_head = lambda: None async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: self.inserts.append((table, rows)) async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if params and "keys" in params: + head = self.read_head() + if head is not None and memory_repo.identity_key(head) in params["keys"]: + return [memory_repo.build_memory_version_row(head)] return [] async def query_one(self, sql: str, params: Any = None) -> Any: @@ -56,7 +62,7 @@ async def query_one(self, sql: str, params: Any = None) -> Any: def _row(agent: str, session_id: str) -> Any: - return SimpleNamespace( + return memory_repo.MemoryRow( org="clickhouse", namespace="jdoe", agent=agent, @@ -66,12 +72,15 @@ def _row(agent: str, session_id: str) -> Any: memory_id="33333333-3333-3333-3333-333333333333", memory_type="semantic", value='"x"', - run_id="", + run_id="-", source_class="extracted", tags=[], entities=[], version=1, - locked=False, + confidence=1.0, + pinned=False, + expires_at=None, + created_at=datetime(2026, 1, 1), ) @@ -83,6 +92,8 @@ def world(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: "ch": _CH(), } + state["ch"].read_head = lambda: state["row"] + async def _get_by_key(identity: Any, key: str, **kw: Any) -> Any: state["lookups"].append((identity.agent, identity.user_id, identity.session_id, key)) if identity.agent == "*": @@ -115,22 +126,6 @@ async def _no_check(*_a: Any, **_k: Any) -> None: monkeypatch.setattr(_restore_mod, "get_client", lambda: state["ch"]) monkeypatch.setattr("server.memory.app.write.get_client", hm.get_client) monkeypatch.setattr("server.memory.app.update.get_client", hm.get_client) - monkeypatch.setattr( - memory_repo, - "build_memory_version_row", - lambda existing, **kw: { - "org": existing.org, - "namespace": existing.namespace, - "memory_key": existing.memory_key, - "agent": existing.agent, - "user_id": existing.user_id, - "session_id": existing.session_id, - **kw, - }, - ) - # The stub builder returns partial rows; the funnel's full-row check is - # pinned in test_rmt_write_is_typed. - monkeypatch.setattr("server.memory.store.inserts.check_memory_rows", lambda rows: None) monkeypatch.setattr(hm._lock, "assert_unlocked", lambda *a, **k: None) return state @@ -153,7 +148,10 @@ async def _run() -> Any: out = asyncio.run(_run()) assert out.tombstoned is True and out.found is True assert world["lookups"][0][0] == "claude-desktop" and world["lookups"][1][0] == "*" - table, rows = world["ch"].inserts[0] + refused, fingerprints = world["ch"].inserts[0] + assert refused == "loom.memory_refusals" and fingerprints + assert all(len(row["digest"]) == 64 for row in fingerprints) + table, rows = next(item for item in world["ch"].inserts if item[0] == "loom.memory") assert table == "loom.memory" assert rows[0]["agent"] == "claude-code" and rows[0]["session_id"] == "sess-65187490" @@ -378,3 +376,62 @@ async def _hold(*key: str): assert held == [ ("clickhouse", "jdoe", "claude-code", "-", "sess-65187490", "user_home_location") ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_name", ["namespace", "own_session"]) +async def test_wildcard_deletion_keeps_the_requested_user_and_session_ceiling( + world, monkeypatch, scope_name +): + from server.identity.domain.read_scope import ReadFilters + from server.memory.api.models_crud import MemoryDeleteInput + + world["row"] = world["row"].model_copy(update={"user_id": "alice"}) + scopes = [] + original = memory_repo.get_by_key + + async def read(identity, key, **kwargs): + scopes.append(kwargs["scope"]) + return await original(identity, key, **kwargs) + + scope = ReadFilters( + scope_name, + "alice" if scope_name == "own_session" else None, + "sess-65187490" if scope_name == "own_session" else None, + ) + monkeypatch.setattr(hm, "request_scope", lambda: scope) + monkeypatch.setattr(memory_repo, "get_by_key", read) + out = await hm.handle_memory_delete( + MemoryDeleteInput( + org="clickhouse", + namespace="jdoe", + agent="*", + user_id="alice", + memory_key="user_home_location", + ) + ) + assert out.tombstoned + assert scopes == [ + ReadFilters( + "own_session" if scope_name == "own_session" else "own_user", "alice", scope.session_id + ) + ] + + +@pytest.mark.asyncio +async def test_a_pinned_user_cannot_wildcard_delete_a_different_users_row(world, monkeypatch): + from server.identity.domain.read_scope import ReadFilters + from server.memory.api.models_crud import MemoryDeleteInput + + monkeypatch.setattr(hm, "request_scope", lambda: ReadFilters("own_user", "bob", None)) + out = await hm.handle_memory_delete( + MemoryDeleteInput( + org="clickhouse", + namespace="jdoe", + agent="*", + user_id="alice", + memory_key="user_home_location", + ) + ) + assert not out.found and not out.tombstoned + assert world["lookups"] == [] and world["ch"].inserts == [] diff --git a/tests/unit/test_delete_version_consent_races.py b/tests/unit/test_delete_version_consent_races.py new file mode 100644 index 0000000000..8b56c82d5b --- /dev/null +++ b/tests/unit/test_delete_version_consent_races.py @@ -0,0 +1,249 @@ +"""A deletion refuses only the exact fact its tombstone actually removes.""" + +from datetime import datetime +from unittest.mock import AsyncMock + +import pytest + +from server.memory.api.models_crud import MemoryDeleteVersionInput +from server.memory.app import handlers +from server.memory.store import pg_memory, rows +from server.memory.store.delete_checks import checked_state_sha256 + +_AFTER_CHECKED = handlers._after_checked + + +def _head(value="first", version=10): + return rows.MemoryRow( + org="org", + namespace="ns", + agent="agent", + user_id="-", + session_id="-", + memory_key="key", + memory_type="semantic", + value=value, + confidence=1.0, + version=version, + created_at=datetime(2026, 1, 1), + memory_id="00000000-0000-0000-0000-000000000001", + pinned=False, + expires_at=None, + ) + + +def _check(): + return dict( + org="org", + namespace="ns", + agent="agent", + user_id="-", + session_id="-", + memory_key="key", + checked=10, + tombstone=20, + checked_state_sha256=checked_state_sha256(_head()), + ) + + +@pytest.fixture +def store(monkeypatch): + import server.memory.store as store + + monkeypatch.setattr(store, "record_delete_check", AsyncMock()) + monkeypatch.setattr(store, "fetch_version_rows", AsyncMock(return_value=[_head()])) + monkeypatch.setattr(store, "get_by_key", AsyncMock(return_value=_head())) + monkeypatch.setattr(handlers, "_after_checked", AsyncMock(return_value=([], []))) + return store + + +@pytest.mark.asyncio +@pytest.mark.parametrize("same_version", [False, True]) +async def test_a_known_pre_send_race_returns_changed_without_refusing_the_old_value( + store, monkeypatch, same_version +): + current = _head("second", 10 if same_version else 11) + if same_version: + store.fetch_version_rows.return_value = [current] + store.get_by_key.side_effect = [_head(), current] + monkeypatch.setattr( + handlers, + "_write_tombstone_row", + AsyncMock(side_effect=pg_memory.AdmissionContendedError("head changed")), + ) + out = await handlers._delete_version_locked( + MemoryDeleteVersionInput( + **{ + k: v + for k, v in _check().items() + if k not in {"checked", "tombstone", "checked_state_sha256"} + }, + version=10, + ), + rows.Identity(org="org", namespace="ns", agent="agent"), + ) + assert not out.tombstoned and out.reason == "version_changed" + assert handlers._write_tombstone_row.await_count == 1 + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_recovery_rechecks_a_second_known_pre_send_race(store, monkeypatch): + store.get_by_key.side_effect = [_head(), _head("second", 11)] + monkeypatch.setattr( + handlers, + "_write_tombstone_row", + AsyncMock(side_effect=pg_memory.AdmissionContendedError("head changed")), + ) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and not out.tombstoned and out.reason == "version_changed" + assert handlers._write_tombstone_row.await_count == 1 + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_durable_recovery_cannot_substitute_a_same_version_historical_row(store, monkeypatch): + store.fetch_version_rows.return_value = [_head("second")] + write = AsyncMock() + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and out.reason == "version_changed" + write.assert_not_awaited() + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_committed_tombstone_supplies_the_content_for_owed_retirement(store, monkeypatch): + tombstone = _head().model_copy( + update={ + "version": 20, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + store.fetch_version_rows.return_value = [_head("second"), tombstone] + finish = AsyncMock(return_value=None) + monkeypatch.setattr(handlers, "finish_delete_check", finish) + await handlers.rerun_delete_check(_check()) + removed = finish.call_args.args[0] + assert removed.value == "first" and removed.version == 10 + + +@pytest.mark.asyncio +async def test_a_retrys_tombstone_supplies_content_when_checked_history_collides( + store, monkeypatch +): + tombstone = _head().model_copy( + update={ + "version": 30, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + store.fetch_version_rows.return_value = [_head("second")] + handlers._after_checked.return_value = ([30], []) + monkeypatch.setattr(handlers, "_version_row", AsyncMock(return_value=tombstone)) + finish = AsyncMock(return_value=None) + monkeypatch.setattr(handlers, "finish_delete_check", finish) + await handlers.rerun_delete_check(_check()) + removed = finish.call_args.args[0] + assert removed.value == "first" and removed.version == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("other_versions", [([30], []), ([], [11])]) +async def test_an_unknown_checked_state_can_reconcile_without_recreating_a_tombstone( + store, monkeypatch, other_versions +): + handlers._after_checked.return_value = other_versions + monkeypatch.setattr( + handlers, + "_version_row", + AsyncMock( + return_value=_head().model_copy( + update={ + "version": 30, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + ), + ) + finish = AsyncMock(return_value=None) + write = AsyncMock() + monkeypatch.setattr(handlers, "finish_delete_check", finish) + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + check = _check() + check.pop("checked_state_sha256") + await handlers.rerun_delete_check(check) + finish.assert_awaited_once() + write.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("uncertain", [False, True]) +async def test_recovery_leaves_a_check_owed_after_bounded_contention_or_an_uncertain_send( + store, monkeypatch, uncertain +): + error = RuntimeError("uncertain send") if uncertain else pg_memory.AdmissionContendedError() + write = AsyncMock(side_effect=error) + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + with pytest.raises(type(error)): + await handlers.rerun_delete_check(_check()) + assert write.await_count == (1 if uncertain else 4) + store.record_delete_check.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_recovery_without_a_checked_state_cannot_refuse_a_historical_value( + store, monkeypatch +): + write = AsyncMock() + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + check = _check() + check.pop("checked_state_sha256") + out = await handlers.rerun_delete_check(check) + assert out is not None and out.reason == "version_changed" + assert store.record_delete_check.call_args.kwargs == {"done": True} + write.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deleted", [False, True]) +async def test_a_foreign_row_at_the_intended_tombstone_version_is_preserved( + store, monkeypatch, deleted +): + foreign = _head("second", 20) + if deleted: + foreign = foreign.model_copy( + update={ + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:9", + } + ) + history = {10: _head(), 20: foreign} + + async def fetch(org, namespace, pairs, **kwargs): + return [history[version] for key, version in pairs if version in history] + + store.fetch_version_rows.side_effect = fetch + monkeypatch.setattr(handlers, "_after_checked", _AFTER_CHECKED) + monkeypatch.setattr( + store, + "versions_above", + AsyncMock( + return_value={ + ("agent", "-", "-", "key"): [(20, deleted, 9 if deleted else None)], + } + ), + ) + lift, retire = AsyncMock(), AsyncMock() + monkeypatch.setattr(handlers, "_write_above", lift) + monkeypatch.setattr(handlers, "_retire_derived", retire) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and not out.tombstoned + assert out.found is (not deleted) + assert out.reason == ("" if deleted else "version_changed") + lift.assert_awaited_once() + assert lift.call_args.args[0].value == "second" + retire.assert_not_awaited() diff --git a/tests/unit/test_delete_version_sql.py b/tests/unit/test_delete_version_sql.py index c4e741579a..2e61556d97 100644 --- a/tests/unit/test_delete_version_sql.py +++ b/tests/unit/test_delete_version_sql.py @@ -18,6 +18,7 @@ import pytest import server.memory.app.handlers as hm +from server.core.hashing import sha256_value from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory import store as memory_repo from server.memory.api.models_crud import ( @@ -27,6 +28,7 @@ ) from server.memory.app import set_commit from server.memory.store import key_write_lock +from server.memory.store.refusals import check_clickhouse as check_consent from server.memory.store.rows import build_first_version_row from tests.unit.clickhouse_local import CLICKHOUSE, KeptCH, ddl, view @@ -54,7 +56,7 @@ def _row(value: str, version: int) -> dict[str, Any]: "memory_type": "procedural", # not embedded: memory.set's plain insert "value": json.dumps(value), "value_json": "{}", - "content_sha256": "", + "content_sha256": sha256_value(value), "actor": "process:okf-sync", "source_class": "imported", "confidence": 1.0, @@ -67,7 +69,7 @@ def _row(value: str, version: int) -> dict[str, Any]: async def _recorded_checks(ch: KeptCH) -> list[dict[str, Any]]: """Every check delete_version recorded, done or not, oldest tombstone first.""" return await ch.query_all( - "SELECT checked, tombstone, done FROM loom.delete_version_checks FINAL ORDER BY tombstone" + "SELECT checked, tombstone, checked_state_sha256, done FROM loom.delete_version_checks FINAL ORDER BY tombstone" ) @@ -77,6 +79,7 @@ async def _times_out(*_a: Any, **_k: Any) -> Any: @pytest.fixture def ch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> KeptCH: + monkeypatch.setattr("server.memory.store.refusals.check_clickhouse", check_consent) ch = KeptCH( tmp_path, ddl("memory"), @@ -111,6 +114,7 @@ def _in_flight_set(version: int) -> set_commit._SetWrite: expires_at=None, ) w.memory_row = _row("Thursdays", version) + w.new_sha = sha256_value("Thursdays") w.memory_id = _MID return w @@ -363,6 +367,7 @@ async def _noop(*_a: Any, **_k: Any) -> int: memory_key=_KEY, memory_type="semantic", content_excerpt=value, + content_sha256=sha256_value(value), version=v, embedding=[0.0] * 3072, embedding_dim=3072, @@ -740,7 +745,13 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: # Run again (its done mark lost), the check answers the same and copies nothing twice. rows_before = await ch.query_all("SELECT version FROM loom.memory_versions ORDER BY version") (done,) = await _recorded_checks(ch) - check = {**_ID, "memory_key": _KEY, "checked": done["checked"], "tombstone": done["tombstone"]} + check = { + **_ID, + "memory_key": _KEY, + "checked": done["checked"], + "tombstone": done["tombstone"], + "checked_state_sha256": done["checked_state_sha256"], + } again = await hm.rerun_delete_check(check) assert again is not None and (again.tombstoned, again.found) == (False, False) assert ( @@ -754,9 +765,18 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: monkeypatch.setattr(restore_mod, "get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Thursdays" @@ -795,15 +815,24 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: assert out.tombstoned is False and out.found is False monkeypatch.setattr(restore_mod, "get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Tuesdays" assert (list(got.tags) if column == "tags" else got.expire_reason) == changed @pytest.mark.asyncio -async def test_a_restored_key_does_not_carry_the_delete_version_marker( +async def test_a_deleted_version_refuses_restore_and_retains_its_marker( ch: KeptCH, monkeypatch: pytest.MonkeyPatch ) -> None: # The marker stays on delete_version's tombstone; the row memory.restore brings back, @@ -817,10 +846,19 @@ async def test_a_restored_key_does_not_carry_the_delete_version_marker( await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) ) - out = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert out.restored is True - got = await _read_back(ch) - assert got is not None and not got.source_trace.startswith("memory.delete_version:") + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) + assert got is not None and got.source_trace == f"memory.delete_version:{head}" @pytest.mark.asyncio @@ -835,11 +873,11 @@ async def test_a_later_write_listed_but_not_read_leaves_the_check_owed( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row + key_moved_on = hm._key_moved_on - async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + async def set_lands_first(existing: Any, **kw: Any) -> Any: + await ch.insert("loom.memory", [w.memory_row]) + return await key_moved_on(existing, **kw) async def nothing(*_a: Any, **_k: Any) -> list[Any]: return [] @@ -847,7 +885,7 @@ async def nothing(*_a: Any, **_k: Any) -> list[Any]: async def no_retire(*_a: Any, **_k: Any) -> None: raise AssertionError("nothing retires before the check runs") - monkeypatch.setattr(hm, "_write_tombstone_row", set_lands_first) + monkeypatch.setattr(hm, "_key_moved_on", set_lands_first) monkeypatch.setattr(hm, "_retire_derived", no_retire) monkeypatch.setattr(memory_repo, "fetch_version_rows", nothing) with pytest.raises(BackendUnavailableError): @@ -887,6 +925,7 @@ async def _noop(*_a: Any, **_k: Any) -> int: memory_key=_KEY, memory_type="semantic", content_excerpt=value, + content_sha256=sha256_value(value), version=v, embedding=[0.0] * 3072, embedding_dim=3072, @@ -1165,12 +1204,12 @@ async def test_a_check_that_fails_after_the_tombstone_is_finished_by_the_worker( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row + key_moved_on = hm._key_moved_on retired: list[Any] = [] async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + await ch.insert("loom.memory", [w.memory_row]) + raise RuntimeError("timeout") retire = hm._retire_derived @@ -1178,10 +1217,8 @@ async def count_retire(*a: Any, **k: Any) -> Any: retired.append(True) return await retire(*a, **k) - monkeypatch.setattr(hm, "_write_tombstone_row", set_lands_first) + monkeypatch.setattr(hm, "_key_moved_on", set_lands_first) monkeypatch.setattr(hm, "_retire_derived", count_retire) - key_moved_on = hm._key_moved_on - monkeypatch.setattr(hm, "_key_moved_on", _times_out) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) @@ -1333,12 +1370,11 @@ async def _live_chunk_contents(ch: KeptCH) -> list[str]: @pytest.mark.asyncio -async def test_a_short_versions_passage_shared_with_the_head_stays_live( +async def test_a_deleted_passage_is_not_restored_through_a_newer_heads_chunks( ch: KeptCH, monkeypatch: pytest.MonkeyPatch ) -> None: - # The removed version is too short to chunk, but its one passage is chunk 0 of the head - # that lands after the delete's check: the delete retires that row with the version's - # passages and puts it back for the head. + # A delayed row shares the deleted passage as its first chunk. Refusal keeps + # that passage retired even when the newer row still carries it. from server.memory.app import set_chunks from server.memory.app.chunk_projection import chunk_text @@ -1363,7 +1399,8 @@ async def head_lands_after_the_check(existing: Any, **kw: Any) -> Any: MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=old) ) assert out.tombstoned is True - assert await _live_chunk_contents(ch) == sorted(r["content"] for r in rows) + assert await _live_chunk_contents(ch) == sorted(r["content"] for r in rows[1:]) + assert rows[0]["content"] not in await _live_chunk_contents(ch) _PAST = datetime(2020, 1, 1) @@ -1581,15 +1618,15 @@ async def _owed_after_a_failed_check( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row - async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + async def late_set_then_timeout(existing: Any, **kw: Any) -> None: + # A previously admitted insert lands after the tombstone; its head check + # times out before it can lift the live write above the delete. + await ch.insert("loom.memory", [w.memory_row]) + raise RuntimeError("timeout") with monkeypatch.context() as m: - m.setattr(hm, "_write_tombstone_row", set_lands_first) - m.setattr(hm, "_key_moved_on", _times_out) + m.setattr(hm, "_key_moved_on", late_set_then_timeout) m.setattr(hm, "next_version", lambda: time.time_ns() + clock_ahead_ns) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( @@ -1664,25 +1701,31 @@ async def _tombstone_held_back(ch: KeptCH, monkeypatch: pytest.MonkeyPatch) -> t queued: the version it named, and the call that commits the insert late.""" head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) - write_row = hm._write_tombstone_row - queued: list[tuple[Any, dict[str, Any]]] = [] + insert = ch.insert + queued: list[tuple[str, list[dict[str, Any]], Any]] = [] - async def times_out_queued(existing: Any, **kw: Any) -> None: - queued.append((existing, kw)) - raise RuntimeError("timeout") + async def times_out_queued(table: str, rows: list[dict[str, Any]], settings=None) -> None: + if table == "loom.memory" and rows[0]["source_trace"].startswith("memory.delete_version:"): + queued.append((table, rows, settings)) + raise RuntimeError("timeout") + await insert(table, rows, settings=settings) async def commit_late() -> None: - ((existing, kw),) = queued - await write_row(existing, **kw) + ((table, rows, settings),) = queued + await insert(table, rows, settings=settings) from server.core.errors import BackendUnavailableError with monkeypatch.context() as m: - m.setattr(hm, "_write_tombstone_row", times_out_queued) + m.setattr(ch, "insert", times_out_queued) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) ) + from server.memory.store import write_fence + + keeper = write_fence._lease_client(ch) + keeper.advance(max((row["ttl"] for row in keeper.keys.values()), default=0) + 1) return head, commit_late @@ -1765,7 +1808,7 @@ async def record(event_type: str, **kw: Any) -> str: finally: request_id_var.reset(rid) setup_receipts = [row for event, row in emitted if event == "memory.set"] - assert len(setup_receipts) == int(failure in {"the check after it", "a retry"}) + assert setup_receipts == [] if setup_receipts: assert setup_receipts[0]["subject"]["memory_key"] == _KEY emitted[:] = [(event, row) for event, row in emitted if event != "memory.set"] @@ -1909,9 +1952,18 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: assert out.tombstoned is False and out.found is False monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) monkeypatch.setattr(restore_mod, "get_client", lambda: ch) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Thursdays" @@ -1923,7 +1975,13 @@ async def test_a_recovered_check_run_again_stands_on_the_tombstone_it_wrote( # stands on that tombstone, answers the same and writes nothing. head, _commit_late = await _tombstone_held_back(ch, monkeypatch) (owed,) = await _recorded_checks(ch) - check = {**_ID, "memory_key": _KEY, "checked": head, "tombstone": owed["tombstone"]} + check = { + **_ID, + "memory_key": _KEY, + "checked": head, + "tombstone": owed["tombstone"], + "checked_state_sha256": owed["checked_state_sha256"], + } first = await hm.rerun_delete_check(check) assert first is not None and first.tombstoned is True assert first.version is not None and first.version > int(owed["tombstone"]) diff --git a/tests/unit/test_digest_chain.py b/tests/unit/test_digest_chain.py index d7bbf4de07..206e87f1ec 100644 --- a/tests/unit/test_digest_chain.py +++ b/tests/unit/test_digest_chain.py @@ -23,6 +23,8 @@ def __init__(self, answers: list[list[dict[str, Any]]] | None = None, fail: bool self.fail = fail async def query_all(self, sql, params=None): + if "FROM loom.memory_refusals" in sql: + return [] if self.fail: raise RuntimeError("boom") self.queries.append((sql, dict(params or {}))) @@ -149,26 +151,33 @@ async def insert(self, table, rows, settings=None): self.inserts.append((table, list(rows))) -def _draft(): - from server.memory.app.session_summary import SessionSummaryDraft +def _wire_writer(monkeypatch, ch): + from types import SimpleNamespace + from unittest.mock import AsyncMock - return SessionSummaryDraft( - summary_text="on 2024-03-12, the user moved.", - embedding=[0.1] * 4, - embedding_model="fake-4", + from server.memory import store + from server.memory.app import session_summary + from tests.unit.test_memory_verify import _row + + source = _row(namespace="n", agent="a", user_id="u", session_id="s2", value="the user moved") + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={store.identity_key(source): source}), ) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", AsyncMock(return_value=[0.1] * 4)) + return _FakeState(written=[{"memory_id": source.memory_id, "version": source.version}]) @pytest.mark.asyncio async def test_digest_insert_then_chain_edges(monkeypatch) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.session_summary import _insert_session_summary + from server.memory.app.session_summary import _write_session_summary ch = _WriterCH(answers=[[{"session_id": "s1"}], []]) - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - await _insert_session_summary(_FakeInput(), _FakeState(), _draft()) + state = _wire_writer(monkeypatch, ch) + await _write_session_summary(_FakeInput(), state) tables = [t for t, _ in ch.inserts] assert tables == ["loom.session_summaries", "loom.operator_edges"] edge = ch.inserts[1][1][0] @@ -184,11 +193,14 @@ async def test_digest_insert_then_chain_edges(monkeypatch) -> None: @pytest.mark.asyncio async def test_chain_failure_leaves_the_digest(monkeypatch) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.session_summary import _insert_session_summary + from server.memory.app.session_summary import _write_session_summary class _NoQuery: + async def query_all(self, sql, params=None): + if "FROM loom.memory_refusals" in sql: + return [] + raise RuntimeError("chain unavailable") + def __init__(self): self.inserts = [] @@ -196,7 +208,6 @@ async def insert(self, table, rows, settings=None): self.inserts.append((table, list(rows))) ch = _NoQuery() # no query_all at all: the chain raises, the digest stays - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - await _insert_session_summary(_FakeInput(), _FakeState(), _draft()) + state = _wire_writer(monkeypatch, ch) + await _write_session_summary(_FakeInput(), state) assert [t for t, _ in ch.inserts] == ["loom.session_summaries"] diff --git a/tests/unit/test_embedding_reemit.py b/tests/unit/test_embedding_reemit.py index 83e7a2b26b..f525c43bb7 100644 --- a/tests/unit/test_embedding_reemit.py +++ b/tests/unit/test_embedding_reemit.py @@ -7,6 +7,7 @@ import asyncio import re import time +from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace @@ -14,6 +15,7 @@ import pytest +from server.core.hashing import sha256_value from server.memory import store as M @@ -33,15 +35,60 @@ async def _none(*_a: Any, **_k: Any) -> dict[Any, int]: class _CH: def __init__(self, rows: list[dict[str, Any]]) -> None: self.rows = rows + self.heads = {} + self.execute_calls = [] + for row in rows: + if row.get("scope") != "shared": + source = _memory_row( + **{ + k: row[k] + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + ) + } + ) + self.heads[M.identity_key(source)] = M.build_memory_version_row(source) self.query_all_calls: list[tuple[str, dict[str, Any]]] = [] self.insert_calls: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [r for k, r in self.heads.items() if k in (params or {}).get("keys", self.heads)] return list(self.rows) async def insert(self, table, rows, settings=None): self.insert_calls.append((table, rows, settings or {})) + if table == "loom.memory": + self.heads.update((M.identity_key(r), dict(r)) for r in rows) + + async def execute(self, sql, params=None, *, settings=None): + self.execute_calls.append((sql, params, settings)) + assert sql.startswith("DELETE FROM loom.memory_embeddings") + columns = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + ) + self.rows = [ + r + for r in self.rows + if not ( + all(r.get(c) == params[c] for c in columns) and r["version"] < params["version"] + ) + ] _LIVE = { @@ -59,6 +106,7 @@ async def insert(self, table, rows, settings=None): "memory_key_embedding": [0.4, 0.5, 0.6], "memory_key_embedding_dim": 3, "content_excerpt": "text", + "content_sha256": sha256_value("text"), "tags": ["t"], "confidence": 0.9, "expires_at": datetime(2026, 1, 1), @@ -130,8 +178,11 @@ async def test_shared_reemit_reads_the_scope_shared_rows_of_the_key() -> None: sql, params = ch.query_all_calls[0] assert "scope = 'shared'" in sql and "FINAL" in sql and "tombstoned_at IS NULL" in sql assert params == {"org": "o", "ns": "n", "k": "k"} - ((_, rows, _),) = ch.insert_calls - assert rows[0]["scope"] == "shared" and rows[0]["tombstoned_at"] == ts + assert not ch.insert_calls and not ch.rows + ((delete_sql, delete_params, settings),) = ch.execute_calls + assert delete_sql.startswith("DELETE FROM loom.memory_embeddings") + assert delete_params["scope"] == "shared" and delete_params["version"] > _LIVE["version"] + assert settings == {"lightweight_deletes_sync": 2} def _of_keys(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]: @@ -156,6 +207,8 @@ def __init__(self, rows: list[dict[str, Any]], entity_rows: list[dict[str, Any]] self.entity_rows = entity_rows async def query_all(self, sql, params=None): + if "memory_refusals" in sql or "FROM loom.memory FINAL" in sql: + return await super().query_all(sql, params) self.query_all_calls.append((sql, params or {})) if "loom.entity_vectors" in sql: return list(self.entity_rows) @@ -268,6 +321,48 @@ class _RMTCH: def __init__(self) -> None: self.emb: list[dict[str, Any]] = [] self.ent: list[dict[str, Any]] = [] + self.heads = {M.identity_key(_memory_row()): M.build_memory_version_row(_memory_row())} + self.execute_calls = [] + + def remember(self, rows): + for row in rows: + key = M.identity_key(row) + base = self.heads.get(key) or M.build_memory_version_row( + _memory_row( + **dict( + zip( + ("org", "namespace", "agent", "user_id", "session_id", "memory_key"), + key, + strict=True, + ) + ) + ) + ) + self.heads[key] = { + **base, + **{k: v for k, v in row.items() if k in M.MEMORY_PERSISTED_COLS}, + } + + async def execute(self, sql, params=None, *, settings=None): + self.execute_calls.append((sql, params, settings)) + assert sql.startswith("DELETE FROM loom.memory_embeddings") + columns = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + ) + self.emb = [ + r + for r in self.emb + if not ( + all(r.get(c) == params[c] for c in columns) and r["version"] < params["version"] + ) + ] @staticmethod def _final(rows: list[dict[str, Any]], key: Any, ver: Any) -> list[dict[str, Any]]: @@ -290,12 +385,50 @@ def live_ent(self) -> list[dict[str, Any]]: ) async def query_all(self, sql, params=None): + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [r for k, r in self.heads.items() if k in (params or {}).get("keys", self.heads)] if "entity_vectors" in sql: return [dict(r) for r in self.live_ent()] return [dict(r) for r in _of_keys(self.live_emb(), params)] async def insert(self, table, rows, settings=None): - (self.ent if table == "loom.entity_vectors" else self.emb).extend(dict(r) for r in rows) + if table == "loom.memory": + self.remember(rows) + elif table == "loom.entity_vectors": + self.ent.extend(dict(r) for r in rows) + elif table == "loom.memory_embeddings": + self.emb.extend(dict(r) for r in rows) + + +def _native_admission(monkeypatch, ch): + from server.core import pg + from server.memory.store import pg_memory + + async def execute(*args, **kwargs): + return None + + async def fetch(*args, **kwargs): + return [] + + async def fetchval(*args, **kwargs): + return None + + @asynccontextmanager + async def transaction(): + yield SimpleNamespace(execute=execute, fetch=fetch, fetchval=fetchval) + + async def heads(keys, **kwargs): + return {k: ch.heads[k] for k in keys if k in ch.heads} + + async def on_connection(conn, keys, **kwargs): + assert kwargs["lock"] + return await heads(keys) + + monkeypatch.setattr(pg, "transaction", transaction) + monkeypatch.setattr(pg_memory, "heads_by_keys", heads) + monkeypatch.setattr(pg_memory, "heads_on_connection", on_connection) class _Head(dict[str, Any]): @@ -320,21 +453,32 @@ async def test_a_reject_copies_the_heads_own_rows_and_retires_the_rest_bare(monk from server.core.ch import sequential_reads_active from server.memory.store import lifecycle, pg_memory, pg_publish - sha = "a" * 64 + sha = sha256_value("text") ch = _RMTCH() + _native_admission(monkeypatch, ch) ent = {"org": "o", "namespace": "n", "agent": "a", "user_id": "u", "memory_key": "k"} ch.ent.append({**ent, "entity_text": "e", "created_at": datetime(2026, 1, 1)}) ch.emb += [ {**_LIVE, "content_sha256": sha}, - {**_LIVE, "embedding_model": "m2", "content_sha256": "b" * 64, "content_excerpt": "old"}, - {**_LIVE, "session_id": "s2", "content_sha256": "c" * 64, "content_excerpt": "secret"}, + { + **_LIVE, + "embedding_model": "m2", + "content_sha256": sha256_value("old"), + "content_excerpt": "old", + }, + { + **_LIVE, + "session_id": "s2", + "content_sha256": sha256_value("secret"), + "content_excerpt": "secret", + }, ] heads = [ {**ent, "session_id": "s", "content_sha256": sha}, { **ent, "session_id": "s2", - "content_sha256": "c" * 64, + "content_sha256": sha256_value("secret"), "redacted_at": datetime(2026, 2, 1), }, ] @@ -343,6 +487,7 @@ async def _heads(*_: object) -> list[dict[str, Any]]: return heads async def _commit(rows: Any, **_: object) -> pg_memory.Committed: + ch.remember(rows) deadline = time.monotonic() + 30 targets = [0] * len(rows) return pg_memory.Committed( @@ -394,7 +539,8 @@ async def test_a_reject_whose_vector_copy_fails_still_stands_and_publishes( from server.memory.store import lifecycle, pg_memory, pg_publish ch = _RMTCH() - ch.emb.append({**_LIVE, "content_sha256": "a" * 64}) + _native_admission(monkeypatch, ch) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text")}) published: list[Any] = [] deadline = 0.0 if failure == "lease ended" else time.monotonic() + 30 @@ -405,9 +551,10 @@ async def _refused(*_: object, **__: object) -> None: monkeypatch.setattr(ch, "insert", _refused) async def _heads(*_: object) -> list[dict[str, Any]]: - return [{**_LIVE, "content_sha256": "a" * 64}] + return [{**_LIVE, "content_sha256": sha256_value("text")}] async def _commit(rows: Any, **_: object) -> pg_memory.Committed: + ch.remember(rows) return pg_memory.Committed(rows=rows, audits=[], seqs=[], targets=[0], deadline=deadline) async def _publish(committed: Any, **_: object) -> None: @@ -451,7 +598,7 @@ async def test_a_rejected_head_with_no_search_document_copies_no_row_of_its_text not landed: the reject copies none of them above it, and retires them with no text.""" from server.memory.store import lifecycle - sha = "a" * 64 + sha = sha256_value("text") ch = _RMTCH() ch.emb.append({**_LIVE, "content_sha256": sha}) rejected = {**_LIVE, "memory_type": "semantic", "content_sha256": sha, "version": 50} @@ -635,6 +782,7 @@ async def test_a_status_retire_writes_the_memory_rows_version_and_keeps_newer_ro {**_LIVE, "status": "superseded", "version": 30}, {**newer, "status": "superseded", "version": 30}, ] + ch.remember(rows) await M.mirror_status_to_embeddings(rows, ch=ch) by_agent = {r["agent"]: r for r in ch.live_emb()} assert (by_agent["a"]["status"], by_agent["a"]["version"]) == ("superseded", 30) @@ -674,13 +822,10 @@ class _SiteCH(_RMTCH): async def query_all(self, sql, params=None): if "embeddings_status" in sql: return [{**_LIVE, "status": "superseded", "invalid_at": None}] - if "entity_vectors" in sql or "memory_embeddings" in sql: - return await super().query_all(sql, params) - return [] + return await super().query_all(sql, params) async def insert(self, table, rows, settings=None): - if table in ("loom.memory_embeddings", "loom.entity_vectors"): - await super().insert(table, rows, settings) + await super().insert(table, rows, settings) def _memory_row(**over: Any) -> Any: @@ -689,7 +834,8 @@ def _memory_row(**over: Any) -> Any: **{c: _LIVE[c] for c in ("org", "namespace", "agent", "user_id", "session_id")}, "memory_key": "k", "memory_type": "episodic", # not searchable: no chunk side paths - "value": "v", + "value": "text", + "content_sha256": sha256_value("text"), "confidence": 0.9, "version": 1, "tags": [], @@ -709,7 +855,10 @@ def _patch_memory_io(mp: pytest.MonkeyPatch, ch: Any, row: Any) -> None: async def get_by_id(*_a, **_k): return row - async def no_insert(*_a, **_k): + ch.remember([M.build_memory_version_row(row)]) + + async def no_insert(rows, *_a, **_k): + ch.remember(rows) if on_inserted := _k.get("on_inserted"): on_inserted() return None @@ -892,6 +1041,7 @@ async def _site_dedup_collapse(ch, mp): from server.memory.api.models_graph import MemoryDedupCollapseInput from server.memory.app import dedup + ch.remember([M.build_memory_version_row(_memory_row(), status="duplicate")]) pair = SimpleNamespace(agent="a", user_id="u", session_id="s", memory_key="k", duplicate_of="c") async def pairs(*_a, **_k): @@ -982,8 +1132,9 @@ async def test_a_copy_under_a_high_or_deleted_head_is_retired_above_itself( async def _heads(ch: Any, keys: Any, sequential: bool) -> dict[Any, str]: return {k: ("high" if k[5] == "hi" else "") for k in keys if k[5] != "gone"} - async def _insert(rows: Any, **_k: Any) -> None: + async def _insert(rows: Any, **_k: Any) -> Any: inserted.extend(rows) + return rows monkeypatch.setattr(M.embeddings, "_head_sensitivity", _heads) monkeypatch.setattr(M.embeddings, "insert_embedding_rows", _insert) @@ -1035,8 +1186,8 @@ async def _check(rows: Any, **_k: Any) -> int: checked.append(list(rows)) return 0 - async def _insert(rows: Any, **_k: Any) -> None: - return None + async def _insert(rows: Any, **_k: Any) -> Any: + return rows monkeypatch.setattr(embeddings, "retire_copies_under_unembedded_heads", _check) monkeypatch.setattr(embeddings, "insert_embedding_rows", _insert) @@ -1095,7 +1246,12 @@ async def query_all(self, sql: str, params: Any = None) -> list[Any]: def _judged_memory_row() -> dict[str, Any]: # A deferred verdict lands at the judged write's version + 1 (`conflicts.judged_write`). - return {**_LIVE, "content_sha256": "a", "status": "duplicate", "version": _LIVE["version"] + 1} + return { + **_LIVE, + "content_sha256": sha256_value("text"), + "status": "duplicate", + "version": _LIVE["version"] + 1, + } @pytest.mark.asyncio @@ -1104,9 +1260,10 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_landed_first() -> N verdict's mirror must not tie with it or skip it.""" ch = _RMTCH() ch.emb += [ - {**_LIVE, "content_sha256": "a"}, - {**_LIVE, "content_sha256": "a", "version": _LIVE["version"] + 1}, + {**_LIVE, "content_sha256": sha256_value("text")}, + {**_LIVE, "content_sha256": sha256_value("text"), "version": _LIVE["version"] + 1}, ] + ch.remember([_judged_memory_row()]) await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) (live,) = ch.live_emb() assert live["status"] == "duplicate" @@ -1115,10 +1272,13 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_landed_first() -> N @pytest.mark.asyncio async def test_a_judged_mirror_outranks_a_metadata_copy_that_lands_after() -> None: ch = _RMTCH() - ch.emb.append({**_LIVE, "content_sha256": "a"}) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text")}) + ch.remember([_judged_memory_row()]) await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) # A copy that read write V before the mirror landed writes V + 1, still active. - ch.emb.append({**_LIVE, "content_sha256": "a", "version": _LIVE["version"] + 1}) + ch.emb.append( + {**_LIVE, "content_sha256": sha256_value("text"), "version": _LIVE["version"] + 1} + ) (live,) = ch.live_emb() assert live["status"] == "duplicate" @@ -1126,11 +1286,11 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_lands_after() -> No @pytest.mark.asyncio async def test_a_judged_mirror_leaves_a_newer_revisions_vector_alone() -> None: ch = _RMTCH() - newer = {**_LIVE, "content_sha256": "b", "version": M.next_version()} - ch.emb += [{**_LIVE, "content_sha256": "a"}, newer] + newer = {**_LIVE, "content_sha256": sha256_value("old"), "version": M.next_version()} + ch.emb += [{**_LIVE, "content_sha256": sha256_value("text")}, newer] assert await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) == 0 (live,) = ch.live_emb() - assert live["content_sha256"] == "b" and live["status"] == "active" + assert live["content_sha256"] == sha256_value("old") and live["status"] == "active" @pytest.mark.asyncio @@ -1138,8 +1298,13 @@ async def test_a_judged_mirror_leaves_a_vector_a_supersede_hid_meanwhile() -> No """Another write's refines verdict superseded A between the settle's memory row and its mirror: the retire stands, A does not come back into search.""" ch = _RMTCH() - retired = {**_LIVE, "content_sha256": "a", "status": "superseded", "version": M.next_version()} - ch.emb += [{**_LIVE, "content_sha256": "a"}, retired] + retired = { + **_LIVE, + "content_sha256": sha256_value("text"), + "status": "superseded", + "version": M.next_version(), + } + ch.emb += [{**_LIVE, "content_sha256": sha256_value("text")}, retired] assert await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) == 0 (live,) = ch.live_emb() assert live["status"] == "superseded" @@ -1153,19 +1318,39 @@ async def test_a_reject_copies_only_rows_of_the_heads_hash() -> None: own, redacted = ("o", "n", "a", "u", "s", "k"), ("o", "n", "a", "u", "s2", "k") ch = _VectorsCH( [ - {**_LIVE, "content_sha256": "b" * 64}, - {**_LIVE, "embedding_model": "m2", "content_sha256": "a" * 64}, + {**_LIVE, "content_sha256": sha256_value("old")}, + {**_LIVE, "embedding_model": "m2", "content_sha256": sha256_value("text")}, {**_LIVE, "embedding_model": "m3", "content_sha256": ""}, - {**_LIVE, "embedding_model": "m4", "content_sha256": "c" * 64, "version": 99}, - {**_LIVE, "embedding_model": "m5", "content_sha256": "a" * 64, "version": 99}, - {**_LIVE, "session_id": "s2", "content_sha256": "a" * 64}, + { + **_LIVE, + "embedding_model": "m4", + "content_sha256": sha256_value("secret"), + "version": 99, + }, + { + **_LIVE, + "embedding_model": "m5", + "content_sha256": sha256_value("text"), + "version": 99, + }, + {**_LIVE, "session_id": "s2", "content_sha256": sha256_value("text")}, ], [], ) - assert await M.reject_embedding_rows([(own, 50, "a" * 64), (redacted, 50, None)], ch=ch) == 4 + ch.heads[own].update(status="rejected", version=50) + assert ( + await M.reject_embedding_rows( + [(own, 50, sha256_value("text")), (redacted, 50, None)], ch=ch + ) + == 4 + ) # One read for the hashed key, and one for the redacted key that fetches no text. - fetched = {params["s"][0]: _selected(sql) for sql, params in ch.query_all_calls} - assert len(ch.query_all_calls) == 2 and sorted(fetched) == ["s", "s2"] + vector_reads = [ + (sql, params) for sql, params in ch.query_all_calls if "FROM loom.memory_embeddings" in sql + ] + fetched = {params["s"][0]: _selected(sql) for sql, params in vector_reads} + assert len(vector_reads) == 2 and sorted(fetched) == ["s", "s2"] + assert len([sql for sql, _ in ch.query_all_calls if "FROM loom.memory FINAL" in sql]) == 2 assert sorted(fetched["s2"]) == sorted( [ "org", @@ -1285,10 +1470,12 @@ async def test_a_merge_retires_only_the_rows_still_holding_the_text_it_read(monk async def written(org, ids, update, *, skip, strong, **_k): asked["strong"] = strong heads = [ - _memory_row(memory_id="m-same", content_sha256="a" * 64), - _memory_row(memory_id="m-rewritten", content_sha256="b" * 64), - _memory_row(memory_id="m-pinned", content_sha256="a" * 64, pinned=True), - _memory_row(memory_id="m-locked", content_sha256="a" * 64, lock_mode="no_delete"), + _memory_row(memory_id="m-same", content_sha256=sha256_value("text")), + _memory_row(memory_id="m-rewritten", content_sha256=sha256_value("old")), + _memory_row(memory_id="m-pinned", content_sha256=sha256_value("text"), pinned=True), + _memory_row( + memory_id="m-locked", content_sha256=sha256_value("text"), lock_mode="no_delete" + ), ] rows = [update(h) for h in heads if not skip(h)] return len(rows) @@ -1306,10 +1493,22 @@ async def no_chunks(**_k): object(), MemoryConsolidateInput(org="o", namespace="n", agent="a", user_id="u", session_id="s"), [ - {"memory_id": "m-same", "memory_key": "k-same", "content_sha256": "a" * 64}, - {"memory_id": "m-rewritten", "memory_key": "k-rewritten", "content_sha256": "a" * 64}, - {"memory_id": "m-pinned", "memory_key": "k-pinned", "content_sha256": "a" * 64}, - {"memory_id": "m-locked", "memory_key": "k-locked", "content_sha256": "a" * 64}, + {"memory_id": "m-same", "memory_key": "k-same", "content_sha256": sha256_value("text")}, + { + "memory_id": "m-rewritten", + "memory_key": "k-rewritten", + "content_sha256": sha256_value("text"), + }, + { + "memory_id": "m-pinned", + "memory_key": "k-pinned", + "content_sha256": sha256_value("text"), + }, + { + "memory_id": "m-locked", + "memory_key": "k-locked", + "content_sha256": sha256_value("text"), + }, ], canonical_key="canon", now_naive=datetime.now(), @@ -1322,11 +1521,16 @@ async def test_a_copy_of_one_content_leaves_another_versions_vectors_alone() -> """`of_content` copies only the rows of the head's hash: a row of another version's text, which a redaction may be purging, is not carried forward.""" ch = _SiteCH() - ch.emb.append({**_LIVE, "content_sha256": "a" * 64, "version": 10}) - ch.emb.append({**_LIVE, "embedding_model": "m2", "content_sha256": "b" * 64, "version": 10}) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text"), "version": 10}) + ch.emb.append( + {**_LIVE, "embedding_model": "m2", "content_sha256": sha256_value("old"), "version": 10} + ) + ch.remember([M.build_memory_version_row(_memory_row(), status="conflicted", version=20)]) n = await M.reemit_embedding_rows_for_keys( - [(_KEY, {"status": "conflicted", "version": 20})], ch=ch, of_content={_KEY: "a" * 64} + [(_KEY, {"status": "conflicted", "version": 20})], + ch=ch, + of_content={_KEY: sha256_value("text")}, ) assert n == 1 copied = [r for r in ch.emb if r["version"] == 20] - assert [r["content_sha256"] for r in copied] == ["a" * 64] + assert [r["content_sha256"] for r in copied] == [sha256_value("text")] diff --git a/tests/unit/test_embedding_worker_fence.py b/tests/unit/test_embedding_worker_fence.py index c844e6b314..4ca63f8f81 100644 --- a/tests/unit/test_embedding_worker_fence.py +++ b/tests/unit/test_embedding_worker_fence.py @@ -84,6 +84,7 @@ async def find_stubs(**kw): async def insert_rows(rows, *a, **kw): inserted.append(list(rows)) + return rows async def none_missing(**kw): return [] diff --git a/tests/unit/test_entity_mirror_heal.py b/tests/unit/test_entity_mirror_heal.py index 3a54ef48ca..6644217df2 100644 --- a/tests/unit/test_entity_mirror_heal.py +++ b/tests/unit/test_entity_mirror_heal.py @@ -7,7 +7,9 @@ import asyncio from typing import Any +from server.core.hashing import sha256_value from server.memory import store as mr +from tests.unit.test_memory_verify import _row class _CH: @@ -15,11 +17,26 @@ class _CH: def __init__(self, answers: list[list[dict[str, Any]]]) -> None: self.answers = list(answers) + self.heads = {} + for pair in answers[0] if answers else []: + source = _row( + org="o", + namespace="n", + agent="a", + memory_key=pair["memory_key"], + value="x", + entities=pair["entities"], + ) + self.heads[mr.identity_key(source)] = mr.build_memory_version_row(source) self.queries: list[str] = [] self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] async def query_all(self, sql, params=None): self.queries.append(" ".join(sql.split())[:80]) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "PREWHERE" in sql: + return [r for key, r in self.heads.items() if key in params["keys"]] return self.answers.pop(0) if self.answers else [] async def insert(self, table, rows, settings=None): @@ -37,6 +54,7 @@ def _emb_row(entities: list[str]) -> dict[str, Any]: "memory_key": "k", "memory_type": "semantic", "content_excerpt": "x", + "content_sha256": sha256_value("x"), "embedding_model": "m", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], diff --git a/tests/unit/test_erasure_no_content_columns.py b/tests/unit/test_erasure_no_content_columns.py index 080ab53bf0..c5a0cd7357 100644 --- a/tests/unit/test_erasure_no_content_columns.py +++ b/tests/unit/test_erasure_no_content_columns.py @@ -24,6 +24,9 @@ # table -> {column: why it is not subject content} _REVIEWED: dict[str, dict[str, str]] = { + "loom.memory_refusals": { + "scope_value": "the namespace or user id whose deletion consent the hash guards" + }, "loom.blacklist": {"reason": "the admin's reason for a refusal pattern"}, "loom.org_model_calls": {"reason": "the org admin's note on switching model calls off"}, "loom.runtime_overrides": {"value": "a rate limit or latency budget as JSON numbers"}, diff --git a/tests/unit/test_erasure_publication_tombstones.py b/tests/unit/test_erasure_publication_tombstones.py new file mode 100644 index 0000000000..d15272faf6 --- /dev/null +++ b/tests/unit/test_erasure_publication_tombstones.py @@ -0,0 +1,404 @@ +"""Delayed retirement cannot recreate plaintext or marker rows after erasure.""" + +from datetime import datetime + +import pytest + +from server.core.errors import BackendUnavailableError, ValidationError +from server.core.hashing import sha256_value +from server.memory.domain.refusals import fingerprint +from server.memory.store import refusals +from server.memory.store.inserts import insert_embedding_rows, insert_memory_versions +from server.memory.store.rows import build_embedding_row, build_memory_version_row +from tests.unit.test_memory_verify import _row + + +class CH: + def __init__(self, head, scope="", *, active=False, failure=False): + self.head = head + self.scope = scope + self.active = active + self.failure = failure + self.inserts = [] + self.deletes = [] + self.patterns = set(fingerprint("retired copper lark")) + + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(self.head)] if self.head else [] + assert "memory_refusals" in sql + if self.failure: + raise BackendUnavailableError("policy unavailable") + if "SELECT DISTINCT" in sql: + return [{"units": -1 if self.active else 3}] + erasure_only = "('identity','erased_identity')" not in sql + match = self.scope and (not erasure_only or self.scope != "identity") + return [{"found": 1}] if match and self.patterns.intersection(params["candidates"]) else [] + + async def insert(self, table, rows, **kwargs): + self.inserts.extend(rows) + + async def execute(self, sql, params=None, *, settings=None): + assert sql.startswith("DELETE FROM loom.memory_embeddings") + assert "created_at" not in sql + assert settings == {"lightweight_deletes_sync": 2} + self.deletes.append(params) + + +def vector(head): + return build_embedding_row( + **{ + c: getattr(head, c) + for c in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + content_excerpt="retired copper lark", + content_sha256=sha256_value("retired copper lark"), + embedding_model="test", + embedding_dim=1, + embedding=[0.5], + created_at=datetime(2026, 1, 1), + version=200, + tombstoned_at=datetime(2026, 1, 2), + ) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +async def test_retirement_preserves_only_permitted_cache_content(scope): + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + ch = CH(head, scope) + row = vector(head) + acknowledged = await insert_embedding_rows([row], ch=ch) + assert len(acknowledged) == 1 + if scope in {"namespace", "user_id", "erased_identity"}: + assert not ch.inserts and len(ch.deletes) == 1 + else: + assert not ch.deletes and len(ch.inserts) == 1 + assert ch.inserts[0]["content_excerpt"] == ( + "" if scope == "identity" else row["content_excerpt"] + ) + + +@pytest.mark.parametrize("scope", ["", "identity", "user_id"]) +async def test_missing_source_is_cleaned_without_inserting_even_a_bare_marker(scope): + ch = CH(None, scope) + row = vector(_row(value="retired copper lark")) + assert await insert_embedding_rows([row], ch=ch) == [row] + assert not ch.inserts and len(ch.deletes) == 1 + assert ch.deletes[0]["version"] == 200 + assert "created_at" not in ch.deletes[0] + + +@pytest.mark.parametrize("active,failure", [(True, False), (False, True)]) +async def test_missing_source_still_requires_available_policy_and_no_active_erasure( + active, failure +): + ch = CH(None, active=active, failure=failure) + row = vector(_row(value="retired copper lark")) + with pytest.raises(BackendUnavailableError): + await insert_embedding_rows([row], ch=ch) + assert not ch.inserts and not ch.deletes + + +@pytest.mark.parametrize("scope", ["identity", "namespace", "user_id", "erased_identity"]) +async def test_canonical_tombstone_preserves_point_history_but_cannot_republish_erased_text(scope): + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + ch = CH(head, scope) + rows = [build_memory_version_row(head)] + if scope == "identity": + await refusals.check_clickhouse(ch, rows) + else: + with pytest.raises(ValidationError, match="refused"): + await refusals.check_clickhouse(ch, rows) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +async def test_native_committed_history_delivers_after_point_deletion_but_not_erasure( + monkeypatch, scope +): + import json + from contextlib import asynccontextmanager + + from server.core import pg + from server.memory.app import extraction_queue + + class Connection: + async def execute(self, *args): + return None + + async def fetch(self, *args): + return [{"units": 3}] + + async def fetchval(self, sql, *args): + erasure_only = "('identity','erased_identity')" not in sql + return bool(scope and (not erasure_only or scope != "identity")) + + conn = Connection() + + @asynccontextmanager + async def transaction(): + yield conn + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + rows = [build_memory_version_row(_row(value="retired copper lark"))] + + async def publish(): + ch = CH(None) + assert await insert_memory_versions(rows, ch=ch, published=True) == 1 + assert ch.inserts == rows + + if scope in {"", "identity"}: + await publish() + else: + with pytest.raises(ValidationError, match="refused"): + await publish() + # A new authority commit still checks point refusals. + if scope: + with pytest.raises(ValidationError, match="refused"): + await refusals.check_postgres(conn, rows) + with pytest.raises(ValidationError, match="refused"): + async with refusals.publication(CH(None), rows): + pytest.fail("Uncommitted content bypassed its refusal") + raw = { + **rows[0], + "content": "retired copper lark", + "payload": json.dumps( + {"messages": [{"role": "user", "content": "retired copper lark"}]} + ), + } + for table in ("loom.memory_evidence", "loom.memory_chunks", "queue"): + ch = CH(None) + with pytest.raises(ValidationError, match="refused"): + if table == "queue": + await extraction_queue._insert_rows(ch, [raw], skip_refused=False, create=True) + else: + await refusals.insert_content_rows(ch, table, [raw]) + assert ch.inserts == [] + else: + await refusals.check_postgres(conn, rows) + + +@pytest.mark.parametrize("scope", ["private", "shared"]) +async def test_orphan_cleanup_preserves_newer_vectors_and_removes_older_months(scope): + head = _row(value="retired copper lark") + requested = vector(head) | {"scope": scope} + old = requested | {"version": 50, "created_at": datetime(2025, 12, 1)} + recent = requested | {"version": 100, "created_at": datetime(2026, 1, 1)} + newer = requested | {"version": 250} + + class State(CH): + def __init__(self): + super().__init__(None) + self.rows = [old, recent, newer] + + async def execute(self, sql, params=None, *, settings=None): + await super().execute(sql, params, settings=settings) + self.rows = [r for r in self.rows if r["version"] >= params["version"]] + + ch = State() + await insert_embedding_rows([requested], ch=ch) + assert ch.rows == [newer] + assert not ch.inserts + + +@pytest.mark.parametrize( + "durable_present,current_present", [(True, False), (False, True), (False, False)] +) +async def test_native_retirement_rechecks_on_its_only_connection( + monkeypatch, durable_present, current_present +): + from contextlib import asynccontextmanager + + from server.core import pg + from server.memory.store import pg_memory + from server.memory.store.rows import identity_key + + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + raw = build_memory_version_row(head) + heads = {identity_key(raw): raw} + borrowed = False + + class Connection: + async def execute(self, *args): + assert borrowed + + async def fetch(self, *args): + assert borrowed + return [] + + async def fetchval(self, *args): + assert borrowed + return None + + connection = Connection() + + async def durable(*args, **kwargs): + assert not borrowed and kwargs["durable"] + return heads if durable_present else {} + + async def current(conn, *args, **kwargs): + assert borrowed and conn is connection and kwargs["lock"] + return heads if current_present else {} + + @asynccontextmanager + async def transaction(): + nonlocal borrowed + assert not borrowed + borrowed = True + try: + yield connection + finally: + borrowed = False + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + monkeypatch.setattr(pg_memory, "heads_by_keys", durable) + monkeypatch.setattr(pg_memory, "heads_on_connection", current) + ch = CH(None) + acknowledged = await insert_embedding_rows([vector(head)], ch=ch) + assert not ch.inserts + assert len(acknowledged) == len(ch.deletes) == int(not durable_present and not current_present) + + +async def test_a_purged_queue_source_is_not_recreated_by_a_state_transition(): + from server.memory.app import extraction_queue + from tests.unit.test_extraction_queue import _stored_row + + class Missing: + async def query_all(self, sql, params=None): + return [] + + async def insert(self, *args, **kwargs): + pytest.fail("A queue mirror recreated its missing source") + + await extraction_queue.mark_done(_stored_row(), written=1, ch=Missing()) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +@pytest.mark.parametrize("present", [False, True]) +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("user", ["alice", "alice/team", ""]) +async def test_shared_canonical_retirement_cannot_recreate_erased_plaintext( + monkeypatch, scope, present, native, user +): + from server.memory.store import shared + + head = _row(value="retired copper lark", user_id=user) + row = { + **dict.fromkeys(shared.SHARED_PERSISTED_COLS), + "org": head.org, + "namespace": head.namespace, + "memory_key": head.memory_key, + "value": '"retired copper lark"', + "version": 200, + "tombstoned_at": datetime(2026, 1, 2), + "promoted_from": [ + "/".join( + str(getattr(head, c)) + for c in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + ) + ], + } + + class SharedCH(CH): + async def query_all(self, sql, params=None): + if params["uid"] != (user or "-"): + return [] + return await super().query_all(sql, params=params) + + async def query_one(self, sql, params=None): + assert "FROM loom.shared" in sql and "select_sequential_consistency" in sql + return {"version": 50} if present else None + + async def execute(self, sql, params=None, *, settings=None): + assert sql.startswith("DELETE FROM loom.shared") + assert "version <" in sql and params["version"] == 200 + assert settings == {"lightweight_deletes_sync": 2} + self.deletes.append(params) + + if native: + from contextlib import asynccontextmanager + + from server.core import pg + + borrowed = False + + class Connection: + async def execute(self, *args): + assert borrowed + + async def fetch(self, *args): + assert borrowed + return [{"units": 3}] if args[3] == (user or "-") else [] + + async def fetchval(self, *args): + assert borrowed + return args[3] == (user or "-") and scope in { + "namespace", + "user_id", + "erased_identity", + } + + @asynccontextmanager + async def transaction(): + nonlocal borrowed + assert not borrowed + borrowed = True + try: + yield Connection() + finally: + borrowed = False + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + + ch = SharedCH(head, scope) + await shared.insert_shared_row(row, ch=ch) + purged = not present or scope in {"namespace", "user_id", "erased_identity"} + assert bool(ch.deletes) == purged + assert ch.inserts == ([] if purged else [row]) + + +async def test_shared_repair_cannot_borrow_an_ambiguous_neighbors_point_consent(): + import json + + from server.memory.domain.excerpt import build_excerpt + from server.memory.store import shared + + source = _row( + org="o", + namespace="n", + agent="a/alice", + user_id="team", + session_id="s", + memory_key="k", + value="retired copper lark", + ) + row = vector(source) | { + "agent": "_shared", + "user_id": "-", + "session_id": "-", + "scope": "shared", + "tombstoned_at": None, + "content_excerpt": build_excerpt(json.dumps(source.value), memory_key="k"), + } + canonical = { + "value": json.dumps(source.value), + "tombstoned_at": None, + "promoted_from": ["o/n/a/alice/team/s/k"], + "version": 100, + } + + class Shared(CH): + async def query_one(self, sql, params=None): + return canonical + + async def query_all(self, sql, params=None): + if "memory_refusals" in sql and params["uid"] != "alice/team": + return [] + return await super().query_all(sql, params=params) + + ch = Shared(source, "identity") + with pytest.raises(ValidationError, match="previous deletion"): + await shared.repair_shared_embeddings([row], ch=ch) + assert not ch.inserts diff --git a/tests/unit/test_erasure_step_order.py b/tests/unit/test_erasure_step_order.py index 588af3483f..1d58f7efba 100644 --- a/tests/unit/test_erasure_step_order.py +++ b/tests/unit/test_erasure_step_order.py @@ -58,6 +58,7 @@ def _push_delete(namespaces: int) -> tuple[str, tuple[tuple[str, str], ...]]: USER_TRACE: list[tuple[str, tuple[tuple[str, str], ...]]] = [ + ("memory.refusals", _SILENT), ("memory.tombstone", ((U, "loom.memory"),)), ("memory_embeddings.tombstone", ((U, "loom.memory_embeddings"),)), ( @@ -424,6 +425,7 @@ def _push_delete(namespaces: int) -> tuple[str, tuple[tuple[str, str], ...]]: # A user purge: the memory rows the later steps read the user's keys from are stamped # first and deleted, with the summaries built from them, in `memory.retire`. USER_PURGE_TRACE: list[tuple[str, tuple[tuple[str, str], ...]]] = [ + ("memory.refusals", _SILENT), ("memory.tombstone", ((U, "loom.memory"),)), ("memory_embeddings.tombstone", ((L, "loom.memory_embeddings"),)), ( @@ -625,6 +627,7 @@ async def test_namespace_saga_trace( run: Any, scope_type: str, scope_value: str, backends: list[str] ) -> None: want = [ + ("memory.refusals", _SILENT), _SOURCES_RETIRE, _push_delete(1 if scope_type == "namespace" else 2), ( @@ -646,6 +649,7 @@ async def test_namespace_purge_saga_trace(run: Any, external: bool) -> None: lightweight DELETE, the narrow ones by an ALTER DELETE. Same steps.""" got = await run("namespace_list", _ns_list(purge=True, external=external)) assert got == [ + ("memory.refusals", _SILENT), _SOURCES_RETIRE, _push_delete(2), ( diff --git a/tests/unit/test_export_evidence_holds_high.py b/tests/unit/test_export_evidence_holds_high.py index 81dfbe664a..8ad351d352 100644 --- a/tests/unit/test_export_evidence_holds_high.py +++ b/tests/unit/test_export_evidence_holds_high.py @@ -18,13 +18,13 @@ class _CH: + def __init__(self, digests): + self.digests = digests + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: if "session_summaries" in sql: - return [ - {"session_id": "s1", "summary_text": "a plain day", "derived_from": ["k"]}, - {"session_id": "s2", "summary_text": "plain", "derived_from": ["vault"]}, - {"session_id": "s3", "summary_text": f"the key is {_KEY}", "derived_from": []}, - ] + return self.digests + return [ {"session_id": "s", "chunk_index": 0, "content": "plain talk", "embedding_dim": 2}, {"session_id": "s", "chunk_index": 1, "content": "about the vault", "embedding_dim": 2}, @@ -44,10 +44,36 @@ async def _links(_ch: Any, **_k: Any) -> dict[tuple[str, int], list[str]]: return {("s", 0): ["k"], ("s", 1): ["vault"], ("s", 2): [], ("s", 3): []} async def _high_keys(_org: str, _ns: str, keys: Any, **_k: Any) -> set[str]: - # `vault` had a high version, and has since been deleted + # `vault` had a high version; its current source is a plain fact. return {"vault"} & set(keys) - monkeypatch.setattr(evidence_transfer, "get_client", lambda: _CH()) + from tests.unit.memory_store import seed_summary_sources + + digests = [ + { + "org": "o", + "namespace": "n", + "session_id": "s1", + "summary_text": "a plain day", + "derived_from": ["k"], + }, + { + "org": "o", + "namespace": "n", + "session_id": "s2", + "summary_text": "plain", + "derived_from": ["vault"], + }, + { + "org": "o", + "namespace": "n", + "session_id": "s3", + "summary_text": f"the key is {_KEY}", + "derived_from": ["credential"], + }, + ] + seed_summary_sources(monkeypatch, digests) + monkeypatch.setattr(evidence_transfer, "get_client", lambda: _CH(digests)) monkeypatch.setattr(chunk_links, "chunk_memory_keys", _links) monkeypatch.setattr(memory_repo, "high_keys", _high_keys) audits: list[list[dict[str, Any]]] = [] @@ -89,7 +115,7 @@ async def test_chunks_behind_a_high_key_holding_a_credential_or_unembedded_stay_ async def test_digests_behind_a_high_key_or_holding_a_credential_stay_home(evidence) -> None: out = await evidence_transfer.handle_memory_export_evidence(_input("digests")) assert [d.session_id for d in out.digests] == ["s1"] - assert out.held_high == 2 + assert out.held_high == 1 @pytest.mark.asyncio @@ -97,10 +123,9 @@ async def test_include_high_releases_held_digests_and_audits_each(evidence) -> N out = await evidence_transfer.handle_memory_export_evidence( _input("digests", include_high=True) ) - assert [d.session_id for d in out.digests] == ["s1", "s2", "s3"] and out.held_high == 0 + assert [d.session_id for d in out.digests] == ["s1", "s2"] and out.held_high == 0 assert evidence == [ [ {"kind": "digests", "session_id": "s2", "memory_keys": ["vault"]}, - {"kind": "digests", "session_id": "s3", "memory_keys": []}, ] ] diff --git a/tests/unit/test_extract_concurrent_projections.py b/tests/unit/test_extract_concurrent_projections.py index 48d4fa1c7a..782d8f296d 100644 --- a/tests/unit/test_extract_concurrent_projections.py +++ b/tests/unit/test_extract_concurrent_projections.py @@ -10,19 +10,20 @@ import asyncio import time +from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException from server.memory.api.models_bulk import MemorySetFromMessagesInput -from server.memory.app.session_summary import SessionSummaryDraft -_DRAFT = SessionSummaryDraft( - summary_text="User moved to Lisbon in 2021.", - embedding=[0.1, 0.2, 0.3], - embedding_model="fake-embed", -) + +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) def _input() -> MemorySetFromMessagesInput: @@ -63,8 +64,7 @@ def _wire( monkeypatch: pytest.MonkeyPatch, log: _Log, *, - draft: Any = _DRAFT, - draft_raises: bool = False, + summary_raises: bool = False, chunks_raise: bool = False, ) -> None: import server.identity.app.namespace_ontology as ont @@ -72,6 +72,10 @@ def _wire( import server.memory.app.pipeline as pipe from server.memory.app import bulk_set as _bulk_set_mod + monkeypatch.setattr( + mx, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + async def _no_evidence(input: Any, msgs: Any, **kw: Any) -> list[str]: return ["ev-1"] @@ -92,14 +96,9 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_write_source_evidence", _no_evidence) monkeypatch.setattr(pipe, "run_pipeline", _pipeline) - monkeypatch.setattr( - mx, - "_draft_session_summary", - log.stage("draft", seconds=0.02, result=draft, raises=draft_raises), - ) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", log.stage("link", result=3)) monkeypatch.setattr(mx, "write_memory_chunks", log.stage("chunks", raises=chunks_raise)) - monkeypatch.setattr(mx, "_write_session_summary", log.stage("summary")) + monkeypatch.setattr(mx, "_write_session_summary", log.stage("summary", raises=summary_raises)) monkeypatch.setattr(mx, "record_input_projection_failure", _failure) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _failure) monkeypatch.setattr(ont, "load_namespace_ontology", _cold) @@ -110,7 +109,7 @@ def _overlap(a: tuple[float, float], b: tuple[float, float]) -> bool: @pytest.mark.asyncio -async def test_draft_runs_during_extraction_and_projections_overlap( +async def test_committed_extraction_precedes_overlapping_projections( monkeypatch: pytest.MonkeyPatch, ) -> None: import server.memory.app.extract.handler as mx @@ -119,14 +118,10 @@ async def test_draft_runs_during_extraction_and_projections_overlap( _wire(monkeypatch, log) out = await mx.handle_memory_set_from_messages(_input()) - # The digest draft started before the pipeline finished. - assert log.spans["draft"][0] < log.spans["pipeline"][1] + assert log.spans["summary"][0] >= log.spans["pipeline"][1] # The three projections ran together, not back to back. assert _overlap(log.spans["link"], log.spans["chunks"]) assert _overlap(log.spans["chunks"], log.spans["summary"]) - # The pre-made draft reached the writer; nothing was drafted twice. - assert log.drafts_written == [_DRAFT] - assert log.calls.count("draft") == 1 assert out.belief_source_links == 3 assert [w.memory_key for w in out.written] == ["home"] assert log.failures == [] @@ -149,67 +144,31 @@ async def test_one_failing_projection_does_not_take_the_others_down( @pytest.mark.asyncio -async def test_failed_draft_is_recorded_and_not_redrafted(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_a_summary_projection_failure_is_recorded(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx log = _Log() - _wire(monkeypatch, log, draft_raises=True) + _wire(monkeypatch, log, summary_raises=True) out = await mx.handle_memory_set_from_messages(_input()) assert log.failures == ["session_summaries"] - assert "summary" not in log.calls # no second model call on the inline path + assert log.calls.count("summary") == 1 assert out.written -@pytest.mark.asyncio -async def test_empty_draft_skips_the_row_without_a_second_call( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import server.memory.app.extract.handler as mx - - log = _Log() - _wire(monkeypatch, log, draft=None) - await mx.handle_memory_set_from_messages(_input()) - - assert log.calls.count("draft") == 1 - assert "summary" not in log.calls - assert log.failures == [] - - -@pytest.mark.asyncio -async def test_extraction_failure_cancels_the_draft(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_extraction_failure_never_starts_a_digest(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe from server.memory.app.extract import ExtractionError log = _Log() _wire(monkeypatch, log) - cancelled = asyncio.Event() - - async def _slow_draft(*args: Any, **kwargs: Any) -> Any: - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - cancelled.set() - raise - async def _pipeline_raises(steps: Any, state: Any) -> Any: - await asyncio.sleep(0) # let the draft task start, as a real model call would + async def fail(*args: Any, **kwargs: Any) -> Any: raise ExtractionError("no JSON object in LLM response") - monkeypatch.setattr(mx, "_draft_session_summary", _slow_draft) - monkeypatch.setattr(pipe, "run_pipeline", _pipeline_raises) - - with pytest.raises(HTTPException) as ei: + monkeypatch.setattr(pipe, "run_pipeline", fail) + with pytest.raises(HTTPException) as error: await mx.handle_memory_set_from_messages(_input()) - assert ei.value.status_code == 502 - await asyncio.wait_for(cancelled.wait(), timeout=1.0) - - -def test_write_session_summary_uses_a_given_draft() -> None: - """The writer's `draft` parameter is the seam the concurrent path relies on.""" - import inspect - - from server.memory.app.session_summary import _write_session_summary - - assert "draft" in inspect.signature(_write_session_summary).parameters + assert error.value.status_code == 502 + assert "summary" not in log.calls diff --git a/tests/unit/test_extract_defer_allowlist.py b/tests/unit/test_extract_defer_allowlist.py index b9c2edec42..53b987049a 100644 --- a/tests/unit/test_extract_defer_allowlist.py +++ b/tests/unit/test_extract_defer_allowlist.py @@ -67,6 +67,7 @@ class _Auth: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_takes_the_queue_path_for_an_allowlisted_agent( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_extract_write_dates.py b/tests/unit/test_extract_write_dates.py index 780c07ccfd..61216e6805 100644 --- a/tests/unit/test_extract_write_dates.py +++ b/tests/unit/test_extract_write_dates.py @@ -84,6 +84,7 @@ async def test_triple_valid_from_null_when_undated() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_schema_mode_entry_defaults_valid_at_from_observation_date( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_extraction_policy.py b/tests/unit/test_extraction_policy.py index 03b2d2b56e..83fb45603d 100644 --- a/tests/unit/test_extraction_policy.py +++ b/tests/unit/test_extraction_policy.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import AsyncMock import pytest @@ -15,6 +16,13 @@ from server.settings import settings from tests.unit.switch_state import policies_read_now, policies_unread + +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + _POLICY = "Never store credentials, payment card data or health data." _MSGS = [ {"role": "user", "content": "I moved to Lisbon in 2021 and I still live there today."}, @@ -182,29 +190,43 @@ async def test_the_typed_value_stage_ends_with_the_policy(monkeypatch) -> None: assert all(_POLICY not in s for s in llm.systems if s not in value_stage) -async def test_the_session_digest_ends_with_the_policy(monkeypatch) -> None: - import server.llm.client as llm_client - from server.memory.app import session_summary +async def test_the_session_digest_uses_committed_policy_filtered_facts_only(monkeypatch) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock - llm = _Provider(text="On 2026-01-01 the user said they live in Lisbon.") - monkeypatch.setattr(llm_client, "get_llm_client", lambda: llm) + from server.memory import store + from server.memory.app import digest_chain, session_summary + from tests.unit.test_memory_verify import _row - class _Embedder: - model_name = "fake" - dim = 1 + source = _row(value="Lives in Lisbon.") + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) - async def embed(self, text: str) -> list[float]: - return [1.0] + async def heads(keys, **kwargs): + from server.memory.store import identity_key - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] + return {identity_key(source): source} - monkeypatch.setattr(session_summary, "get_embedder", lambda: _Embedder()) - draft = await session_summary._draft_session_summary( - object(), _MSGS, "2026-01-01", operator_policy=_POLICY + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", heads) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", AsyncMock(return_value=[1.0])) + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + monkeypatch.setattr(digest_chain, "link_digest", AsyncMock()) + inp = SimpleNamespace( + org=source.org, + namespace=source.namespace, + agent=source.agent, + user_id=source.user_id, + session_id=source.session_id, + ) + state = SimpleNamespace( + written=[{"memory_id": source.memory_id, "version": source.version}], + extracted_facts=[{"value": "forbidden raw detail"}], + operator_policy=_POLICY, + observation_date=None, ) - assert draft is not None - assert llm.systems == [with_policy(session_summary._SESSION_SUMMARY_SYSTEM, _POLICY)] + await session_summary._write_session_summary(inp, state) + assert ch.insert.call_args.args[1][0]["summary_text"] == "Lives in Lisbon." async def test_trajectory_events_and_notes_end_with_the_policy( @@ -595,7 +617,7 @@ def who(org: str, *role: Role, ns: tuple[str, ...] = ()) -> AuthInfo: def _wire(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: """The handler with every write stubbed: what reached the pipeline and the - digest draft, and whether the evidence write ran.""" + committed state, and whether the evidence write ran.""" import server.identity.app.namespace_ontology as ont import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe @@ -610,9 +632,6 @@ async def _pipeline(steps: Any, state: Any) -> Any: seen["state_policy"] = state.operator_policy return state - async def _draft(*args: Any, **kw: Any) -> None: - seen["draft_policy"] = kw.get("operator_policy") - async def _nothing(*args: Any, **kw: Any) -> Any: return 0 @@ -621,7 +640,6 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_write_source_evidence", _evidence) monkeypatch.setattr(pipe, "run_pipeline", _pipeline) - monkeypatch.setattr(mx, "_draft_session_summary", _draft) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _nothing) monkeypatch.setattr(mx, "write_memory_chunks", _nothing) monkeypatch.setattr(mx, "_write_session_summary", _nothing) @@ -654,7 +672,6 @@ async def test_set_from_messages_runs_under_the_stored_policy_whatever_it_asks( ) effective = f"{_POLICY}\n\nNo addresses." assert seen["state_policy"] == effective - assert seen["draft_policy"] == effective assert out.under_operator_policy is True policies_read_now({}) assert (await mx.handle_memory_set_from_messages(_request())).under_operator_policy is False @@ -1164,7 +1181,6 @@ async def no_conflict_policy(state: Any) -> bool: no_delete_in_flight(monkeypatch) monkeypatch.setattr(pipe, "_newest_wins", no_conflict_policy) monkeypatch.setattr(mx, "_write_source_evidence", no_evidence) - monkeypatch.setattr(mx, "_draft_session_summary", nothing) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", nothing) monkeypatch.setattr(mx, "write_memory_chunks", nothing) monkeypatch.setattr(mx, "_write_session_summary", nothing) diff --git a/tests/unit/test_extraction_queue.py b/tests/unit/test_extraction_queue.py index 83f093b488..4f359d0adb 100644 --- a/tests/unit/test_extraction_queue.py +++ b/tests/unit/test_extraction_queue.py @@ -49,6 +49,11 @@ async def insert(self, table, rows, settings=None): async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if params and "source_version" in params: + # Transition tests supply a stored source; missing and changed sources have dedicated cases. + return [{"found": 1}] return self.query_all_returns.pop(0) if self.query_all_returns else [] async def query_one(self, sql, params=None): @@ -130,6 +135,7 @@ async def test_enqueue_writes_one_full_pending_row() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_defer_short_circuits_before_extraction( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -153,6 +159,7 @@ async def _boom(*a, **k): @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_defer_enqueue_failure_is_503(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx @@ -430,7 +437,9 @@ class _Out: assert res.failures == ["bad: failed"] assert all(inp.defer is False for inp in seen) # inserts: 1 claim (2 rows) + done + failed - statuses = [r["status"] for _, rows, _ in ch.inserts[1:] for r in rows] + statuses = [ + r["status"] for _, rows, _ in ch.inserts for r in rows if r["status"] != q.STATUS_RUNNING + ] assert statuses == [q.STATUS_DONE, q.STATUS_FAILED] snap = get_registry().snapshot()["extraction_queue"] assert snap["last_outcome"]["done"] == 1 @@ -994,6 +1003,11 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> Any: params = params or {} + if "memory_refusals" in sql: + return [] + if "source_version" in params: + row = self.rows.get(params["queue_id"]) + return [{"found": 1}] if row and row["version"] == params["source_version"] else [] self.sql.append(sql) now = datetime.now(UTC) held = { @@ -1070,7 +1084,7 @@ async def test_a_payload_only_enqueuer_waits_without_holding_another_enqueuer( pytest.skip("clickhouse binary not installed") a, _, _ = q.payload_for(_input(), scope=WHOLE_NAMESPACE, enqueued_by="process:sa:t:agent-a") b, _, _ = q.payload_for(_input(), scope=WHOLE_NAMESPACE, enqueued_by="process:sa:t:agent-b") - setup = [ddl("memory_extraction_queue")] + setup = [ddl("memory_extraction_queue"), ddl("memory_refusals"), ddl("erasure_requests")] for qid, payload, status, lease, age, column in ( ("a-held", a, "running", "now64(3) + INTERVAL 10 MINUTE", 3, held_column), ("a-next", a, "pending", "toDateTime64(0, 3)", 2, ""), @@ -1119,7 +1133,7 @@ async def write_drops(items: list[Any], stage: str, reason: str, *, ch: Any = No ([("o", "n", "h")], "extraction_queue", "held_past_ttl"), ([("o", "n", "p")], "extraction_queue", "waited_past_ttl"), ] - marked = ch.inserts[-1][1] + marked = [r for _, rows, _ in ch.inserts for r in rows] assert [r["status"] for r in marked] == [q.STATUS_FAILED, q.STATUS_FAILED] assert "(held: refused)" in marked[0]["last_error"] @@ -1152,7 +1166,7 @@ async def write_drops(items: list[Any], stage: str, reason: str, *, ch: Any = No monkeypatch.setattr(drops, "write_drops", write_drops) assert await q.drop_expiring(ch=ch) == 4 - quoted = [r["last_error"].rsplit(" (", 1)[1] for r in ch.inserts[-1][1]] + quoted = [r["last_error"].rsplit(" (", 1)[1] for _, rows, _ in ch.inserts for r in rows] assert quoted == [ "RuntimeError)", "unreachable: ConnectError)", diff --git a/tests/unit/test_extraction_retry_mine.py b/tests/unit/test_extraction_retry_mine.py index d9a66de971..a850acdc88 100644 --- a/tests/unit/test_extraction_retry_mine.py +++ b/tests/unit/test_extraction_retry_mine.py @@ -37,8 +37,24 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: self.reads.append((sql, dict(params or {}))) - if "system.mutations" in sql: + if "system.mutations" in sql or "FROM loom.memory_refusals" in sql: return [] + if params and "source_version" in params: + from server.core.hashing import sha256_text + + return ( + [{"found": 1}] + if any( + all( + row.get(k) == params[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + ) + and row["version"] == params["source_version"] + and sha256_text(row.get("payload") or "").upper() == params["payload_sha"] + for row in self.rows + ) + else [] + ) return [dict(r) for r in self.rows] async def execute(self, sql: str, params: dict[str, Any] | None = None) -> None: @@ -157,6 +173,7 @@ def test_a_failed_replay_ends_failed_again(max_attempts: int) -> None: ch = _StubCH([_failed_row(attempts=max_attempts)]) _requeue(ch, ["q1"], max_attempts=max_attempts) row = _failed_row() | {"status": "pending", "attempts": ch.commands[0][1]["attempts"]} + ch.rows = [row] # The in-place retry's current stored revision. status = asyncio.run(q.mark_error(row, error="again", max_attempts=max_attempts, ch=ch)) assert status == "failed" and ch.inserted[-1]["attempts"] == max_attempts diff --git a/tests/unit/test_file_rules_shared.py b/tests/unit/test_file_rules_shared.py index 6ca494d174..9312927c0d 100644 --- a/tests/unit/test_file_rules_shared.py +++ b/tests/unit/test_file_rules_shared.py @@ -124,16 +124,25 @@ def test_version_row_builders_carry_files() -> None: class _CH: - def __init__(self) -> None: + def __init__(self, existing=None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} + if existing is not None: + row = memory_repo.build_memory_version_row(existing) + self.heads[memory_repo.identity_key(row)] = row async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update((memory_repo.identity_key(r), dict(r)) for r in rows) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql, params=None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql: + keys = set((params or {}).get("keys", self.heads)) + return [r for key, r in self.heads.items() if key in keys] return [] async def execute(self, *_: Any, **__: Any) -> None: @@ -151,7 +160,7 @@ def _wire(monkeypatch: pytest.MonkeyPatch, existing: memory_repo.MemoryRow | Non from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod - ch = _CH() + ch = _CH(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_high_rows_sql.py b/tests/unit/test_high_rows_sql.py index ad77d64921..e6f335f574 100644 --- a/tests/unit/test_high_rows_sql.py +++ b/tests/unit/test_high_rows_sql.py @@ -661,9 +661,12 @@ def row( "period_digest_s26_s30", "period_digest_s31_s35", } - assert await period_digest._skipped_period_keys(ch, ident, rewrite=False) == every + candidates = sorted(every | {"period_digest_s16_s20", "period_digest_s21_s25", "summary:s1"}) + assert await period_digest._skipped_period_keys(ch, ident, candidates, rewrite=False) == every taken_out = {"period_digest_s1_s5", "period_digest_s6_s10", "period_digest_s31_s35"} - assert await period_digest._skipped_period_keys(ch, ident, rewrite=True) == taken_out + assert ( + await period_digest._skipped_period_keys(ch, ident, candidates, rewrite=True) == taken_out + ) @pytest.mark.asyncio @@ -779,24 +782,44 @@ async def _heads(keys, **_k): @pytest.mark.asyncio async def test_memory_digests_never_serves_a_stored_credential_key_fact(monkeypatch) -> None: - # A digest an earlier build wrote may hold one: the read leaves it out. + # One unsafe source withholds the whole digest; a proven neighbor remains available. from server.memory.api.models_temporal import MemoryDigestsInput from server.memory.app import curation token = "ghp_" + "a" * 36 - ch = _LocalCH( - "SET enable_quantized_codec = 1", # the summary embedding column's codec - ddl("session_summaries"), - "INSERT INTO loom.session_summaries (org, namespace, agent, user_id, session_id, " - "summary_text, key_facts, summary_embedding) VALUES ('o', 'n', 'a', '-', 's', " - f"'a session', ['the token is {token}', 'ran the tests'], " - "arrayWithConstant(3072, toFloat32(0)))", - ) + import json + + from clickhouse_connect.driver.binding import format_query_value + + statements = ["SET enable_quantized_codec = 1", ddl("session_summaries"), ddl("memory")] + for session, key, text in ( + ("s", "vault", f"the token is {token}"), + ("s", "tests", "ran the tests"), + ("safe", "safe", "safe fact"), + ): + statements.append( + "INSERT INTO loom.memory (org, namespace, agent, user_id, session_id, memory_key, " + "value, memory_type, version) VALUES ('o','n','a','-'," + f"{format_query_value(session)}, {format_query_value(key)}, " + f"{format_query_value(json.dumps(text))}, 'semantic', 1)" + ) + for session, facts, keys in ( + ("s", [f"the token is {token}", "ran the tests"], ["vault", "tests"]), + ("safe", ["safe fact"], ["safe"]), + ): + statements.append( + "INSERT INTO loom.session_summaries (org, namespace, agent, user_id, session_id, " + "summary_text, key_facts, derived_from, summary_embedding) VALUES ('o','n','a','-'," + f"{format_query_value(session)}, {format_query_value(chr(10).join(facts))}, " + f"{format_query_value(facts)}, {format_query_value(keys)}, " + "arrayWithConstant(3072, toFloat32(0)))" + ) + ch = _LocalCH(*statements) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) out = await curation.handle_memory_digests( MemoryDigestsInput(org="o", namespace="n", agent="a") ) - assert [r.key_facts for r in out.rows] == [["ran the tests"]] + assert [(r.session_id, r.key_facts) for r in out.rows] == [("safe", ["safe fact"])] @pytest.mark.asyncio diff --git a/tests/unit/test_import_evidence_refuses_blacklisted.py b/tests/unit/test_import_evidence_refuses_blacklisted.py index ce388dc42c..7bdca47946 100644 --- a/tests/unit/test_import_evidence_refuses_blacklisted.py +++ b/tests/unit/test_import_evidence_refuses_blacklisted.py @@ -24,6 +24,11 @@ class _CH: def __init__(self) -> None: self.inserts: dict[str, list[dict[str, Any]]] = {} + async def query_all(self, sql, params=None): + if "memory_refusals" in sql: + return [] + raise AssertionError(sql) + async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.setdefault(table, []).extend(rows) @@ -33,7 +38,21 @@ def wired(monkeypatch: pytest.MonkeyPatch) -> tuple[_CH, list[str]]: from server import embedding from server.memory.app import blacklist_guard as blacklist from server.memory.app import evidence_transfer as mod - + from tests.unit.memory_store import seed_summary_sources + + seed_summary_sources( + monkeypatch, + [ + { + "org": "acme", + "namespace": "ns", + "agent": "a", + "user_id": "-", + "session_id": "s1", + "summary_text": "a parade happened", + } + ], + ) ch = _CH() hits: list[str] = [] @@ -85,7 +104,12 @@ async def test_blacklisted_chunks_and_digests_are_dropped_not_written(wired) -> ), ], digests=[ - EvidenceDigest(session_id="s1", summary_text="a parade happened"), + EvidenceDigest( + session_id="s1", + summary_text="a parade happened", + key_facts=["a parade happened"], + derived_from=["source:s1"], + ), EvidenceDigest(session_id="s2", summary_text=f"key {SECRET} shared"), # key_facts are stored on the row too, so they are checked as well. EvidenceDigest(session_id="s3", summary_text="ok", key_facts=[SECRET]), @@ -121,3 +145,22 @@ async def test_a_chunk_the_blacklist_could_not_check_is_counted_apart(wired) -> ) ) assert (out.chunks_written, out.skipped, out.unchecked) == (0, 2, 1) + + +async def test_an_unproven_plain_digest_is_counted_as_skipped(wired): + from server.memory.api.models_evidence_transfer import EvidenceDigest, MemoryImportEvidenceInput + from server.memory.app.evidence_transfer import handle_memory_import_evidence + + ch, hits = wired + output = await handle_memory_import_evidence( + MemoryImportEvidenceInput( + org="acme", + namespace="ns", + agent="a", + digests=[ + EvidenceDigest(session_id="missing", summary_text="A plain but unproven passage") + ], + ) + ) + assert (output.digests_written, output.skipped) == (0, 1) + assert not ch.inserts and not hits diff --git a/tests/unit/test_in_use_coverage.py b/tests/unit/test_in_use_coverage.py index 079eb227ed..7f1e4cc7c5 100644 --- a/tests/unit/test_in_use_coverage.py +++ b/tests/unit/test_in_use_coverage.py @@ -24,6 +24,7 @@ _TRANSIENT = "a short-lived claim or projection in Keeper" EXEMPT: dict[str, str] = { + "loom.memory_refusals": "hash-only refusal policy intentionally survives namespace reuse; no content or outside target", "loom.namespaces": "the registry row, which every path reads before the check", "loom.analysis_canary": "the analysis probe's one row, at org '' and namespace '', a scope no tenant uses", # derived @@ -96,6 +97,7 @@ } PG_EXEMPT: dict[str, str] = { + "loom.memory_refusals": "hash-only refusal policy intentionally survives namespace reuse; no content or outside target", "loom.memory_deliveries": "which targets a revision still owes; beside its revision", "loom.erased_memories": "evidence that rows were erased; holds no content", "loom.erased_documents": "evidence that documents were erased; holds no content", diff --git a/tests/unit/test_key_axis_text.py b/tests/unit/test_key_axis_text.py index 956fcefbea..db178ef7ae 100644 --- a/tests/unit/test_key_axis_text.py +++ b/tests/unit/test_key_axis_text.py @@ -100,6 +100,11 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: class _CH: rows: list[dict[str, Any]] = [] + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [row for key, row in heads.items() if key in params["keys"]] + return [] + async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.rows = rows @@ -115,6 +120,9 @@ async def _one_group(rows: Any) -> dict[Any, list[int]]: stored = { "org": "o", "namespace": "n", + "agent": "a", + "user_id": "-", + "session_id": "-", "embedding_model": "m1", "embedding_dim": 2, "memory_key_embedding": [0.0, 0.0], @@ -125,6 +133,15 @@ async def _one_group(rows: Any) -> dict[Any, list[int]]: {**stored, "memory_key": digest, "content_excerpt": '"Prefers dark roast."'}, {**stored, "memory_key": "coffee_order", "content_excerpt": '"Dark." (#coffee_order)'}, ] + from server.core.hashing import sha256_value + from server.memory.store.rows import build_memory_version_row, identity_key + from tests.unit.test_memory_verify import _row + + heads = {} + for row, value in zip(chunk, ("Prefers dark roast.", "Dark."), strict=True): + source = _row(org="o", namespace="n", agent="a", memory_key=row["memory_key"], value=value) + heads[identity_key(source)] = build_memory_version_row(source) + row["content_sha256"] = sha256_value(value) emb, ch = _Emb(), _CH() await reembed._rewrite_page(ch, emb, "memory_embeddings", chunk, key_only=False) assert emb.batches == [['"Prefers dark roast."', '"Dark." (#coffee_order)'], ["coffee order"]] diff --git a/tests/unit/test_legacy_vector_retirement.py b/tests/unit/test_legacy_vector_retirement.py new file mode 100644 index 0000000000..e6768f9e16 --- /dev/null +++ b/tests/unit/test_legacy_vector_retirement.py @@ -0,0 +1,59 @@ +"""Retirement removes unknown legacy content without treating it as a publishable copy.""" + +import pytest + +from server.core.hashing import sha256_value +from server.memory.store import embeddings +from server.memory.store.rows import build_embedding_row, build_memory_version_row +from tests.unit.test_memory_verify import _row + + +@pytest.mark.parametrize("status", ["superseded", "rejected", "duplicate"]) +@pytest.mark.parametrize("vector_version", [100, 250]) +@pytest.mark.parametrize("hash_kind", ["missing", "different", "bytes"]) +async def test_a_retirement_tombstones_unhashed_vectors_below_its_version( + status, vector_version, hash_kind +): + source = _row(value="current fact", status=status, version=200) + canonical = build_memory_version_row(source) + vector = build_embedding_row( + **{ + k: canonical[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + content_excerpt="legacy text", + content_sha256=( + "" + if hash_kind == "missing" + else sha256_value("legacy text").encode() + if hash_kind == "bytes" + else sha256_value("legacy text") + ), + embedding_model="legacy", + embedding_dim=1, + embedding=[0.5], + version=vector_version, + ) + + class CH: + def __init__(self): + self.inserted = [] + + async def query_all(self, sql, params=None): + if "FROM loom.memory_embeddings" in sql: + return [vector] + if params and "keys" in params: + return [canonical] + return [] + + async def insert(self, table, rows, **kwargs): + assert table == "loom.memory_embeddings" + self.inserted.extend(rows) + + ch = CH() + count = await embeddings.mirror_status_to_embeddings([canonical], ch=ch, strict=True) + assert count == len(ch.inserted) == int(vector_version < 200) + if ch.inserted: + (retired,) = ch.inserted + assert retired["tombstoned_at"] is not None and retired["version"] == 200 + assert retired["content_excerpt"] == "" and retired["embedding"] == [] diff --git a/tests/unit/test_lock_guard_consistent_read.py b/tests/unit/test_lock_guard_consistent_read.py index ea9465b394..1f4da85e92 100644 --- a/tests/unit/test_lock_guard_consistent_read.py +++ b/tests/unit/test_lock_guard_consistent_read.py @@ -39,10 +39,10 @@ def _lagging_replica(monkeypatch: pytest.MonkeyPatch) -> tuple[_CH, list[dict[st stale = _row() # the version before the lock locked = _row(version=101, lock_mode="read_only", lock_reason="hold", locked_by="human:o") - async def head_versions(*args: Any, **kw: Any) -> list[dict[str, Any]]: + async def head_versions(sql, params=None) -> list[dict[str, Any]]: from server.memory.store import build_memory_version_row - return [build_memory_version_row(locked)] + return [build_memory_version_row(locked)] if "FROM loom.memory FINAL" in sql else [] monkeypatch.setattr(ch, "query_all", head_versions) reads: list[dict[str, Any]] = [] @@ -195,6 +195,13 @@ async def test_a_write_right_after_unlock_sees_the_unlock(monkeypatch: pytest.Mo ch = _CH() locked = _row(lock_mode="read_only", lock_reason="hold", locked_by="human:o") unlocked = _row(version=102) + + async def head_versions(sql, params=None): + from server.memory.store import build_memory_version_row + + return [build_memory_version_row(unlocked)] if "FROM loom.memory FINAL" in sql else [] + + monkeypatch.setattr(ch, "query_all", head_versions) reads: list[dict[str, Any]] = [] monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_memory_apply.py b/tests/unit/test_memory_apply.py index 8b4ddb59ba..7bdaa42572 100644 --- a/tests/unit/test_memory_apply.py +++ b/tests/unit/test_memory_apply.py @@ -114,6 +114,7 @@ def __init__(self, filter_rows: list[dict[str, Any]] | None = None) -> None: self.executes: list[tuple[str, dict[str, Any]]] = [] self.query_alls: list[tuple[str, dict[str, Any]]] = [] self.filter_rows = filter_rows or [] + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) @@ -125,6 +126,10 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None, **_: Any ) -> list[dict[str, Any]]: self.query_alls.append((sql, params or {})) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] return self.filter_rows async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: @@ -132,6 +137,9 @@ async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: def _stub(monkeypatch: pytest.MonkeyPatch, rows_by_key: dict[str, MemoryRow], ch: _CH) -> None: + from server.memory.store.rows import build_memory_version_row, identity_key + + ch.heads = {identity_key(row): build_memory_version_row(row) for row in rows_by_key.values()} monkeypatch.setattr("server.core.ch.get_client", lambda: ch) monkeypatch.setattr("server.memory.app.handlers.get_client", lambda: ch) monkeypatch.setattr("server.memory.app.stack.get_client", lambda: ch) @@ -174,8 +182,9 @@ async def test_enforcing_delete_goes_through_the_handler(monkeypatch: pytest.Mon ) ) assert out.applied == 1 and out.rows[0].status == "applied" and out.rows[0].version - table, rows = ch.inserts[0] + table, rows = next((table, rows) for table, rows in ch.inserts if table == "loom.memory") assert table == "loom.memory" and rows[0]["tombstoned_at"] is not None + assert ch.inserts[0][0] == "loom.memory_refusals" @pytest.mark.asyncio diff --git a/tests/unit/test_memory_ask_deep_evidence.py b/tests/unit/test_memory_ask_deep_evidence.py index c1fd6df1e6..aff9f7b377 100644 --- a/tests/unit/test_memory_ask_deep_evidence.py +++ b/tests/unit/test_memory_ask_deep_evidence.py @@ -472,145 +472,13 @@ async def test_fetch_with_no_keys_does_not_query() -> None: assert ch.params is None -# ---- session-summary hits open their session, windowed by the question ------ - -from server.memory.app.deep_evidence import ( # noqa: E402 - fetch_session_turns, - session_ids_of, - window_turns_by_question, -) - - -def test_session_ids_of_reads_summary_keys_only() -> None: - assert session_ids_of(["m1", "summary:s1", "summary:s1", "summary:", "summary:s2"]) == [ - "s1", - "s2", - ] - - -def _turn(i: int, content: str) -> dict[str, Any]: - return { - "session_id": "s1", - "role": "user", - "observed_at": f"2026-01-01 00:00:{i:02d}", - "content": content, - } - - -def test_window_keeps_scoring_turns_with_neighbours_in_order() -> None: - rows = [ - _turn(0, "hello there"), - _turn(1, "we booked the Denver trip for March"), - _turn(2, "great, bring a jacket"), - _turn(3, "unrelated chatter about lunch"), - _turn(4, "also the concert in Denver was Brandon Flowers"), - _turn(5, "nice"), - ] - kept = window_turns_by_question(rows, "What did I do in Denver?", max_chars=10_000) - idx = [rows.index(r) for r in kept] - assert idx == [0, 1, 2, 3, 4, 5] or idx == sorted(idx) # chronological - assert 1 in idx and 4 in idx # both scoring turns - assert 3 in idx # neighbour of turn 4 - - -def test_window_respects_the_character_cap() -> None: - rows = [_turn(i, f"Denver turn {i} " + "x" * 200) for i in range(10)] - kept = window_turns_by_question(rows, "Denver", max_chars=700) - assert 0 < len(kept) <= 3 - assert sum(len(r["content"]) for r in kept) <= 700 - - -def test_window_returns_nothing_without_a_scoring_turn() -> None: - rows = [_turn(0, "hello"), _turn(1, "goodbye")] - assert window_turns_by_question(rows, "Denver trip", max_chars=1000) == [] - - -class _OneReadCH: - """A single statement answered with the rows given (the session-turn read).""" - - def __init__(self, rows: list[dict[str, Any]]) -> None: - self.rows = rows - self.params: dict[str, Any] | None = None - - async def query_all( - self, sql: str, params: dict[str, Any] | None = None - ) -> list[dict[str, Any]]: - self.params = params - return self.rows - - -async def test_fetch_session_turns_labels_with_the_summary_key_and_caps() -> None: - ch = _OneReadCH( - [ - { - "session_id": "s1", - "role": "user", - "observed_at": "2026-01-01 00:00:01", - "content": "Denver plan " + "a" * 100, - }, - { - "session_id": "s1", - "role": "assistant", - "observed_at": "2026-01-01 00:00:02", - "content": "ok", - }, - { - "session_id": "s2", - "role": "user", - "observed_at": "2026-01-02 00:00:01", - "content": "Denver again " + "b" * 100, - }, - ] - ) - turns = await fetch_session_turns( - org="o", - namespace="ns", - session_ids=["s1", "s2"], - question="Denver", - max_chars_per_session=1000, - max_chars_total=150, - ch=ch, - ) - assert ch.params["sids"] == ["s1", "s2"] - assert turns and turns[0].memory_key == "summary:s1" and turns[0].support_type == "session" - # 150 total: s1's window (114 + 2 chars) fits, s2 does not. - assert [t.memory_key for t in turns] == ["summary:s1", "summary:s1"] - - -async def test_deep_pass_opens_a_picked_session_summary(env) -> None: - """End to end through memory.ask: the selector picks the summary hit, its - session is fetched and rendered under the summary key.""" - env.search_output = _output( - [ - _hit("summary:s9", "Session about the Denver trip and a concert.", 0.9), - _hit("m1", "User's favorite color is blue.", 0.8), - ] - ) - env.set_llm( - _ScriptedLLM( - ["Blue [m1].", "Brandon Flowers in Denver [summary:s9]."], picks='{"picks": [0]}' - ) - ) - - async def fake_sessions(**kw: Any) -> list[EvidenceTurn]: - env.session_calls = kw - return [ - EvidenceTurn( - "summary:s9", - "user", - "2026-01-02 10:00:00", - "saw Brandon Flowers in Denver", - "session", - ) - ] - - env.monkeypatch.setattr(deep, "fetch_session_turns", fake_sessions) - out = await _run(query="who did I see in Denver", depth="raw") - assert env.session_calls["session_ids"] == ["s9"] - assert env.session_calls["question"] == "who did I see in Denver" - assert env.fetch_calls[0]["memory_keys"] == [] # no fact keys were picked - assert out.depth == "raw" - assert "[summary:s9] (user) saw Brandon Flowers" in env.llm.synth_calls[-1][1] +async def test_a_summary_hit_does_not_open_unproven_session_text(env) -> None: + env.search_output = _output([_hit("summary:s9", "a verified digest", 0.9)]) + env.set_llm(_ScriptedLLM(["A fact [summary:s9]."], picks='{"picks": [0]}')) + env.turns = [] + await _run(query="what happened", depth="raw") + assert env.fetch_calls[0]["memory_keys"] == [] + assert len(env.llm.synth_calls) == 1 # ---- the temporal trigger's selector is a setting ---- diff --git a/tests/unit/test_memory_clear.py b/tests/unit/test_memory_clear.py index df32308efd..bb8cdeccaf 100644 --- a/tests/unit/test_memory_clear.py +++ b/tests/unit/test_memory_clear.py @@ -530,15 +530,20 @@ def page(sql, params): .on("SELECT max(version) AS v", [{"v": 0}]) .on("SELECT count() AS n FROM loom.memory FINAL", [{"n": 0}], times=1) ) + ch.on("FROM loom.memory_refusals", []) ch.on("SELECT count() AS n", [{"n": 2}]) ch.on("SELECT * FROM loom.memory FINAL", page) ch.on("SELECT DISTINCT cityHash64", []) - ch.on( - "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)", - lambda sql, params: [ - r for r in ch.inserted("memory") if lifecycle.identity_key(r) in params["keys"] - ], - ) + + def heads(sql, params): + current = {} + for row in [*rows, *ch.inserted("memory")]: + key = lifecycle.identity_key(row) + if key not in current or row["version"] >= current[key]["version"]: + current[key] = row + return [current[key] for key in params["keys"] if key in current] + + ch.on("PREWHERE (org, namespace, agent, user_id, session_id, memory_key)", heads) monkeypatch.setattr(lifecycle, "reemit_embedding_rows_for_keys", AsyncMock(return_value=0)) return ch.install(monkeypatch) diff --git a/tests/unit/test_memory_consolidate_handler.py b/tests/unit/test_memory_consolidate_handler.py index 96237e7278..c81cdd38ea 100644 --- a/tests/unit/test_memory_consolidate_handler.py +++ b/tests/unit/test_memory_consolidate_handler.py @@ -463,6 +463,7 @@ def test_tombstone_embeddings_for_key_mirrors_every_column_and_sets_the_stamp() async def fake_insert(rows, *, ch=None): inserted.extend(rows) + return list(rows) ts = datetime(2026, 9, 11, 12, 0, 0) with patch.object(embeddings, "insert_embedding_rows", new=fake_insert): diff --git a/tests/unit/test_memory_delete_tombstones_all_types.py b/tests/unit/test_memory_delete_tombstones_all_types.py index a6dc82748d..22f3bfca9b 100644 --- a/tests/unit/test_memory_delete_tombstones_all_types.py +++ b/tests/unit/test_memory_delete_tombstones_all_types.py @@ -21,7 +21,12 @@ from server.memory.app import handlers as mem_handlers from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod -from server.memory.store import EMBEDDING_PERSISTED_COLS, MemoryRow +from server.memory.store import ( + EMBEDDING_PERSISTED_COLS, + MemoryRow, + build_memory_version_row, + identity_key, +) _NOW = datetime(2026, 9, 18, 12, 0, 0) @@ -43,11 +48,11 @@ def _mk_row(**overrides: Any) -> MemoryRow: "created_at": _NOW - timedelta(days=1), "expires_at": None, "valid_at": None, - "run_id": "", + "run_id": "-", "tags": ["alert"], "entities": [], "temporal_anchor": None, - "reinforcement_count": 0, + "reinforcement_count": 1, "last_reinforced_at": None, "source_resource_id": "", } @@ -98,6 +103,7 @@ def __init__( emb_rows: list[dict[str, Any]] | None = None, fail_inserts_into: tuple[str, ...] = (), ) -> None: + self.source: MemoryRow | None = None self.emb_rows = emb_rows or [] self.fail_inserts_into = fail_inserts_into self.insert_calls: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] @@ -107,6 +113,9 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if params and "keys" in params and self.source is not None: + if identity_key(self.source) in params["keys"]: + return [build_memory_version_row(self.source)] if "FROM loom.memory_embeddings" in sql: return [dict(r) for r in self.emb_rows] return [] @@ -126,6 +135,7 @@ def rows_for(self, table: str) -> list[dict[str, Any]]: async def _delete(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow, ch: _StubCH) -> Any: + ch.source = existing monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_memory_digests_op.py b/tests/unit/test_memory_digests_op.py index 90d7de96cc..eed7a67bec 100644 --- a/tests/unit/test_memory_digests_op.py +++ b/tests/unit/test_memory_digests_op.py @@ -24,6 +24,30 @@ async def query_all( return self.rows[: int(self.params["lim"])] +def _wire_sources(monkeypatch, rows): + from tests.unit.test_memory_verify import _row + + for row in rows: + row.update(agent="a", user_id="-", derived_from=["k1"], key_facts=[row["summary_text"]]) + + async def read(keys, **kwargs): + from server.memory.store.rows import identity_key + + parents = [ + _row( + namespace="n", + agent="a", + session_id=r["session_id"], + memory_key="k1", + value=r["summary_text"], + ) + for r in rows + ] + return {identity_key(r): r for r in parents if identity_key(r) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + + def test_op_is_registered_for_viewers() -> None: op = next(o for o in ALL_OPERATIONS if o.name == "memory.digests") assert op.required_role.name == "VIEWER" @@ -45,6 +69,7 @@ async def test_wildcard_lists_all_sessions_and_pages(monkeypatch: pytest.MonkeyP } for i in range(3) ] + _wire_sources(monkeypatch, rows) ch = _FakeCH(rows) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) out = await cur.handle_memory_digests( @@ -52,7 +77,7 @@ async def test_wildcard_lists_all_sessions_and_pages(monkeypatch: pytest.MonkeyP ) assert "session_id = {s:String}" not in ch.sql and ch.params["lim"] == 3 assert [r.session_id for r in out.rows] == ["ns:session_0", "ns:session_1"] and out.truncated - assert out.rows[0].key_facts == ["a"] and out.rows[0].derived_from == ["k1"] + assert out.rows[0].key_facts == ["digest 0"] and out.rows[0].derived_from == ["k1"] @pytest.mark.asyncio @@ -67,7 +92,7 @@ async def test_concrete_session_filters(monkeypatch: pytest.MonkeyPatch) -> None @pytest.mark.asyncio -async def test_a_digest_that_reads_as_a_credential_is_listed_without_its_text( +async def test_a_digest_that_reads_as_a_credential_is_withheld( monkeypatch: pytest.MonkeyPatch, ) -> None: from tests.unit._fake_credentials import OPENAI_KEY @@ -76,6 +101,7 @@ async def test_a_digest_that_reads_as_a_credential_is_listed_without_its_text( {"session_id": "ns:s1", "summary_text": f"The user pasted {OPENAI_KEY}.", "key_facts": []}, {"session_id": "ns:s2", "summary_text": "Planned the retreat.", "key_facts": []}, ] + _wire_sources(monkeypatch, rows) monkeypatch.setattr("server.core.ch.get_client", lambda: _FakeCH(rows)) out = await cur.handle_memory_digests(MemoryDigestsInput(org="o", namespace="n", agent="a")) - assert [r.summary_text for r in out.rows] == ["", "Planned the retreat."] + assert [r.summary_text for r in out.rows] == ["Planned the retreat."] diff --git a/tests/unit/test_memory_drops.py b/tests/unit/test_memory_drops.py index af9bfe1ed3..3e71f945b0 100644 --- a/tests/unit/test_memory_drops.py +++ b/tests/unit/test_memory_drops.py @@ -303,6 +303,7 @@ async def handle_consolidate(inp): @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_schema_mode_records_stubbed_fields(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx from server.llm.client import LLMResult diff --git a/tests/unit/test_memory_expiry.py b/tests/unit/test_memory_expiry.py index e258229c4e..3e78d92709 100644 --- a/tests/unit/test_memory_expiry.py +++ b/tests/unit/test_memory_expiry.py @@ -83,6 +83,8 @@ def __init__(self) -> None: async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: assert sequential_reads_active() or "select_sequential_consistency = 1" in sql + if "FROM loom.memory_refusals" in sql: + return [] return [self.heads[tuple(k)] for k in params["keys"] if tuple(k) in self.heads] async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: @@ -477,6 +479,7 @@ async def test_batched_reemit_reads_and_inserts_once_per_batch( async def _insert(rows, *, ch=None): ch_.inserts.append(rows) + return list(rows) ch_ = ch monkeypatch.setattr(embeddings, "insert_embedding_rows", _insert) @@ -771,8 +774,9 @@ async def _live(ch, keys, **_: Any) -> list[dict[str, Any]]: inserted: list[dict[str, Any]] = [] - async def _insert(rows, *, ch=None) -> None: + async def _insert(rows, *, ch=None): inserted.extend(rows) + return list(rows) async def _none(*_a: Any, **_k: Any) -> int: return 0 diff --git a/tests/unit/test_memory_fence_publication.py b/tests/unit/test_memory_fence_publication.py index 39c36ab9c2..ffaa4567ec 100644 --- a/tests/unit/test_memory_fence_publication.py +++ b/tests/unit/test_memory_fence_publication.py @@ -45,6 +45,8 @@ async def test_contended_value_write_publishes_no_embedding( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, rows, **kw): @@ -129,6 +131,8 @@ async def test_unanswered_high_memory_send_still_fences_search(monkeypatch): class LostAnswer: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [] async def insert(self, *args, **kw): @@ -167,6 +171,8 @@ async def test_high_write_cancelled_during_holder_release_still_fences_search( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, batch, **kw): @@ -214,6 +220,8 @@ async def test_cancelled_high_send_fences_and_preserves_the_cancellation(monkeyp class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [] async def insert(self, *args, **kw): @@ -245,6 +253,8 @@ async def test_copy_commit_mark_and_followups_survive_holder_release_cancellatio class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): @@ -334,6 +344,8 @@ async def follow(name, *args, **kw): class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, batch, **kw): @@ -397,6 +409,8 @@ async def insert(self, table, batch, **kw): raise RuntimeError("the server's insert answer was lost") async def query_all(self, sql, params): + if "FROM loom.memory_refusals" in sql: + return [] if "FROM loom.memory_refusals" in sql: return [] if "PREWHERE (org, namespace" in sql: @@ -462,6 +476,8 @@ async def test_acknowledged_copy_finishes_followups_under_repeated_cancellation( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): @@ -518,6 +534,8 @@ async def test_bulk_embedding_failure_still_announces_acknowledged_memory(monkey class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, rows, **kw): @@ -547,6 +565,8 @@ async def test_bulk_followup_finishes_under_repeated_cancellation(monkeypatch, c class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): diff --git a/tests/unit/test_memory_metadata_contention.py b/tests/unit/test_memory_metadata_contention.py index beb167e39b..4f53d5e5f7 100644 --- a/tests/unit/test_memory_metadata_contention.py +++ b/tests/unit/test_memory_metadata_contention.py @@ -32,6 +32,11 @@ async def insert(self, table: str, batch: list[dict[str, Any]], **kw: Any) -> No self.table.extend(batch) async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + from server.core.ch import sequential_reads_active + + assert sequential_reads_active() + return [] assert "loom.memory" in sql assert "select_sequential_consistency = 1" in sql heads = {} @@ -245,9 +250,11 @@ async def insert(self, *args: Any, **kw: Any) -> None: with pytest.raises(TimeoutError): await insert_memory_versions(batch, ch=LostAnswer(table)) - assert set(memory_insert_keeper.keys) == { - write_fence.lease_name(rows.identity_key(row)) for row in batch - } + from server.memory.store.refusals import fence_names + + assert set(memory_insert_keeper.keys) == ( + {write_fence.lease_name(rows.identity_key(row)) for row in batch} | set(fence_names(batch)) + ) with pytest.raises(lease_writes.FenceUnavailableError): await insert_memory_versions(batch[:1], ch=Replica(table)) assert len(table) == 2 @@ -386,11 +393,17 @@ async def pause(delay): monkeypatch.setattr(lease_writes.leases, "release_shielded", release) monkeypatch.setattr(write_fence, "_admission_wait_s", lambda: wait_s) row = memory_repo.build_memory_version_row(_row()) + from server.memory.store import refusals + + names = {write_fence.lease_name(rows.identity_key(row)), *refusals.fence_names([row])} + count = len(names) + assert count > 1 # Key and consent admission share this deadline. if admitted: await insert_memory_versions([row], ch=client) else: with pytest.raises(lease_writes.FenceUnavailableError): await insert_memory_versions([row], ch=client) assert client.insert.await_count == int(admitted) - assert attempts == (2 if stage == "retry" and admitted else 1) - assert release.await_count == int(admitted or stage == "take") + assert attempts == count * (2 if stage == "retry" and admitted else 1) + expected_releases = count if stage == "take" else count - 1 + (count if admitted else 0) + assert release.await_count == expected_releases diff --git a/tests/unit/test_memory_move.py b/tests/unit/test_memory_move.py index 445048acb7..aa35736b33 100644 --- a/tests/unit/test_memory_move.py +++ b/tests/unit/test_memory_move.py @@ -22,7 +22,7 @@ import server.memory.app.handlers as mem_handlers import server.memory.app.move as mv -from server.core.hashing import sha256_value_json +from server.core.hashing import sha256_value, sha256_value_json from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory import store as memory_repo from server.memory.api.models_crud import MemoryGetInput, MemorySetOutput @@ -100,6 +100,7 @@ def _source(**over: Any) -> MemoryRow: "session_id": "-", "memory_key": _FROM, "content_excerpt": "ReplicaLag ch-3 42 (#alert_3f9c1a2b)", + "content_sha256": sha256_value(_OLD_VALUE), "embedding_model": "fake-test", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], @@ -142,17 +143,24 @@ def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) self.query_all_sqls: list[str] = [] self.routes = routes or [] self.fail_inserts_into: tuple[str, ...] = () + self.memory_heads: dict[Any, Any] = {} async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: if table in self.fail_inserts_into: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + from server.memory.store.rows import identity_key + + self.memory_heads.update({identity_key(row): row for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + return [self.memory_heads[key] for key in params["keys"] if key in self.memory_heads] for needle, result in self.routes: if needle in sql: return [dict(r) for r in result] @@ -187,6 +195,10 @@ def __init__( set_out: MemorySetOutput | None = None, ) -> None: self.ch = ch or _ch_with_mirror_and_chunks() + if source is not None: + from server.memory.store.rows import identity_key + + self.ch.memory_heads[identity_key(source)] = source.model_dump() self.set_inputs: list[Any] = [] self.delete_inputs: list[Any] = [] self.get_by_key_calls: list[tuple[memory_repo.Identity, str]] = [] @@ -217,7 +229,7 @@ async def _delete(inp: Any) -> Any: monkeypatch.setattr( "server.memory.app.write.handle_memory_set", write_mod.handle_memory_set ) - monkeypatch.setattr(mem_handlers, "handle_memory_delete", _delete) + monkeypatch.setattr(mv, "_tombstone_source", _delete) monkeypatch.setattr(mv, "get_client", lambda: self.ch) @@ -342,7 +354,7 @@ def test_keep_redirect_false_takes_the_delete_path_and_writes_no_redirect( d = h.delete_inputs[0] assert (d.memory_key, d.agent, d.user_id, d.session_id) == (_FROM, "sre", "-", "-") assert out.redirected is False and out.memory_id == _DEST_ID - assert h.ch.inserts == [] # the delete path (stubbed) owns the source's rows + assert h.ch.inserts == [] # Internal retirement (stubbed) owns the source's rows. def test_a_merge_that_wrote_nothing_still_redirects_to_the_live_destination_row( @@ -622,3 +634,42 @@ def test_the_op_is_writer_audited_not_idempotent_and_listed_under_all_only() -> assert "memory.move" not in _MCP_CODING_OPS assert "memory.move" not in _MCP_ATOMIC_OPS assert op_toolsets("memory.move") == ["all"] + + +@pytest.mark.parametrize("cascade_fails", [False, True]) +async def test_internal_move_retirement_announces_its_landed_tombstone(monkeypatch, cascade_fails): + from unittest.mock import AsyncMock + + source = _source() + write = AsyncMock() + retire = AsyncMock(side_effect=RuntimeError("cascade unavailable") if cascade_fails else None) + announce = AsyncMock(return_value=False) + monkeypatch.setattr(mem_handlers, "_write_tombstone_row", write) + monkeypatch.setattr(mem_handlers, "_retire_derived", retire) + monkeypatch.setattr(mv.webhooks, "announce", announce) + if cascade_fails: + with pytest.raises(RuntimeError, match="cascade unavailable"): + await mv._tombstone_source(source) + else: + await mv._tombstone_source(source) + assert write.await_args.kwargs["exact"] is True + assert not write.await_args.kwargs.get("refuse_value", False) + assert announce.await_args.args[0][0][0] == "memory.tombstoned" + + +def test_a_move_preserves_a_source_written_while_the_destination_was_created(monkeypatch): + from server.memory.store.pg_memory import AdmissionContendedError + from server.memory.store.rows import build_memory_version_row, identity_key + + source = _source() + h = _Harness(monkeypatch, source=source) + newer = source.model_copy(update={"value": "a newer fact", "version": source.version + 100}) + + async def set_destination(inp): + h.ch.memory_heads[identity_key(newer)] = build_memory_version_row(newer) + return h.set_out + + monkeypatch.setattr(write_mod, "handle_memory_set", set_destination) + with pytest.raises(AdmissionContendedError): + _move(_inp()) + assert h.ch.rows_for("loom.memory") == [] diff --git a/tests/unit/test_memory_outline_backlinks_op.py b/tests/unit/test_memory_outline_backlinks_op.py index 613f06f143..a5d0796ef1 100644 --- a/tests/unit/test_memory_outline_backlinks_op.py +++ b/tests/unit/test_memory_outline_backlinks_op.py @@ -35,6 +35,16 @@ async def query_all( return [] +@pytest.fixture +def qualified_outline_sources(monkeypatch): + """Outline formatting receives qualified rows; source proof has separate cases.""" + + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + def test_ops_are_registered_for_viewers_and_classed_as_reads() -> None: from server.insights.app.op_mix import classify @@ -49,12 +59,13 @@ def test_ops_are_registered_for_viewers_and_classed_as_reads() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_outline_sources") async def test_outline_composes_counts_sessions_hubs_communities_and_profile( monkeypatch: pytest.MonkeyPatch, ) -> None: ch = _RoutedCH( { - "loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL ORDER BY": [ + "coalesce(valid_at, created_at)": [ { "session_id": "ns:s4", "summary_text": "Discussed the parish workshop. Then planned the retreat.", @@ -118,6 +129,15 @@ async def test_outline_skips_the_profile_read_for_the_dash_user( assert not any("memory_type = 'profile'" in sql for sql, _ in ch.calls) +async def test_outline_withholds_a_legacy_digest_without_source_proof(monkeypatch): + ch = _RoutedCH( + {"coalesce(valid_at, created_at)": [{"session_id": "s", "summary_text": "An old claim"}]} + ) + monkeypatch.setattr("server.core.ch.get_client", lambda: ch) + out = await cur.handle_memory_outline(MemoryOutlineInput(org="o", namespace="n", agent="a")) + assert out.sessions == [] + + def test_headline_takes_the_first_sentence_or_clips() -> None: assert cur._headline("One. Two.") == "One." assert cur._headline("x" * 300) == "x" * 160 @@ -187,6 +207,7 @@ async def test_backlinks_of_an_unknown_key_is_empty_not_an_error( @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_outline_sources") async def test_outline_shows_no_headline_for_a_digest_that_reads_as_a_credential( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_memory_pipeline.py b/tests/unit/test_memory_pipeline.py index f2838e1ed6..aec697e37f 100644 --- a/tests/unit/test_memory_pipeline.py +++ b/tests/unit/test_memory_pipeline.py @@ -240,6 +240,7 @@ async def test_run_pipeline_aborts_on_extraction_error( fail-open one: downstream steps do not run, the error reaches the caller (the handler maps it to 502).""" import server.memory.app.extract as mx + from server.memory.app import pipeline from server.memory.app.pipeline import register_step, run_pipeline from server.memory.app.pipeline_state import PipelineStep @@ -253,6 +254,7 @@ async def _sentinel(state, config): ran.append("after") return state + monkeypatch.setattr(pipeline, "_REGISTRY", dict(pipeline._REGISTRY)) register_step("test.after_extract", _sentinel) state = WorkflowState( org="dev", namespace="ns", agent="a", messages=[{"role": "user", "content": "x"}] diff --git a/tests/unit/test_memory_refusals.py b/tests/unit/test_memory_refusals.py new file mode 100644 index 0000000000..df201ec864 --- /dev/null +++ b/tests/unit/test_memory_refusals.py @@ -0,0 +1,462 @@ +"""Deleted content cannot return through another key or a containing passage.""" + +import pytest + +from server.memory.domain.refusals import fingerprint, matches, tokens + + +@pytest.mark.parametrize("native", [False, True]) +def test_erasure_captures_clickhouse_history_under_both_authorities(monkeypatch, native): + from server.core import pg + from server.erasure.store import refusals + + monkeypatch.setattr(pg, "enabled", lambda: native) + assert "loom.memory_versions" in dict(refusals._sources()) + + +@pytest.mark.parametrize("user_id", [None, "alice"]) +async def test_native_erasure_keeps_cleared_revision_identity_without_fingerprinting_null( + monkeypatch, user_id +): + from types import SimpleNamespace + + from server.erasure.store import consent, refusals, steps_pg + + identity = {"org": "o", "namespace": "n", "agent": "a", "user_id": "alice", "memory_key": "k"} + payload = {**identity, "value": '"current fact"', "value_json": "{}"} + recorded, dependents = [], [] + + class Connection: + async def cursor(self, sql, *args): + assert "coalesce(r.memory_key, r.row->>'memory_key'" in sql + yield {**identity, "row": None} + yield {**identity, "row": payload} + + async def record(conn, rows, **kwargs): + assert all(isinstance(row, dict) for row in rows) + recorded.extend(rows) + + async def capture(conn, rows, uid): + assert uid == "alice" + dependents.extend(rows) + + monkeypatch.setattr(consent, "current_policy", lambda: SimpleNamespace(record_postgres=record)) + monkeypatch.setattr(refusals, "capture_native_chunks", capture) + await steps_pg._capture_history(Connection(), "r.org=$1", "org=$1", ["o"], user_id) + assert recorded == [payload] + assert dependents == ([identity, identity] if user_id else []) + + +@pytest.mark.parametrize("value", [None, "", 0, False, {"b": 2, "a": 1}, [1, 2]]) +def test_canonical_values_are_refused(value): + assert matches(value, fingerprint(value)) + assert matches({"wrapped": value}, fingerprint(value)) + assert not matches("unrelated", fingerprint(value)) + + +def test_normalized_literal_is_refused_inside_later_text(): + policy = fingerprint("My code is ABC-123") + assert matches("Reminder: MY code is abc-123. Do not repeat it.", policy) + assert not matches("My code is abc-124", policy) + + +def test_oversized_scan_refuses_instead_of_allowing_a_prefix(): + from server.core.errors import ValidationError + + with pytest.raises(ValidationError): + matches("other " * 250_001, fingerprint("forgotten value")) + + +@pytest.mark.parametrize("value", ["alpha", None, False, 17, 0, [], {}, {"k": "v"}]) +def test_persisted_scalar_and_object_values_use_the_served_column(value): + import json + + from server.core.hashing import served_row_value + + stored = { + "value": json.dumps(value), + "value_json": json.dumps(value) if isinstance(value, dict) else "{}", + } + assert served_row_value(stored) == value + assert matches(served_row_value(stored), fingerprint(value)) + + +def test_absent_raw_metadata_does_not_reintroduce_a_deleted_null(): + from server.core.hashing import served_row_value + from server.memory.store.refusals import content_policy_row + + row = content_policy_row({"org": "o", "namespace": "n"}, ["an unrelated passage"]) + assert not matches(served_row_value(row), fingerprint(None)) + + +@pytest.mark.parametrize("metadata", [None, "", "{}", " { } ", {}, [], "null"]) +def test_default_raw_metadata_does_not_freeze_unrelated_intake(metadata): + from server.memory.store.refusals import _raw_content_values + + values = _raw_content_values({"content": "an unrelated passage", "metadata": metadata}) + assert values == ["an unrelated passage"] + + +@pytest.mark.parametrize("content", ["null", "{}", "[]"]) +def test_erasure_keeps_literal_content_distinct_from_empty_metadata(monkeypatch, content): + import json + + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = {"org": "o", "namespace": "n", "value_json": "{}", "value": json.dumps(content)} + captured = policy._policy([row]) + patterns = [(r["kind"], r["units"], r["digest"]) for r in captured] + assert matches(policy._raw_content_values({"content": content}), patterns) + + +@pytest.mark.parametrize("metadata", [" { } ", " [ ] ", "null", '""', ""]) +def test_erasure_ignores_only_decoded_empty_metadata(monkeypatch, metadata): + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": metadata, + "capture_kind": "metadata", + } + assert policy._policy([row]) == [] + + +def test_erased_structured_metadata_matches_later_metadata(monkeypatch): + import json + + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + metadata = json.dumps({"secret": "copper lark"}) + row = { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": metadata, + "capture_kind": "metadata", + } + captured = policy._policy([row]) + patterns = [(r["kind"], r["units"], r["digest"]) for r in captured] + assert matches( + policy._raw_content_values({"content": "unrelated", "metadata": metadata}), patterns + ) + + +async def test_candidate_lookups_fit_the_http_parameter_limit(): + from clickhouse_connect.driver.binding import format_bind_value + + from server.core.ch_params import FIELD_BYTES + from server.memory.store import refusals as policy + + batches = [] + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 1}] + batch = params["candidates"] + assert len(format_bind_value(batch).encode()) <= FIELD_BYTES + batches.append(batch) + return [] + + value = " ".join(f"word{i}" for i in range(2100)) + await policy.check_clickhouse( + CH(), [policy.content_policy_row({"org": "o", "namespace": "n"}, value)] + ) + assert len(batches) >= 3 + assert sum(len(batch) for batch in batches) >= 2100 + + +async def test_a_current_source_hash_cannot_publish_a_refused_excerpt(monkeypatch): + from server.core.errors import ValidationError + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + + source = policy.content_policy_row({"org": "o", "namespace": "n"}, "current fact") + excerpt = "retired secret" + row = {**source, "content_sha256": sha256_value("current fact"), "content_excerpt": excerpt} + patterns = set(fingerprint(excerpt)) + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 2}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def heads(*args, **kwargs): + return [row], [source] + + monkeypatch.setattr(policy, "_embedding_sources", heads) + with pytest.raises(ValidationError, match="refused"): + async with policy.publication(CH(), [row], embeddings=True): + pytest.fail("The refused excerpt reached its publisher") + + +@pytest.mark.parametrize("native_bytes", [False, True]) +async def test_embedding_admission_accepts_the_current_fixed_string_hash(monkeypatch, native_bytes): + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + from server.memory.store.rows import identity_key + from tests.unit.test_memory_verify import _row + + source = _row(value="current fact") + digest = sha256_value(source.value) + vector = { + **source.model_dump(), + "content_sha256": digest.encode() if native_bytes else digest, + } + + async def heads(*args, **kwargs): + return {identity_key(source): source} + + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) + admitted, sources = await policy._embedding_sources(object(), [vector]) + assert admitted == [vector] and len(sources) == 1 + + +@pytest.mark.parametrize( + "change", [{"value": "a replacement"}, {"status": "rejected"}, {"tombstoned_at": "2026-01-01"}] +) +async def test_embedding_admission_withholds_a_changed_or_deleted_source(monkeypatch, change): + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + from server.memory.store.rows import identity_key + from tests.unit.test_memory_verify import _row + + source = _row(value="current fact") + vector = {**source.model_dump(), "content_sha256": sha256_value(source.value)} + current = source.model_copy(update=change) + + async def heads(*args, **kwargs): + return {identity_key(current): current} + + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) + assert await policy._embedding_sources(object(), [vector]) == ([], []) + + +@pytest.mark.parametrize("entry", ["handler", "queue"]) +async def test_prior_transcript_messages_cannot_bypass_refusal(monkeypatch, entry): + from server.core.errors import ValidationError + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.app import extraction_queue + from server.memory.app.extract import handler + + secret = "The retired phrase is copper lark" + patterns = set(fingerprint(secret)) + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": len(tokens(secret))}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def insert(self, *args, **kwargs): + pytest.fail("Refused context reached persistent storage") + + ch = CH() + monkeypatch.setattr(handler, "get_client", lambda: ch) + request = MemorySetFromMessagesInput( + org="o", + namespace="n", + agent="a", + defer=True, + messages=[{"role": "user", "content": "hello"}], + last_messages=[{"role": "user", "content": secret}], + ) + with pytest.raises(ValidationError, match="refused"): + if entry == "handler": + await handler.handle_memory_set_from_messages(request) + else: + await extraction_queue.enqueue(request, scope=WHOLE_NAMESPACE, ch=ch) + + +def test_every_extraction_field_has_an_explicit_content_classification(): + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.domain.refusals import EXTRACTION_CONTENT_FIELDS, EXTRACTION_PROTOCOL_FIELDS + + assert not EXTRACTION_CONTENT_FIELDS & EXTRACTION_PROTOCOL_FIELDS + assert EXTRACTION_CONTENT_FIELDS | EXTRACTION_PROTOCOL_FIELDS == set( + MemorySetFromMessagesInput.model_fields + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("extraction_instructions", "retired copper lark"), + ("session_values", ["retired copper lark"]), + ("extra_tags", ["retired copper lark"]), + ("recalled_keys", ["retired copper lark"]), + ("record_schema", {"choice": ["retired copper lark"]}), + ("pipeline_steps", [{"step": "custom", "note": "retired copper lark"}]), + ("observation_date", "retired copper lark"), + ("messages", [{"role": "user", "name": "retired copper lark", "content": "hello"}]), + ("messages", [{"role": "retired copper lark", "content": "hello"}]), + ], +) +@pytest.mark.parametrize("entry", ["queue", "transition", "capture"]) +async def test_all_queued_text_obeys_the_same_refusal_policy(monkeypatch, field, value, entry): + import json + + from server.core.errors import ValidationError + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.app import extraction_queue as queue + from server.memory.store import refusals as policy + from tests.unit.test_extraction_queue import _stored_row + + patterns = set(fingerprint("retired copper lark")) + + class CH: + def __init__(self): + self.ended = [] + + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 3}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def insert(self, *args, **kwargs): + pytest.fail("Refused queue text was copied") + + async def execute(self, sql, params=None): + assert "UPDATE status='failed'" in sql + assert "retired copper lark" not in sql + repr(params) + self.ended.append(params) + + request = MemorySetFromMessagesInput( + **{ + "org": "o", + "namespace": "n", + "agent": "a", + "messages": [{"role": "user", "content": "hello"}], + field: value, + } + ) + ch = CH() + if entry == "capture": + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + rows = policy._policy( + [ + { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": request.model_dump_json(), + "capture_kind": "payload", + } + ] + ) + assert patterns <= {(r["kind"], r["units"], r["digest"]) for r in rows} + elif entry == "transition": + await queue.mark_done( + _stored_row(payload=json.dumps(request.model_dump())), written=0, ch=ch + ) + assert len(ch.ended) == 1 + else: + with pytest.raises(ValidationError, match="refused"): + await queue.enqueue(request, scope=WHOLE_NAMESPACE, ch=ch) + + +async def test_erasure_records_an_indexed_refusal_for_a_derived_rows_own_identity(monkeypatch): + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = policy.content_policy_row( + {"org": "o", "namespace": "n", "user_id": "bob"}, "retired copper lark" + ) + recorded = policy._recording_policy([row], scope_kind="user_id", scope_value="alice") + assert {(r["scope_kind"], r["scope_value"], r["user_id"]) for r in recorded} == { + ("user_id", "alice", "bob"), + ("erased_identity", "n", "bob"), + } + + +def test_queue_protocol_defaults_do_not_freeze_unrelated_intake(): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + old = { + "messages": [{"role": "user", "content": "old passage"}], + "record_schema": {"old_field": "string"}, + "pipeline_steps": [{"step_id": "loom.llm_extract"}], + } + patterns = payload_fingerprints(old) + unrelated = { + "messages": [{"role": "user", "content": "unrelated passage"}], + "record_schema": {"new_field": "string"}, + "pipeline_steps": [{"step_id": "loom.llm_extract"}], + } + assert not matches(transcript_content(unrelated), patterns) + for default in ("user", "string", "loom.llm_extract", "role", "content", "step_id"): + assert not matches(default, patterns) + + +@pytest.mark.parametrize( + "value", [4417, False, {"private": [4417, "copper lark"]}, [4417, "copper lark"]] +) +def test_typed_queue_arguments_are_checked_and_captured_as_values(value): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [{"step_id": "loom.llm_extract", "config": {"arg": value}}]} + assert matches(transcript_content(payload), fingerprint(value)) + assert set(fingerprint(value)) <= payload_fingerprints(payload) + + +async def test_a_refused_stale_queue_transition_cannot_stop_a_newer_source_revision(): + from server.core.hashing import sha256_text + from server.memory.app import extraction_queue as queue + from tests.unit.test_extraction_queue import _stored_row + + original = _stored_row(version=10) + stale = queue._mirror(original, last_error="retired copper lark") + current = {**original, "version": 15} + + class CH: + async def execute(self, sql, params=None): + assert "version={source_version:UInt64}" in sql + assert "hex(SHA256(payload))={payload_sha:String}" in sql + assert params["source_version"] == 10 + assert params["payload_sha"] == sha256_text(original["payload"]).upper() + if current["version"] == params["source_version"]: + current["status"] = "failed" + + await queue._drop_refused(CH(), stale) + assert current["status"] == original["status"] + + +def test_known_pipeline_ids_are_classified_as_protocol(): + from server.memory.app.pipeline import registered_step_ids + from server.memory.domain.refusals import EXTRACTION_PROTOCOL_STEPS + + assert EXTRACTION_PROTOCOL_STEPS == set(registered_step_ids()) + + +@pytest.mark.parametrize( + "step", + [ + {"step_id": "retired copper lark"}, + {"step_id": "loom.hash_dedup", "retired copper lark": "ordinary"}, + ], +) +def test_unknown_step_ids_and_caller_argument_names_remain_content(step): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [step]} + patterns = set(fingerprint("retired copper lark")) + assert matches(transcript_content(payload), patterns) + assert patterns <= payload_fingerprints(payload) + + +@pytest.mark.parametrize("value", [None, False, 0, [], {}, ""]) +@pytest.mark.parametrize("field", ["step_id", "config"]) +def test_explicit_empty_and_typed_step_fields_are_not_protocol_defaults(field, value): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [{field: value}]} + assert matches(transcript_content(payload), fingerprint(value)) + assert set(fingerprint(value)) <= payload_fingerprints(payload) diff --git a/tests/unit/test_memory_reindex_mirror.py b/tests/unit/test_memory_reindex_mirror.py index d58ef5cd5d..cfd6acd38e 100644 --- a/tests/unit/test_memory_reindex_mirror.py +++ b/tests/unit/test_memory_reindex_mirror.py @@ -13,6 +13,7 @@ import pytest +from server.core.hashing import sha256_value from server.embedding._provider import MalformedEmbedding from server.memory import store as mr @@ -27,6 +28,26 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: self.sql.append(sql) self.params.append(params or {}) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store import MemoryRow, build_memory_version_row, identity_key + + heads = [ + MemoryRow.from_snapshot( + { + **r, + "value": r["content_excerpt"], + "memory_id": "11111111-1111-1111-1111-111111111111", + } + ) + for r in self.rows + ] + return [ + build_memory_version_row(row) + for row in heads + if identity_key(row) in params["keys"] + ] if "max(version) AS ev" in sql: return [] # no embedding row of the key yet return self.rows @@ -57,6 +78,7 @@ def _memory_mirror_row() -> dict[str, Any]: "memory_key": "k", "memory_type": "preference", "content_excerpt": "x" * 600, + "content_sha256": sha256_value("x" * 600), "run_id": "r", "entities": ["e"], "tags": ["t"], @@ -125,6 +147,23 @@ def test_a_reindexed_row_carries_its_memory_rows_columns(monkeypatch: Any) -> No assert row["embedding_model"] == "fake-model" +def test_a_withheld_reindex_is_not_counted_or_indexed(monkeypatch: Any) -> None: + ch = _CH([_memory_mirror_row()]) + indexed: list[Any] = [] + + async def withheld(rows: Any, **_: Any) -> list[Any]: + assert len(rows) == 1 + return [] + + async def index(rows: Any, **_: Any) -> None: + indexed.extend(rows) + + monkeypatch.setattr(mr, "insert_embedding_rows", withheld) + monkeypatch.setattr(mr, "index_late_rows", index) + assert _run(monkeypatch, ch).processed == 0 + assert indexed == [] and ch.inserted == [] + + def test_the_read_skips_high_sensitivity_rows_and_current_keys_unless_forced( monkeypatch: Any, ) -> None: @@ -170,7 +209,15 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: def test_reindex_embeds_memory_sets_texts_and_pairs_the_key_vector(monkeypatch: Any) -> None: from server.memory.domain.embed_text import embedding_texts - rows = [dict(_memory_mirror_row(), memory_key=k, content_excerpt=f"v {k}") for k in "ab"] + rows = [ + dict( + _memory_mirror_row(), + memory_key=k, + content_excerpt=f"v {k}", + content_sha256=sha256_value(f"v {k}"), + ) + for k in "ab" + ] ch = _CH(rows) emb = _RecordingEmbedder() _run_with(monkeypatch, ch, emb) @@ -286,3 +333,12 @@ async def _index(rows: Any, **_k: Any) -> int: ch = _CH([dict(_memory_mirror_row())]) assert _run(monkeypatch, ch).processed == 1 assert len(handed) == 1 and handed[0][0]["memory_key"] == "k" + + +def test_reindex_repairs_an_unhashed_canonical_source_without_attesting_it(monkeypatch): + source = _memory_mirror_row() | {"content_sha256": "", "verified": 1} + ch = _CH([source]) + assert _run(monkeypatch, ch).processed == 1 + row = ch.inserted[0][1][0] + assert row["content_sha256"] == sha256_value(source["content_excerpt"]) + assert row["verified"] == 0 diff --git a/tests/unit/test_memory_set_chunks.py b/tests/unit/test_memory_set_chunks.py index a14e701f84..c3735111f0 100644 --- a/tests/unit/test_memory_set_chunks.py +++ b/tests/unit/test_memory_set_chunks.py @@ -30,6 +30,7 @@ from server.memory.domain.conflicts import ConflictResult from server.memory.domain.excerpt import EXCERPT_LIMIT from server.memory.store import MemoryRow +from server.memory.store.rows import build_memory_version_row, identity_key from server.settings import settings @@ -69,17 +70,24 @@ def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) self.query_all_sqls: list[str] = [] self.routes = routes or [] self.fail_inserts_into: tuple[str, ...] = () + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: if table in self.fail_inserts_into: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, rows, settings or {})) + if table == "loom.memory": + self.heads.update({identity_key(row): dict(row) for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] for needle, result in self.routes: if needle in sql: return result @@ -463,6 +471,7 @@ async def test_memory_delete_tombstones_the_keys_chunks(monkeypatch: pytest.Monk async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) out = await mem_handlers.handle_memory_delete( MemoryDeleteInput(org="test", namespace="ops-alerts", agent="sre", memory_key=_KEY) @@ -507,6 +516,7 @@ async def test_memory_delete_of_an_episodic_row_reads_no_chunks( async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) await mem_handlers.handle_memory_delete( MemoryDeleteInput(org="test", namespace="ops-alerts", agent="sre", memory_key=_KEY) @@ -662,6 +672,7 @@ async def _announce(changes: Any) -> None: async def _retire(*_: Any, **__: Any) -> tuple[int, int]: raise RuntimeError("summary downgrade failed") + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) monkeypatch.setattr("server.memory.app.webhooks.announce", _announce) monkeypatch.setattr(mem_handlers, "_retire_derived", _retire) @@ -752,7 +763,12 @@ async def test_a_writer_retires_only_its_own_and_legacy_chunks() -> None: ch=ch, embedder=_BatchEmbedder(), ) - assert [p["uid"] for p in ch.params if "uid" in p] == ["u-7"] + chunk_params = [ + params + for sql, params in zip(ch.query_all_sqls, ch.params, strict=True) + if "FROM loom.memory_chunks FINAL" in sql + ] + assert [params["uid"] for params in chunk_params] == ["u-7"] @pytest.mark.asyncio @@ -785,6 +801,7 @@ async def test_memory_delete_retires_the_deleted_rows_users_chunks( async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) await mem_handlers.handle_memory_delete( MemoryDeleteInput( @@ -811,6 +828,9 @@ async def _tombstone(**kw: Any) -> int: return 1 class _CH: + async def query_all(self, *args, **kwargs): + return [] + async def insert(self, *_a: Any, **_k: Any) -> None: return None @@ -847,6 +867,9 @@ async def _tombstone(**kw: Any) -> int: return 1 class _CH: + async def query_all(self, *args, **kwargs): + return [] + async def insert(self, *_a: Any, **_k: Any) -> None: return None diff --git a/tests/unit/test_memory_set_verdict.py b/tests/unit/test_memory_set_verdict.py index 7ce9de5a73..f0501cba83 100644 --- a/tests/unit/test_memory_set_verdict.py +++ b/tests/unit/test_memory_set_verdict.py @@ -25,6 +25,7 @@ from server.memory.app import write as write_mod from server.memory.domain.conflicts import ConflictPeer, ConflictResult from server.memory.store import MemoryRow +from server.memory.store.rows import identity_key from server.observability.event_args import project_result _NOW = datetime(2026, 9, 14, 12, 0, 0) @@ -34,16 +35,21 @@ class _CH: def __init__(self) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: """The graph-write marks a scheduled projection flushes are not the write under test.""" if table not in ("loom.namespace_write_marks", "loom.graph_projections"): self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update({identity_key(row): dict(row) for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql: str, params: Any = None, **__: Any) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] return [] async def execute(self, *_: Any, **__: Any) -> None: diff --git a/tests/unit/test_memory_update_conflict.py b/tests/unit/test_memory_update_conflict.py index 188b65bf1e..399477e399 100644 --- a/tests/unit/test_memory_update_conflict.py +++ b/tests/unit/test_memory_update_conflict.py @@ -70,6 +70,8 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None self.inserts.append((table, rows)) async def query_all(self, sql: str, params: Any = None, **kwargs: Any) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + return [] if ( "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql diff --git a/tests/unit/test_memory_verify.py b/tests/unit/test_memory_verify.py index 18905f76d6..ef13ff4bf1 100644 --- a/tests/unit/test_memory_verify.py +++ b/tests/unit/test_memory_verify.py @@ -453,9 +453,9 @@ async def boom(r: MemoryRow, *, verified: bool) -> None: async def test_the_mirror_re_emits_every_live_row_of_the_key( monkeypatch: pytest.MonkeyPatch, ) -> None: - """One row per (scope, embedding model): each is re-emitted whole with `verified` set, + """One private row per embedding model is re-emitted whole with `verified` set, keeping the columns the verify input never carried (reinforcement, mention times).""" - from server.memory.store import EMBEDDING_PERSISTED_COLS + from server.memory.store import EMBEDDING_PERSISTED_COLS, build_memory_version_row row = _row() stored = [ @@ -467,6 +467,7 @@ async def test_the_mirror_re_emits_every_live_row_of_the_key( "user_id": row.user_id, "session_id": row.session_id, "memory_key": row.memory_key, + "content_sha256": _sha(row.value), "scope": scope, "embedding_model": model, "embedding_dim": 2, @@ -475,12 +476,16 @@ async def test_the_mirror_re_emits_every_live_row_of_the_key( "mentioned_at": datetime(2026, 1, 3), "version": 7, } - for scope, model in (("private", "m1"), ("shared", "m2")) + for scope, model in (("private", "m1"), ("private", "m2")) ] inserted: list[dict[str, Any]] = [] class _CH: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> Any: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(row)] assert "FROM loom.memory_embeddings FINAL" in sql return [dict(r) for r in stored] @@ -492,7 +497,7 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None await mv._mirror_verified_to_embeddings(row, verified=True) assert [(r["scope"], r["embedding_model"]) for r in inserted] == [ ("private", "m1"), - ("shared", "m2"), + ("private", "m2"), ] assert all(r["verified"] == 1 and r["version"] > 7 for r in inserted) assert all(r["mentioned_at"] == datetime(2026, 1, 3) for r in inserted) diff --git a/tests/unit/test_model_call_customer.py b/tests/unit/test_model_call_customer.py index 28f26927a9..8d1820f605 100644 --- a/tests/unit/test_model_call_customer.py +++ b/tests/unit/test_model_call_customer.py @@ -366,19 +366,29 @@ def test_the_fakes_check_the_customer_as_a_metered_provider_would(metered) -> No asyncio.run(FakeNGramEmbedder(dim=8).embed_many(["text"])) -def test_a_reflection_is_written_for_the_org_of_its_pair(metered) -> None: +def test_a_reflection_is_written_for_the_org_of_its_pair(metered, monkeypatch) -> None: # The contrast call and the reflection's write (which embeds) are both that org's. from server.memory.app import session_outcomes as so seen: list[tuple[str, str]] = [] - neg = {"org": "acme", "namespace": "ns", "session_id": "s1", "summary_embedding": [0.1]} + from tests.unit.memory_store import seed_summary_sources + + neg = { + "org": "acme", + "namespace": "ns", + "session_id": "s1", + "summary_embedding": [0.1], + "summary_text": "failed step", + } + pos = {**neg, "session_id": "s2", "similarity": 0.99, "summary_text": "ok"} + seed_summary_sources(monkeypatch, [neg, pos]) class _CH: async def query_all(self, sql: str, params: Any) -> list[Any]: return [neg] async def query_one(self, sql: str, params: Any) -> Any: - return {"session_id": "s2", "similarity": 0.99, "summary_text": "ok"} + return pos class _LLM: async def complete(self, *a: Any, **k: Any) -> Any: @@ -1084,11 +1094,15 @@ async def link(**kw: Any) -> None: return None monkeypatch.setattr("server.memory.app.evidence.link_belief_sources", link) + from tests.unit.memory_store import seed_summary_sources + + digests = _digests(1, 5) + seed_summary_sources(monkeypatch, digests) cfg = PeriodDigestConfig(sessions_per_period=5, namespace_filter="ns", require_opt_in=False) res = asyncio.run( run_period_digest( cfg, - ch=_FakeCH(_digests(1, 5)), + ch=_FakeCH(digests), llm=_MeteredLLM("Between 2023-07-01 and 2023-07-05, Caroline painted."), embedder=FakeNGramEmbedder(dim=3), ) diff --git a/tests/unit/test_model_call_holds.py b/tests/unit/test_model_call_holds.py index 8fa461293e..9f36a6bad0 100644 --- a/tests/unit/test_model_call_holds.py +++ b/tests/unit/test_model_call_holds.py @@ -82,12 +82,13 @@ async def find(self, *, batch_size: int, **_: Any) -> Any: candidates = sorted(live, key=lambda r: -r["version"])[: batch_size * 4] return [dict(r) for r in sorted(candidates, key=lambda r: r["created_at"])[:batch_size]] - async def insert(self, rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(self, rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: self.embedded.update(r["memory_key"] for r in rows if r["embedding_dim"] > 0) for row in rows: self.rows = [ dict(row) if old["memory_key"] == row["memory_key"] else old for old in self.rows ] + return rows async def writers(self, org: str, keys: Any, **_: Any) -> dict[Any, Any]: # a* were written by agent-a, b* by agent-b, into the shared namespace diff --git a/tests/unit/test_model_call_recovery.py b/tests/unit/test_model_call_recovery.py index 0e840741ce..4e4b5cee8b 100644 --- a/tests/unit/test_model_call_recovery.py +++ b/tests/unit/test_model_call_recovery.py @@ -30,7 +30,25 @@ async def embed_many(self, texts): seen.append(customer.headers(await customer.require())) return [[1.0, 0.0] for _ in texts] + from server.core.hashing import sha256_value + from server.memory.store.rows import build_memory_version_row + from tests.unit.test_memory_verify import _row + + source = _row( + org="acme", + namespace=AGENT_NS, + agent="a", + user_id="u1", + memory_key="k1", + value="agent-8 wrote this", + ) + class CH: + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(source)] + return [] + async def insert(self, *args, **kwargs): pass @@ -42,6 +60,7 @@ async def insert(self, *args, **kwargs): session_id="-", memory_key="k1", content_excerpt="agent-8 wrote this", + content_sha256=sha256_value(source.value), embedding_model="old", embedding=[1.0, 0.0], memory_key_embedding=[1.0, 0.0], diff --git a/tests/unit/test_model_call_writers.py b/tests/unit/test_model_call_writers.py index 74142524cb..f39d19cb79 100644 --- a/tests/unit/test_model_call_writers.py +++ b/tests/unit/test_model_call_writers.py @@ -307,6 +307,10 @@ def test_a_reflection_judges_a_session_for_its_writer(writers: _Directory, monke "key_facts": [], } + from tests.unit.memory_store import seed_summary_sources + + seed_summary_sources(monkeypatch, [summary]) + class _CH: async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: return [summary] diff --git a/tests/unit/test_model_calls_held_mid_pass.py b/tests/unit/test_model_calls_held_mid_pass.py index 314104e50d..7a0aa9b518 100644 --- a/tests/unit/test_model_calls_held_mid_pass.py +++ b/tests/unit/test_model_calls_held_mid_pass.py @@ -280,13 +280,11 @@ def test_the_period_digest_records_no_failure_when_its_call_is_held(monkeypatch, async def idents(ch: Any, cfg: Any) -> list[dict[str, Any]]: return [{"org": "acme", "namespace": "n", "agent": "a", "user_id": "u"}] - async def nothing(*a: Any, **k: Any) -> Any: - return set() + async def planned(*a: Any, **k: Any) -> Any: + return [period], 0 monkeypatch.setattr(pd, "_fetch_namespaces", idents) - monkeypatch.setattr(pd, "_fetch_digests", nothing) - monkeypatch.setattr(pd, "_skipped_period_keys", nothing) - monkeypatch.setattr(pd, "plan_periods", lambda *a, **k: ([period], 0)) + monkeypatch.setattr(pd, "_plan_digest_pages", planned) monkeypatch.setattr(pd, "build_period_prompt", lambda p: ("system", "user")) llm = SimpleNamespace(complete=_raise(error)) result = asyncio.run(pd.run_period_digest(ch=object(), llm=llm, embedder=object())) diff --git a/tests/unit/test_model_calls_switch.py b/tests/unit/test_model_calls_switch.py index 8f902b2127..eb95a51245 100644 --- a/tests/unit/test_model_calls_switch.py +++ b/tests/unit/test_model_calls_switch.py @@ -435,8 +435,9 @@ async def find_stubs(*, exclude_orgs: frozenset[str] = frozenset(), **_: Any) -> dict(s) for s in stubs if s["org"] not in exclude_orgs and s["memory_key"] not in done ] - async def insert_rows(rows: list[dict[str, Any]], *a: Any, **kw: Any) -> None: + async def insert_rows(rows: list[dict[str, Any]], *a: Any, **kw: Any) -> list[dict[str, Any]]: inserted.extend(rows) + return rows async def none(**_: Any) -> list[Any]: return [] @@ -662,6 +663,24 @@ def __init__(self, rows: list[dict[str, Any]], *, fail_drops: bool = False) -> N self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] async def query_all(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + return [] + if "source_version" in params: + from server.core.hashing import sha256_text + + return ( + [{"found": 1}] + if any( + all( + row.get(k) == params[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + ) + and row["version"] == params["source_version"] + and sha256_text(row.get("payload") or "").upper() == params["payload_sha"] + for row in self.rows + ) + else [] + ) self.sql, self.params = sql, params return self.rows @@ -671,7 +690,19 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None self.inserts.append((table, rows)) -_WAITING = [{"org": "acme", "namespace": "n", "queue_id": "q1", "status": "pending"}] +_WAITING = [ + { + "org": "acme", + "namespace": "n", + "agent": "a", + "user_id": "-", + "session_id": "-", + "version": 1, + "payload": '{"messages":[]}', + "queue_id": "q1", + "status": "pending", + } +] def test_a_dropped_request_is_marked_failed_after_its_drop_record_is_written() -> None: diff --git a/tests/unit/test_model_calls_switch_workers.py b/tests/unit/test_model_calls_switch_workers.py index de049a53b2..27fcf98293 100644 --- a/tests/unit/test_model_calls_switch_workers.py +++ b/tests/unit/test_model_calls_switch_workers.py @@ -303,8 +303,9 @@ async def picked(**kw: Any) -> list[dict[str, Any]]: async def served(rows: list[dict[str, Any]], *a: Any) -> tuple[list[Any], list[Any]]: return rows, [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def nothing(*a: Any, **k: Any) -> None: return None @@ -384,8 +385,9 @@ def test_a_stale_switch_stops_the_embedding_worker_before_any_write(monkeypatch) written: list[Any] = [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def none(*a: Any, **k: Any) -> list[Any]: return [] @@ -438,8 +440,9 @@ async def picked(**kw: Any) -> list[dict[str, Any]]: async def served(rows: list[dict[str, Any]], *a: Any) -> tuple[list[Any], list[Any]]: return rows, [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def nothing(*a: Any, **k: Any) -> None: return None diff --git a/tests/unit/test_namespace_rollback_unembedded.py b/tests/unit/test_namespace_rollback_unembedded.py index 671c98824c..1125554a3e 100644 --- a/tests/unit/test_namespace_rollback_unembedded.py +++ b/tests/unit/test_namespace_rollback_unembedded.py @@ -84,8 +84,9 @@ async def _memory(rows: list[dict[str, Any]], **_: Any) -> None: if on_inserted := _.get("on_inserted"): on_inserted() - async def _stubs(rows: list[dict[str, Any]], **_: Any) -> None: + async def _stubs(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: seen["stubs"].extend(rows) + return rows async def _retire(items: list[tuple[Any, int]], **_: Any) -> int: seen["retire_calls"].append(items) @@ -465,10 +466,15 @@ async def _published(*_a: Any, **_k: Any) -> None: async def _complete(seqs: list[Any], done: int, **_: Any) -> None: settled.append((list(seqs), done, list(calls["stubs"]))) + async def _held(org: str, ns: str, ids: list[Any], **_: Any) -> set[Any]: + assert (org, ns) == ("test", "personal") + return {pg_publish.document_id(row) for row in calls["stubs"]} & set(ids) + _authority_holds(monkeypatch, calls) monkeypatch.setattr(pg_publish, "commit", _commit) monkeypatch.setattr(pg_publish, "publish", _published) monkeypatch.setattr(pg_memory, "complete", _complete) + monkeypatch.setattr(pg_publish, "held_documents", _held) plan = RollbackPlan( reinstate=[_action("plain", "reinstate", 10), _action("secret", "reinstate", 11)], remove=[], diff --git a/tests/unit/test_negative_escalation.py b/tests/unit/test_negative_escalation.py index ce1af3ea63..ade8d41186 100644 --- a/tests/unit/test_negative_escalation.py +++ b/tests/unit/test_negative_escalation.py @@ -13,7 +13,10 @@ import pytest import server.memory.app.ops as ops +from server.core.datetimes import naive_utc_now +from server.core.hashing import sha256_value from server.memory.api.models_temporal import MemoryOutcomeInput +from server.memory.store.rows import MemoryRow, _hydrate, build_memory_version_row, identity_key @pytest.fixture(autouse=True) @@ -60,6 +63,21 @@ class _QuarantineCH(_StubCH): def __init__(self, ids): super().__init__([]) self.ids = ids + self.heads = {} + + def current(self): + heads = { + identity_key(h): build_memory_version_row(h) + for h in self.heads.values() + if h is not None + } + for table, rows in self.inserts: + if table == "loom.memory": + for row in rows: + key = identity_key(row) + if key not in heads or int(heads[key]["version"]) < int(row["version"]): + heads[key] = row + return heads async def query_one(self, sql, params=None): if "event_type = 'rejection'" in sql: @@ -70,7 +88,11 @@ async def query_one(self, sql, params=None): async def query_all(self, sql, params=None): if "FROM loom.memory FINAL" in sql: - return [{"memory_id": mid} for mid in self.ids] + return [ + r + for key, r in self.current().items() + if not params or "keys" not in params or key in params["keys"] + ] if "FROM loom.memory_embeddings FINAL" in sql: p = params or {} return [ @@ -86,6 +108,7 @@ async def query_all(self, sql, params=None): "embedding_dim": 2, "status": "active", "version": 2, # copied at read + 1 since its memory row + "content_sha256": sha256_value("session fact"), } for o, n, a, u, s, k in zip( p["o"], p["n"], p["a"], p["u"], p["s"], p["k"], strict=True @@ -95,7 +118,13 @@ async def query_all(self, sql, params=None): def _session_row(session_id, lock_mode="", version=1): - return SimpleNamespace( + return MemoryRow( + value="session fact", + pinned=False, + confidence=1, + created_at=naive_utc_now(), + expires_at=None, + content_sha256=sha256_value("session fact"), org="o", namespace="n", agent="a", @@ -121,6 +150,7 @@ def _wire_quarantine(monkeypatch, heads, outcome_reads=None): from server.settings import settings ch = _QuarantineCH(ids=list(heads)) + ch.heads = heads first = outcome_reads or heads async def get_by_id(memory_id, **kw): @@ -129,28 +159,17 @@ async def get_by_id(memory_id, **kw): async def get_first(memory_id, **kw): return first[memory_id] - def build(row, **overrides): - keys = ("org", "namespace", "agent", "user_id", "session_id", "memory_key", "status") - return {**{k: getattr(row, k) for k in keys}, "invalid_at": None, **overrides} - monkeypatch.setattr(settings, "memory_negative_demote_threshold", 3) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_id", get_first) monkeypatch.setattr(lifecycle, "get_by_id", get_by_id) - monkeypatch.setattr(lifecycle, "build_memory_version_row", build) async def current_heads(keys, **kwargs): - columns = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") - return { - tuple(getattr(head, c) for c in columns): head - for head in heads.values() - if tuple(getattr(head, c) for c in columns) in keys - } + return {key: _hydrate(row) for key, row in ch.current().items() if key in keys} monkeypatch.setattr( "server.memory.store.write_fence.memory_rows_by_identity_keys", current_heads ) - monkeypatch.setattr("server.memory.store.inserts.check_memory_rows", lambda rows: None) return ch @@ -194,8 +213,8 @@ async def test_a_locked_memory_is_not_quarantined(monkeypatch, mode): async def test_a_session_copy_never_quarantines_the_namespace_definition(monkeypatch): copy = _session_row("s1") shared = _session_row("-") - for row in (copy, shared): - row.agent, row.memory_type = "-", "definition" + copy = copy.model_copy(update={"agent": "-", "memory_type": "definition"}) + shared = shared.model_copy(update={"agent": "-", "memory_type": "definition"}) ch = _wire_quarantine(monkeypatch, {"mid-s1": copy, "mid--": shared}) out = await ops.handle_memory_outcome(_negative()) @@ -207,8 +226,8 @@ async def test_a_session_copy_never_quarantines_the_namespace_definition(monkeyp async def test_rating_the_namespace_definition_quarantines_nothing(monkeypatch): shared = _session_row("-") copy = _session_row("s1") - for row in (copy, shared): - row.agent, row.memory_type = "-", "definition" + copy = copy.model_copy(update={"agent": "-", "memory_type": "definition"}) + shared = shared.model_copy(update={"agent": "-", "memory_type": "definition"}) ch = _wire_quarantine(monkeypatch, {"mid--": shared, "mid-s1": copy}) out = await ops.handle_memory_outcome( diff --git a/tests/unit/test_observation_date_default.py b/tests/unit/test_observation_date_default.py index c92e070fa2..f9fe7a5b48 100644 --- a/tests/unit/test_observation_date_default.py +++ b/tests/unit/test_observation_date_default.py @@ -5,12 +5,19 @@ from datetime import UTC, datetime from typing import Any +from unittest.mock import AsyncMock import pytest from server.memory.domain.observation_date import resolve_observation_date +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + def test_caller_value_wins_and_is_trimmed() -> None: assert resolve_observation_date(" 2023-05-30 ") == ("2023-05-30", "caller") assert resolve_observation_date("2023-05-30T10:00:00", default="none") == ( @@ -62,7 +69,6 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _zero) monkeypatch.setattr(mx, "write_memory_chunks", _none) monkeypatch.setattr(mx, "_write_session_summary", _none) - monkeypatch.setattr(mx, "_draft_session_summary", _none) monkeypatch.setattr(mx, "record_input_projection_failure", _none) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _none) monkeypatch.setattr(ont, "load_namespace_ontology", _cold) diff --git a/tests/unit/test_passage_sensitivity.py b/tests/unit/test_passage_sensitivity.py index 05d8346efc..051d83d216 100644 --- a/tests/unit/test_passage_sensitivity.py +++ b/tests/unit/test_passage_sensitivity.py @@ -489,7 +489,7 @@ def test_a_private_key_header_without_a_body_holds_only_its_own_chunk() -> None: @pytest.mark.asyncio -async def test_import_evidence_stores_a_credential_chunk_and_digest_without_a_vector( +async def test_import_holds_a_credential_chunk_and_withholds_an_unproven_digest( monkeypatch: pytest.MonkeyPatch, ) -> None: import server.embedding as embmod @@ -525,9 +525,8 @@ async def not_refused(*_: Any, **__: Any) -> bool: assert set(chunks) == {0, 1} assert (chunks[0]["embedding_dim"], chunks[1]["embedding_dim"]) == (2, 0) assert chunks[1]["content"] == f"key {OPENAI_KEY}" # stored as written - [digest] = ch.inserts["loom.session_summaries"] - assert digest["embedding_dim"] == 0 and digest["summary_text"] == f"Shared {AWS_KEY_ID}." - assert (out.chunks_written, out.digests_written, out.skipped) == (2, 1, 1) + assert "loom.session_summaries" not in ch.inserts + assert (out.chunks_written, out.digests_written, out.skipped) == (2, 0, 2) assert out.embedding_dim == 2 assert emb.seen == [_PLAIN] @@ -659,24 +658,33 @@ def test_a_chunk_key_with_a_credential_version_is_held_back() -> None: @pytest.mark.asyncio -async def test_the_summary_leg_serves_no_digest_that_reads_as_a_credential() -> None: +async def test_the_summary_leg_serves_no_digest_that_reads_as_a_credential(monkeypatch) -> None: + from unittest.mock import AsyncMock + + from server.memory import store from server.memory.app.retrieval.summaries import fetch_session_summaries + from tests.unit.test_memory_verify import _row + from tests.unit.test_summary_sources import digest + + secret = _row( + namespace="n", + agent="a", + user_id="u", + memory_key="secret", + session_id="s1", + value="x" * 1195 + f" {AWS_KEY_ID}.", + ) + plain = _row( + namespace="n", agent="a", user_id="u", memory_key="plain", session_id="s2", value=_PLAIN + ) + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={store.identity_key(r): r for r in [secret, plain]}), + ) class _Rows: async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: - return [ - { - # The key straddles the served excerpt's 1200-character cut. - "session_id": "s1", - "summary_text": "x" * 1195 + f" {AWS_KEY_ID}.", - "distance": 0.1, - }, - { - "session_id": "s2", - "summary_text": _PLAIN, - "distance": 0.2, - }, - ] + return [digest(secret), digest(plain, distance=0.2)] hits = await fetch_session_summaries( _Rows(), @@ -710,29 +718,30 @@ async def find(*, limit: int, **_: Any) -> list[dict[str, Any]]: assert asked == [2 + _HELD_MARGIN] -@pytest.mark.asyncio -async def test_a_digest_that_reads_as_a_credential_is_drafted_without_a_vector( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import server.llm.client as llm_client - from server.memory.app import session_summary +async def test_a_credential_source_never_enters_a_digest(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import AsyncMock - class _LLM: - async def complete(self, *_: Any, **__: Any) -> Any: - return SimpleNamespace(text=f"The user pasted {OPENAI_KEY}.", tokens_in=1, tokens_out=1) - - emb = _Embedder() - monkeypatch.setattr(llm_client, "get_llm_client", lambda: _LLM()) - monkeypatch.setattr(session_summary, "get_embedder", lambda: emb) - draft = await session_summary._draft_session_summary( - SimpleNamespace(session_id="s"), - [{"role": "user", "content": "x"}], - None, - operator_policy="", + from server.memory import store + from server.memory.app import session_summary + from tests.unit.test_memory_verify import _row + + source = _row(value=f"The user pasted {OPENAI_KEY}.", sensitivity="") + ch = SimpleNamespace(insert=AsyncMock()) + vector = AsyncMock() + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "document_vector", vector) + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + inp = SimpleNamespace( + org=source.org, + namespace=source.namespace, + agent=source.agent, + user_id=source.user_id, + session_id=source.session_id, ) - assert draft is not None - assert draft.embedding == [] and emb.seen == [] - assert OPENAI_KEY in draft.summary_text # stored as written + state = SimpleNamespace(written=[{"memory_id": source.memory_id, "version": source.version}]) + await session_summary._write_session_summary(inp, state) + vector.assert_not_awaited() + ch.insert.assert_not_awaited() @pytest.mark.asyncio @@ -801,31 +810,6 @@ async def test_the_deep_evidence_pass_opens_no_turn_that_reads_as_a_credential() assert [t.content for t in turns] == [_PLAIN] -@pytest.mark.asyncio -async def test_a_summary_hit_opens_no_session_turn_that_reads_as_a_credential() -> None: - from server.memory.app.deep_evidence import fetch_session_turns - - rows = [ - {"session_id": "s1", "role": "user", "observed_at": "", "content": c} - for c in (f"the standup key is {AWS_KEY_ID}", _PLAIN) - ] - - class _CH: - async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: - return rows - - turns = await fetch_session_turns( - org="o", - namespace="n", - session_ids=["s1"], - question="When is the standup?", - max_chars_per_session=1000, - max_chars_total=5000, - ch=_CH(), - ) - assert [t.content for t in turns] == [_PLAIN] - - @pytest.mark.asyncio async def test_the_when_evidence_pass_recalls_no_turn_that_reads_as_a_credential( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_period_digest.py b/tests/unit/test_period_digest.py index f177c25a12..696227e3a1 100644 --- a/tests/unit/test_period_digest.py +++ b/tests/unit/test_period_digest.py @@ -28,6 +28,16 @@ ) +@pytest.fixture(autouse=True) +def verified_period_inputs(monkeypatch): + """Period-policy tests start after source qualification, covered in test_summary_sources.""" + + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + def _digest(n: int, agent: str = "a") -> dict[str, Any]: return { "org": "o", @@ -149,13 +159,25 @@ def __init__(self, digests: list[dict[str, Any]], existing: set[str] | None = No self.link_calls: list[dict[str, Any]] = [] async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store.rows import identity_key + + heads = {} + for table, rows in self.inserts: + if table == "loom.memory": + heads.update({identity_key(row): row for row in rows}) + return [heads[key] for key in params["keys"] if key in heads] if "uniqExact(s.session_id)" in sql: sessions = len({d["session_id"] for d in self._digests}) return [ {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-", "sessions": sessions} ] if "FROM loom.session_summaries FINAL" in sql: - return list(self._digests) + ordered = sorted(self._digests, key=lambda d: (d["at"], d["session_id"])) + after = params.get("after") + if after: + ordered = [d for d in ordered if (d["at"], d["session_id"]) > after] + return ordered[: params["limit"]] if "startsWith(memory_key" in sql: return [{"memory_key": k, "taken_out": False} for k in self._existing] if "memory_key IN" in sql: @@ -335,7 +357,7 @@ async def _heads(keys: Any, **_: Any) -> Any: # every source has a newer head async def test_a_failed_source_read_fails_the_period_before_its_model_call() -> None: class _SourcesDown(_FakeCH): async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: - if "memory_key IN" in sql: + if "memory_key IN" in sql and "startsWith" not in sql: raise ConnectionError("clickhouse unreachable") return await super().query_all(sql, params) @@ -693,3 +715,69 @@ async def _run(cfg: Any, **_: Any) -> Any: ) assert app._dispatch(args) == 0 assert seen == ["process:cli:period-digest"] + + +async def test_unqualified_session_text_never_reaches_the_period_model(monkeypatch): + from unittest.mock import AsyncMock + + from server.memory.app.period_digest import _plan_digest_pages + + qualification = AsyncMock(return_value=[]) + monkeypatch.setattr( + "server.memory.store.summary_sources.current_summary_sources", qualification + ) + ch = _FakeCH(_digests(1, 5)) + assert await _plan_digest_pages( + ch, {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"}, _CFG + ) == ([], 0) + qualification.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_streamed_windows_skip_old_keys_before_proof_and_reach_later_pages(monkeypatch): + from datetime import timedelta + from unittest.mock import AsyncMock + + from server.memory.app.period_digest import _plan_digest_pages + + digests = [ + dict(_digest(1), session_id=f"ns:session_{n}", at=datetime(2023, 1, 1) + timedelta(days=n)) + for n in range(1, 401) + ] + existing = {period_key(f"ns:session_{n}", f"ns:session_{n + 4}") for n in range(1, 301, 5)} + qualified = AsyncMock(side_effect=lambda ch, rows, **kw: rows) + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + ident = {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"} + cfg = replace(_CFG, max_periods_per_identity=2) + periods, skipped = await _plan_digest_pages(_FakeCH(digests, existing), ident, cfg) + assert [p.memory_key for p in periods] == [ + period_key("ns:session_301", "ns:session_305"), + period_key("ns:session_306", "ns:session_310"), + ] + assert skipped == 60 + qualified.assert_awaited_once() + checked = qualified.call_args.args[1] + assert len(checked) == 100 and all( + int(d["session_id"].rsplit("_", 1)[1]) > 300 for d in checked + ) + existing.update(p.memory_key for p in periods) + later, _ = await _plan_digest_pages(_FakeCH(digests, existing), ident, cfg) + assert later[0].memory_key == period_key("ns:session_311", "ns:session_315") + + +@pytest.mark.asyncio +async def test_an_unproven_digest_withholds_its_whole_window_without_shifting_later_windows( + monkeypatch, +): + from server.memory.app.period_digest import _plan_digest_pages + + async def qualify(ch, rows, **kwargs): + return [d for d in rows if d["session_id"] != "ns:session_2"] + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualify) + periods, _ = await _plan_digest_pages( + _FakeCH(_digests(1, 10)), + {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"}, + _CFG, + ) + assert [p.memory_key for p in periods] == [_KEY_6_10] diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 3a4ae4f60a..9dd0cc75f0 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -1365,6 +1365,118 @@ def _claim(seq: int, targets: int, *, ns: str = "n", attempts: int = 1) -> pg_me ) +async def test_delivery_retains_only_the_document_admission_withheld(monkeypatch) -> None: + from dataclasses import replace + + from server.memory.app import after_write, pg_delivery + from server.memory.store import pg_publish + + claims = [ + replace( + _claim(i, pg_memory.SEARCH_DOC), + lease_token=10 + i, + deadline=time.monotonic() + 60, + row={ + "org": "o", + "namespace": "n", + "agent": "a", + "user_id": "u", + "session_id": "s", + "memory_key": f"k{i}", + "version": i, + }, + ) + for i in range(1, 5) + ] + completed, released, indexed = [], [], [] + + async def none(*a, **kw): + return None + + async def held(*a, **kw): + return {pg_publish.document_id(claims[0].row)} + + async def heads(owed, deadline): + assert [c.seq for c in owed] == [2, 3, 4] + return [replace(claims[2], row={**claims[2].row, "version": 30}), claims[3]] + + async def documents(built, **kw): + return [dict(c.row) for c in built] + + async def insert(rows, **kw): + assert [r["version"] for r in rows] == [30, 4] + return rows[:1] + + async def complete(seqs, target, *, tokens): + completed.append((seqs, target, tokens)) + + async def release(owed): + released.extend(c.seq for c in owed) + + monkeypatch.setattr("server.core.ch.get_client", lambda: object()) + monkeypatch.setattr(pg_publish, "publish_rows", none) + monkeypatch.setattr(pg_publish, "held_documents", held) + monkeypatch.setattr(pg_publish, "retire_decision_records", none) + monkeypatch.setattr(pg_delivery, "_still_heads", heads) + monkeypatch.setattr(pg_delivery, "_search_documents", documents) + monkeypatch.setattr(pg_delivery, "_retire_old_vectors", none) + monkeypatch.setattr("server.memory.store.insert_embedding_rows", insert) + monkeypatch.setattr(after_write, "index_late_rows", lambda rows, **kw: indexed.extend(rows)) + monkeypatch.setattr(pg_memory, "complete", complete) + monkeypatch.setattr(pg_memory, "release", release) + assert await pg_delivery._publish(claims) == 3 + assert completed == [ + ([("o", "n", 1), ("o", "n", 2), ("o", "n", 3)], pg_memory.SEARCH_DOC, [11, 12, 13]) + ] + assert released == [4] and [r["version"] for r in indexed] == [30] + + +async def test_a_delivery_withheld_by_admission_counts_no_success_or_failure(monkeypatch) -> None: + from server.memory.app import pg_delivery + + async def renew(claims): + return claims + + async def withheld(claims, **kw): + return 0 + + async def unexpected(*a, **kw): + raise AssertionError("withholding counts no failure") + + monkeypatch.setattr(pg_memory, "renew", renew) + monkeypatch.setattr(pg_memory, "record_failure", unexpected) + monkeypatch.setattr(pg_delivery, "_publish", withheld) + assert await pg_delivery.publish_claims([_claim(1, pg_memory.SEARCH_DOC)]) == (0, 0) + + +@pytest.mark.parametrize("present", [False, True]) +async def test_a_copy_settles_only_an_exact_document_held_in_clickhouse(monkeypatch, present): + from server.memory.store import lifecycle, pg_publish + + row = { + "org": "o", + "namespace": "n", + "agent": "a", + "user_id": "u", + "session_id": "s", + "memory_key": "k", + "version": 7, + } + seq, completed = ("o", "n", 3), [] + + async def held(org, ns, ids, **kw): + assert (org, ns, ids) == ("o", "n", [pg_publish.document_id(row)]) + return set(ids) if present else {(*ids[0][:-1], 6)} + + async def complete(seqs, target): + completed.append((seqs, target)) + + monkeypatch.setattr(pg_publish, "held_documents", held) + monkeypatch.setattr(pg_memory, "complete", complete) + await lifecycle._settle_documents(SimpleNamespace(rows=[row], seqs=[seq]), [True], ch=object()) + assert completed == [([seq] if present else [], pg_memory.SEARCH_DOC)] + + @pytest.mark.asyncio async def test_a_revision_clickhouse_keeps_refusing_holds_back_only_itself( monkeypatch: pytest.MonkeyPatch, @@ -1376,10 +1488,11 @@ async def test_a_revision_clickhouse_keeps_refusing_holds_back_only_itself( published: list[tuple[list[int], bool]] = [] failed: list[int] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 2 for c in claims) and not stub: raise RuntimeError("refused") published.append(([c.seq for c in claims], stub)) + return len(claims) async def _record_failure( claims: list[pg_memory.Claimed], error: BaseException, *, park: bool = False @@ -1406,10 +1519,11 @@ async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: assert published == [([2], True)] # A stub refused too parks that claim alone; the rest of the pass is published. - async def _refuse_2(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _refuse_2(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 2 for c in claims): raise RuntimeError("refused") published.append(([c.seq for c in claims], stub)) + return len(claims) monkeypatch.setattr(pg_delivery, "_publish", _refuse_2) published.clear() @@ -1459,10 +1573,14 @@ def test_the_authoritys_erasure_step_leads_the_erasing_steps_and_runs_only_under from server.settings import settings monkeypatch.setattr(settings, "storage_authority", "postgres") - assert authority_steps(USER_STEPS)[0][0] == "pg_memory.delete" + assert [name for name, _ in authority_steps(USER_STEPS)][:2] == [ + "memory.refusals", + "pg_memory.delete", + ] # A namespace's sources stop first, so no sync turn writes into what the steps erase. names = [name for name, _ in authority_steps(NAMESPACE_STEPS)] - assert names[:4] == [ + assert names[:5] == [ + "memory.refusals", "knowledge_sources.retire", "knowledge_source_push.delete", "pg_memory.delete", @@ -1859,10 +1977,11 @@ async def test_a_refused_statement_is_the_claims_attempt_and_the_rest_still_publ published: list[int] = [] refused = pg.PostgresRefusedError(asyncpg.exceptions.CheckViolationError("check")) - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 1 for c in claims): raise refused published.extend(c.seq for c in claims) + return len(claims) async def _record( claims: list[pg_memory.Claimed], error: BaseException, *, park: bool = False @@ -1925,8 +2044,9 @@ async def test_a_namespace_whose_claims_are_all_gone_leaves_the_rest_of_the_pass published: list[str] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: published.extend(c.namespace for c in claims) + return len(claims) async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: return [c for c in claims if c.namespace != "erased"] @@ -1952,11 +2072,12 @@ async def test_a_last_attempt_its_lease_ran_out_on_publishes_its_stub_under_a_ne stubs: list[pg_memory.Claimed] = [] renewals: list[int] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: pg_lease.check_lease(claims[0].deadline) if not stub: raise pg_lease.PublicationPending() # the embedder outlasted the lease stubs.extend(claims) + return len(claims) async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: renewals.append(len(claims)) @@ -2057,10 +2178,11 @@ async def _held(*_: object, **__: object) -> set[object]: return set() async def _docs(claims: list[object], *, stub: bool = False) -> list[dict[str, object]]: - return [{"memory_key": "k"}] + return [{"org": "o", "namespace": "n", "memory_key": "k"}] - async def _insert(docs: list[object], ch: object = None) -> None: + async def _insert(docs: list[object], ch: object = None) -> list[object]: steps.append("documents") + return docs async def _complete(seqs: object, done: int, *, tokens: object = 1) -> None: steps.append(f"complete {done}") @@ -3001,9 +3123,13 @@ async def execute(self, sql: str, *args: object) -> None: return None async def fetch(self, sql: str, *args: Any) -> list[dict[str, Any]]: + if "loom.memory_refusals" in sql: + return [] return [{"org": "o", "scope": "n", "generation": 5}] async def fetchval(self, sql: str, *args: Any) -> bool: + if "loom.memory_refusals" in sql: + return False asked.append(list(args[2])) return uuid.UUID(erased) in args[2] @@ -3136,9 +3262,10 @@ async def _renew(batch: list[Any]) -> list[Any]: # New objects, as the store's `replace` returns. return [SimpleNamespace(**vars(c)) for c in batch] - async def _publish(batch: list[Any], **_: Any) -> None: + async def _publish(batch: list[Any], **_: Any) -> int: seen["published"].append([c.seq for c in batch]) await publish(batch) + return len(batch) async def _release(batch: list[Any]) -> None: seen["released"].extend(c.seq for c in batch) @@ -5821,6 +5948,8 @@ async def execute(self, sql: str, *args: object) -> None: return None async def fetch(self, sql: str, *args: Any) -> list[dict[str, Any]]: + if "loom.memory_refusals" in sql: + return [] if "loom.memory_current c" in sql: return [{**row, "row": pg_memory.encode_row(head)}] return [{"org": "o", "scope": "n", "generation": 5}] diff --git a/tests/unit/test_pg_batch.py b/tests/unit/test_pg_batch.py index bb7d1344bf..9f2bf87581 100644 --- a/tests/unit/test_pg_batch.py +++ b/tests/unit/test_pg_batch.py @@ -56,6 +56,7 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch, *, interval_ms: int = 0) -> self.release = asyncio.Event() self.alone: BaseException | None = None self.refuse_documents: BaseException | None = None + self.withhold_documents: set[str] = set() self.refuse_retirement = False # Each retirement's keys; `retire_alone` is what a retirement of one commit raises. self.retired: list[list[str]] = [] @@ -91,11 +92,13 @@ async def _memory(rows: Any, *, ch: Any, published: bool) -> int: self.memory.append(keys) return len(rows) - async def _documents(rows: Any, *, ch: Any) -> None: + async def _documents(rows: Any, *, ch: Any) -> list[dict[str, Any]]: if self.refuse_documents is not None: raise self.refuse_documents + rows = [r for r in rows if r["memory_key"] not in self.withhold_documents] self.documents.append([r["memory_key"] for r in rows]) self.document_rows += rows + return rows async def _audits(rows: Any, *, ch: Any, retried: bool) -> None: self.audits.append(len(rows)) @@ -387,6 +390,47 @@ async def test_a_document_whose_key_moved_on_is_not_sent_and_stays_owed( assert batch.handed_over == [("o", "n", 1)] +async def test_a_document_withheld_after_the_heads_read_stays_owed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + batch = _Batching(monkeypatch) + batch.withhold_documents = {"k1"} + await asyncio.gather(batch.submit("k1", 1), batch.submit("k2", 2)) + assert batch.documents == [["k2"]] + assert sorted(batch.landed) == [ + (("o", "n", 1), MEMORY_ROW | AUDIT_ROW), + (("o", "n", 2), _FULL), + ] + assert batch.handed_over == [("o", "n", 1)] + + +@pytest.mark.parametrize("field", ["org", "namespace", "agent", "user_id", "session_id"]) +async def test_a_receipt_cannot_settle_another_identity(monkeypatch, field) -> None: + from server.memory.store.rows import identity_key + + batch = _Batching(monkeypatch) + first, other = batch.committed("k", 1), batch.committed("k", 2) + other.rows[0][field] = "other" + other.seqs[0] = (other.rows[0]["org"], other.rows[0]["namespace"], 2) + rows = [first.rows[0], other.rows[0]] + + async def heads(keys, **kw): + return {identity_key(row): (row, 0) for row in rows} + + async def insert(documents, **kw): + assert len(documents) == 2 + return documents[:1] + + monkeypatch.setattr(pg_memory, "heads_owing", heads) + monkeypatch.setattr("server.memory.store.inserts.insert_embedding_rows", insert) + await asyncio.gather( + pg_batch.submit(first, documents=rows[:1], ch=batch.ch), + pg_batch.submit(other, documents=rows[1:], ch=batch.ch), + ) + assert batch.landed == [(first.seqs[0], _FULL), (other.seqs[0], MEMORY_ROW | AUDIT_ROW)] + assert batch.handed_over == other.seqs + + @pytest.mark.parametrize("head_owes", [0, SEARCH_DOC]) async def test_a_document_whose_row_was_copied_is_sent_onto_the_copy( monkeypatch: pytest.MonkeyPatch, head_owes: int diff --git a/tests/unit/test_pg_durable_reads.py b/tests/unit/test_pg_durable_reads.py index 750e1ba6c6..2a7c7a6c70 100644 --- a/tests/unit/test_pg_durable_reads.py +++ b/tests/unit/test_pg_durable_reads.py @@ -112,8 +112,11 @@ "server/memory/store/records.py": "a retirement reads its writers durably; " "`live_writer_record` filters a request's read, which commits through `memory.set`", "server/memory/store/redaction.py": "decides a redaction, which commits", - "server/memory/store/source_review.py": "holds the exact source after a durable read " - "through shared publication", + "server/memory/store/refusals.py": "reads durable sources before publication, then " + "rechecks their heads on the admission transaction's connection", + "server/memory/store/shared.py": "qualifies a durable source before its fenced promotion", + "server/memory/store/summary_sources.py": "reads durable source proofs before publication, " + "then rechecks them under the same transaction's admission locks", "server/memory/workers/pg_payload_clear.py": "clears through its own commit, and only " "revisions delivered a horizon ago that ClickHouse holds", } diff --git a/tests/unit/test_pipeline_actor_sensitivity_gauges.py b/tests/unit/test_pipeline_actor_sensitivity_gauges.py index 8dd948450e..477ecd16c2 100644 --- a/tests/unit/test_pipeline_actor_sensitivity_gauges.py +++ b/tests/unit/test_pipeline_actor_sensitivity_gauges.py @@ -241,6 +241,8 @@ def __init__(self, rows: list[dict[str, Any]] | None = None) -> None: async def query_all(self, sql, params=None): self.calls.append((sql, params or {})) + if "FROM loom.memory_refusals" in sql: + return [] return list(self.rows) async def query_one(self, sql, params=None): @@ -341,6 +343,7 @@ async def _live(ch, **ident): async def _ins(rows, *, ch=None): inserted.append(rows) + return rows monkeypatch.setattr(memory_repo, "retire_vectors_for_keys", _retire) monkeypatch.setattr(memory_repo, "live_embedding_rows_for_key", _live) @@ -1292,6 +1295,7 @@ async def test_integrity_counts_and_the_dim0_repair(monkeypatch) -> None: async def _ins(rows, *, ch=None): inserted.append(rows) + return rows monkeypatch.setattr(memory_repo, "insert_embedding_rows", _ins) rows = [ @@ -1490,7 +1494,7 @@ async def _live(ch, **ident): return [{**ident, "tags": ["t"], "embedding": [0.1], "embedding_dim": 1, "version": 3}] async def _ins(rows, *, ch=None): - return None + return rows async def _check(rows, *, ch=None): checked.append(rows) diff --git a/tests/unit/test_profile_leg.py b/tests/unit/test_profile_leg.py index f8b6b1caa8..9c9a62eb60 100644 --- a/tests/unit/test_profile_leg.py +++ b/tests/unit/test_profile_leg.py @@ -230,8 +230,9 @@ def test_the_stub_embeds_label_and_value_where_the_old_vector_lives() -> None: class _Repo: pass - async def _insert(rows: list, ch: Any = None) -> None: + async def _insert(rows: list, ch: Any = None) -> list: written.extend(rows) + return rows import server.memory.store as mr diff --git a/tests/unit/test_promote_definition.py b/tests/unit/test_promote_definition.py index 3b91481d4d..ec25be3a6e 100644 --- a/tests/unit/test_promote_definition.py +++ b/tests/unit/test_promote_definition.py @@ -21,6 +21,7 @@ from server.memory.app import promote from server.memory.domain.definitions import definition_names, fold_name from server.memory.store import MemoryRow, shared +from server.memory.store.rows import identity_key from server.settings import settings from tests.unit.clickhouse_local import CLICKHOUSE, LocalCH, ddl @@ -50,7 +51,7 @@ def _row(**over: Any) -> MemoryRow: @pytest.fixture -def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: +def store(monkeypatch: pytest.MonkeyPatch, no_blacklisted_values: None) -> dict[str, Any]: """The source row, the live shared definitions (key -> value) and what was written, with the claim: `free` (False: another holds it; None: the lease store does not answer), `claims` (the names taken) and `held` (inside it); `heads` and @@ -69,6 +70,10 @@ def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: async def get_by_key(*_a: Any, **_k: Any) -> MemoryRow: return state["row"] + async def sources(*args, **kwargs): + row = state["row"] + return {identity_key(row): row} + async def definitions(org: str, namespace: str, limit: int, **_k: Any) -> list[dict]: # The two fields the store extracts, as its statement reads them. assert state["held"], "the check runs under the namespace's claim" @@ -99,6 +104,7 @@ async def holding(name: str, **_k: Any) -> AsyncIterator[bool]: state["held"] = False monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", sources) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(promote, "writer_actor", lambda fallback="": "user:admin") monkeypatch.setattr(promote, "naive_utc_now", lambda: NOW) @@ -176,6 +182,20 @@ def test_a_definitions_names(value: Any, names: set[str] | None) -> None: assert definition_names(value) == names +async def test_definition_claim_precedes_the_source_connection(store, monkeypatch): + """A demote holds the claim while borrowing a connection; promotion uses that order.""" + original = promote.refusals.source_publication + + @asynccontextmanager + async def admission(*args, **kwargs): + assert store["held"], "the definition claim must precede the source connection" + async with original(*args, **kwargs) as mark: + yield mark + + monkeypatch.setattr(promote.refusals, "source_publication", admission) + await promote.handle_memory_promote(_input()) + + async def test_an_approved_definition_is_promoted_by_the_tokens_actor( store: dict[str, Any], ) -> None: diff --git a/tests/unit/test_promote_refuses_high.py b/tests/unit/test_promote_refuses_high.py index 0e87de20fa..37e609b11e 100644 --- a/tests/unit/test_promote_refuses_high.py +++ b/tests/unit/test_promote_refuses_high.py @@ -73,8 +73,10 @@ async def get(*args, **kwargs): ) +@pytest.mark.usefixtures("no_blacklisted_values") async def test_promotion_rechecks_the_source_after_preparing_its_projection(monkeypatch): from server.memory.store.pg_memory import AdmissionContendedError + from server.memory.store.rows import identity_key source = _row("") changed = source.model_copy( @@ -89,7 +91,11 @@ async def get(*args, **kwargs): async def write(*args, **kwargs): writes.append(args) + async def heads(*args, **kwargs): + return {identity_key(changed): changed} + monkeypatch.setattr(memory_repo, "get_by_key", get) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(shared, "insert_shared_row", write) with pytest.raises(AdmissionContendedError): diff --git a/tests/unit/test_promote_source_identity.py b/tests/unit/test_promote_source_identity.py index db5dc8a790..f8f40c7e7b 100644 --- a/tests/unit/test_promote_source_identity.py +++ b/tests/unit/test_promote_source_identity.py @@ -15,6 +15,7 @@ @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_the_copy_names_the_row_it_was_read_from(monkeypatch: pytest.MonkeyPatch) -> None: """A lookup by `agent="*"` reads one row; the copy names that row's identity, by which an erasure finds it, not the wildcard.""" @@ -41,7 +42,14 @@ async def _get(*_a: Any, **_k: Any) -> MemoryRow: async def _write(row: dict[str, Any]) -> None: rows.append(row) + async def sources(*args, **kwargs): + from server.memory.store.rows import identity_key + + row = await _get() + return {identity_key(row): row} + monkeypatch.setattr(memory_repo, "get_by_key", _get) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", sources) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(shared, "insert_shared_row", _write) await promote.handle_memory_promote( diff --git a/tests/unit/test_promotion_source.py b/tests/unit/test_promotion_source.py index 9371c3ffdf..681d4eaafd 100644 --- a/tests/unit/test_promotion_source.py +++ b/tests/unit/test_promotion_source.py @@ -30,3 +30,61 @@ def test_an_identity_part_holding_a_slash_is_read_into_the_key() -> None: parsed = parse_promotion_source("o/n/-/-/run/7/definition/arr") assert parsed is not None assert (parsed["session_id"], parsed["memory_key"]) == ("run", "7/definition/arr") + + +@pytest.mark.parametrize("field", ["agent", "user_id", "session_id", "memory_key"]) +def test_consent_covers_every_ambiguous_user_interpretation(field): + from server.memory.domain.promotion_source import promotion_users + + identity = { + "org": "o/div", + "namespace": "n/team", + "agent": "bot", + "user_id": "alice", + "session_id": "session", + "memory_key": "definition/arr", + } + identity[field] += "/segment" + users = promotion_users([promotion_source(**identity)], identity["org"], identity["namespace"]) + assert users is not None and identity["user_id"] in users + if field == "user_id": + assert "alice" in users and "alice/segment" in users + + +@pytest.mark.parametrize( + "entries", [[], ["wrong/prefix/a/alice/s/k"], ["o/n/a/alice/s/"], ["o/n/" + "x/" * 1000 + "k"]] +) +def test_incomplete_source_denial_proof_cannot_authorize_plaintext(entries): + from server.memory.domain.promotion_source import promotion_users + + assert promotion_users(entries, "o", "n") is None + + +@pytest.mark.parametrize( + "field", ["org", "namespace", "agent", "user_id", "session_id", "memory_key"] +) +def test_positive_proof_finds_the_actual_identity_despite_slashes(field): + from server.memory.domain.promotion_source import promotion_keys + + identity = dict( + org="o", + namespace="n", + agent="bot", + user_id="alice", + session_id="s", + memory_key="definition/arr", + ) + identity[field] += "/team" + keys = promotion_keys([promotion_source(**identity)], identity["org"], identity["namespace"]) + assert keys is not None and tuple(identity.values()) in keys + + +@pytest.mark.parametrize("key", ["path/", "/", "a//", "//", "a//b/"]) +@pytest.mark.parametrize("user", ["", "alice/team", "alice"]) +def test_complete_source_key_may_end_in_a_slash(key, user): + from server.memory.domain.promotion_source import promotion_keys, promotion_users + + identity = ("o", "n", "agent", user, "session", key) + entry = promotion_source(*identity) + assert user in promotion_users([entry], "o", "n") + assert identity in promotion_keys([entry], "o", "n") diff --git a/tests/unit/test_proposal_commit_review.py b/tests/unit/test_proposal_commit_review.py index a267318e0e..26a3148b33 100644 --- a/tests/unit/test_proposal_commit_review.py +++ b/tests/unit/test_proposal_commit_review.py @@ -163,6 +163,8 @@ async def test_native_commit_checks_decoded_review_state_before_publishing(monke class Conn: async def fetch(self, sql, *args): + if "loom.memory_refusals" in sql: + return [] if "loom.memory_current c" in sql: return [{**head, "row": head}] return [{"org": source.org, "scope": source.namespace, "generation": 5}] diff --git a/tests/unit/test_proposals.py b/tests/unit/test_proposals.py index 2f23f40175..5235d09979 100644 --- a/tests/unit/test_proposals.py +++ b/tests/unit/test_proposals.py @@ -82,6 +82,9 @@ def _proposal_row(key: str, status: str, conflict_with: list[str] | None = None) class _CH: + async def query_all(self, *args, **kwargs): + return [] + def __init__(self) -> None: self.inserts: list[list[dict[str, Any]]] = [] diff --git a/tests/unit/test_read_scope_admission.py b/tests/unit/test_read_scope_admission.py index 61bee06cf1..fec2803bc0 100644 --- a/tests/unit/test_read_scope_admission.py +++ b/tests/unit/test_read_scope_admission.py @@ -614,17 +614,6 @@ async def test_deep_evidence_opens_only_the_scopes_links_turns_and_sessions() -> # The links come only from the memory rows the scope reaches. assert "memory_id IN (" in ch.calls[0][0] _bound_to_u1(ch) - ch = _CH() - await deep_evidence.fetch_session_turns( - org="o", - namespace="n", - session_ids=["sess-1"], - question="q", - max_chars_per_session=99, - max_chars_total=99, - ch=ch, - ) - _bound_to_u1(ch) def _peer(mid: str, uid: str, *, pinned: bool = False) -> Any: diff --git a/tests/unit/test_read_your_write_inserts.py b/tests/unit/test_read_your_write_inserts.py index ec14afffcf..2d33987614 100644 --- a/tests/unit/test_read_your_write_inserts.py +++ b/tests/unit/test_read_your_write_inserts.py @@ -147,7 +147,7 @@ def test_default_read_path_tables_are_written_synchronously(rel_path: str, table ("server/observability/audit.py", "loom.audit_log", "switched"), ("server/observability/events.py", "loom.agent_events", "switched"), ("server/memory/store/webhooks.py", "loom.memory_webhook_outbox", "switched"), - ("server/memory/app/evidence.py", "loom.memory_evidence", "switched"), + ("server/memory/store/refusals.py", "loom.memory_evidence", "switched"), ("server/memory/app/evidence.py", "loom.memory_belief_sources", "switched"), ("server/core/rmt_carry.py", "loom.carry_acks", "request"), ("server/auth/attempts.py", "loom.auth_attempts", "request"), diff --git a/tests/unit/test_reembed_in_database.py b/tests/unit/test_reembed_in_database.py index 6c55cc8393..2e196b3f00 100644 --- a/tests/unit/test_reembed_in_database.py +++ b/tests/unit/test_reembed_in_database.py @@ -132,21 +132,72 @@ async def _one(sql, params=None): return {"n": 7} async def _all(sql, params=None): - return [{"name": n} for n in COLS] + return [ + {"name": n} + for n in ("org", "entity_text", "entity_embedding", "embedding_model", "embedding_dim") + ] monkeypatch.setattr(c, "query_one", _one) monkeypatch.setattr(c, "query_all", _all) monkeypatch.setattr(ch_mod, "guard_pending_schema", lambda *a, **k: _noop()) n = await rb.reembed_table_in_database( - c, "memory_embeddings", model="m2", dim=4, credentials="coll", log=lambda *_: None + c, "entity_vectors", model="m2", dim=4, credentials="coll", log=lambda *_: None ) assert n == 7 and len(sent) == 1 got = sent[0] - assert got["sql"].startswith("INSERT INTO loom.memory_embeddings") + assert got["sql"].startswith("INSERT INTO loom.entity_vectors") assert got["settings"]["allow_experimental_ai_functions"] == 1 assert got["settings"]["ai_function_max_api_calls_per_query"] == 1000 +async def test_database_memory_vectors_publish_through_admission_and_count_only_acks(monkeypatch): + import server.memory.store as store + + rows = [ + {"org": org, "memory_key": "fact", "embedding_model": "m2", "_k": (org,) * 8} + for org in ("kept", "refused") + ] + queries = [] + + class CH: + async def query_one(self, sql, params=None): + return {"n": 2} + + async def query_all(self, sql, params=None, *, settings=None): + if "system.columns" in sql: + return [{"name": name} for name in COLS] + queries.append(sql) + assert settings["allow_experimental_ai_functions"] == 1 + assert params["page"] == 200 + assert "AS embedding_model" in sql.split(" FROM (", 1)[0] + assert "embedding_model != 'm2'" in sql.split(" FROM (", 1)[1] + assert "LIMIT {page:UInt32}" in sql + return rows + + async def execute(self, *args, **kwargs): + pytest.fail("A memory vector bypassed protected publication") + + published = [] + + async def admit(out, *, ch): + published.extend(out) + return out[:1] + + monkeypatch.setattr(store, "insert_embedding_rows", admit) + held = set() + written = await rb.reembed_table_in_database( + CH(), + "memory_embeddings", + model="m2", + dim=4, + credentials="coll", + left_orgs=held, + log=lambda *_: None, + ) + assert written == 1 and held == {"refused"} + assert len(queries) == 1 and all("_k" not in row for row in published) + + async def _noop() -> None: return None diff --git a/tests/unit/test_reembed_paging.py b/tests/unit/test_reembed_paging.py index 75b09d8fe4..992212f8b0 100644 --- a/tests/unit/test_reembed_paging.py +++ b/tests/unit/test_reembed_paging.py @@ -10,7 +10,9 @@ import pytest from server.core.ch import sequential_reads_active +from server.core.hashing import sha256_value from server.memory.app import reembed as rb +from server.memory.store.rows import MemoryRow, build_memory_version_row from tests.unit._fake_credentials import OPENAI_KEY from tests.unit.clickhouse_local import CLICKHOUSE, EMBEDDINGS, VECTOR, run @@ -22,6 +24,19 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: return [[0.1, 0.2] for _ in texts] +def _canonical(row): + return build_memory_version_row( + MemoryRow.from_snapshot( + { + **row, + "memory_id": "11111111-1111-1111-1111-111111111111", + "memory_type": "fact", + "value": row["content_excerpt"], + } + ) + ) + + def key(r: dict[str, Any]) -> tuple[str, ...]: return (r["org"], r["namespace"], r["embedding_model"], r["thread_id"], r["msg_id"]) @@ -137,6 +152,7 @@ async def test_a_memory_row_is_re_embedded_whatever_its_text() -> None: "embedding_model": "m1", "memory_key": "fixture_key", "content_excerpt": excerpt, + "content_sha256": sha256_value(excerpt), "tags": ["sensitivity:low"], "embedding": [0.0], "embedding_dim": 1, @@ -147,6 +163,10 @@ async def test_a_memory_row_is_re_embedded_whatever_its_text() -> None: class _MemCH: async def query_all(self, sql: str, params: Any = None) -> list[dict]: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [_canonical(row)] if "system.columns" in sql: return [{"name": c} for c in row] if "keys" in (params or {}): # by the key the row was read with, not the new model's @@ -324,6 +344,7 @@ def row(k: str) -> dict[str, Any]: "embedding_model": "m2", "memory_key": "k", "content_excerpt": "x", + "content_sha256": sha256_value("x"), "embedding": [0.0, 0.0], "embedding_dim": 2, "memory_key_embedding": [0.0, 0.0], @@ -340,6 +361,10 @@ def __init__(self, stored_now: dict[str, Any]) -> None: self.written: list[dict[str, Any]] = [] async def query_all(self, sql: str, params: Any = None) -> list[dict]: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [_canonical(self.stored_now)] if "system.columns" in sql: return [{"name": c} for c in _MEMORY_ROW] if "keys" in (params or {}): @@ -407,3 +432,18 @@ async def test_a_key_only_rewrite_keeps_the_value_vector_stored_now() -> None: n = await rb.reembed_table(ch, _Emb(), "memory_embeddings", key_only=True, log=lambda *_: None) assert n == 1 and ch.written[0]["embedding"] == [0.5, 0.5] and ch.written[0]["version"] == 8 assert ch.written[0]["memory_key_embedding"] == [0.1, 0.2] + + +async def test_a_withheld_publication_is_not_counted_or_deleted_as_replaced(monkeypatch): + from server.memory import store + + async def hold(rows, **kwargs): + return [] + + monkeypatch.setattr(store, "insert_embedding_rows", hold) + ch = _KeyOnlyCH(dict(_MEMORY_ROW)) + held = set() + count = await rb.reembed_table( + ch, _Emb(), "memory_embeddings", key_only=True, left_orgs=held, log=lambda *_: None + ) + assert count == 0 and held == {"o"} and ch.written == [] diff --git a/tests/unit/test_restore_mirrors_all_types.py b/tests/unit/test_restore_mirrors_all_types.py index ffa21f44b9..63cc57e51a 100644 --- a/tests/unit/test_restore_mirrors_all_types.py +++ b/tests/unit/test_restore_mirrors_all_types.py @@ -18,6 +18,7 @@ import pytest from server.core import leases +from server.core.hashing import sha256_value from server.memory import store as memory_repo from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import handlers as mem_handlers @@ -67,6 +68,7 @@ def _row(memory_type: str, **over: Any) -> MemoryRow: "memory_key": "alert:disk-full:host-7", "memory_type": memory_type, "value": {"state": "firing"}, + "content_sha256": sha256_value({"state": "firing"}), "value_text": '{"state": "firing"}', "confidence": 1.0, "version": 100, @@ -101,6 +103,7 @@ def _row(memory_type: str, **over: Any) -> MemoryRow: class _CH: def __init__(self, emb_rows: list[dict[str, Any]], *, fail: tuple[str, ...] = ()) -> None: + self.heads = {} self.emb_rows = emb_rows self.fail = fail self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] @@ -112,6 +115,8 @@ async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.sqls.append(sql) + if params and "keys" in params and "FROM loom.memory FINAL" in sql: + return [self.heads[k] for k in params["keys"] if k in self.heads] if "FROM loom.memory_embeddings" in sql: return [dict(r) for r in self.emb_rows] return [] @@ -120,6 +125,8 @@ async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = N if table in self.fail: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + self.heads.update({memory_repo.identity_key(r): dict(r) for r in rows}) async def execute(self, *_: Any, **__: Any) -> None: return None @@ -151,6 +158,9 @@ def _query_one_row(existing: MemoryRow, **over: Any) -> dict[str, Any]: def _wire(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow, ch: _CH) -> None: + ch.heads[memory_repo.identity_key(existing)] = memory_repo.build_memory_version_row(existing) + for row in ch.emb_rows: + row.setdefault("content_sha256", existing.content_sha256) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_schema_baseline.py b/tests/unit/test_schema_baseline.py index 76fbf08655..f0aff2c8dd 100644 --- a/tests/unit/test_schema_baseline.py +++ b/tests/unit/test_schema_baseline.py @@ -170,6 +170,7 @@ def test_the_journal_files() -> None: "112_code_repo_git_proxy_identity.sql", "113_oauth_verified_email.sql", "114_memory_proposal_review.sql", + "115_memory_refusals.sql", ] diff --git a/tests/unit/test_session_digest_prompt.py b/tests/unit/test_session_digest_prompt.py index b3640c69ca..a8b8b6758b 100644 --- a/tests/unit/test_session_digest_prompt.py +++ b/tests/unit/test_session_digest_prompt.py @@ -1,76 +1,53 @@ -"""The session digest's prompt and output budget, and the transcript cap -behind `memory_session_digest_max_input_chars`.""" +"""Digest text is extractive, bounded and independent of model-generated prose.""" from __future__ import annotations -from types import SimpleNamespace +from datetime import datetime from typing import Any import pytest -from server.memory.app import session_summary as ss +from server.memory.domain.summary_provenance import render_summary, source_text +from tests.unit import test_session_summaries +from tests.unit.test_memory_verify import _row +writer = test_session_summaries.writer -class _LLM: - def __init__(self) -> None: - self.calls: list[tuple[str, str, int]] = [] - async def complete(self, system: str, user: str, max_tokens: int = 0, **kw: Any): - self.calls.append((system, user, max_tokens)) - return SimpleNamespace( - text="Events:\n- on 2024-03-11 the user ran a 5k", tokens_in=10, tokens_out=5 - ) +def test_renderer_preserves_source_text_without_inventing_facts() -> None: + facts = ["Lives in Lisbon.", "Deployments use Tuesday and Thursday."] + assert render_summary(facts) == "Lives in Lisbon.\nDeployments use Tuesday and Thursday." + assert render_summary([]) == "" -class _Emb: - model_name = "fake-embed" - dim = 2 - - async def embed(self, text: str) -> list[float]: - return [0.1, 0.2] - - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] +def test_structured_source_values_have_a_stable_rendering() -> None: + first = _row(value={"city": "Lisbon", "age": 30}) + second = _row(value={"age": 30, "city": "Lisbon"}) + now = datetime.now() + assert ( + source_text(first, now=now) + == source_text(second, now=now) + == '{"age": 30, "city": "Lisbon"}' + ) -@pytest.fixture -def env(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: - import server.llm.client as llm_client - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.settings import settings +async def test_digest_generation_never_calls_a_model(writer: Any, monkeypatch: Any) -> None: + from server.memory.app.session_summary import _write_session_summary - llm = _LLM() - monkeypatch.setattr(llm_client, "get_llm_client", lambda: llm) - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: _Emb()) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: _Emb()) - monkeypatch.setattr(settings, "memory_session_digest_max_input_chars", 3000) - return SimpleNamespace(llm=llm, settings=settings, monkeypatch=monkeypatch) + def unexpected() -> Any: + raise AssertionError("a model could introduce text without a source") + monkeypatch.setattr("server.llm.client.get_llm_client", unexpected) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_awaited_once() -def _msgs(n: int = 3, size: int = 100) -> list[dict[str, str]]: - return [{"role": "user", "content": ("x" * size)} for _ in range(n)] +@pytest.mark.parametrize("basis", ["created_at", "temporal_anchor", "valid_at"]) +def test_a_source_cannot_lend_text_before_its_valid_time_lower_bound(basis): + from datetime import timedelta -@pytest.mark.asyncio -async def test_the_digest_is_dated_prose_under_its_budget(env) -> None: - draft = await ss._draft_session_summary( - SimpleNamespace(session_id="s"), _msgs(), "2024-03-11", operator_policy="" - ) - system, user, max_tokens = env.llm.calls[0] - assert system == ss._SESSION_SUMMARY_SYSTEM and max_tokens == 160 - assert "Session date: 2024-03-11" in user - assert draft is not None and draft.embedding_model == "fake-embed" - - -def test_transcript_cap_follows_the_setting(env) -> None: - long = _msgs(n=10, size=1000) # ~10k chars - p = ss._build_session_summary_user_prompt(long, "2024-01-01") - assert len(p) < 3000 + 200 - env.monkeypatch.setattr(env.settings, "memory_session_digest_max_input_chars", 12000) - p2 = ss._build_session_summary_user_prompt(long, "2024-01-01") - assert len(p2) > 9000 - env.monkeypatch.setattr(env.settings, "memory_session_digest_max_input_chars", 0) - assert len(ss._build_session_summary_user_prompt(long, "2024-01-01")) < 3000 + 200, ( - "0 = default" - ) + now = datetime(2026, 1, 2) + source = _row(value="a later fact", created_at=now) + if basis != "created_at": + source = source.model_copy(update={basis: now, "created_at": now - timedelta(days=2)}) + assert source_text(source, now=now, as_of=now - timedelta(days=1)) is None diff --git a/tests/unit/test_session_outcomes.py b/tests/unit/test_session_outcomes.py index c9067f85c0..6fd1e74c0c 100644 --- a/tests/unit/test_session_outcomes.py +++ b/tests/unit/test_session_outcomes.py @@ -16,6 +16,16 @@ from server.memory.store.session_outcomes import OUTCOMES_TABLE +@pytest.fixture +def qualified_sessions(monkeypatch): + """Outcome policy cases start after source proof, tested separately below.""" + + async def qualified(ch, rows): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.verified_summary_batch", qualified) + + class FakeCH: def __init__(self) -> None: self.all_returns: list[list[dict[str, Any]]] = [] @@ -140,6 +150,7 @@ def test_build_outcome_row_validates_and_stamps_a_version() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_label_sessions_judges_unlabelled_summaries_and_isolates_failures() -> None: ch = FakeCH() ch.all_returns.append( @@ -237,6 +248,7 @@ def _neg(sid="bad", conf=0.9): @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_distil_pairs_a_failed_session_with_its_nearest_success_and_writes_one_reflection() -> ( None ): @@ -295,9 +307,10 @@ async def writer(**kw): assert "s.user_id = {uid:String}" in sql and params["uid"] == "u" assert "o.agent = s.agent" in sql and "o.user_id = s.user_id" in sql assert params["q"] == [0.1, 0.2, 0.3] - # the contrast prompt carried both digests and the judged reason + # The contrast prompt carries proven digests; an unproven judged reason stays out. system, user = llm.calls[0] - assert "Never ran the tests." in user and "ran the suite" in user and "declared done" in user + assert "Never ran the tests." not in user + assert "ran the suite" in user and "declared done" in user w = written[0] assert (w["neg_session"], w["pos_session"]) == ("bad", "good") assert w["lessons"] == ["Run the test suite before declaring a parser change done."] @@ -307,6 +320,7 @@ async def writer(**kw): @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_distil_skips_when_no_neighbour_is_close_enough_or_nothing_was_learned() -> None: ch = FakeCH() ch.all_returns.append([_neg("a"), _neg("b"), _neg("c")]) @@ -497,3 +511,19 @@ async def _scoped(table, *, org, scope_where, params, ch, lightweight=False): dict(saga_steps.USER_STEPS)["session_outcomes.delete"] is memory_steps.delete_session_outcomes_by_user ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("contrast", [False, True]) +async def test_an_unproven_session_never_reaches_judgement_or_contrast(contrast): + ch = FakeCH() + ch.all_returns.append([_neg()]) + llm = FakeLLM(["{}"]) + result = so.DistillResult() + if contrast: + await so.distil_contrasts(ch, llm, so.DistillConfig(), result) + assert result.contrast_candidates == 0 + else: + await so.label_sessions(ch, llm, so.DistillConfig(), result) + assert result.sessions_scanned == 0 + assert llm.calls == [] and ch.inserted == [] diff --git a/tests/unit/test_session_summaries.py b/tests/unit/test_session_summaries.py index a192155144..5df49df8d6 100644 --- a/tests/unit/test_session_summaries.py +++ b/tests/unit/test_session_summaries.py @@ -12,9 +12,10 @@ class (a write omits a column -> FINAL surfaces the schema DEFAULT over a from __future__ import annotations -from dataclasses import dataclass, field from datetime import datetime +from types import SimpleNamespace from typing import Any +from unittest.mock import ANY, AsyncMock import pytest @@ -81,264 +82,156 @@ def test_defaults_match_schema_for_omitted_columns() -> None: assert row["user_id"] == "-" -# ---------------------------------------------------------------------- -# Pure helpers — _top_key_facts + _build_session_summary_user_prompt -# ---------------------------------------------------------------------- - - -def test_top_key_facts_never_carry_a_credential(monkeypatch: pytest.MonkeyPatch) -> None: - # The digest is listed by memory.digests and sent to the outcome judge; a fact whose - # text reads as high stays out of its key facts, whatever the autodetect setting. - from server.memory.app.extract.handler import _top_key_facts - from server.settings import settings - - monkeypatch.setattr(settings, "memory_sensitivity_autodetect", False) - token = "ghp_" + "a" * 36 - facts = [ - {"value": f"the deploy token is {token}"}, - {"value": {"note": "deploy token:\n" + token}}, # JSON-escaped, the newline hides it - {"value": "user lives in Berlin"}, - ] - assert _top_key_facts(facts) == ["user lives in Berlin"] - - -def test_the_judge_prompt_drops_a_stored_credential_key_fact() -> None: - # Digests an earlier build wrote may hold one; the outcome judge's prompt drops it. - from server.memory.app.session_outcomes import judge_prompt - - token = "ghp_" + "a" * 36 - _system, user = judge_prompt("a session", [f"the token is {token}", "ran the tests"]) - assert token not in user and "ran the tests" in user - - -def test_top_key_facts_dedups_and_caps() -> None: - from server.memory.app.extract.handler import ( - _SESSION_SUMMARY_MAX_KEY_FACTS, - _top_key_facts, +@pytest.fixture +def writer(monkeypatch: Any) -> Any: + from server.memory import store + from server.memory.app import digest_chain, session_summary + from tests.unit.test_memory_verify import _row + + source = _row( + namespace="n", + agent="a", + user_id="u", + session_id="s", + memory_key="known", + value="current fact", ) - - facts = [ - {"value": "user lives in Berlin"}, - {"value": "user lives in Berlin"}, # dup -> dropped - {"value": {"city": "Berlin"}}, # dict -> json string surface - {"value": ""}, # empty -> dropped - {"value": "fact a"}, - {"value": "fact b"}, - {"value": "fact c"}, - {"value": "fact d"}, - {"value": "fact e"}, - ] - out = _top_key_facts(facts) - assert len(out) <= _SESSION_SUMMARY_MAX_KEY_FACTS - assert out[0] == "user lives in Berlin" - # The dict value coerces to a JSON string surface, not dropped. - assert any("Berlin" in s and "{" in s for s in out) - - -def test_user_prompt_carries_date_and_truncates() -> None: - from server.memory.app.extract.handler import ( - _SESSION_SUMMARY_MAX_INPUT_CHARS, - _build_session_summary_user_prompt, + state = SimpleNamespace( + written=[ + { + "memory_id": source.memory_id, + "memory_key": source.memory_key, + "version": source.version, + } + ], + extracted_facts=[{"memory_key": "uncommitted", "value": "retired transcript fact"}], + messages=[{"role": "user", "content": "retired transcript fact"}], + observation_date=None, ) + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) + from server.memory.store.rows import identity_key - msgs = [{"role": "user", "content": "x" * 10000}] - prompt = _build_session_summary_user_prompt(msgs, "2024-03-11") - assert "Session date: 2024-03-11" in prompt - assert "[user]" in prompt - # The transcript body is bounded; the header + framing add a little. - assert len(prompt) < _SESSION_SUMMARY_MAX_INPUT_CHARS + 200 - - -def test_user_prompt_unknown_date() -> None: - from server.memory.app.extract.handler import _build_session_summary_user_prompt - - prompt = _build_session_summary_user_prompt([{"role": "user", "content": "hi"}], None) - assert "Session date: unknown" in prompt + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={identity_key(source): source}), + ) + read = AsyncMock(return_value=source) + vector = AsyncMock(return_value=[1.0, 0.0]) + link = AsyncMock() + monkeypatch.setattr(store, "get_by_id", read) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", vector) + monkeypatch.setattr(digest_chain, "link_digest", link) + inp = SimpleNamespace( + org="o", namespace="n", agent="a", user_id="u", session_id="s", run_id="run" + ) + return SimpleNamespace( + input=inp, state=state, source=source, ch=ch, read=read, vector=vector, link=link + ) -# ---------------------------------------------------------------------- -# _write_session_summary — end-to-end with stubs -# ---------------------------------------------------------------------- +async def test_digest_contains_only_committed_current_sources(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + await _write_session_summary(writer.input, writer.state) + table, rows = writer.ch.insert.call_args.args + assert table == "loom.session_summaries" + row = rows[0] + assert set(row) == set(SESSION_SUMMARY_PERSISTED_COLS) + assert row["summary_text"] == "current fact" + assert row["key_facts"] == ["current fact"] + assert row["derived_from"] == ["known"] + assert "retired transcript fact" not in str(row) + assert row["run_id"] == "run" + writer.vector.assert_awaited_once_with(ANY, "current fact") + writer.link.assert_awaited_once() + + +@pytest.mark.parametrize( + "change", + [ + {"status": "rejected"}, + {"status": "proposed"}, + {"sensitivity": "high"}, + {"version": 101}, + {"session_id": "another-session"}, + {"value_truncated": True}, + ], +) +async def test_unqualified_sources_are_not_embedded_or_written(writer: Any, change: Any) -> None: + from server.memory.app.session_summary import _write_session_summary -@dataclass -class _FakeInput: - org: str = "o" - namespace: str = "n" - agent: str = "a" - user_id: str = "u" - session_id: str = "sess-1" + writer.read.return_value = writer.source.model_copy(update=change) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.vector.assert_not_awaited() -@dataclass -class _FakeState: - extracted_facts: list[dict[str, Any]] = field(default_factory=list) - observation_date: str | None = None - operator_policy: str = "" +async def test_no_committed_fact_produces_no_digest(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + writer.state.written = [] + await _write_session_summary(writer.input, writer.state) + writer.read.assert_not_awaited() + writer.ch.insert.assert_not_awaited() -class _FakeLLM: - provider = "fake" - model = "fake-echo-1" - def __init__(self, text: str = "on 2024-03-11, the user moved to Berlin.") -> None: - self._text = text - self.calls: list[tuple[str, str, int]] = [] +async def test_a_source_read_failure_is_reported_to_the_projection_caller(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary - async def complete(self, system, user, max_tokens=1024, response_format=None): - from server.llm.client import LLMResult + writer.read.side_effect = TimeoutError("authority unavailable") + with pytest.raises(TimeoutError): + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() - self.calls.append((system, user, max_tokens)) - return LLMResult( - text=self._text, - tokens_in=10, - tokens_out=5, - provider=self.provider, - model=self.model, - ) +async def test_digest_cap_keeps_whole_facts_instead_of_partial_provenance( + writer: Any, monkeypatch: Any +) -> None: + from server.memory.app.session_summary import _write_session_summary + from server.settings import settings -class _FakeEmbedder: - def __init__(self, vec: list[float] | None = None) -> None: - self.model_name = "fake-ngram-8" - self.dim = 8 - self._vec = [0.1] * 8 if vec is None else vec + monkeypatch.setattr(settings, "memory_session_digest_max_input_chars", 500) + writer.read.return_value = writer.source.model_copy(update={"value": "x" * 501}) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.vector.assert_not_awaited() - async def embed(self, text: str) -> list[float]: - return list(self._vec) - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] +async def test_a_missing_observation_date_stays_undated(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + await _write_session_summary(writer.input, writer.state) + assert writer.ch.insert.call_args.args[1][0]["valid_at"] is None -class _FakeCH: - def __init__(self) -> None: - self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] - async def insert(self, table, rows, settings=None): - self.inserts.append((table, rows, settings or {})) +async def test_a_source_removed_while_embedding_is_not_published(writer, monkeypatch): + from server.memory.app.session_summary import _write_session_summary + async def removed_during_embedding(*args, **kwargs): + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={}), + ) + return [1.0, 0.0] -@pytest.fixture -def _patch_clients(monkeypatch): - """Patch get_llm_client / get_embedder / get_client used by the writer.""" - import server.llm.client as llm_mod - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - - llm = _FakeLLM() - embedder = _FakeEmbedder() - ch = _FakeCH() - # _write_session_summary imports get_llm_client lazily from server.llm.client - monkeypatch.setattr(llm_mod, "get_llm_client", lambda: llm) - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: embedder) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: embedder) - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - return llm, embedder, ch - - -@pytest.mark.asyncio -async def test_write_summary_inserts_full_row(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - llm, embedder, ch = _patch_clients - state = _FakeState( - extracted_facts=[{"value": "user moved to Berlin"}, {"value": "likes jazz"}], - observation_date="2024-03-11", + monkeypatch.setattr( + "server.memory.app.session_summary.document_vector", removed_during_embedding ) - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert len(ch.inserts) == 1 - table, rows, settings = ch.inserts[0] - assert table == "loom.session_summaries" - # Synchronous insert so the immediate post-ingest read sees it. - assert settings.get("async_insert") == 0 - assert settings.get("wait_for_async_insert") == 1 - assert len(rows) == 1 - row = rows[0] - # Mirror-every-column: every persisted column is present in the insert dict. - assert set(row) == set(SESSION_SUMMARY_PERSISTED_COLS) - assert row["summary_text"] - assert row["summary_embedding"] == [0.1] * 8 - assert row["embedding_model"] == "fake-ngram-8" - assert row["embedding_dim"] == 8 - # key_facts derived from extracted_facts (no extra LLM call). - assert row["key_facts"] == ["user moved to Berlin", "likes jazz"] - # valid_at carries the parsed session date (LABEL). - assert isinstance(row["valid_at"], datetime) - assert row["valid_at"].year == 2024 and row["valid_at"].month == 3 - # Exactly one cheap LLM call, max_tokens bounded. - assert len(llm.calls) == 1 - assert llm.calls[0][2] == 160 - - -@pytest.mark.asyncio -async def test_write_summary_null_valid_at_when_undated(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - _llm, _embedder, ch = _patch_clients - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date=None) - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - row = ch.inserts[0][1][0] - assert row["valid_at"] is None - + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.link.assert_not_awaited() -@pytest.mark.asyncio -async def test_write_summary_skips_when_no_facts(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - llm, _embedder, ch = _patch_clients - state = _FakeState(extracted_facts=[], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) +async def test_a_current_source_after_many_missing_candidates_can_still_form_a_digest(writer): + from server.memory.app.session_summary import _write_session_summary - # Nothing to summarize -> no LLM call, no insert. - assert llm.calls == [] - assert ch.inserts == [] - - -@pytest.mark.asyncio -async def test_write_summary_skips_on_empty_digest(monkeypatch, _patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - llm, _embedder, ch = _patch_clients - llm._text = " " # whitespace-only digest - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert ch.inserts == [] # empty digest -> no row - - -@pytest.mark.asyncio -@pytest.mark.parametrize("vec", [[], [0.1], [float("nan")] * 8, [float("inf")] * 8]) -async def test_write_summary_skips_on_invalid_embedding(monkeypatch, _patch_clients, vec) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.extract.handler import _write_session_summary - - llm, _embedder, ch = _patch_clients - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: _FakeEmbedder(vec=vec)) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: _FakeEmbedder(vec=vec)) - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert ch.inserts == [] # Invalid vectors never become summary rows. - - -@pytest.mark.asyncio -async def test_write_summary_propagates_for_caller_failopen(monkeypatch, _patch_clients) -> None: - """The writer itself raises on LLM error; the CALLER wraps it (the - set_from_messages try/except records a projection failure). So the writer's - contract is: raise, don't swallow — the fail-open boundary is one level up.""" - import server.llm.client as llm_mod - from server.memory.app.extract.handler import _write_session_summary - - class _BoomLLM: - async def complete(self, *a, **k): - raise RuntimeError("llm down") - - monkeypatch.setattr(llm_mod, "get_llm_client", lambda: _BoomLLM()) - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - with pytest.raises(RuntimeError, match="llm down"): - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) + committed = writer.state.written[0] + writer.state.written = [dict(memory_id=f"missing-{i}", version=100) for i in range(48)] + [ + committed + ] + writer.read.side_effect = [None] * 48 + [writer.source] + await _write_session_summary(writer.input, writer.state) + assert writer.read.await_count == 49 + assert writer.ch.insert.call_args.args[1][0]["key_facts"] == [writer.source.value] diff --git a/tests/unit/test_session_summary_retrieval.py b/tests/unit/test_session_summary_retrieval.py index e25c5645e4..927eb4ee93 100644 --- a/tests/unit/test_session_summary_retrieval.py +++ b/tests/unit/test_session_summary_retrieval.py @@ -18,6 +18,15 @@ ) +@pytest.fixture(autouse=True) +def _qualified_sources(monkeypatch): + # These cases isolate ranking and hit mapping; source proof has its own suite. + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + class StubCH: def __init__(self, returns: list[dict[str, Any]] | None = None, raise_on_query: bool = False): self.returns = returns or [] diff --git a/tests/unit/test_set_from_messages_fail_closed.py b/tests/unit/test_set_from_messages_fail_closed.py index f175782886..d39a96479a 100644 --- a/tests/unit/test_set_from_messages_fail_closed.py +++ b/tests/unit/test_set_from_messages_fail_closed.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException @@ -19,6 +20,12 @@ from server.memory.api.models_bulk import REFUSED, MemorySetFromMessagesInput, _BulkSetResult +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + def _stub_projections(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx from server.memory.app import bulk_set as _bulk_set_mod @@ -36,7 +43,6 @@ async def _none(*args, **kwargs): monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _zero) monkeypatch.setattr(mx, "write_memory_chunks", _none) monkeypatch.setattr(mx, "_write_session_summary", _none) - monkeypatch.setattr(mx, "_draft_session_summary", _none) monkeypatch.setattr(mx, "record_input_projection_failure", _none) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _none) @@ -153,37 +159,25 @@ async def _fake_typed( assert bool(out.degraded) is not projected -@pytest.mark.asyncio -async def test_any_pipeline_failure_stops_the_digest_draft( +async def test_any_pipeline_failure_prevents_digest_generation( monkeypatch: pytest.MonkeyPatch, ) -> None: - """The digest draft (an LLM call and an embed) runs beside the extractor; a pipeline - failure that is not an ExtractionError must stop it too, not leave it spending.""" - import asyncio + from unittest.mock import AsyncMock import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe _stub_projections(monkeypatch) - cancelled: list[bool] = [] + summary = AsyncMock() - async def _slow_draft(*args, **kwargs): - try: - await asyncio.sleep(30) - except asyncio.CancelledError: - cancelled.append(True) - raise - - async def _pipeline_raises(steps, state): - await asyncio.sleep(0) # the draft has started + async def fail(*args, **kwargs): raise RuntimeError("bulk write refused") - monkeypatch.setattr(mx, "_draft_session_summary", _slow_draft) - monkeypatch.setattr(pipe, "run_pipeline", _pipeline_raises) + monkeypatch.setattr(mx, "_write_session_summary", summary) + monkeypatch.setattr(pipe, "run_pipeline", fail) with pytest.raises(RuntimeError, match="bulk write refused"): await mx.handle_memory_set_from_messages(_input()) - await asyncio.sleep(0) - assert cancelled == [True] + summary.assert_not_awaited() @pytest.mark.asyncio diff --git a/tests/unit/test_set_from_messages_refusals.py b/tests/unit/test_set_from_messages_refusals.py index 95ad782017..e89b169025 100644 --- a/tests/unit/test_set_from_messages_refusals.py +++ b/tests/unit/test_set_from_messages_refusals.py @@ -12,11 +12,18 @@ async def test_a_schema_request_without_a_key_writes_no_evidence( ) -> None: """The 400 comes before the evidence write starts, so a refused request leaves no evidence rows behind.""" + from types import SimpleNamespace + from unittest.mock import AsyncMock + from fastapi import HTTPException from server.memory.api.models_bulk import MemorySetFromMessagesInput from server.memory.app.extract import handler as hm + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + started: list[object] = [] async def _evidence(*a: object, **k: object) -> list[str]: diff --git a/tests/unit/test_shared_and_documents_list_repo.py b/tests/unit/test_shared_and_documents_list_repo.py index 5f40c82053..1629a0d723 100644 --- a/tests/unit/test_shared_and_documents_list_repo.py +++ b/tests/unit/test_shared_and_documents_list_repo.py @@ -10,14 +10,18 @@ from __future__ import annotations +from contextlib import asynccontextmanager from datetime import UTC, datetime from typing import Any import pytest +from server.core.hashing import sha256_value from server.core.tag_sql import TagFilter, tag_where from server.knowledge.store import documents as documents_repo from server.memory.store import shared as shared_repo +from server.memory.store.rows import build_memory_version_row +from tests.unit.test_memory_verify import _row # The any-case tag matches, as the list reads render them. _FOLDED_ANY = tag_where(TagFilter(("x",), "folded_any"), {}) @@ -26,6 +30,7 @@ class StubCH: def __init__(self) -> None: + self.source = None self.query_one_calls: list[tuple[str, dict[str, Any]]] = [] self.query_one_returns: list[dict[str, Any] | None] = [] self.query_all_calls: list[tuple[str, dict[str, Any]]] = [] @@ -39,15 +44,30 @@ async def query_one(self, sql, params=None): async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(self.source)] if self.source else [] return self.query_all_returns.pop(0) if self.query_all_returns else [] - async def execute(self, sql, params=None): + async def execute(self, sql, params=None, *, settings=None): self.execute_calls.append((sql, params or {})) async def insert(self, table, rows, settings=None): self.insert_calls.append((table, rows, settings or {})) +@asynccontextmanager +async def _approved_source(ch, value="source fact"): + from server.memory.store.refusals import source_publication + + ch.source = _row( + org="clickhouse", namespace="ns", agent="author", memory_key="source", value=value + ) + async with source_publication(ch, ch.source, "k"): + yield ch.source + + # ---------------------------------------------------------------------- # shared_repo # ---------------------------------------------------------------------- @@ -56,15 +76,17 @@ async def insert(self, table, rows, settings=None): @pytest.mark.asyncio async def test_insert_shared_row_forces_synchronous_visibility() -> None: ch = StubCH() - await shared_repo.insert_shared_row( - { - **dict.fromkeys(shared_repo.SHARED_PERSISTED_COLS), - "org": "clickhouse", - "namespace": "ns", - "memory_key": "k", - }, - ch=ch, - ) + async with _approved_source(ch, "source fact"): + await shared_repo.insert_shared_row( + { + **dict.fromkeys(shared_repo.SHARED_PERSISTED_COLS), + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "value": "source fact", + }, + ch=ch, + ) table, _, settings = ch.insert_calls[0] assert table == "loom.shared" # search-visibility contract: next FINAL read sees the row. @@ -98,10 +120,18 @@ async def test_insert_shared_embedding_uses_memory_embeddings_table() -> None: """The shared embedding lives on loom.memory_embeddings (the same table as private embeddings) — scope='shared' is the marker.""" ch = StubCH() - await shared_repo.insert_shared_embedding( - {"scope": "shared", "agent": "_shared"}, - ch=ch, - ) + async with _approved_source(ch, "source fact"): + await shared_repo.insert_shared_embedding( + { + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "content_sha256": sha256_value("source fact"), + "scope": "shared", + "agent": "_shared", + }, + ch=ch, + ) table, rows, _ = ch.insert_calls[0] assert table == "loom.memory_embeddings" assert rows[0]["agent"] == "_shared" @@ -116,10 +146,19 @@ async def test_insert_shared_embedding_writes_a_complete_builder_row() -> None: from server.memory.store.rows import EMBEDDING_PERSISTED_COLS ch = StubCH() - await shared_repo.insert_shared_embedding( - {"scope": "shared", "agent": "_shared", "content_excerpt": "dogs were barking"}, - ch=ch, - ) + async with _approved_source(ch, "dogs were barking"): + await shared_repo.insert_shared_embedding( + { + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "content_sha256": sha256_value("dogs were barking"), + "scope": "shared", + "agent": "_shared", + "content_excerpt": "dogs were barking", + }, + ch=ch, + ) row = ch.insert_calls[0][1][0] assert set(row) == set(EMBEDDING_PERSISTED_COLS) assert row["content_excerpt_lemma"] == "dog be bark" @@ -140,13 +179,16 @@ async def test_promote_carries_the_source_provenance_onto_the_shared_embedding( from server.memory.app import promote as _memory_promote written: list[dict[str, Any]] = [] + ch = StubCH() + insert_shared_embedding = shared_repo.insert_shared_embedding + monkeypatch.setattr(_memory_promote, "get_client", lambda: ch) class _Emb: model_name = "fake" dim = 2 - async def embed(self, text: str) -> list[float]: - return [0.1, 0.2] + async def embed_many(self, texts: list[str]) -> list[list[float]]: + return [[0.1, 0.2] for _ in texts] async def _no_peers(**kw: Any) -> list[Any]: return [] @@ -155,6 +197,7 @@ async def _shared_row(row: dict[str, Any], **kw: Any) -> None: return None async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: + await insert_shared_embedding(row, ch=ch) written.append(row) monkeypatch.setattr(_memory_promote, "get_embedder", lambda: _Emb()) @@ -162,13 +205,13 @@ async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: monkeypatch.setattr(shared_repo, "insert_shared_row", _shared_row) monkeypatch.setattr(shared_repo, "insert_shared_embedding", _shared_emb) - for verified_sha, want in (("a" * 64, 1), ("b" * 64, 0)): + for verified_sha, want in ((sha256_value({"v": "external fact"}), 1), ("b" * 64, 0)): src = memory_repo.MemoryRow.from_snapshot( dict( memory_id="11111111-1111-1111-1111-111111111111", sensitivity="", org="o", - namespace="team", + namespace="n", agent="a", user_id="-", session_id="-", @@ -180,7 +223,7 @@ async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: source_class="external", verified_by="human:alice", verified_sha256=verified_sha, - content_sha256="a" * 64, + content_sha256=sha256_value({"v": "external fact"}), ) ) @@ -188,6 +231,7 @@ async def _get(*a: Any, _src: Any = src, **kw: Any) -> Any: return _src monkeypatch.setattr(memory_repo, "get_by_key", _get) + ch.source = src written.clear() await _memory_promote.handle_memory_promote( MemoryPromoteInput(org="o", namespace="n", agent="a", memory_key="k") @@ -216,13 +260,12 @@ async def test_fetch_shared_none_when_missing() -> None: @pytest.mark.asyncio -async def test_tombstone_shared_embedding_re_emits_a_versioned_row() -> None: +async def test_tombstone_shared_embedding_deletes_older_cached_versions() -> None: """memory.search(scope='shared') filters on tombstoned_at IS NULL; the demote path lands the tombstone as a committed re-emit of the live shared rows (every column carried, tombstoned_at set, fresh version) so the next search does not see the row — and never as an ALTER UPDATE, whose patch part would cost the fact leg its codes-first read.""" - from server.memory.store import EMBEDDING_PERSISTED_COLS ch = StubCH() when = datetime(2026, 5, 17, tzinfo=UTC) @@ -243,19 +286,14 @@ async def test_tombstone_shared_embedding_re_emits_a_versioned_row() -> None: ] ) await shared_repo.tombstone_shared_embedding("clickhouse", "ns", "k", when, version=9, ch=ch) - assert ch.execute_calls == [] + assert len(ch.execute_calls) == 1 and not ch.insert_calls read_sql, read_params = ch.query_all_calls[0] assert "scope = 'shared'" in read_sql and "tombstoned_at IS NULL" in read_sql assert "FINAL" in read_sql and read_params["k"] == "k" - ((table, rows, settings),) = ch.insert_calls - assert table == "loom.memory_embeddings" and settings == { - "async_insert": 0, - "wait_for_async_insert": 1, - } - (row,) = rows - assert set(row) >= set(EMBEDDING_PERSISTED_COLS) - assert row["tombstoned_at"] == when and row["scope"] == "shared" - assert row["embedding"] == [0.1, 0.2, 0.3] and row["version"] == 9 # the demote's + delete_sql, delete_params = ch.execute_calls[0] + assert delete_sql.startswith("DELETE FROM loom.memory_embeddings") + assert "version < {version:UInt64}" in delete_sql + assert delete_params["scope"] == "shared" and delete_params["version"] == 9 @pytest.mark.asyncio @@ -368,3 +406,26 @@ async def test_list_with_chunk_counts_requires_every_tag_when_asked() -> None: sql, params = ch.query_all_calls[0] assert _FOLDED_ALL in sql assert params["tags"] == ["Specs", "onboarding"] + + +def test_promoted_vector_binds_verification_to_the_complete_served_value(): + from types import SimpleNamespace + + from server.memory.app.promote import _shared_vector_row + + source = _row( + value="complete served fact", + content_sha256="a" * 64, + verified_by="human:reviewer", + verified_sha256="a" * 64, + ) + row = _shared_vector_row( + source, + "k", + "display excerpt", + SimpleNamespace(model_name="m", dim=1), + [0.5], + datetime.now(UTC), + ) + assert row["content_sha256"] == sha256_value(source.value) + assert row["verified"] == 0 diff --git a/tests/unit/test_status_mirror.py b/tests/unit/test_status_mirror.py index 22b899216a..964956648d 100644 --- a/tests/unit/test_status_mirror.py +++ b/tests/unit/test_status_mirror.py @@ -9,21 +9,40 @@ import pytest +from server.core.hashing import sha256_value from server.memory import store as memory_repo from server.memory.store import lifecycle +@pytest.fixture +def mirror_publication(monkeypatch): + """Reconcile selection uses an acknowledged publisher; consent has separate cases.""" + + async def publish(rows, *, ch): + await ch.insert("loom.memory_embeddings", rows) + return rows + + monkeypatch.setattr("server.memory.store.embeddings.insert_embedding_rows", publish) + + class _CH: """Records inserts; answers the embeddings read with one live row per key.""" def __init__(self, emb_status: str = "active") -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] self.emb_status = emb_status + self.heads: dict[Any, Any] = {} async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: p = params or {} + if "FROM loom.memory FINAL" in sql: + return [ + memory_repo.build_memory_version_row(self.heads[key]) + for key in p.get("keys", []) + if key in self.heads + ] if "FROM loom.memory_embeddings FINAL" in sql: # the keyed read: one identity per position of the six arrays return [ @@ -42,6 +61,7 @@ async def query_all( "status": self.emb_status, "version": 1, "content_excerpt": "x", + "content_sha256": sha256_value("x"), } for o, n, a, u, s_, k in zip( p["o"], p["n"], p["a"], p["u"], p["s"], p["k"], strict=True @@ -53,6 +73,13 @@ async def insert( self, table: str, rows: list[dict[str, Any]], settings: dict[str, Any] | None = None ) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + from server.memory.store.rows import identity_key + + for row in rows: + key = identity_key(row) + if key in self.heads: + self.heads[key] = self.heads[key].model_copy(update=row) def _existing(status: str = "active") -> Any: @@ -72,6 +99,13 @@ def _existing(status: str = "active") -> Any: def _wire(monkeypatch: pytest.MonkeyPatch, ch: _CH, existing: Any) -> None: + from server.memory.store.rows import MemoryRow, identity_key + + source = MemoryRow.from_snapshot( + {**vars(existing), "value": "x", "content_sha256": sha256_value("x")} + ) + ch.heads[identity_key(source)] = source + async def _get_by_id( memory_id: str, *, @@ -159,6 +193,7 @@ async def test_already_mirrored_row_is_not_rewritten(monkeypatch: pytest.MonkeyP @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_holds_back_a_key_the_authority_has_moved_past( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -229,6 +264,7 @@ async def _reconcile(*_: Any, **__: Any) -> dict[str, int]: @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_counts_on_dry_run_and_mirrors_otherwise() -> None: class _RCH(_CH): async def query_all( @@ -278,6 +314,7 @@ async def query_all( @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_reads_the_mirror_by_the_full_identity() -> None: """One memory_key lives under many identities, each with its own status. The divergence is computed per identity row and @@ -322,6 +359,7 @@ async def query_all( @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_reads_the_mirror_in_one_statement_per_batch() -> None: """`loom reconcile-status` hands up to 5,000 divergent identities to the mirror: one keyed read per 500 of them, not one read per identity.""" @@ -643,9 +681,12 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: if "FROM loom.memory FINAL" in sql: + if "keys" in (params or {}): + return await super().query_all(sql, params) assert (params or {})["key"] == "home" return [{"memory_id": "m1"}, {"memory_id": "m2"}] - self.vector_reads.append(sql) + if "FROM loom.memory_embeddings FINAL" in sql: + self.vector_reads.append(sql) rows = await super().query_all(sql, params) # Each vector row was copied at read + 1 since its memory row (version 1). return [{**r, "version": 2} for r in rows] @@ -668,10 +709,20 @@ async def _get_by_id(memory_id: str, **_: Any) -> Any: return heads[memory_id] async def _current(keys, **kwargs): - from server.memory.store.rows import MemoryRow, identity_key - rows = [MemoryRow.from_snapshot(vars(row)) for row in heads.values()] - return {identity_key(row): row for row in rows if identity_key(row) in keys} + return {key: row for key, row in ch.heads.items() if key in keys} + + from server.memory.store.rows import MemoryRow, identity_key + + ch.heads = { + identity_key(row): row + for row in ( + MemoryRow.from_snapshot( + {**vars(head), "value": "x", "content_sha256": sha256_value("x")} + ) + for head in heads.values() + ) + } monkeypatch.setattr(lifecycle, "get_by_id", _get_by_id) monkeypatch.setattr("server.memory.store.write_fence.memory_rows_by_identity_keys", _current) diff --git a/tests/unit/test_summary_leg_links.py b/tests/unit/test_summary_leg_links.py index 2d3a2dcbf8..b596e1ab48 100644 --- a/tests/unit/test_summary_leg_links.py +++ b/tests/unit/test_summary_leg_links.py @@ -5,6 +5,17 @@ from typing import Any +import pytest + + +@pytest.fixture(autouse=True) +def _qualified_sources(monkeypatch): + # These cases isolate ranking and hit mapping; source proof has its own suite. + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + def test_digest_row_carries_links_and_session_into_the_hit() -> None: """The summary leg's row → SearchHit mapping (memory.app.retrieval).""" diff --git a/tests/unit/test_summary_sources.py b/tests/unit/test_summary_sources.py index 536493dd7a..64ddd41c84 100644 --- a/tests/unit/test_summary_sources.py +++ b/tests/unit/test_summary_sources.py @@ -1,36 +1,185 @@ -"""An unchecked source never makes a session digest eligible for retrieval.""" +"""A digest is served only when every byte can be reconstructed from live facts.""" from __future__ import annotations +import json +from datetime import timedelta +from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock +import pytest + +from server.core.datetimes import naive_utc_now +from server.memory import store from server.memory.app.retrieval.summaries import fetch_session_summaries +from server.memory.store.summary_sources import current_summary_sources +from tests.unit.test_memory_verify import _row + + +def digest(source: Any, **overrides: Any) -> dict[str, Any]: + return { + "agent": source.agent, + "user_id": source.user_id, + "session_id": source.session_id, + "derived_from": [source.memory_key], + "key_facts": [source.value], + "summary_text": source.value, + "version": 200, + "distance": 0.1, + **overrides, + } + + +@pytest.fixture +def sources(monkeypatch: Any) -> dict[str, Any]: + values = {"source": _row(value="current fact", memory_key="source")} + + async def read(keys, **kwargs): + from server.memory.store.rows import identity_key + + assert kwargs["sequential"] + return {identity_key(r): r for r in values.values() if identity_key(r) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + return values + + +@pytest.mark.parametrize("lineage", [None, []]) +async def test_a_summary_without_source_lineage_is_not_retrievable(lineage: Any) -> None: + row = {"summary_text": "unknown fact", "key_facts": ["unknown fact"]} + if lineage is not None: + row["derived_from"] = lineage + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +@pytest.mark.parametrize( + "change", + [ + {"status": "rejected"}, + {"status": "proposed"}, + {"status": "duplicate"}, + {"status": "superseded"}, + {"sensitivity": "high"}, + {"value": "replacement fact"}, + {"value_truncated": True}, + {"requires_review": True}, + {"user_id": "another-user"}, + {"session_id": "another-session"}, + {"agent": "another-agent"}, + ], +) +async def test_changed_or_unserved_sources_withhold_the_digest(sources: Any, change: Any) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy(update=change) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +@pytest.mark.parametrize("field", ["expires_at", "valid_to", "tombstoned_at"]) +async def test_retired_sources_withhold_the_digest(sources: Any, field: str) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy( + update={field: naive_utc_now() - timedelta(seconds=1)} + ) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +async def test_a_touch_does_not_invalidate_verifiable_unchanged_text(sources: Any) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy(update={"version": 999, "pinned": True}) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [row] + + +async def test_a_missing_source_and_a_forged_sentence_do_not_hide_a_valid_neighbour( + sources: Any, +) -> None: + valid = digest(sources["source"]) + missing = digest(sources["source"], derived_from=["missing"]) + forged = digest(sources["source"], summary_text="current fact\nretired fact") + assert await current_summary_sources( + None, [missing, forged, valid], org="o", namespace="ns" + ) == [valid] + + +async def test_source_state_failure_withholds_the_summary(monkeypatch: Any) -> None: + source = _row(value="current fact", memory_key="source") + + async def unavailable(*args: Any, **kwargs: Any) -> Any: + raise TimeoutError("source state unavailable") + + class CH: + async def query_all(self, *args: Any, **kwargs: Any) -> Any: + return [digest(source)] + + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", unavailable + ) + assert ( + await fetch_session_summaries( + CH(), + org="o", + namespace="ns", + agent=source.agent, + user_id="-", + query_vec=[1.0, 0.0], + embedding_model="test-model", + ) + == [] + ) + + +async def test_postgres_authority_overrides_a_stale_clickhouse_source(monkeypatch: Any) -> None: + from server.memory.store import pg_memory + from server.settings import settings + + source = _row(value="current fact", memory_key="source") + rejected = store.build_memory_version_row(source, status="rejected", version=201) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr( + pg_memory, "heads_by_keys", AsyncMock(return_value={store.identity_key(source): rejected}) + ) + ch = SimpleNamespace(query_all=AsyncMock(side_effect=AssertionError("must read the authority"))) + assert await current_summary_sources(ch, [digest(source)], org="o", namespace="ns") == [] + ch.query_all.assert_not_awaited() + + +@pytest.mark.parametrize("fact", ["文" * 600, "x" * 3000, "🌱" * 600]) +async def test_encoded_source_size_does_not_drop_a_complete_fact(monkeypatch, fact): + from server.memory.store.rows import identity_key + + source = _row(value=fact, memory_key="source") + + async def read(keys, **kwargs): + assert kwargs["max_value_chars"] >= len(json.dumps(fact)) + return {identity_key(source): source} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + row = digest(source) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [row] + + +@pytest.mark.parametrize("count", [6, 50]) +@pytest.mark.parametrize("retired_last", [False, True]) +async def test_many_sources_are_qualified_in_one_exact_key_read(monkeypatch, count, retired_last): + from server.memory.store.rows import identity_key + + sources = [_row(value="文" * 400, memory_key=f"source-{n}") for n in range(count)] + calls = [] + async def read(keys, **kwargs): + calls.append(keys) + return {identity_key(source): source for source in sources} -async def test_source_state_failure_withholds_the_summary() -> None: - class UnavailableSources: - async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: - if "FROM loom.memory FINAL" in sql: - raise TimeoutError("source state unavailable") - return [ - { - "agent": "writer", - "user_id": "reader", - "session_id": "session", - "version": 200, - "derived_from": ["source"], - "summary_text": "old fact", - "distance": 0.1, - } - ] - - hits = await fetch_session_summaries( - UnavailableSources(), - org="org", - namespace="namespace", - agent="writer", - user_id="reader", - query_vec=[1.0, 0.0], - embedding_model="test-model", + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + facts = [source.value for source in sources] + row = digest( + sources[0], + derived_from=[source.memory_key for source in sources], + key_facts=facts, + summary_text="\n".join(facts), ) - assert hits == [] + if retired_last: + sources[-1] = sources[-1].model_copy(update={"status": "superseded"}) + expected = [] if retired_last else [row] + assert await current_summary_sources(None, [row], org="o", namespace="ns") == expected + assert len(calls) == 1 and len(calls[0]) == count diff --git a/tests/unit/test_value_updates.py b/tests/unit/test_value_updates.py index 776d5867b4..275c9f1a08 100644 --- a/tests/unit/test_value_updates.py +++ b/tests/unit/test_value_updates.py @@ -202,8 +202,15 @@ async def boom(*a: Any, **k: Any) -> int: def test_enabled_precedence_request_over_contextvar_over_setting( monkeypatch: pytest.MonkeyPatch, ) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + from server.memory.app import bulk_set as bulk_set from server.memory.app.extract import handler as hm + + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) from server.settings import settings monkeypatch.setattr(settings, "memory_extract_value_updates", False) @@ -242,6 +249,9 @@ async def test_set_from_messages_starts_its_link_count_at_zero( ) -> None: """A worker replays queued requests one after another in one task: a request that writes nothing must not report the previous request's value-update links.""" + from types import SimpleNamespace + from unittest.mock import AsyncMock + from server.memory.api.models_bulk import ( MemorySetFromMessagesInput, MemorySetFromMessagesOutput, @@ -249,6 +259,10 @@ async def test_set_from_messages_starts_its_link_count_at_zero( from server.memory.app import bulk_set as bulk_set from server.memory.app.extract import handler as hm + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + async def _enqueued(_inp: object, _rule_agent: str) -> MemorySetFromMessagesOutput: return MemorySetFromMessagesOutput(written=[]) diff --git a/tests/unit/test_version_row_value_bytes.py b/tests/unit/test_version_row_value_bytes.py index 1fd1cd8ab4..6846041f04 100644 --- a/tests/unit/test_version_row_value_bytes.py +++ b/tests/unit/test_version_row_value_bytes.py @@ -87,16 +87,21 @@ def _same_bytes_and_hash(written: str, stored: str, parsed: Any) -> None: class _CH: def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] + self.heads = {} self.routes = routes or [] self.query_one_result: dict[str, Any] | None = None async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + self.heads.update({mr.identity_key(r): dict(r) for r in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return self.query_one_result async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: + if params and "keys" in params and "FROM loom.memory FINAL" in sql: + return [self.heads[k] for k in params["keys"] if k in self.heads] for needle, result in self.routes: if needle in sql: return [dict(r) for r in result] @@ -120,7 +125,15 @@ def rows_for(self, table: str) -> list[dict[str, Any]]: def _wire(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow) -> _CH: - ch = _CH(routes=[("FROM loom.memory_embeddings", [dict(_EMB_ROW)])]) + ch = _CH( + routes=[ + ( + "FROM loom.memory_embeddings", + [dict(_EMB_ROW, content_sha256=existing.content_sha256)], + ) + ] + ) + ch.heads[mr.identity_key(existing)] = mr.build_memory_version_row(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_workers_poll_the_work.py b/tests/unit/test_workers_poll_the_work.py index 03f46b9fad..5be1087d37 100644 --- a/tests/unit/test_workers_poll_the_work.py +++ b/tests/unit/test_workers_poll_the_work.py @@ -7,8 +7,7 @@ exact, reads the full rows by exact key only for what the scan returned, writes one insert per table per round, and scans from 0 on its slow clock; - entity_extractor reads the cost cap once per pass for every namespace of - its rows and writes the pass's rows in one batch (six statements, not - six per row); the batch write emits the same rows as N single writes; + its rows and writes the pass's rows in batches with bounded source checks; the batch write emits the same rows as N single writes; - the cost-cap verdict for N namespaces is one statement, fail-open; - embedding_worker marks itself idle only when all three legs found nothing; - every pass records its counts as outcome scalars. @@ -23,6 +22,7 @@ import pytest from server.code.store.index import CodeRepo as _RealCodeRepo +from server.core.hashing import sha256_value from server.settings import settings from server.workers import pass_context from server.workers import supervisor as sup @@ -565,6 +565,32 @@ def __init__(self, answers: list[list[dict[str, Any]]]) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] self.heads: dict[Any, Any] = {} + for batch in answers: + for row in batch: + if "entities" in row and "embedding_model" not in row: + from server.memory.store.rows import identity_key + + head = {**_mem_row(row["memory_key"]), **row} + self.heads[identity_key(head)] = head + + async def query_one(self, sql, params=None): + import json + + from server.core.hashing import served_row_value + from server.memory.domain.promotion_source import promotion_source + + self.queries.append(" ".join(sql.split())) + assert "FROM loom.shared FINAL" in sql + key = (params["org"], params["ns"], "a", "-", "-", params["memory_key"]) + source = self.heads.get(key) + if source is None: + return None + return { + "value": json.dumps(served_row_value(source)), + "tombstoned_at": None, + "promoted_from": [promotion_source(*key)], + } + async def query_all(self, sql, params=None): from server.memory.store.rows import identity_key @@ -606,6 +632,7 @@ def _mem_row(key: str) -> dict[str, Any]: "memory_type": "semantic", "value": '{"a": 1}', "value_json": "{}", + "content_sha256": sha256_value({"a": 1}), "confidence": 1.0, "version": 1, "pinned": 0, @@ -619,7 +646,10 @@ def _mem_row(key: str) -> dict[str, Any]: def _emb_row(key: str, scope: str = "private") -> dict[str, Any]: + import json + from server.memory import store as mr + from server.memory.domain.excerpt import build_excerpt return { **mr.EMBEDDING_PERSISTED_COLS, @@ -631,7 +661,10 @@ def _emb_row(key: str, scope: str = "private") -> dict[str, Any]: "memory_key": key, "memory_type": "semantic", "scope": scope, - "content_excerpt": "x", + "content_excerpt": build_excerpt(json.dumps({"a": 1}), memory_key=key) + if scope == "shared" + else "x", + "content_sha256": sha256_value({"a": 1}), "embedding_model": "m", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], @@ -656,9 +689,7 @@ def _item( } -def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_rows_as_n_single_writes() -> ( - None -): +def test_apply_entities_batch_bounds_reads_and_preserves_every_mirror() -> None: from server.memory import store as mr ch = _CH( @@ -703,27 +734,34 @@ def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_ro mr.apply_entities_batch([_item("k1", ["alice"]), _item("k2", ["bob", "eve"])], ch=ch) ) assert written == {"memory": 2, "memory_embeddings": 3} - assert len(ch.queries) == 4 and [t for t, _ in ch.inserts] == [ + assert len(ch.queries) == 17 and [t for t, _ in ch.inserts] == [ "loom.memory", "loom.memory_embeddings", + "loom.memory_embeddings", ] + mirrors = [q for q in ch.queries if "FROM loom.memory_embeddings" in q] + heads = [q for q in ch.queries if "FROM loom.memory FINAL" in q] + assert len(mirrors) == 2 and len(heads) == 5 + assert len([q for q in ch.queries if "FROM loom.shared FINAL" in q]) == 2 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 8 assert ( ch.queries[0].startswith("SELECT") and "FROM loom.memory FINAL PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN" in ch.queries[0] ) - assert ( - "PREWHERE (org, namespace" in ch.queries[1] - ) # Review state is checked under the write fence. - assert ch.queries[2].startswith( + assert "PREWHERE (org, namespace" in heads[1] # Review state is checked under the write fence. + assert mirrors[0].startswith( "SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key, scope, embedding_model FROM loom.memory_embeddings" ) - assert ch.queries[3].startswith( + assert mirrors[1].startswith( "SELECT * FROM loom.memory_embeddings FINAL PREWHERE (org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key) IN" ) mem = {r["memory_key"]: r for r in ch.inserts[0][1]} assert mem["k1"]["entities"] == ["alice"] and mem["k2"]["entities"] == ["bob", "eve"] - emb = ch.inserts[1][1] + emb = sorted( + (r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows), + key=lambda r: (r["memory_key"], r["scope"]), + ) assert [(r["memory_key"], r["scope"], r["entities"]) for r in emb] == [ ("k1", "private", ["alice"]), ("k2", "private", ["bob", "eve"]), @@ -736,7 +774,11 @@ def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_ro assert all(set(mr.EMBEDDING_PERSISTED_COLS) <= set(r) for r in emb), ( "every persisted column mirrored" ) - assert all(r["embedding"] == [0.1, 0.2, 0.3] and r["content_excerpt"] == "x" for r in emb) + assert all( + r["embedding"] == [0.1, 0.2, 0.3] + and r["content_excerpt"] == _emb_row(r["memory_key"], r["scope"])["content_excerpt"] + for r in emb + ) def test_apply_entities_batch_writes_only_onto_the_head_the_entities_came_from() -> None: @@ -771,12 +813,12 @@ def test_apply_entities_batch_keeps_its_work_over_a_copy_of_the_same_text() -> N takes the entities; one whose text changed does not.""" from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) ch = _CH( answers=[ [ _mem_row("k1") | {"version": 3, "content_sha256": read}, - _mem_row("k2") | {"version": 3, "content_sha256": "b" * 64}, + _mem_row("k2") | {"version": 3, "content_sha256": sha256_value({"a": 2})}, ] ] ) @@ -796,7 +838,7 @@ def test_a_copy_of_the_same_text_redacted_or_made_high_since_keeps_no_entities( ) -> None: from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) ch = _CH(answers=[[_mem_row("k1") | {"version": 3, "content_sha256": read} | since]]) written = asyncio.run( mr.apply_entities_batch([_item("k1", ["alice"], content_sha256=read)], ch=ch) @@ -818,7 +860,8 @@ def test_apply_entities_batch_copies_every_search_document_of_a_head_written() - ) written = asyncio.run(mr.apply_entities_batch([_item("k1", ["alice"])], ch=ch)) assert written == {"memory": 1, "memory_embeddings": 2} - assert [(r["scope"], r["version"]) for r in ch.inserts[1][1]] == [("private", 2), ("shared", 3)] + vectors = [r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows] + assert sorted((r["scope"], r["version"]) for r in vectors) == [("private", 2), ("shared", 3)] def test_apply_entities_batch_leaves_a_newer_writes_search_document( @@ -835,7 +878,7 @@ async def sleep(s: float) -> None: slept.append(s) monkeypatch.setattr(entities.asyncio, "sleep", sleep) - head, newer = "a" * 64, "b" * 64 + head, newer = sha256_value({"a": 1}), sha256_value({"a": 2}) ch = _CH( answers=[ [_mem_row("k1") | {"content_sha256": head}], @@ -875,7 +918,7 @@ def test_a_restatement_with_its_own_entities_keeps_them() -> None: pass's to tag.""" from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) restated = _mem_row("k1") | {"version": 3, "content_sha256": read, "entities": ["acme"]} ch = _CH(answers=[[restated]]) written = asyncio.run( @@ -890,7 +933,7 @@ def test_apply_entities_batch_writes_above_a_copy_newer_than_its_draw() -> None: from server.memory import store as mr from server.memory.store.rows import next_version - read = "a" * 64 + read = sha256_value({"a": 1}) far = next_version() + 10**9 # a lock from a replica whose clock is a second ahead ch = _CH(answers=[[_mem_row("k1") | {"version": far, "content_sha256": read}]]) asyncio.run(mr.apply_entities_batch([_item("k1", ["alice"], content_sha256=read)], ch=ch)) @@ -1053,7 +1096,10 @@ async def fake_sleep(s): written = asyncio.run(mr.apply_entities_batch([_item("k1", ["a"]), _item("k2", ["b"])], ch=ch)) assert slept == [mr.ENTITY_MIRROR_RETRY_DELAY_S], "exactly one bounded retry for the batch" assert written == {"memory": 2, "memory_embeddings": 2} - assert len(ch.queries) == 6 and len(ch.inserts) == 2 + assert len(ch.queries) == 11 and len(ch.inserts) == 2 + assert len([q for q in ch.queries if "FROM loom.memory_embeddings" in q]) == 4 + assert len([q for q in ch.queries if "FROM loom.memory FINAL" in q]) == 3 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 4 assert [r["memory_key"] for r in ch.inserts[1][1]] == ["k1", "k2"] @@ -1075,7 +1121,7 @@ async def fake_sleep(s): assert asyncio.run(mr.apply_entities_batch([], ch=ch)) == {"memory": 0, "memory_embeddings": 0} -def test_heal_reads_every_pair_in_one_keyed_final_read_and_one_insert() -> None: +def test_heal_batches_mirror_reads_and_checks_shared_sources() -> None: from server.memory import store as mr pairs = [ @@ -1100,9 +1146,12 @@ def test_heal_reads_every_pair_in_one_keyed_final_read_and_one_insert() -> None: ] ch = _CH(answers=[pairs, [_emb_row("k1"), _emb_row("k2"), _emb_row("k2", "shared")]]) healed = asyncio.run(mr.heal_divergent_entities(limit=50, ch=ch)) - assert healed == 3 and len(ch.queries) == 2 and len(ch.inserts) == 1 + assert healed == 3 and len(ch.queries) == 13 and len(ch.inserts) == 2 + assert len([q for q in ch.queries if "FROM loom.shared FINAL" in q]) == 2 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 6 assert "IN {keys:Array(Tuple(String, String, String, String, String, String))}" in ch.queries[1] - assert [(r["memory_key"], r["entities"]) for r in ch.inserts[0][1]] == [ + vectors = [r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows] + assert sorted((r["memory_key"], r["entities"]) for r in vectors) == [ ("k1", ["x"]), ("k2", ["y"]), ("k2", ["y"]), @@ -1172,6 +1221,7 @@ async def find_stubs(**kw): async def insert_rows(rows, *a, **kw): env["inserted"].append(list(rows)) + return [] if env.get("withheld") else rows async def orphans(**kw): return [dict(o) for o in env["orphans"]] @@ -1361,3 +1411,42 @@ async def query_all(self, sql, params=None): ) assert params["keys"] == [("o", "n1", "m", "p", "c")] assert out == [{"org": "o", "namespace": "n1", "path": "p", "chunk_id": "c", "version": 7}] + + +async def test_embedding_worker_counts_only_acknowledged_fills(emb_env): + from server.memory.workers import embedding as worker + from server.workers import status + + emb_env.update(stubs=[_stub_row()], withheld=True) + result = await worker.run_pass(worker.EmbeddingWorkerConfig(batch_size=10)) + assert result.embedded_count == 0 and result.reconciled == [] + assert status.get_registry().snapshot()["embedding_worker"]["last_outcome"]["rows_written"] == 0 + + +@pytest.mark.copy_head_check +async def test_embedding_worker_withholds_an_unbacked_shared_stub_before_embedding( + emb_env, monkeypatch +): + from server.memory.store import shared + from server.memory.workers import embedding as worker + + seen = [] + + async def missing(row, **kwargs): + seen.append(row) + return None + + class Provider: + model_name, dim = "fake-ngram-3", 3 + + async def embed_many(self, texts): + pytest.fail("An unbacked shared stub reached the provider") + + async def embed(self, text): + pytest.fail("An unbacked shared stub reached the provider") + + emb_env["stubs"] = [_stub_row() | {"scope": "shared"}] + monkeypatch.setattr(shared, "shared_embedding_source", missing) + monkeypatch.setattr(worker, "get_embedder", lambda: Provider()) + result = await worker.run_pass(worker.EmbeddingWorkerConfig(batch_size=10)) + assert len(seen) == 1 and result.embedded_count == 0 and not emb_env["inserted"] diff --git a/tests/unit/tunable_constants.txt b/tests/unit/tunable_constants.txt index 737af69e9b..eae60243ee 100644 --- a/tests/unit/tunable_constants.txt +++ b/tests/unit/tunable_constants.txt @@ -430,7 +430,6 @@ server/memory/app/search/query_spelling.py::_MAX_DOUBLINGS server/memory/app/search_llm.py::_QUERY_ENTITY_CACHE_MAX server/memory/app/search_llm.py::_VERB_SYNONYM_CACHE_MAX server/memory/app/session_brief.py::_MAX_SOURCE_SESSIONS -server/memory/app/session_summary.py::_SESSION_SUMMARY_MAX_INPUT_CHARS server/memory/app/session_summary.py::_SESSION_SUMMARY_MAX_KEY_FACTS server/memory/app/set_chunks.py::_MAX_CHUNKS_FALLBACK server/memory/app/set_entities.py::_SET_ENTITY_MAX_CANDIDATES From 568b5618ea071dae106d812e578f73096f797829 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 19:46:14 +0530 Subject: [PATCH 2/5] Digest listings scan past withheld source facts Fill listing pages from proven sources so stale candidates cannot hide current facts or falsely report completeness. Bound keyset scans and refuse an unfinished scan; cover listing, outline and evidence export against the database. Signed-off-by: Lareb Zafar --- bench/conflict_resolver_scenario.py | 3 +- docs/architecture/memory-consent.md | 5 + server/memory/app/curation.py | 36 +++--- server/memory/app/evidence_transfer.py | 22 ++-- server/memory/app/period_digest.py | 7 +- server/memory/store/summary_pages.py | 74 +++++++++++ tests/analyzer/cases/workers.py | 39 ++++++ tests/analyzer/shapes.txt | 12 +- .../test_digest_listing_sources.py | 86 +++++++++++++ tests/unit/test_memory_digests_op.py | 39 +++++- tests/unit/test_memory_fence_publication.py | 2 - tests/unit/test_summary_pages.py | 116 ++++++++++++++++++ 12 files changed, 394 insertions(+), 47 deletions(-) create mode 100644 server/memory/store/summary_pages.py create mode 100644 tests/integration/test_digest_listing_sources.py create mode 100644 tests/unit/test_summary_pages.py diff --git a/bench/conflict_resolver_scenario.py b/bench/conflict_resolver_scenario.py index 34bc04adc2..d503fb11c8 100644 --- a/bench/conflict_resolver_scenario.py +++ b/bench/conflict_resolver_scenario.py @@ -196,8 +196,7 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: SESSIONS = 12 SOLO = {f.key for f in FACTS if f.at[0][0].startswith("solo")} -# The key deleted in a session, and the session whose Undo brings it back (after -# each session's writes). +# The key deleted in a session, and the later session that checks its continued absence. DELETE_IN, CHECK_DELETED_IN = {4: "badge"}, {6: "badge"} DELETED = set(DELETE_IN.values()) diff --git a/docs/architecture/memory-consent.md b/docs/architecture/memory-consent.md index e9879658d3..f8e0a126e8 100644 --- a/docs/architecture/memory-consent.md +++ b/docs/architecture/memory-consent.md @@ -89,6 +89,11 @@ from the authoritative current heads and withhold any digest whose source is mis changed, retired, expired, sensitive, incomplete or awaiting review. A metadata touch keeps an unchanged fact valid. A legacy digest without a complete proof is withheld. +Digest listing, outline and evidence export scan past withheld candidates to fill their +requested rows. Each query reads at most 256 candidates. After 256 full queries without +enough verified rows, the operation returns `BACKEND_UNAVAILABLE` rather than claiming +the listing is complete. + Period and contrast synthesis remain proposals. A model-generated restatement therefore waits for independent review before ordinary fact retrieval serves it. diff --git a/server/memory/app/curation.py b/server/memory/app/curation.py index b59081a182..ce454bc367 100644 --- a/server/memory/app/curation.py +++ b/server/memory/app/curation.py @@ -143,17 +143,11 @@ async def handle_memory_digests(input: MemoryDigestsInput) -> MemoryDigestsOutpu where += " AND session_id = {s:String}" params["s"] = input.session_id where += scope_and(request_scope(), params) - rows = await ch.query_all( - "SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, " - "valid_at, created_at, " - "version FROM loom.session_summaries FINAL WHERE " - + where - + " ORDER BY created_at, session_id LIMIT {lim:UInt32}", - params, - ) - from server.memory.store.summary_sources import current_summary_sources + from server.memory.store.summary_pages import verified_summary_page - rows = await current_summary_sources(ch, rows, org=input.org, namespace=input.namespace) + rows = await verified_summary_page( + ch, org=input.org, namespace=input.namespace, where=where, params=params, limit=input.limit + ) page = rows[: int(input.limit)] # Checking every digest's text is CPU work; it runs off the event loop. return MemoryDigestsOutput( @@ -209,13 +203,16 @@ async def handle_memory_outline(input: MemoryOutlineInput) -> MemoryOutlineOutpu sessions: list[_OutlineSession] = [] sessions_total = 0 if int(input.sessions_limit) > 0: - digests = await ch.query_all( - "SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, " - "length(key_facts) AS kf, length(derived_from) AS df, " - "coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL " - "WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL " - "ORDER BY d DESC, session_id LIMIT {lim:UInt32}", - {**base, "lim": int(input.sessions_limit) + 1}, + from server.memory.store.summary_pages import verified_summary_page + + digests = await verified_summary_page( + ch, + org=input.org, + namespace=input.namespace, + where="org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL", + params=base, + limit=input.sessions_limit, + order="newest", ) total = await ch.query_all( "SELECT count() AS c FROM loom.session_summaries FINAL " @@ -223,11 +220,6 @@ async def handle_memory_outline(input: MemoryOutlineInput) -> MemoryOutlineOutpu base, ) sessions_total = int((total or [{}])[0].get("c") or 0) - from server.memory.store.summary_sources import current_summary_sources - - digests = await current_summary_sources( - ch, digests, org=input.org, namespace=input.namespace - ) page = digests[: int(input.sessions_limit)] # Checking every digest's text is CPU work; it runs off the event loop. headlines = await asyncio.to_thread( diff --git a/server/memory/app/evidence_transfer.py b/server/memory/app/evidence_transfer.py index 96607eae9d..b4a1622ea7 100644 --- a/server/memory/app/evidence_transfer.py +++ b/server/memory/app/evidence_transfer.py @@ -134,21 +134,17 @@ async def handle_memory_export_evidence( params["sid"] = input.session_id if input.kind == "digests": - rows = await ch.query_all( - f""" - SELECT session_id, agent, user_id, summary_text, key_facts, derived_from, - valid_at, created_at - FROM loom.session_summaries FINAL - WHERE {" AND ".join(where)} - ORDER BY session_id ASC, version DESC - LIMIT 1 BY session_id - LIMIT {{lim:UInt32}} - """, + from server.memory.store.summary_pages import verified_summary_page + + rows = await verified_summary_page( + ch, + org=input.org, + namespace=input.namespace, + where=" AND ".join(where), params=params, + limit=input.limit, + order="session", ) - from server.memory.store.summary_sources import current_summary_sources - - rows = await current_summary_sources(ch, rows, org=input.org, namespace=input.namespace) digests = [ EvidenceDigest( session_id=str(r["session_id"]), diff --git a/server/memory/app/period_digest.py b/server/memory/app/period_digest.py index 3d05173f29..7be6f54e77 100644 --- a/server/memory/app/period_digest.py +++ b/server/memory/app/period_digest.py @@ -22,10 +22,9 @@ Under Postgres the authority's heads decide which periods are written, so one committed but not yet in ClickHouse is not written again. Nothing is tombstoned. -The rows are plain memories: they compete in the vector legs like any fact, -keep out of the session-digest leg (whose keys start with `summary:`) and -out of the digest chain, and read back through `memory.list` -and `memory.evidence` like every other derived row. +The rows are proposals. Independent approval admits them to ordinary fact retrieval; +explicit proposal reads can inspect them before approval. They keep out of the +session-digest leg (whose keys start with `summary:`) and out of the digest chain. """ from __future__ import annotations diff --git a/server/memory/store/summary_pages.py b/server/memory/store/summary_pages.py new file mode 100644 index 0000000000..688147b39d --- /dev/null +++ b/server/memory/store/summary_pages.py @@ -0,0 +1,74 @@ +"""Fill digest listings from eligible sources rather than raw candidate limits.""" + +from __future__ import annotations + +from typing import Any, Literal + +from server.core.ch import set_query_tag +from server.core.errors import BackendUnavailableError +from server.core.repo_base import _CHClient +from server.memory.store import summary_sources + +_PAGE_SIZE = 256 +_MAX_PAGES = 256 +Order = Literal["oldest", "newest", "session"] + + +def _ordering(order: Order) -> tuple[str, str, str]: + if order == "session": + return "session_id", "session_id, version DESC, agent, user_id, embedding_model", ">" + date = "d" if order == "newest" else "created_at" + key = f"({date}, session_id, agent, user_id, embedding_model)" + return key, key + (" DESC" if order == "newest" else " ASC"), "<" if order == "newest" else ">" + + +async def verified_summary_page( + ch: _CHClient, + *, + org: str, + namespace: str, + where: str, + params: dict[str, Any], + limit: int, + order: Order = "oldest", +) -> list[dict[str, Any]]: + """Return up to limit+1 proven rows; refuse an incomplete bounded scan.""" + key, ordering, comparison = _ordering(order) + after: Any = None + found: list[dict[str, Any]] = [] + for page in range(_MAX_PAGES): + set_query_tag("memory.verified_digest_page") + size = min(_PAGE_SIZE, limit + 1) if page == 0 else _PAGE_SIZE + cursor_type = ( + "String" if order == "session" else "Tuple(DateTime,String,String,String,String)" + ) + cursor = f" AND {key} {comparison} {{after:{cursor_type}}}" if after is not None else "" + rows = await ch.query_all( + "SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, " + "derived_from, valid_at, created_at, version, length(key_facts) AS kf, " + "length(derived_from) AS df, coalesce(valid_at, created_at) AS d " + "FROM loom.session_summaries FINAL WHERE " + + where + + cursor + + " ORDER BY " + + ordering + + (" LIMIT 1 BY session_id" if order == "session" else "") + + " LIMIT {lim:UInt32}", + params={**params, "lim": size, "after": after}, + ) + found.extend( + await summary_sources.current_summary_sources(ch, rows, org=org, namespace=namespace) + ) + if len(found) > limit or len(rows) < size: + return found[: limit + 1] + last = rows[-1] + after = ( + str(last["session_id"]) + if order == "session" + else ( + last["d" if order == "newest" else "created_at"], + *(str(last.get(c, "-")) for c in ("session_id", "agent", "user_id")), + str(last.get("embedding_model") or ""), + ) + ) + raise BackendUnavailableError("Digest source verification exceeded its scan budget") diff --git a/tests/analyzer/cases/workers.py b/tests/analyzer/cases/workers.py index 5fdfc76abd..b71bea0b03 100644 --- a/tests/analyzer/cases/workers.py +++ b/tests/analyzer/cases/workers.py @@ -275,6 +275,38 @@ async def _resolver_cluster(ch: RecordingCH, c: dict[str, Any]) -> None: ) +async def _summary_pages(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.memory.store.summary_pages import verified_summary_page + + await verified_summary_page( + ch, + org=ORG, + namespace=NS, + where="org={org:String} AND namespace={ns:String}", + params={"org": ORG, "ns": NS}, + limit=1, + order=c["order"], + ) + + +def _summary_page_rows(sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + if params.get("after") is not None: + return [] + return [ + dict( + agent="bot", + user_id="alice", + session_id="s1", + embedding_model="fake", + created_at=datetime(2026, 1, 1), + d=datetime(2026, 1, 1), + summary_text="legacy", + key_facts=[], + derived_from=[], + ) + ] * params["lim"] + + async def _period_sources(ch: RecordingCH, c: dict[str, Any]) -> None: """The facts a period's digests compressed: their ids and entities.""" from types import SimpleNamespace @@ -446,6 +478,13 @@ async def _refusal_publication(ch: RecordingCH, c: dict[str, Any]) -> None: def cases() -> list[Case]: return [ + Case( + "memory.summary_pages", + "server.memory.store.summary_pages", + _summary_pages, + {"order": ("oldest", "newest", "session")}, + rows=_summary_page_rows, + ), Case( "memory.refusal_publication", "server.memory.store.refusals", _refusal_publication, {} ), diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index a86c073dbf..b10e66d920 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -401,7 +401,6 @@ 2ed1e4c14bc5 SELECT path, scope, symbol_ids, kind, start_line, end_line, content, shingles, uniqExact(path) OVER () AS admitted_files FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chu 2ed709d1c719 SELECT count() AS n FROM loom.memory_belief_sources FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 2eeae7d3e832 SELECT path, scope, kind, start_line, end_line, content FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND -2f64af3a35d9 SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombston 2f87519d6c67 SELECT count() AS n FROM loom.memory_records FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 2fab97e447f4 SELECT count() AS n FROM loom.entity_vectors FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 2fc0cc735773 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a @@ -625,6 +624,7 @@ 47d6216caf2e SELECT number, state, author, title, labels, opened_at, closed_at, updated_at, files, title_hits, file_hit FROM ( SELECT number, state, author, title, labels, opened_at, closed_at, updated_at, files, 47db1deeabfd SELECT DISTINCT namespace FROM ( SELECT namespace FROM loom.memory WHERE org = {org:String} AND (namespace, agent, user_id, session_id, memory_key) IN ( SELECT namespace, agent, user_id, session_id, m 47e13d18aa4f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ +47ea41c2e2e8 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 482347c34a21 SELECT memory_key, groupUniqArray(?)(ent) AS shared FROM ( SELECT other.memory_key AS memory_key, other.entity_a AS ent FROM loom.entity_edge_memories AS other WHERE other.org = {o:String} AND other.n 48419f5696b5 SELECT count() AS n FROM loom.consolidation_runs WHERE org = {org:String} AND user_id = {uid:String} AND ? 4888e1579c2e SELECT count() AS n FROM loom.context_blobs FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} @@ -762,6 +762,7 @@ 5852405dd8a6 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom 5862d4ef7b73 ALTER TABLE loom.session_brief_revisions DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 586a6ce18c21 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user +58cdc04fc385 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 58d8e597b166 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 58dd61abeb39 WITH m AS ( SELECT org, namespace, agent, user_id, session_id, memory_key, argMax(status, version) AS ms, argMax(memory_id, version) AS mid FROM loom.memory WHERE org = {org:String} AND namespace = {n 58f4b29e4e3a SELECT org, namespace, policy_id, effect, description, actions, condition, enabled, mode, created_by, created_at, updated_at, tombstoned_at FROM loom.namespace_policies FINAL WHERE org = {org:String} @@ -1200,6 +1201,7 @@ 89f40a73fa83 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, version, value, toString(value_json) AS object_text FROM loom.memory FINAL WHERE tombstoned_at IS NULL AND sensitivity = ? AND 8a1fca063e7f SELECT org, agent, count() AS calls, countIf(if(consumer = ?, args[?], consumer) NOT IN {untagged:Array(String)}) AS tagged, calls - tagged AS untagged, round(tagged / calls, ?) AS tagged_ratio, anyIf 8a44f1b04017 SELECT count() AS n FROM loom.entity_triple_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND memory_id NOT IN (SELECT memory_id FROM loom.memory FINAL WHERE org = {org:String} AND +8a5fb93b3273 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 8a725f267f59 SELECT count() AS n FROM loom.memory_reinforcement WHERE org = {org:String} AND namespace = {ns:String} AND ? 8a8b47f8466b SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 8a8bd6cc9556 SELECT toString(memory_id) AS memory_id, org, namespace, agent, user_id, session_id, memory_key, value, actor, version, redacted_at, toString(coalesce(valid_at, updated_at)) AS since FROM loom.memory @@ -1241,6 +1243,7 @@ 8ea78684e9ed SELECT entities, sensitivity FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {uid:String} AND session_id = {sid:String} AND memory_ 8ee4f33d0c2f SELECT count() AS n FROM loom.entity_summaries FINAL WHERE org = {org:String} AND namespace = {ns:String} 8ee5e12ce3f8 SELECT memory_type, retention_days, dedup_threshold, hot_quota, conflict_policy FROM loom.namespace_types FINAL WHERE org = {org:String} AND namespace = {ns:String} ORDER BY memory_type +8f11758986c7 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 8f5abdba34be SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 8fa56eef0912 SELECT count() AS n FROM loom.task_events WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 8fabc8412cfc SELECT subject, role, granted_by FROM loom.namespace_grants FINAL WHERE org = {org:String} AND namespace = {ns:String} AND revoked_at IS NULL AND status = ? @@ -1301,6 +1304,7 @@ 9648988c847c SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} 96a4e124aaa7 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 96cd9708162d SELECT count() AS n FROM loom.memory_reinforcement WHERE org = {org:String} AND user_id = {uid:String} +96e6221f7f9f SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 96ecb876ec8c SELECT * FROM loom.scim_users FINAL WHERE id = {id:String} 971a9c534152 SELECT entity, substring(summary_text, ?, ?) AS content_excerpt, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- alias must NOT be `valid_from`: it would shadow the base column - 9761b739c189 SELECT m.org AS org, m.namespace AS namespace, m.user_id AS user_id, count() AS n FROM loom.memory AS m FINAL INNER JOIN loom.namespaces AS n FINAL ON m.org = n.org AND m.namespace = n.namespace WHERE @@ -1671,7 +1675,6 @@ bccb3b408622 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = { bcd0d6a01148 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND namespace IN {ns:Array(String)} bcf5fd2c0231 SELECT count() AS n FROM ( SELECT evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND hasAnyTokens(content_lc, {tokens:Array(String)}) ORDER B bd095b55bf5d SELECT count() AS n FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND session_id = {s:String} AND memory_key = {k:Str -bd1ff98ced0a SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL WHE bd4266cbfe15 SELECT count() AS n FROM loom.conflict_allowlist FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} bda43b1f632f WITH arrayConcat(splitByNonAlpha(context_lc), splitByNonAlpha(content_lc)) AS words SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, cosineDistance bdde77a5c044 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni @@ -1733,6 +1736,7 @@ c46246e37ab2 SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String c464d4b1053c SELECT count() AS n FROM loom.knowledge_uploads WHERE org = {org:String} AND user_id = {uid:String} AND ? c466198a7fda WITH (SELECT toFloat64(count()) FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL) AS n_keys, (SELECT mapFromArrays(groupArray(s), groupA c4831b86cc45 SELECT now64(?) AS now, groupArray((token, started_at, finished)) AS keys FROM loom.graph_projections WHERE token IN {tokens:Array(String)} +c48726998045 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create c4cee4797736 SELECT count() AS sources, countIf(j.has_document = ? AND NOT j.replaced) AS without_document, countIf(j.replaced) AS replaced, sum(j.pending) AS pending, sum(j.processing) AS processing, sum(j.ready) c4d11f8d928a SELECT count() AS n FROM loom.memory_record_fields FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} c4d8e27d9015 SELECT tier, qualified_name, symbol_id, name, path, start_line, end_line FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespace = {ns:String} AND (tier, qualified_name, symbol_id) IN ( SE @@ -2019,6 +2023,7 @@ e094b0459706 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = e0a2fdcb10f9 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND namespace = {ns:String} AND ? e0a394a9b663 SELECT pattern_id, org, namespace, pattern, pattern_type, scope, reason, created_by, hit_count, last_hit_at, created_at FROM loom.blacklist FINAL WHERE tombstoned_at IS NULL AND ((org = {org:String} A e0ce2526325b SELECT count() AS n FROM loom.operator_edges FINAL WHERE org = {org:String} AND namespace = {ns:String} +e0f548d3f7a8 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e0f90e7e0243 ALTER TABLE loom.memory_embeddings UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_ e107d944ecff SELECT n.namespace AS namespace, n.description AS description, n.kind AS kind, n.owner_email AS owner_email, n.requires_approval AS requires_approval, toString(n.visibility) AS visibility, toString(n. e10df0e1e1d9 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} @@ -2071,6 +2076,7 @@ e5d3479fff14 SELECT agent, user_id, session_id, memory_key, cosineDistance(embed e5df581175ec SELECT count() AS n FROM loom.knowledge_gaps WHERE org = {org:String} AND namespace IN {ns:Array(String)} e5e482e9d615 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g e60347067264 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, +e62faca61816 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e67a89c4f6c6 SELECT group_id, members FROM loom.principal_groups FINAL WHERE org = {org:String} AND NOT deleted e68412b65969 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value e6990fcc8c82 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam @@ -2083,6 +2089,7 @@ e77afbb3fb28 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND n e77b590b9f54 SELECT org, namespace, name, source_id, provider, generation, fingerprint, provider_id, status, secret_enc, agent, user_id, session_id, actor FROM loom.knowledge_source_push FINAL WHERE org = {org:Str e77c1e05f3c4 SELECT count() AS n FROM loom.blob_uploads WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? e7d4c583cc0e ALTER TABLE loom.procedure_runs DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? +e7ef19d41a90 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e7f1d693d5b1 SELECT count() AS n FROM loom.entity_vectors WHERE org = {org:String} AND (user_id = {uid:String} OR (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) e8009bd53bc4 SELECT number, head_sha FROM loom.code_gh_items FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND is_pr AND state = ? AND head_sha != ? ORDER BY updated_at DESC, e82345a88eca SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -2287,4 +2294,3 @@ ff438b330e4f SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} ffa5c94364aa SELECT DISTINCT namespace FROM ( SELECT namespace FROM loom.memory WHERE org = {org:String} AND (namespace, agent, user_id, session_id, memory_key) IN ( SELECT namespace, agent, user_id, session_id, m ffa9b9e1eddc SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU ffbcd8654cca SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value -ffd2717a1e1a SELECT agent, user_id, session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombston diff --git a/tests/integration/test_digest_listing_sources.py b/tests/integration/test_digest_listing_sources.py new file mode 100644 index 0000000000..67c2b7a7d4 --- /dev/null +++ b/tests/integration/test_digest_listing_sources.py @@ -0,0 +1,86 @@ +"""Stale digest candidates cannot hide a later page of proven facts.""" + +from __future__ import annotations + +from datetime import timedelta + +import httpx +import pytest + +from server.core.datetimes import naive_utc_now +from tests.integration._resources import unique_org + + +@pytest.mark.parametrize("op", ["memory.digests", "memory.outline", "memory.export_evidence"]) +async def test_digest_listings_scan_past_withheld_sources( + control_plane_url, mint_test_token, ch_client, op +): + org = unique_org("digest-pages") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = dict(org=org, namespace="facts", agent="a", user_id="subject") + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for i in (2, 3): + response = await client.post( + "/v1/memory.set", + json={**identity, "session_id": f"s{i}", "memory_key": "k", "value": f"fact {i}"}, + ) + assert response.status_code == 200, response.text + now = naive_utc_now() + columns = [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "embedding_model", + "summary_text", + "key_facts", + "derived_from", + "created_at", + "version", + "summary_embedding", + ] + ch_client.insert( + "loom.session_summaries", + [ + [ + org, + "facts", + "a", + "subject", + f"s{i}", + "test", + f"fact {i}", + [f"fact {i}"], + ["k"] if i >= 2 else [], + now + timedelta(seconds=-i if op == "memory.outline" else i), + 1, + [0.0] * 3072, + ] + for i in range(4) + ], + column_names=columns, + settings={"async_insert": 0}, + ) + fields = ( + {"sessions_limit": 1, "hubs_limit": 0, "communities_limit": 0} + if op == "memory.outline" + else {"limit": 1} + ) + if op == "memory.export_evidence": + fields["kind"] = "digests" + response = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert response.status_code == 200, response.text + result = response.json() + field = { + "memory.digests": "rows", + "memory.outline": "sessions", + "memory.export_evidence": "digests", + }[op] + assert [row["session_id"] for row in result[field]] == ["s2"], result + if op == "memory.digests": + assert result["truncated"] is True + if op == "memory.export_evidence": + assert result["has_more"] is True diff --git a/tests/unit/test_memory_digests_op.py b/tests/unit/test_memory_digests_op.py index eed7a67bec..4e9fd0f213 100644 --- a/tests/unit/test_memory_digests_op.py +++ b/tests/unit/test_memory_digests_op.py @@ -21,7 +21,21 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: self.sql, self.params = sql, dict(params or {}) - return self.rows[: int(self.params["lim"])] + rows = self.rows + if after := self.params.get("after"): + rows = [ + row + for row in rows + if ( + row["created_at"], + row["session_id"], + row.get("agent", "-"), + row.get("user_id", "-"), + row.get("embedding_model", ""), + ) + > after + ] + return rows[: int(self.params["lim"])] def _wire_sources(monkeypatch, rows): @@ -105,3 +119,26 @@ async def test_a_digest_that_reads_as_a_credential_is_withheld( monkeypatch.setattr("server.core.ch.get_client", lambda: _FakeCH(rows)) out = await cur.handle_memory_digests(MemoryDigestsInput(org="o", namespace="n", agent="a")) assert [r.summary_text for r in out.rows] == ["Planned the retreat."] + + +@pytest.mark.parametrize("withheld", [2, 270]) +@pytest.mark.parametrize("qualified", [1, 2]) +async def test_withheld_pages_do_not_hide_current_digests(monkeypatch, withheld, qualified): + rows = [ + { + "session_id": f"s{i:04}", + "agent": "a", + "user_id": "-", + "embedding_model": "test", + "created_at": datetime(2026, 1, 1), + "summary_text": f"fact {i}", + } + for i in range(withheld + qualified) + ] + _wire_sources(monkeypatch, rows[withheld:]) + monkeypatch.setattr("server.core.ch.get_client", lambda: _FakeCH(rows)) + out = await cur.handle_memory_digests( + MemoryDigestsInput(org="o", namespace="n", agent="a", limit=1) + ) + assert [row.session_id for row in out.rows] == [rows[withheld]["session_id"]] + assert out.truncated is (qualified > 1) diff --git a/tests/unit/test_memory_fence_publication.py b/tests/unit/test_memory_fence_publication.py index ffaa4567ec..93699cdf66 100644 --- a/tests/unit/test_memory_fence_publication.py +++ b/tests/unit/test_memory_fence_publication.py @@ -409,8 +409,6 @@ async def insert(self, table, batch, **kw): raise RuntimeError("the server's insert answer was lost") async def query_all(self, sql, params): - if "FROM loom.memory_refusals" in sql: - return [] if "FROM loom.memory_refusals" in sql: return [] if "PREWHERE (org, namespace" in sql: diff --git a/tests/unit/test_summary_pages.py b/tests/unit/test_summary_pages.py new file mode 100644 index 0000000000..631d7e9ecc --- /dev/null +++ b/tests/unit/test_summary_pages.py @@ -0,0 +1,116 @@ +"""Source filtering cannot make an unfinished digest scan look complete.""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + +from server.core.errors import BackendUnavailableError +from server.memory.store import summary_pages +from server.memory.store.rows import identity_key +from tests.unit.test_memory_verify import _row + + +class Pages: + def __init__(self, rows, order): + self.rows, self.order, self.calls = rows, order, [] + + def key(self, row): + if self.order == "session": + return row["session_id"] + return ( + row["created_at"], + row["session_id"], + row["agent"], + row["user_id"], + row["embedding_model"], + ) + + async def query_all(self, sql, params): + self.calls.append((sql, params)) + assert "org={o:String}" in sql and params["o"] == "o" + assert params["uid"] == "subject" and "user_id={uid:String}" in sql + rows = self.rows + if params["after"] is not None: + assert "{after:" in sql + rows = [ + row + for row in rows + if ( + self.key(row) < params["after"] + if self.order == "newest" + else self.key(row) > params["after"] + ) + ] + return rows[: params["lim"]] + + +async def listing(ch, limit=1): + return await summary_pages.verified_summary_page( + ch, + org="o", + namespace="n", + limit=limit, + order=ch.order, + where="org={o:String} AND namespace={n:String} AND user_id={uid:String}", + params={"o": "o", "n": "n", "uid": "subject"}, + ) + + +def candidates(count, order): + rows = [ + dict( + agent="a", + user_id="subject", + session_id=f"s{i:04}", + embedding_model="test", + created_at=datetime(2026, 1, 1), + d=datetime(2026, 1, 1), + summary_text=f"fact {i}", + key_facts=[f"fact {i}"], + derived_from=["k"], + ) + for i in range(count) + ] + return rows[::-1] if order == "newest" else rows + + +@pytest.mark.parametrize("order", ["oldest", "newest", "session"]) +@pytest.mark.parametrize("withheld", [2, 270]) +@pytest.mark.parametrize("qualified", [0, 1, 2]) +async def test_pages_fill_from_current_sources(monkeypatch, order, withheld, qualified): + rows = candidates(withheld + qualified, order) + heads = [ + _row( + namespace="n", + agent="a", + user_id="subject", + session_id=row["session_id"], + memory_key="k", + value=row["summary_text"], + ) + for row in rows[withheld:] + ] + + async def read(keys, **kwargs): + return {identity_key(row): row for row in heads if identity_key(row) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + ch = Pages(rows, order) + result = await listing(ch) + assert result == rows[withheld:] + assert len(ch.calls) >= 2 + if order == "session": + assert all("LIMIT 1 BY session_id" in sql for sql, _ in ch.calls) + + +async def test_scan_budget_refuses_instead_of_reporting_no_more_digests(monkeypatch): + monkeypatch.setattr(summary_pages, "_MAX_PAGES", 2) + rows = candidates(300, "oldest") + for row in rows: + row["derived_from"] = [] + ch = Pages(rows, "oldest") + with pytest.raises(BackendUnavailableError, match="scan budget"): + await listing(ch) + assert len(ch.calls) == 2 From 56c653aa379abbeba28f65b4bccba6a53318da54 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 19:56:21 +0530 Subject: [PATCH 3/5] Period synthesis verifies each selected session digest Select one latest live digest per session before applying the page cursor, and retain qualification per row so another model cannot admit stale text. Expose listing scan bounds as settings instead of hidden tuning constants. Signed-off-by: Lareb Zafar --- docs/architecture/memory-consent.md | 11 +++- docs/reference/configuration.md | 2 + loom.example.yaml | 8 +++ schemas/loom-settings.schema.json | 13 ++++ server/memory/app/period_digest.py | 19 +++--- server/memory/store/summary_pages.py | 9 +-- server/settings/catalog.py | 4 ++ tests/analyzer/shapes.txt | 4 +- .../test_digest_listing_sources.py | 59 +++++++++++++++++++ tests/unit/test_period_digest.py | 23 +++++++- tests/unit/test_summary_pages.py | 2 +- tests/unit/test_summary_sources.py | 20 +++++++ 12 files changed, 153 insertions(+), 21 deletions(-) diff --git a/docs/architecture/memory-consent.md b/docs/architecture/memory-consent.md index f8e0a126e8..ee563a01b4 100644 --- a/docs/architecture/memory-consent.md +++ b/docs/architecture/memory-consent.md @@ -90,9 +90,14 @@ changed, retired, expired, sensitive, incomplete or awaiting review. A metadata keeps an unchanged fact valid. A legacy digest without a complete proof is withheld. Digest listing, outline and evidence export scan past withheld candidates to fill their -requested rows. Each query reads at most 256 candidates. After 256 full queries without -enough verified rows, the operation returns `BACKEND_UNAVAILABLE` rather than claiming -the listing is complete. +requested rows. `memory_digest_listing_page_size` bounds candidates per query and +`memory_digest_listing_max_pages` bounds queries per call; both default to 256. If that +budget cannot find enough verified rows or prove exhaustion, the operation returns +`BACKEND_UNAVAILABLE` rather than claiming the listing is complete. + +Period planning selects the newest live digest per session before pagination. Source +verification applies to each digest row; a valid row cannot admit another model's stale +digest from the same session. Period and contrast synthesis remain proposals. A model-generated restatement therefore waits for independent review before ordinary fact retrieval serves it. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index cafd4edc51..78adae3700 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1105,6 +1105,8 @@ The memory plane outside search/ask/extraction: writes, conflicts, lifecycle, wo | `LOOM_MEMORY_REINFORCEMENT_CLOSER_CACHE_SECONDS` | `int` | `300` | | Seconds one (org, consumer) closer answer is cached per worker. | | `LOOM_MEMORY_LIST_VIEWER_LIMIT` | `int` | `100` | | memory.list rows per call for a token below writer on the namespace; it pages with `offset` (wire cap stays 1000). 0 = no viewer cap. | | `LOOM_MEMORY_SESSION_DIGEST_MAX_INPUT_CHARS` | `int` | `3000` | | Characters of complete source facts included in an extractive session digest; 0 = the built-in 3000. Facts are never clipped to fit. | +| `LOOM_MEMORY_DIGEST_LISTING_PAGE_SIZE` | `int` | `256` | | Raw digest candidates per query while filling listing, outline and export rows. | +| `LOOM_MEMORY_DIGEST_LISTING_MAX_PAGES` | `int` | `256` | | Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE. | | `LOOM_MEMORY_STRUCTURED_ANSWER_EXCERPT_LIMIT` | `int` | `4096` | | content_excerpt ceiling for answer-bearing facts (question_topic set) when memory_capture_structured_answers is on, so an enumerated answer is not clipped mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). | | `LOOM_MEMORY_VALUE_ADVISORY_CHARS` | `int` | `30000` | | A searchable memory.set value serializing longer than this many chars gets one advisory line (the write still lands): only the first EXCERPT_LIMIT chars are embedded, and past the embedder's 8,191-token cap nothing is. 30,000 chars ≈ 8k tokens. 0 disables. | diff --git a/loom.example.yaml b/loom.example.yaml index 55bec8b0d5..26ef156602 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -3241,6 +3241,14 @@ settings: # the built-in 3000. Facts are never clipped to fit. [restart] # memory_session_digest_max_input_chars: 3000 + # Raw digest candidates per query while filling listing, outline and export rows. + # [restart] + # memory_digest_listing_page_size: 256 + + # Queries allowed to find enough proven digests; exhaustion answers + # BACKEND_UNAVAILABLE. [restart] + # memory_digest_listing_max_pages: 256 + # content_excerpt ceiling for answer-bearing facts (question_topic set) when # memory_capture_structured_answers is on, so an enumerated answer is not clipped # mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 52d15f0da0..c40a02f2c9 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -4593,6 +4593,19 @@ "type": "integer", "description": "Characters one extraction read takes: a shorter document is read whole, a longer one in windows of whole sections. Sized so a dense window's reply fits the decision_extract cap." }, + "memory_digest_listing_max_pages": { + "default": 256, + "minimum": 1, + "type": "integer", + "description": "Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE." + }, + "memory_digest_listing_page_size": { + "default": 256, + "maximum": 2000, + "minimum": 1, + "type": "integer", + "description": "Raw digest candidates per query while filling listing, outline and export rows." + }, "memory_durability_verifier_max_facts_per_call": { "default": 60, "minimum": 1, diff --git a/server/memory/app/period_digest.py b/server/memory/app/period_digest.py index 7be6f54e77..e4cd012257 100644 --- a/server/memory/app/period_digest.py +++ b/server/memory/app/period_digest.py @@ -351,22 +351,21 @@ async def _fetch_digests( page_size: int = 256, after: tuple[datetime, str] | None = None, ) -> AsyncIterator[list[dict[str, Any]]]: - """Stream ordered raw windows without retaining the identity's whole history.""" + """Page one latest live digest per session before cutting period windows.""" from server.core.ch import set_query_tag while True: set_query_tag("cli.period_digest.sources") - cursor = ( - " AND (coalesce(valid_at, created_at), session_id) > {after:Tuple(DateTime,String)}" - if after - else "" - ) + cursor = " WHERE (at, session_id) > {after:Tuple(DateTime,String)}" if after else "" rows = await ch.query_all( - "SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, " - "derived_from, coalesce(valid_at, created_at) AS at " + "SELECT org, namespace, agent, user_id, session_id, embedding_model, version, " + "summary_text, key_facts, derived_from, at FROM (" + "SELECT org, namespace, agent, user_id, session_id, embedding_model, version, " + "summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at " "FROM loom.session_summaries FINAL " "WHERE org={org:String} AND namespace={ns:String} " "AND agent={agent:String} AND user_id={uid:String} AND tombstoned_at IS NULL " + "ORDER BY session_id, version DESC, embedding_model LIMIT 1 BY session_id)" + cursor + " ORDER BY at, session_id LIMIT {limit:UInt32}", params={ @@ -652,8 +651,8 @@ async def _plan_identity( org=ident["org"], namespace=ident["namespace"], ) - allowed = {str(d["session_id"]) for d in qualified} - verified = [p for p in writable if all(str(d["session_id"]) in allowed for d in p.digests)] + allowed = {id(d) for d in qualified} + verified = [p for p in writable if all(id(d) in allowed for d in p.digests)] return verified[: max(0, cfg.max_periods_per_identity)], skipped diff --git a/server/memory/store/summary_pages.py b/server/memory/store/summary_pages.py index 688147b39d..71d1cdb74d 100644 --- a/server/memory/store/summary_pages.py +++ b/server/memory/store/summary_pages.py @@ -8,9 +8,8 @@ from server.core.errors import BackendUnavailableError from server.core.repo_base import _CHClient from server.memory.store import summary_sources +from server.settings import settings -_PAGE_SIZE = 256 -_MAX_PAGES = 256 Order = Literal["oldest", "newest", "session"] @@ -36,9 +35,11 @@ async def verified_summary_page( key, ordering, comparison = _ordering(order) after: Any = None found: list[dict[str, Any]] = [] - for page in range(_MAX_PAGES): + for page in range(settings.memory_digest_listing_max_pages): set_query_tag("memory.verified_digest_page") - size = min(_PAGE_SIZE, limit + 1) if page == 0 else _PAGE_SIZE + size = settings.memory_digest_listing_page_size + if page == 0: + size = min(size, limit + 1) cursor_type = ( "String" if order == "session" else "Tuple(DateTime,String,String,String,String)" ) diff --git a/server/settings/catalog.py b/server/settings/catalog.py index d1734acb01..c775f0a266 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -2783,6 +2783,10 @@ class MemorySettings(BaseModel): # Characters of complete source facts included in an extractive session digest; # 0 = the built-in 3000. Facts are never clipped to fit. memory_session_digest_max_input_chars: int = 3000 + # Raw digest candidates per query while filling listing, outline and export rows. + memory_digest_listing_page_size: int = Field(default=256, ge=1, le=2000) + # Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE. + memory_digest_listing_max_pages: int = Field(default=256, ge=1) # content_excerpt ceiling for answer-bearing facts (question_topic set) # when memory_capture_structured_answers is on, so an enumerated answer # is not clipped mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index b10e66d920..997dd957e5 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -507,7 +507,6 @@ 3b05a28d96f2 SELECT count() AS cnt, sum(numeric_value) AS s, avg(numeric_value) AS a, min(occurred_at) AS first_at, max(occurred_at) AS last_at FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns: 3b1abae99635 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 3b1ae278b7f6 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { -3b32757af980 SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at FROM loom.session_summaries FINAL WHERE org={org:String} AND namespace={n 3b4b4580cc9a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 3ba235828531 ALTER TABLE loom.documents UPDATE superseded_at = {now:DateTime} WHERE (org = {org:String} AND user_id != {uid:String} AND superseded_at IS NULL AND cityHash64(namespace, source_uri) IN {docs:Array(UI 3bab9ca71efb SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND ((user_id = {uid:String} OR (user_id = ? AND (namespace, parent_memory_key) IN (SELECT namespace, memory_key FROM loom.me @@ -1125,6 +1124,7 @@ 80615d70d925 SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND namespace = {ns:String} AND ? 809ffd3a9a82 SELECT tag, count() AS rows FROM (SELECT arrayJoin(tags) AS tag FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL) GROUP BY tag HAVING rows >= {min: 80ac0a0e1397 SELECT toYYYYMM(ts) AS partition, ts, event_type, org, namespace, actor, subject, request_id, row_hash FROM loom.audit_log WHERE ts >= toDateTime64(now() - INTERVAL {lb:UInt32} SECOND, ?, ?) AND (ts, +80cfa2f097df SELECT org, namespace, agent, user_id, session_id, embedding_model, version, summary_text, key_facts, derived_from, at FROM (SELECT org, namespace, agent, user_id, session_id, embedding_model, version 8107af48e1df SELECT count() AS n FROM ( SELECT evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND user_id = {uid:String} AND session_id = {sid:String} AND 81147eeb107a SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND namespace = {ns:String} 812c33d11f88 ALTER TABLE loom.memory_projection_failures DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations @@ -1651,7 +1651,6 @@ ba28d3f48cd1 SELECT ts, org, namespace, agent, user_id, session_id, operation, r ba3aa58d4a58 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g ba505cefcdfd SELECT subject, namespace, role, granted_at, pending_role, is_owner FROM ( SELECT subject, namespace, toString(role) AS role, granted_at, pending_role, ? AS is_owner FROM loom.namespace_grants FINAL W ba7611490c4d ALTER TABLE loom.search_misses DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -ba79c569a2f7 SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at FROM loom.session_summaries FINAL WHERE org={org:String} AND namespace={n bb02b0e3dee7 SELECT symbol_id, qualified_name, name, signature, path, start_line, end_line, tier, kind FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND bb04945353ff SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g bb4566fa0eb8 SELECT namespace, toString(owner_email) AS owner, active, tags, password_hash = ? AS sso FROM loom.namespaces FINAL WHERE org = {org:String} AND contact_email = {ident:String} @@ -2018,6 +2017,7 @@ dfab46389b75 SELECT count() AS n FROM loom.memory_record_versions WHERE org = {o dfb389c3eaaa SELECT count() AS n FROM loom.memory_records FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL e01b3f138368 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:String} AND hasAny(arrayConcat(window_keys, cited_keys), {keys:Array(String)})) AND ({erasure_mutation_tag:String} = {erasure_mutation_t e026c7f5b6eb SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { +e05f46166d69 SELECT org, namespace, agent, user_id, session_id, embedding_model, version, summary_text, key_facts, derived_from, at FROM (SELECT org, namespace, agent, user_id, session_id, embedding_model, version e08e70804055 SELECT version, toStringCutToZero(content_sha256) AS sha FROM loom.memory_versions WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {u:String} AND session_ e094b0459706 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND user_id = {uid:String} e0a2fdcb10f9 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND namespace = {ns:String} AND ? diff --git a/tests/integration/test_digest_listing_sources.py b/tests/integration/test_digest_listing_sources.py index 67c2b7a7d4..e8ff023b3d 100644 --- a/tests/integration/test_digest_listing_sources.py +++ b/tests/integration/test_digest_listing_sources.py @@ -84,3 +84,62 @@ async def test_digest_listings_scan_past_withheld_sources( assert result["truncated"] is True if op == "memory.export_evidence": assert result["has_more"] is True + + +@pytest.mark.parametrize("page_size", [1, 2]) +async def test_period_pages_choose_latest_model_before_applying_the_cursor(ch_client, page_size): + from server.core.ch import get_client + from server.memory.app.period_digest import _fetch_digests + + org, now = unique_org("period-pages"), naive_utc_now() + columns = [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "embedding_model", + "summary_text", + "key_facts", + "derived_from", + "created_at", + "version", + "summary_embedding", + ] + rows = [ + [ + org, + "facts", + "a", + "subject", + sid, + model, + text, + [text], + ["k"], + now + timedelta(seconds=offset), + version, + [0.0] * 3072, + ] + for sid, model, text, offset, version in ( + ("s1", "old", "retired fact", 10, 1), + ("s1", "middle", "intermediate fact", 5, 2), + ("s1", "current", "current fact", 0, 3), + ("s2", "current", "second fact", 1, 1), + ("s3", "current", "third fact", 2, 1), + ) + ] + ch_client.insert( + "loom.session_summaries", rows, column_names=columns, settings={"async_insert": 0} + ) + ch = get_client() + await ch.connect() + try: + identity = dict(org=org, namespace="facts", agent="a", user_id="subject") + found = [ + row async for page in _fetch_digests(ch, identity, page_size=page_size) for row in page + ] + finally: + await ch.disconnect() + assert [row["session_id"] for row in found] == ["s1", "s2", "s3"] + assert found[0]["summary_text"] == "current fact" and found[0]["version"] == 3 diff --git a/tests/unit/test_period_digest.py b/tests/unit/test_period_digest.py index 696227e3a1..e01bfd4809 100644 --- a/tests/unit/test_period_digest.py +++ b/tests/unit/test_period_digest.py @@ -173,7 +173,15 @@ async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-", "sessions": sessions} ] if "FROM loom.session_summaries FINAL" in sql: - ordered = sorted(self._digests, key=lambda d: (d["at"], d["session_id"])) + rows = self._digests + if "LIMIT 1 BY session_id" in sql: + latest = {} + for row in sorted( + rows, key=lambda d: (-d.get("version", 1), d.get("embedding_model", "")) + ): + latest.setdefault(row["session_id"], row) + rows = list(latest.values()) + ordered = sorted(rows, key=lambda d: (d["at"], d["session_id"])) after = params.get("after") if after: ordered = [d for d in ordered if (d["at"], d["session_id"]) > after] @@ -733,6 +741,19 @@ async def test_unqualified_session_text_never_reaches_the_period_model(monkeypat qualification.assert_awaited_once() +async def test_period_paging_selects_one_latest_digest_per_session_before_its_cursor(): + from server.memory.app.period_digest import _fetch_digests + + rows = [dict(_digest(1), embedding_model=f"m{i}", version=i) for i in (1, 2, 3)] + rows.extend(_digests(2, 3)) + identity = {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"} + found = [ + row async for page in _fetch_digests(_FakeCH(rows), identity, page_size=2) for row in page + ] + assert [row["session_id"] for row in found] == [f"ns:session_{i}" for i in (1, 2, 3)] + assert found[0]["embedding_model"] == "m3" + + @pytest.mark.asyncio async def test_streamed_windows_skip_old_keys_before_proof_and_reach_later_pages(monkeypatch): from datetime import timedelta diff --git a/tests/unit/test_summary_pages.py b/tests/unit/test_summary_pages.py index 631d7e9ecc..10e75e1c6d 100644 --- a/tests/unit/test_summary_pages.py +++ b/tests/unit/test_summary_pages.py @@ -106,7 +106,7 @@ async def read(keys, **kwargs): async def test_scan_budget_refuses_instead_of_reporting_no_more_digests(monkeypatch): - monkeypatch.setattr(summary_pages, "_MAX_PAGES", 2) + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_max_pages", 2) rows = candidates(300, "oldest") for row in rows: row["derived_from"] = [] diff --git a/tests/unit/test_summary_sources.py b/tests/unit/test_summary_sources.py index 64ddd41c84..12dc71a5f1 100644 --- a/tests/unit/test_summary_sources.py +++ b/tests/unit/test_summary_sources.py @@ -101,6 +101,26 @@ async def test_a_missing_source_and_a_forged_sentence_do_not_hide_a_valid_neighb ) == [valid] +async def test_a_current_model_digest_does_not_qualify_a_stale_model_row(sources): + from server.memory.app.period_digest import PeriodDigestConfig, _plan_identity + + source = sources["source"] = sources["source"].model_copy(update={"session_id": "session"}) + scope = {c: getattr(source, c) for c in ("org", "namespace")} + stale = digest( + source, + **scope, + embedding_model="old", + summary_text="retired fact", + key_facts=["retired fact"], + ) + current = digest(source, **scope, embedding_model="current") + ch = SimpleNamespace(query_all=AsyncMock(return_value=[])) + identity = {c: getattr(source, c) for c in ("org", "namespace", "agent", "user_id")} + assert await _plan_identity( + ch, [stale, current], identity, PeriodDigestConfig(sessions_per_period=2) + ) == ([], 0) + + async def test_source_state_failure_withholds_the_summary(monkeypatch: Any) -> None: source = _row(value="current fact", memory_key="source") From eb704a010dea9e9280758c8781aaa7da9130e4fd Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 20:18:48 +0530 Subject: [PATCH 4/5] Native publication orders canonical rows before their vectors Both content sends acquire the same consent fences, so concurrent acquisition can split their claims and stall the flush. Keep audit publication concurrent and preserve independent receipts after a failed canonical send. Pin the new listing settings and cover their page size. Signed-off-by: Lareb Zafar --- docs/architecture/memory-consent.md | 5 ++++ server/memory/store/pg_batch.py | 10 +++++-- tests/unit/snapshots/settings_defaults.json | 2 ++ tests/unit/test_delete_version_sql.py | 6 +--- tests/unit/test_digest_chain.py | 4 +-- .../test_extract_concurrent_projections.py | 10 ++----- tests/unit/test_extraction_queue.py | 1 - tests/unit/test_pg_batch.py | 28 +++++++++++++++++++ tests/unit/test_session_summaries.py | 6 ++-- .../test_shared_and_documents_list_repo.py | 6 +--- tests/unit/test_summary_pages.py | 5 +++- 11 files changed, 56 insertions(+), 27 deletions(-) diff --git a/docs/architecture/memory-consent.md b/docs/architecture/memory-consent.md index ee563a01b4..aa6af666ab 100644 --- a/docs/architecture/memory-consent.md +++ b/docs/architecture/memory-consent.md @@ -54,6 +54,11 @@ owed for another pass. Receipts match the complete identity and published versio a newer metadata copy settles its original revision's delivery. Copy follow-ups confirm the exact document in ClickHouse before settling debt. +Each native publication batch finishes its canonical send before sending its vectors. +Both sends acquire the same consent fences; ordering them prevents competing partial +claims from blocking the batch. Audit publication runs alongside them. A failed canonical +send still permits the independently checked vector send and preserves each target's debt. + Shared canonical retirement holds admission through its final send. It checks the promoted source identities against erasure policy and requires the shared head to remain present. Missing heads, erased content and unproven legacy sources are removed below the diff --git a/server/memory/store/pg_batch.py b/server/memory/store/pg_batch.py index 98ecc28ce7..b384a0f5f3 100644 --- a/server/memory/store/pg_batch.py +++ b/server/memory/store/pg_batch.py @@ -438,12 +438,18 @@ async def _publish(group: list[_Entry]) -> None: rows = [r for e in group if not e.rows_sent for r in e.committed.rows] docs = [d for e in group for d in served.get(id(e), {}).values()] acknowledged: set[tuple[Any, ...]] = set() + rows_finished = asyncio.Event() async def send_rows() -> None: - for block in partition_blocks(rows, created_month): - await inserts.insert_memory_versions(block, ch=ch, published=True) + try: + for block in partition_blocks(rows, created_month): + await inserts.insert_memory_versions(block, ch=ch, published=True) + finally: + rows_finished.set() async def send_documents() -> None: + # Both sends acquire the same consent fences; competing can split their claims. + await rows_finished.wait() for block in partition_blocks(docs, created_month): admitted = await inserts.insert_embedding_rows(block, ch=ch) acknowledged.update(_doc_key(row) for row in admitted) diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index a44ad873f5..088ea4195a 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -810,6 +810,8 @@ "memory_decision_extract_known_max": 30, "memory_decision_extract_restatement_top_k": 20, "memory_decision_extract_window_chars": 24000, + "memory_digest_listing_max_pages": 256, + "memory_digest_listing_page_size": 256, "memory_durability_verifier_max_facts_per_call": 60, "memory_embed_answer_topic": true, "memory_embeddings_patch_parts_probe_seconds": 60, diff --git a/tests/unit/test_delete_version_sql.py b/tests/unit/test_delete_version_sql.py index 2e61556d97..1b69924959 100644 --- a/tests/unit/test_delete_version_sql.py +++ b/tests/unit/test_delete_version_sql.py @@ -835,8 +835,7 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: async def test_a_deleted_version_refuses_restore_and_retains_its_marker( ch: KeptCH, monkeypatch: pytest.MonkeyPatch ) -> None: - # The marker stays on delete_version's tombstone; the row memory.restore brings back, - # and every version after it, drops it. + # Refused restoration preserves the deletion marker. from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import restore as restore_mod @@ -1809,9 +1808,6 @@ async def record(event_type: str, **kw: Any) -> str: request_id_var.reset(rid) setup_receipts = [row for event, row in emitted if event == "memory.set"] assert setup_receipts == [] - if setup_receipts: - assert setup_receipts[0]["subject"]["memory_key"] == _KEY - emitted[:] = [(event, row) for event, row in emitted if event != "memory.set"] calls = 2 if failure == "a retry" else 1 assert len(emitted) == calls expected = MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head).model_dump( diff --git a/tests/unit/test_digest_chain.py b/tests/unit/test_digest_chain.py index 206e87f1ec..dfe8dd01d9 100644 --- a/tests/unit/test_digest_chain.py +++ b/tests/unit/test_digest_chain.py @@ -195,7 +195,7 @@ async def test_digest_insert_then_chain_edges(monkeypatch) -> None: async def test_chain_failure_leaves_the_digest(monkeypatch) -> None: from server.memory.app.session_summary import _write_session_summary - class _NoQuery: + class _ChainUnavailable: async def query_all(self, sql, params=None): if "FROM loom.memory_refusals" in sql: return [] @@ -207,7 +207,7 @@ def __init__(self): async def insert(self, table, rows, settings=None): self.inserts.append((table, list(rows))) - ch = _NoQuery() # no query_all at all: the chain raises, the digest stays + ch = _ChainUnavailable() state = _wire_writer(monkeypatch, ch) await _write_session_summary(_FakeInput(), state) assert [t for t, _ in ch.inserts] == ["loom.session_summaries"] diff --git a/tests/unit/test_extract_concurrent_projections.py b/tests/unit/test_extract_concurrent_projections.py index 782d8f296d..4078d8b3c4 100644 --- a/tests/unit/test_extract_concurrent_projections.py +++ b/tests/unit/test_extract_concurrent_projections.py @@ -1,9 +1,6 @@ -"""The session digest is drafted while the extractor runs, and -the post-extraction projections are awaited together. +"""Extraction commits before overlapping projection writes. -CH-free and model-free: every projection is a stub that records when it ran. -The assertions are about ORDER and OVERLAP, not about row shapes (those are -covered by test_session_summaries.py and the chunk/evidence tests). +Projection stubs record execution order, overlap and isolated failures. """ from __future__ import annotations @@ -43,7 +40,6 @@ def __init__(self) -> None: self.spans: dict[str, tuple[float, float]] = {} self.calls: list[str] = [] self.failures: list[str] = [] - self.drafts_written: list[Any] = [] def stage(self, name: str, *, seconds: float = 0.05, result: Any = None, raises: bool = False): async def _run(*args: Any, **kwargs: Any) -> Any: @@ -53,8 +49,6 @@ async def _run(*args: Any, **kwargs: Any) -> Any: self.spans[name] = (t0, time.perf_counter()) if raises: raise RuntimeError(f"{name} exploded") - if name == "summary": - self.drafts_written.append(kwargs.get("draft")) return result return _run diff --git a/tests/unit/test_extraction_queue.py b/tests/unit/test_extraction_queue.py index 4f359d0adb..767ee7b418 100644 --- a/tests/unit/test_extraction_queue.py +++ b/tests/unit/test_extraction_queue.py @@ -436,7 +436,6 @@ class _Out: assert res.done_count == 1 and res.written_memories == 3 assert res.failures == ["bad: failed"] assert all(inp.defer is False for inp in seen) - # inserts: 1 claim (2 rows) + done + failed statuses = [ r["status"] for _, rows, _ in ch.inserts for r in rows if r["status"] != q.STATUS_RUNNING ] diff --git a/tests/unit/test_pg_batch.py b/tests/unit/test_pg_batch.py index 9f2bf87581..7c9d36fedf 100644 --- a/tests/unit/test_pg_batch.py +++ b/tests/unit/test_pg_batch.py @@ -178,6 +178,34 @@ async def submit( await pg_batch.submit(committed, documents=docs, ch=self.ch, wait=wait) +async def test_canonical_and_vector_sends_do_not_compete_for_the_same_fences(monkeypatch): + batch = _Batching(monkeypatch) + memory_send = pg_batch.inserts.insert_memory_versions + document_send = pg_batch.inserts.insert_embedding_rows + publishing_memory = False + overlap = [] + + async def memory(rows, **kwargs): + nonlocal publishing_memory + publishing_memory = True + try: + await asyncio.sleep(0) + return await memory_send(rows, **kwargs) + finally: + publishing_memory = False + + async def documents(rows, **kwargs): + overlap.append(publishing_memory) + return await document_send(rows, **kwargs) + + monkeypatch.setattr(pg_batch.inserts, "insert_memory_versions", memory) + monkeypatch.setattr(pg_batch.inserts, "insert_embedding_rows", documents) + await batch.submit("k", 1) + assert overlap == [False] + assert batch.memory == batch.documents == [["k"]] + assert batch.landed == [(("o", "n", 1), _FULL)] + + async def test_commits_queued_together_are_one_insert_per_table( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_session_summaries.py b/tests/unit/test_session_summaries.py index 5df49df8d6..76f4ba8c2b 100644 --- a/tests/unit/test_session_summaries.py +++ b/tests/unit/test_session_summaries.py @@ -5,9 +5,9 @@ loom.session_summaries INSERT rows. Guards the RMT(version) silent-data-loss class (a write omits a column -> FINAL surfaces the schema DEFAULT over a prior real value). - - _write_session_summary — the inline ingest writer. Stubs the LLM, embedder, - and CH client so the row shape (mirror-every-column), the date-label carriage, - and the fail-open skips run CH-free / network-free. + - _write_session_summary — the inline ingest writer. Stubs authoritative source + heads, the embedder and the CH client to check complete fact rendering, + row shape, date labels and skipped writes without network access. """ from __future__ import annotations diff --git a/tests/unit/test_shared_and_documents_list_repo.py b/tests/unit/test_shared_and_documents_list_repo.py index 1629a0d723..1ac7b835d1 100644 --- a/tests/unit/test_shared_and_documents_list_repo.py +++ b/tests/unit/test_shared_and_documents_list_repo.py @@ -261,11 +261,7 @@ async def test_fetch_shared_none_when_missing() -> None: @pytest.mark.asyncio async def test_tombstone_shared_embedding_deletes_older_cached_versions() -> None: - """memory.search(scope='shared') filters on tombstoned_at IS NULL; the - demote path lands the tombstone as a committed re-emit of the live shared - rows (every column carried, tombstoned_at set, fresh version) so the next - search does not see the row — and never as an ALTER UPDATE, whose patch - part would cost the fact leg its codes-first read.""" + """Shared retirement deletes older cache versions without publishing another copy.""" ch = StubCH() when = datetime(2026, 5, 17, tzinfo=UTC) diff --git a/tests/unit/test_summary_pages.py b/tests/unit/test_summary_pages.py index 10e75e1c6d..03bf7ce270 100644 --- a/tests/unit/test_summary_pages.py +++ b/tests/unit/test_summary_pages.py @@ -79,7 +79,10 @@ def candidates(count, order): @pytest.mark.parametrize("order", ["oldest", "newest", "session"]) @pytest.mark.parametrize("withheld", [2, 270]) @pytest.mark.parametrize("qualified", [0, 1, 2]) -async def test_pages_fill_from_current_sources(monkeypatch, order, withheld, qualified): +@pytest.mark.parametrize("page_size", [1, 256]) +async def test_pages_fill_from_current_sources(monkeypatch, order, withheld, qualified, page_size): + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_page_size", page_size) + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_max_pages", 512) rows = candidates(withheld + qualified, order) heads = [ _row( From 8d3d306e70f9867013a0c7c0b01723dafa1ccaef Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 22:18:25 +0530 Subject: [PATCH 5/5] Postgres storage and isolation run in separate CI jobs The serial suites reach the shared job deadline even when their tests pass. Independent profiles retain every command, both Postgres versions and the same timeout; restore drills remain last in storage and log artifacts name both matrix dimensions. Signed-off-by: Lareb Zafar --- .github/workflows/ci.yml | 22 +++++++++++-------- docs/development/testing.md | 42 ++++++++++++++++++------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22e025ecfe..bad7d30157 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -926,7 +926,7 @@ jobs: extractor.log integration-postgres: - name: pytest integration (Postgres ${{ matrix.postgres }} memory authority) + name: pytest integration (Postgres ${{ matrix.postgres }}, ${{ matrix.profile }}) needs: [validate-ref, clickhouse] runs-on: ubuntu-latest timeout-minutes: 60 @@ -935,6 +935,7 @@ jobs: fail-fast: false matrix: postgres: ["16", "18"] + profile: [storage, isolation] env: LOOM_CLICKHOUSE_IMAGE: ${{ needs.clickhouse.outputs.image }} LOOM_CH_HOST: localhost @@ -1016,6 +1017,7 @@ jobs: tail -100 server.log exit 1 - name: pytest (Postgres authority, memory, knowledge, isolation and the SQL surface) + if: ${{ matrix.profile == 'storage' }} run: >- pytest -q tests/integration/test_pg_authority.py @@ -1096,6 +1098,7 @@ jobs: tests/integration/test_marked_durable_folds_native.py tests/integration/test_marked_sql_surface_native.py - name: pytest online-index durability on a private physical standby + if: ${{ matrix.profile == 'storage' }} timeout-minutes: 8 env: LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} @@ -1124,6 +1127,7 @@ jobs: docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby pytest -q tests/integration/test_pg_online_index_durability.py - name: pytest placement durability on a private physical standby + if: ${{ matrix.profile == 'storage' }} timeout-minutes: 8 env: LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} @@ -1162,12 +1166,12 @@ jobs: export LOOM_CELL_PLACE_DRILL_DSN="postgres://loom:loomdev@127.0.0.1:$port/loom" docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby pytest -q tests/integration/test_cell_place_durability.py - # The tenant-isolation gates again, against a second server that enforces auth: + # The tenant-isolation profile uses a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. - # They run whether or not the memory suites passed (`!cancelled()`). + # Its matrix job runs independently of the storage profile's outcome. - name: Boot control plane with auth on id: auth_on - if: ${{ !cancelled() && steps.issuer.outcome == 'success' }} + if: ${{ !cancelled() && matrix.profile == 'isolation' && steps.issuer.outcome == 'success' }} env: LOOM_AUTH_REQUIRED: "true" LOOM_INSTANCE_LABEL: "ci:integration-postgres-auth" @@ -1187,7 +1191,7 @@ jobs: # extraction queue, projection failures) hold no Postgres row: they run on the # ClickHouse job. - name: pytest isolation gates (Postgres authority, auth on) - if: ${{ !cancelled() && steps.auth_on.outcome == 'success' }} + if: ${{ !cancelled() && matrix.profile == 'isolation' && steps.auth_on.outcome == 'success' }} env: LOOM_AUTH_REQUIRED: "true" LOOM_BASE_URL: http://localhost:7778 @@ -1209,16 +1213,16 @@ jobs: tests/integration/test_pg_authority.py -k "not test_a_duplicate_never_resolves_to_a_canonical_outside_the_scope and not test_a_superseded_hit_never_names_a_successor_outside_the_scope" - # Last, on its own: `loom reconcile` acts on the whole database, every other test's - # rows included, so it runs after them all and in its file's order. + # Last in the storage profile: `loom reconcile` acts on the whole database, + # the storage tests' rows included, so it runs after them in its file's order. - name: pytest (loom reconcile drills) - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.profile == 'storage' }} run: pytest -q -p no:randomly tests/integration/test_restore_reconcile.py - name: Upload server log on failure if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: server-log-postgres-${{ matrix.postgres }} + name: server-log-postgres-${{ matrix.postgres }}-${{ matrix.profile }} path: | server.log server-auth.log diff --git a/docs/development/testing.md b/docs/development/testing.md index 913881ea58..b23f8e791d 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -325,27 +325,27 @@ failure on one merge commit is never hidden by the next. Every action is pinned commit SHA (its version in a comment), and the workflow grants its jobs read-only repository access (`permissions: contents: read`). -| Job | Runs | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `clickhouse` | resolves the ClickHouse release `dev/clickhouse_release.sh` names, so `unit`, `schema-check` and `integration` test the same one | -| `lint` | `ruff check`, `ruff format --check`, the `--check` of `sdk_py_gen.py`, `sdk_py_client_gen.py`, `sdk_py_admin_gen.py`, `agent_plugin_sync.py`, `settings_reference.py`, `settings_schema.py`, `docs_reference.py` and `op_reach_census.py`, `mypy --strict` on the SDK, the mypy ratchet, the function-length and secret gates | -| `unit` | `pytest tests/unit` in six parts (`LOOM_TEST_SHARD`), each on its own runner, in random order on every core with coverage | -| `unit-coverage` | the parts' coverage combined (floor 40%) and the per-module floors, then `pytest bench` and the generic integration sample (`integrations/generic/test.sh`); its `shard-weights` artifact is the run's seconds per unit file, to commit as `tests/shard_weights.json` when the parts drift apart | -| `sandbox` | the hostile repository-job sandbox suite under real bubblewrap, where a skip fails (`LOOM_TEST_REQUIRE_SANDBOX=1`) | -| `schema-check` | a fresh ClickHouse (`dev/ci_start_clickhouse.sh`), every migration, the strict dry run, `make schema-check` | -| `ts-typecheck` | the dashboard build (`tsc -b` + Vite) and its bundle budget (`npm run budget`) | -| `format` | Biome, Prettier, Taplo and shfmt checks at pinned versions, and Biome's accessibility lint on `frontend/src` | -| `helm` | `helm lint --strict` and kubeconform on every overlay, kubeconform on `deploy/okf-sync/cronjob.yaml`, the values the chart must refuse, and `scripts/gates/helm_extractor_isolation.py`, `helm_metadata_egress.py` and `helm_assistant_isolation.py` | -| `agent-plugins` | `node --test` for the plugin hooks and the grep shim, the Cursor hook sample (`integrations/cursor/test.sh`), a parse check of every hook, library and `bin/` helper script of both plugins and the shim, and a check that the grep shim's entry points are executable | -| `frontend-vitest` | vitest (unit, component, axe), in parts on separate runners | -| `frontend-e2e` | Playwright, in parts on separate runners | -| `integration` | parts, each with its own ClickHouse, migrations, reference extractor and booted server (fake providers, workers off): the numbered parts split `pytest tests/integration` (`LOOM_TEST_SHARD`), the isolation gates part the provider staging tests, the auth-on gates and `pytest tests/analyzer` | -| `integration-postgres` | the memory, knowledge, isolation, idempotency, erasure and restored-id suites and the auth-on gates against a server with `LOOM_STORAGE_AUTHORITY=postgres`, on Postgres 16 and 18 with a synchronous standby ([Postgres](../operations/postgres.md)) | -| `mixed-version` | `scripts/gates/mixed_version.py`: builds N-1 and N side by side on one ClickHouse, N-1 being `HEAD~1`: on a pull request the first parent of its merge commit, the branch it targets as the merge was made, and on a push the previous commit | -| `sdk-typescript` | `sdk_ts_gen.py --check`, `npm run check` (lint, types, version, README examples, tests), `npm run check:dist` | -| `sdk-rust` | dependency resolution with Rust-version fallback, `cargo fmt --check`, `cargo test --locked` (response, HTTP contract and documentation tests), `cargo clippy --locked --all-targets -- -D warnings`, `cargo doc --locked --no-deps` with warnings refused, `cargo package --locked` and packaged license verification; Rust 1.84.1 and stable | -| `sdk-python-adapters` | `make test-sdk-adapters`: for each adapter whose framework is an SDK extra, in a venv of its own with that framework, outside `uv.lock`, its tests and `mypy --strict` on its module and on the adapters that need no extra (`__init__.py`, `langgraph.py`, `strands.py`); on Python 3.12, and on 3.11 with every direct requirement (the SDK's own, the extra's, pytest) at its lower bound | -| `sdk-python-package` | builds the SDK wheel with uv, installs it into a clean venv and runs `scripts/gates/sdk_wheel_smoke.py`, on Python 3.11, 3.12 and 3.14, plus 3.11 and 3.14 with every declared dependency at its lower bound | +| Job | Runs | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `clickhouse` | resolves the ClickHouse release `dev/clickhouse_release.sh` names, so `unit`, `schema-check` and `integration` test the same one | +| `lint` | `ruff check`, `ruff format --check`, the `--check` of `sdk_py_gen.py`, `sdk_py_client_gen.py`, `sdk_py_admin_gen.py`, `agent_plugin_sync.py`, `settings_reference.py`, `settings_schema.py`, `docs_reference.py` and `op_reach_census.py`, `mypy --strict` on the SDK, the mypy ratchet, the function-length and secret gates | +| `unit` | `pytest tests/unit` in six parts (`LOOM_TEST_SHARD`), each on its own runner, in random order on every core with coverage | +| `unit-coverage` | the parts' coverage combined (floor 40%) and the per-module floors, then `pytest bench` and the generic integration sample (`integrations/generic/test.sh`); its `shard-weights` artifact is the run's seconds per unit file, to commit as `tests/shard_weights.json` when the parts drift apart | +| `sandbox` | the hostile repository-job sandbox suite under real bubblewrap, where a skip fails (`LOOM_TEST_REQUIRE_SANDBOX=1`) | +| `schema-check` | a fresh ClickHouse (`dev/ci_start_clickhouse.sh`), every migration, the strict dry run, `make schema-check` | +| `ts-typecheck` | the dashboard build (`tsc -b` + Vite) and its bundle budget (`npm run budget`) | +| `format` | Biome, Prettier, Taplo and shfmt checks at pinned versions, and Biome's accessibility lint on `frontend/src` | +| `helm` | `helm lint --strict` and kubeconform on every overlay, kubeconform on `deploy/okf-sync/cronjob.yaml`, the values the chart must refuse, and `scripts/gates/helm_extractor_isolation.py`, `helm_metadata_egress.py` and `helm_assistant_isolation.py` | +| `agent-plugins` | `node --test` for the plugin hooks and the grep shim, the Cursor hook sample (`integrations/cursor/test.sh`), a parse check of every hook, library and `bin/` helper script of both plugins and the shim, and a check that the grep shim's entry points are executable | +| `frontend-vitest` | vitest (unit, component, axe), in parts on separate runners | +| `frontend-e2e` | Playwright, in parts on separate runners | +| `integration` | parts, each with its own ClickHouse, migrations, reference extractor and booted server (fake providers, workers off): the numbered parts split `pytest tests/integration` (`LOOM_TEST_SHARD`), the isolation gates part the provider staging tests, the auth-on gates and `pytest tests/analyzer` | +| `integration-postgres` | separate storage and authenticated-isolation profiles on Postgres 16 and 18 with a synchronous standby and `LOOM_STORAGE_AUTHORITY=postgres`. Storage covers memory, knowledge, idempotency, erasure, restored ids and durability, followed by whole-database restore drills. Isolation covers the auth-on gates. Each profile uses its own stores and keeps the one-hour job limit ([Postgres](../operations/postgres.md)) | +| `mixed-version` | `scripts/gates/mixed_version.py`: builds N-1 and N side by side on one ClickHouse, N-1 being `HEAD~1`: on a pull request the first parent of its merge commit, the branch it targets as the merge was made, and on a push the previous commit | +| `sdk-typescript` | `sdk_ts_gen.py --check`, `npm run check` (lint, types, version, README examples, tests), `npm run check:dist` | +| `sdk-rust` | dependency resolution with Rust-version fallback, `cargo fmt --check`, `cargo test --locked` (response, HTTP contract and documentation tests), `cargo clippy --locked --all-targets -- -D warnings`, `cargo doc --locked --no-deps` with warnings refused, `cargo package --locked` and packaged license verification; Rust 1.84.1 and stable | +| `sdk-python-adapters` | `make test-sdk-adapters`: for each adapter whose framework is an SDK extra, in a venv of its own with that framework, outside `uv.lock`, its tests and `mypy --strict` on its module and on the adapters that need no extra (`__init__.py`, `langgraph.py`, `strands.py`); on Python 3.12, and on 3.11 with every direct requirement (the SDK's own, the extra's, pytest) at its lower bound | +| `sdk-python-package` | builds the SDK wheel with uv, installs it into a clean venv and runs `scripts/gates/sdk_wheel_smoke.py`, on Python 3.11, 3.12 and 3.14, plus 3.11 and 3.14 with every declared dependency at its lower bound | `.github/workflows/plugin-platforms.yml` runs when a change touches `integrations/`, a plugin marketplace, `.gitattributes` or the workflow itself: the plugin and grep-shim